<commit_before># coding: utf-8
"""
StaSh - Pythonista Shell

https://github.com/ywangd/stash
"""

__version__ = '0.6.11a1'

import os
import sys
from ConfigParser import ConfigParser
from StringIO import StringIO
import imp as pyimp  # rename to avoid name conflict with objc_util
import logging
import logging.handlers

# noinspection PyPep8Naming
from system.shcommon import IN_PYTHONISTA, ON_IPAD
from system.shcommon import _STASH_ROOT, _STASH_CONFIG_FILES, _SYS_STDOUT
from system.shcommon import Graphics as graphics, Control as ctrl, Escape as esc
from system.shuseractionproxy import ShUserActionProxy
from system.shiowrapper import enable as enable_io_wrapper, disable as disable_io_wrapper
from system.shparsers import ShParser, ShExpander, ShCompleter
from system.shruntime import ShRuntime
from system.shstreams import ShMiniBuffer, ShStream
from system.shscreens import ShSequentialScreen, ShSequentialRenderer
from system.shui import ShUI
from system.shio import ShIO

# Setup logging
LOGGER = logging.getLogger('StaSh')

# Debugging constants
_DEBUG_STREAM = 200
_DEBUG_RENDERER = 201
_DEBUG_MAIN_SCREEN = 202
_DEBUG_MINI_BUFFER = 203
_DEBUG_IO = 204
_DEBUG_UI = 300
_DEBUG_TERMINAL = 301
_DEBUG_TV_DELEGATE = 302
_DEBUG_RUNTIME = 400
_DEBUG_PARSER = 401
_DEBUG_EXPANDER = 402
_DEBUG_COMPLETER = 403

# Default configuration (can be overridden by external configuration file)
_DEFAULT_CONFIG = """[system]
rcfile=.stashrc
py_traceback=0
py_pdb=0
input_encoding_utf8=1
ipython_style_history_search=1
thread_type=ctypes

[display]
TEXT_FONT_SIZE={text_size}
BUTTON_FONT_SIZE=14
BACKGROUND_COLOR=(0.0, 0.0, 0.0)
TEXT_COLOR=(1.0, 1.0, 1.0)
TINT_COLOR=(0.0, 0.0, 1.0)
INDICATOR_STYLE=white
HISTORY_MAX=50
BUFFER_MAX=150
AUTO_COMPLETION_MAX=50
VK_SYMBOLS=~/.-*|>$'=!&_"\?`
""".format(text_size=14 if ON_IPAD else 12)

class StaSh(object):
    """
    Main application class. It initialize and wires the components and provide
    utility interfaces to running scripts.
    """

    def __init__(self, debug=(), log_setting=None,
                 no_cfgfile=False, no_rcfile=False, no_historyfile=False):
        self.__version__ = __version__

        # Intercept IO
        enable_io_wrapper()

        self.config = self._load_config(no_cfgfile=no_cfgfile)
        self.logger = self._config_logging(log_setting)

        self.user_action_proxy = ShUserActionProxy(self)

        # Tab handler for running scripts
        self.external_tab_handler = None

        # Wire the components
        self.main_screen = ShSequentialScreen(self,
                                              nlines_max=self.config.getint('display', 'BUFFER_MAX'),
                                              debug=_DEBUG_MAIN_SCREEN in debug)

        self.mini_buffer = ShMiniBuffer(self,
                                        self.main_screen,
                                        debug=_DEBUG_MINI_BUFFER in debug)

        self.stream = ShStream(self,
                               self.main_screen,
                               debug=_DEBUG_STREAM in debug)

        self.io = ShIO(self, debug=_DEBUG_IO in debug)

        self.terminal = None  # will be set during UI initialisation
        self.ui = ShUI(self, debug=_DEBUG_UI in debug)
        self.renderer = ShSequentialRenderer(self.main_screen, self.terminal,
                                             debug=_DEBUG_RENDERER in debug)

        parser = ShParser(debug=_DEBUG_PARSER in debug)
        expander = ShExpander(self, debug=_DEBUG_EXPANDER in debug)
        self.runtime = ShRuntime(self, parser, expander, no_historyfile=no_historyfile,
                                 debug=_DEBUG_RUNTIME in debug)
        self.completer = ShCompleter(self, debug=_DEBUG_COMPLETER in debug)

        # Navigate to the startup folder
        if IN_PYTHONISTA:
            os.chdir(self.runtime.state.environ_get('HOME2'))
        self.runtime.load_rcfile(no_rcfile=no_rcfile)
        self.io.write(self.text_style('StaSh v%s\n' % self.__version__,
                                      {'color': 'blue', 'traits': ['bold']},
                                      always=True))
        # Load shared libraries
        self._load_lib()

        # Show tip of the day (this calls script_will_end)
        self('$STASH_ROOT/bin/totd.py', add_to_history=False, persistent_level=0)

    def __call__(self, input_, persistent_level=2, *args, **kwargs):
        """ This function is to be called by external script for
         executing shell commands """
        worker = self.runtime.run(
            input_, persistent_level=persistent_level, *args, **kwargs)
        worker.join()
        return worker

    @staticmethod
    def _load_config(no_cfgfile=False):
        config = ConfigParser()
        config.optionxform = str  # make it preserve case
        # defaults
        config.readfp(StringIO(_DEFAULT_CONFIG))

        # update from config file
        if not no_cfgfile:
            config.read(os.path.join(_STASH_ROOT, f) for f in _STASH_CONFIG_FILES)

        return config

    @staticmethod
    def _config_logging(log_setting):

        logger = logging.getLogger('StaSh')

        _log_setting = {
            'level': 'DEBUG',
            'stdout': True,
        }

        _log_setting.update(log_setting or {})

        level = {
            'CRITICAL': logging.CRITICAL,
            'ERROR': logging.ERROR,
            'WARNING': logging.WARNING,
            'INFO': logging.INFO,
            'DEBUG': logging.DEBUG,
            'NOTEST': logging.NOTSET,
        }.get(_log_setting['level'], logging.DEBUG)

        logger.setLevel(level)

        if not logger.handlers:
            if _log_setting['stdout']:
                _log_handler = logging.StreamHandler(_SYS_STDOUT)
            else:
                _log_handler = logging.handlers.RotatingFileHandler('stash.log', mode='w')
            _log_handler.setLevel(level)
            _log_handler.setFormatter(logging.Formatter(
                '[%(asctime)s] [%(levelname)s] [%(threadName)s] [%(name)s] [%(funcName)s] [%(lineno)d] - %(message)s'
            ))
            logger.addHandler(_log_handler)

        return logger

    def _load_lib(self):
        """
        Load library files as modules and save each of them as attributes
        """
        lib_path = os.path.join(_STASH_ROOT, 'lib')
        os.environ['STASH_ROOT'] = _STASH_ROOT  # libcompleter needs this value
        try:
            for f in os.listdir(lib_path):
                if f.startswith('lib') and f.endswith('.py') \
                        and os.path.isfile(os.path.join(lib_path, f)):
                    name, _ = os.path.splitext(f)
                    try:
                        self.__dict__[name] = pyimp.load_source(name, os.path.join(lib_path, f))
                    except Exception as e:
                        self.write_message('%s: failed to load library file (%s)' % (f, repr(e)))
        finally:  # do not modify environ permanently
            os.environ.pop('STASH_ROOT')

    def write_message(self, s):
        self.io.write('stash: %s\n' % s)

    def launch(self, style='panel'):
        self.ui.present(style)
        self.terminal.begin_editing()

    def cleanup(self):
        disable_io_wrapper()

    def get_workers(self):
        return [worker for worker in self.runtime.worker_registry]

    # noinspection PyProtectedMember
    @staticmethod
    def text_style(s, style, always=False):
        """
        Style the given string with ASCII escapes.

        :param str s: String to decorate
        :param dict style: A dictionary of styles
        :param bool always: If true, style will be applied even for pipes.
        :return:
        """
        # No color for pipes, files and Pythonista console
        if not always and (isinstance(sys.stdout, StringIO)
                           or isinstance(sys.stdout, file)
                           or sys.stdout.write.im_self is _SYS_STDOUT):
            return s

        fmt_string = u'%s%%d%s%%s%s%%d%s' % (ctrl.CSI, esc.SGR, ctrl.CSI, esc.SGR)
        for style_name, style_value in style.items():
            if style_name == 'color':
                color_id = graphics._SGR.get(style_value.lower())
                if color_id is not None:
                    s = fmt_string % (color_id, s, graphics._SGR['default'])
            elif style_name == 'bgcolor':
                color_id = graphics._SGR.get('bg-' + style_value.lower())
                if color_id is not None:
                    s = fmt_string % (color_id, s, graphics._SGR['default'])
            elif style_name == 'traits':
                for val in style_value:
                    val = val.lower()
                    if val == 'bold':
                        s = fmt_string % (graphics._SGR['+bold'], s, graphics._SGR['-bold'])
                    elif val == 'italic':
                        s = fmt_string % (graphics._SGR['+italics'], s, graphics._SGR['-italics'])
                    elif val == 'underline':
                        s = fmt_string % (graphics._SGR['+underscore'], s, graphics._SGR['-underscore'])
                    elif val == 'strikethrough':
                        s = fmt_string % (graphics._SGR['+strikethrough'], s, graphics._SGR['-strikethrough'])

        return s

    def text_color(self, s, color_name='default', **kwargs):
        return self.text_style(s, {'color': color_name}, **kwargs)

    def text_bgcolor(self, s, color_name='default', **kwargs):
        return self.text_style(s, {'bgcolor': color_name}, **kwargs)

    def text_bold(self, s, **kwargs):
        return self.text_style(s, {'traits': ['bold']}, **kwargs)

    def text_italic(self, s, **kwargs):
        return self.text_style(s, {'traits': ['italic']}, **kwargs)

    def text_bold_italic(self, s, **kwargs):
        return self.text_style(s, {'traits': ['bold', 'italic']}, **kwargs)

    def text_underline(self, s, **kwargs):
        return self.text_style(s, {'traits': ['underline']}, **kwargs)

    def text_strikethrough(self, s, **kwargs):
        return self.text_style(s, {'traits': ['strikethrough']}, **kwargs)

if __name__ == '__main__':
    _stash = StaSh()
    _stash.launch()
<commit_msg>Version bump<commit_after># coding: utf-8
"""
StaSh - Pythonista Shell

https://github.com/ywangd/stash
"""

__version__ = '0.6.11'

import os
import sys
from ConfigParser import ConfigParser
from StringIO import StringIO
import imp as pyimp  # rename to avoid name conflict with objc_util
import logging
import logging.handlers

# noinspection PyPep8Naming
from system.shcommon import IN_PYTHONISTA, ON_IPAD
from system.shcommon import _STASH_ROOT, _STASH_CONFIG_FILES, _SYS_STDOUT
from system.shcommon import Graphics as graphics, Control as ctrl, Escape as esc
from system.shuseractionproxy import ShUserActionProxy
from system.shiowrapper import enable as enable_io_wrapper, disable as disable_io_wrapper
from system.shparsers import ShParser, ShExpander, ShCompleter
from system.shruntime import ShRuntime
from system.shstreams import ShMiniBuffer, ShStream
from system.shscreens import ShSequentialScreen, ShSequentialRenderer
from system.shui import ShUI
from system.shio import ShIO

# Setup logging
LOGGER = logging.getLogger('StaSh')

# Debugging constants
_DEBUG_STREAM = 200
_DEBUG_RENDERER = 201
_DEBUG_MAIN_SCREEN = 202
_DEBUG_MINI_BUFFER = 203
_DEBUG_IO = 204
_DEBUG_UI = 300
_DEBUG_TERMINAL = 301
_DEBUG_TV_DELEGATE = 302
_DEBUG_RUNTIME = 400
_DEBUG_PARSER = 401
_DEBUG_EXPANDER = 402
_DEBUG_COMPLETER = 403

# Default configuration (can be overridden by external configuration file)
_DEFAULT_CONFIG = """[system]
rcfile=.stashrc
py_traceback=0
py_pdb=0
input_encoding_utf8=1
ipython_style_history_search=1
thread_type=ctypes

[display]
TEXT_FONT_SIZE={text_size}
BUTTON_FONT_SIZE=14
BACKGROUND_COLOR=(0.0, 0.0, 0.0)
TEXT_COLOR=(1.0, 1.0, 1.0)
TINT_COLOR=(0.0, 0.0, 1.0)
INDICATOR_STYLE=white
HISTORY_MAX=50
BUFFER_MAX=150
AUTO_COMPLETION_MAX=50
VK_SYMBOLS=~/.-*|>$'=!&_"\?`
""".format(text_size=14 if ON_IPAD else 12)

class StaSh(object):
    """
    Main application class. It initialize and wires the components and provide
    utility interfaces to running scripts.
    """

    def __init__(self, debug=(), log_setting=None,
                 no_cfgfile=False, no_rcfile=False, no_historyfile=False):
        self.__version__ = __version__

        # Intercept IO
        enable_io_wrapper()

        self.config = self._load_config(no_cfgfile=no_cfgfile)
        self.logger = self._config_logging(log_setting)

        self.user_action_proxy = ShUserActionProxy(self)

        # Tab handler for running scripts
        self.external_tab_handler = None

        # Wire the components
        self.main_screen = ShSequentialScreen(self,
                                              nlines_max=self.config.getint('display', 'BUFFER_MAX'),
                                              debug=_DEBUG_MAIN_SCREEN in debug)

        self.mini_buffer = ShMiniBuffer(self,
                                        self.main_screen,
                                        debug=_DEBUG_MINI_BUFFER in debug)

        self.stream = ShStream(self,
                               self.main_screen,
                               debug=_DEBUG_STREAM in debug)

        self.io = ShIO(self, debug=_DEBUG_IO in debug)

        self.terminal = None  # will be set during UI initialisation
        self.ui = ShUI(self, debug=_DEBUG_UI in debug)
        self.renderer = ShSequentialRenderer(self.main_screen, self.terminal,
                                             debug=_DEBUG_RENDERER in debug)

        parser = ShParser(debug=_DEBUG_PARSER in debug)
        expander = ShExpander(self, debug=_DEBUG_EXPANDER in debug)
        self.runtime = ShRuntime(self, parser, expander, no_historyfile=no_historyfile,
                                 debug=_DEBUG_RUNTIME in debug)
        self.completer = ShCompleter(self, debug=_DEBUG_COMPLETER in debug)

        # Navigate to the startup folder
        if IN_PYTHONISTA:
            os.chdir(self.runtime.state.environ_get('HOME2'))
        self.runtime.load_rcfile(no_rcfile=no_rcfile)
        self.io.write(self.text_style('StaSh v%s\n' % self.__version__,
                                      {'color': 'blue', 'traits': ['bold']},
                                      always=True))
        # Load shared libraries
        self._load_lib()

        # Show tip of the day (this calls script_will_end)
        self('$STASH_ROOT/bin/totd.py', add_to_history=False, persistent_level=0)

    def __call__(self, input_, persistent_level=2, *args, **kwargs):
        """ This function is to be called by external script for
         executing shell commands """
        worker = self.runtime.run(
            input_, persistent_level=persistent_level, *args, **kwargs)
        worker.join()
        return worker

    @staticmethod
    def _load_config(no_cfgfile=False):
        config = ConfigParser()
        config.optionxform = str  # make it preserve case
        # defaults
        config.readfp(StringIO(_DEFAULT_CONFIG))

        # update from config file
        if not no_cfgfile:
            config.read(os.path.join(_STASH_ROOT, f) for f in _STASH_CONFIG_FILES)

        return config

    @staticmethod
    def _config_logging(log_setting):

        logger = logging.getLogger('StaSh')

        _log_setting = {
            'level': 'DEBUG',
            'stdout': True,
        }

        _log_setting.update(log_setting or {})

        level = {
            'CRITICAL': logging.CRITICAL,
            'ERROR': logging.ERROR,
            'WARNING': logging.WARNING,
            'INFO': logging.INFO,
            'DEBUG': logging.DEBUG,
            'NOTEST': logging.NOTSET,
        }.get(_log_setting['level'], logging.DEBUG)

        logger.setLevel(level)

        if not logger.handlers:
            if _log_setting['stdout']:
                _log_handler = logging.StreamHandler(_SYS_STDOUT)
            else:
                _log_handler = logging.handlers.RotatingFileHandler('stash.log', mode='w')
            _log_handler.setLevel(level)
            _log_handler.setFormatter(logging.Formatter(
                '[%(asctime)s] [%(levelname)s] [%(threadName)s] [%(name)s] [%(funcName)s] [%(lineno)d] - %(message)s'
            ))
            logger.addHandler(_log_handler)

        return logger

    def _load_lib(self):
        """
        Load library files as modules and save each of them as attributes
        """
        lib_path = os.path.join(_STASH_ROOT, 'lib')
        os.environ['STASH_ROOT'] = _STASH_ROOT  # libcompleter needs this value
        try:
            for f in os.listdir(lib_path):
                if f.startswith('lib') and f.endswith('.py') \
                        and os.path.isfile(os.path.join(lib_path, f)):
                    name, _ = os.path.splitext(f)
                    try:
                        self.__dict__[name] = pyimp.load_source(name, os.path.join(lib_path, f))
                    except Exception as e:
                        self.write_message('%s: failed to load library file (%s)' % (f, repr(e)))
        finally:  # do not modify environ permanently
            os.environ.pop('STASH_ROOT')

    def write_message(self, s):
        self.io.write('stash: %s\n' % s)

    def launch(self, style='panel'):
        self.ui.present(style)
        self.terminal.begin_editing()

    def cleanup(self):
        disable_io_wrapper()

    def get_workers(self):
        return [worker for worker in self.runtime.worker_registry]

    # noinspection PyProtectedMember
    @staticmethod
    def text_style(s, style, always=False):
        """
        Style the given string with ASCII escapes.

        :param str s: String to decorate
        :param dict style: A dictionary of styles
        :param bool always: If true, style will be applied even for pipes.
        :return:
        """
        # No color for pipes, files and Pythonista console
        if not always and (isinstance(sys.stdout, StringIO)
                           or isinstance(sys.stdout, file)
                           or sys.stdout.write.im_self is _SYS_STDOUT):
            return s

        fmt_string = u'%s%%d%s%%s%s%%d%s' % (ctrl.CSI, esc.SGR, ctrl.CSI, esc.SGR)
        for style_name, style_value in style.items():
            if style_name == 'color':
                color_id = graphics._SGR.get(style_value.lower())
                if color_id is not None:
                    s = fmt_string % (color_id, s, graphics._SGR['default'])
            elif style_name == 'bgcolor':
                color_id = graphics._SGR.get('bg-' + style_value.lower())
                if color_id is not None:
                    s = fmt_string % (color_id, s, graphics._SGR['default'])
            elif style_name == 'traits':
                for val in style_value:
                    val = val.lower()
                    if val == 'bold':
                        s = fmt_string % (graphics._SGR['+bold'], s, graphics._SGR['-bold'])
                    elif val == 'italic':
                        s = fmt_string % (graphics._SGR['+italics'], s, graphics._SGR['-italics'])
                    elif val == 'underline':
                        s = fmt_string % (graphics._SGR['+underscore'], s, graphics._SGR['-underscore'])
                    elif val == 'strikethrough':
                        s = fmt_string % (graphics._SGR['+strikethrough'], s, graphics._SGR['-strikethrough'])

        return s

    def text_color(self, s, color_name='default', **kwargs):
        return self.text_style(s, {'color': color_name}, **kwargs)

    def text_bgcolor(self, s, color_name='default', **kwargs):
        return self.text_style(s, {'bgcolor': color_name}, **kwargs)

    def text_bold(self, s, **kwargs):
        return self.text_style(s, {'traits': ['bold']}, **kwargs)

    def text_italic(self, s, **kwargs):
        return self.text_style(s, {'traits': ['italic']}, **kwargs)

    def text_bold_italic(self, s, **kwargs):
        return self.text_style(s, {'traits': ['bold', 'italic']}, **kwargs)

    def text_underline(self, s, **kwargs):
        return self.text_style(s, {'traits': ['underline']}, **kwargs)

    def text_strikethrough(self, s, **kwargs):
        return self.text_style(s, {'traits': ['strikethrough']}, **kwargs)

if __name__ == '__main__':
    _stash = StaSh()
    _stash.launch()
<|endoftext|>


<commit_before>from __future__ import division
import numpy as np
from numpy.random import random
from numpy import newaxis as na
import scipy.stats as stats
import scipy.linalg

# TODO write cholesky versions

def getdatasize(data):
    assert (isinstance(data,np.ndarray) and data.ndim > 0) or \
            (isinstance(data,list) and
                    all((isinstance(d,np.ndarray) and d.ndim > 0) for d in data))

    if isinstance(data,np.ndarray):
        return data.shape[0]
    else:
        return sum(d.shape[0] for d in data)

def combinedata(datas):
    assert all(
            (isinstance(data,np.ndarray) and data.ndim > 0) or \
                    (isinstance(data,list) and
                        all((isinstance(d,np.ndarray) and d.ndim > 0) for d in data))
                    for data in datas)

    ret = []
    for data in datas:
        if isinstance(data,np.ndarray):
            ret.append(data)
        else:
            ret.extend(data)
    return ret

### Sampling functions

def sample_discrete(foo,size=[],dtype=np.int):
    '''samples from a one-dimensional finite pmf'''
    assert (foo >=0).all() and foo.ndim == 1
    cumvals = np.cumsum(foo)
    return np.sum(random(size)[...,na] * cumvals[-1] > cumvals, axis=-1,dtype=dtype)

def sample_discrete_from_log(p_log,axis=0,dtype=np.int):
    '''samples log probability array along specified axis'''
    cumvals = np.exp(p_log - np.expand_dims(p_log.max(axis),axis)).cumsum(axis)
    randvals = np.expand_dims(random(np.delete(p_log.shape,axis)),axis) \
            * cumvals[[slice(None) if idx is not axis else -1 for idx in range(len(p_log.shape))]]
    return np.sum(randvals > cumvals,axis=axis,dtype=dtype)

def sample_niw(mu,lmbda,kappa,nu):
    '''
    Returns a sample from the normal/inverse-wishart distribution, conjugate
    prior for (simultaneously) unknown mean and unknown covariance in a
    Gaussian likelihood model. Returns covariance.
    '''
    # code is based on Matlab's method
    # reference: p. 87 in Gelman's Bayesian Data Analysis

    # first sample Sigma ~ IW(lmbda,nu)
    lmbda = sample_invwishart(lmbda,nu)
    # then sample mu | Lambda ~ N(mu, Lambda/kappa)
    mu = np.random.multivariate_normal(mu,lmbda / kappa)

    return mu, lmbda

def sample_invwishart(lmbda,dof):
    # TODO make a version that returns the cholesky
    # TODO allow passing in chol/cholinv of matrix parameter lmbda
    n = lmbda.shape[0]
    chol = np.linalg.cholesky(lmbda)

    if (dof <= 81+n) and (dof == np.round(dof)):
        x = np.random.randn(dof,n)
    else:
        x = np.diag(np.sqrt(stats.chi2.rvs(dof-np.arange(n))))
        x[np.triu_indices_from(x,1)] = np.random.randn(n*(n-1)/2)
    R = np.linalg.qr(x,'r')
    T = scipy.linalg.solve_triangular(R.T,chol.T,lower=True).T
    return np.dot(T,T.T)

def sample_wishart(sigma, dof):
    n = sigma.shape[0]
    chol = np.linalg.cholesky(sigma)

    # use matlab's heuristic for choosing between the two different sampling schemes
    if (dof <= 81+n) and (dof == round(dof)):
        # direct
        X = np.dot(chol,np.random.normal(size=(n,dof)))
    else:
        A = np.diag(np.sqrt(np.random.chisquare(dof - np.arange(n))))
        A[np.tri(n,k=-1,dtype=bool)] = np.random.normal(size=(n*(n-1)/2.))
        X = np.dot(chol,A)

    return np.dot(X,X.T)

### Predictive

def multivariate_t_loglik(y,nu,mu,lmbda):
    # returns the log value
    # TODO check... gelman says lmbda but emily says nulmbda
    d = len(mu)
    yc = np.array(y-mu,ndmin=2)
    return scipy.special.gammaln((nu+d)/2.) - scipy.special.gammaln(nu/2.) - (d/2.)*np.log(nu*np.pi) - (1./2.)*np.log(np.linalg.det(lmbda)) - (nu+d)/2.*np.log1p(1./nu*np.dot(yc,np.linalg.solve(lmbda,yc.T)).diagonal()) # TODO get rid of diagonal business

def beta_predictive(priorcounts,newcounts):
    prior_nsuc, prior_nfail = priorcounts
    nsuc, nfail = newcounts

    numer = scipy.special.gammaln(np.array([nsuc+prior_nsuc, nfail+prior_nfail, prior_nsuc+prior_nfail])).sum()
    denom = scipy.special.gammaln(np.array([prior_nsuc, prior_nfail, prior_nsuc+prior_nfail+nsuc+nfail])).sum()
    return numer - denom

<commit_msg>fixed a bug i somehow introduced in sample_discrete_from_log<commit_after>from __future__ import division
import numpy as np
from numpy.random import random
from numpy import newaxis as na
import scipy.stats as stats
import scipy.linalg

# TODO write cholesky versions

def getdatasize(data):
    assert (isinstance(data,np.ndarray) and data.ndim > 0) or \
            (isinstance(data,list) and
                    all((isinstance(d,np.ndarray) and d.ndim > 0) for d in data))

    if isinstance(data,np.ndarray):
        return data.shape[0]
    else:
        return sum(d.shape[0] for d in data)

def combinedata(datas):
    assert all(
            (isinstance(data,np.ndarray) and data.ndim > 0) or \
                    (isinstance(data,list) and
                        all((isinstance(d,np.ndarray) and d.ndim > 0) for d in data))
                    for data in datas)

    ret = []
    for data in datas:
        if isinstance(data,np.ndarray):
            ret.append(data)
        else:
            ret.extend(data)
    return ret

### Sampling functions

def sample_discrete(foo,size=[],dtype=np.int):
    '''samples from a one-dimensional finite pmf'''
    assert (foo >=0).all() and foo.ndim == 1
    cumvals = np.cumsum(foo)
    return np.sum(random(size)[...,na] * cumvals[-1] > cumvals, axis=-1,dtype=dtype)

def sample_discrete_from_log(p_log,axis=0,dtype=np.int):
    '''samples log probability array along specified axis'''
    cumvals = np.exp(p_log - np.expand_dims(p_log.max(axis),axis)).cumsum(axis) # cumlogaddexp
    randvals = np.expand_dims(random(size=np.delete(p_log.shape,axis)) \
            * cumvals[[slice(None) if idx is not axis else -1 for idx in range(p_log.ndim)]],axis)
    return np.sum(randvals > cumvals,axis=axis,dtype=dtype)

def sample_niw(mu,lmbda,kappa,nu):
    '''
    Returns a sample from the normal/inverse-wishart distribution, conjugate
    prior for (simultaneously) unknown mean and unknown covariance in a
    Gaussian likelihood model. Returns covariance.
    '''
    # code is based on Matlab's method
    # reference: p. 87 in Gelman's Bayesian Data Analysis

    # first sample Sigma ~ IW(lmbda,nu)
    lmbda = sample_invwishart(lmbda,nu)
    # then sample mu | Lambda ~ N(mu, Lambda/kappa)
    mu = np.random.multivariate_normal(mu,lmbda / kappa)

    return mu, lmbda

def sample_invwishart(lmbda,dof):
    # TODO make a version that returns the cholesky
    # TODO allow passing in chol/cholinv of matrix parameter lmbda
    n = lmbda.shape[0]
    chol = np.linalg.cholesky(lmbda)

    if (dof <= 81+n) and (dof == np.round(dof)):
        x = np.random.randn(dof,n)
    else:
        x = np.diag(np.sqrt(stats.chi2.rvs(dof-np.arange(n))))
        x[np.triu_indices_from(x,1)] = np.random.randn(n*(n-1)/2)
    R = np.linalg.qr(x,'r')
    T = scipy.linalg.solve_triangular(R.T,chol.T,lower=True).T
    return np.dot(T,T.T)

def sample_wishart(sigma, dof):
    n = sigma.shape[0]
    chol = np.linalg.cholesky(sigma)

    # use matlab's heuristic for choosing between the two different sampling schemes
    if (dof <= 81+n) and (dof == round(dof)):
        # direct
        X = np.dot(chol,np.random.normal(size=(n,dof)))
    else:
        A = np.diag(np.sqrt(np.random.chisquare(dof - np.arange(n))))
        A[np.tri(n,k=-1,dtype=bool)] = np.random.normal(size=(n*(n-1)/2.))
        X = np.dot(chol,A)

    return np.dot(X,X.T)

### Predictive

def multivariate_t_loglik(y,nu,mu,lmbda):
    # returns the log value
    # TODO check... gelman says lmbda but emily says nulmbda
    d = len(mu)
    yc = np.array(y-mu,ndmin=2)
    return scipy.special.gammaln((nu+d)/2.) - scipy.special.gammaln(nu/2.) - (d/2.)*np.log(nu*np.pi) - (1./2.)*np.log(np.linalg.det(lmbda)) - (nu+d)/2.*np.log1p(1./nu*np.dot(yc,np.linalg.solve(lmbda,yc.T)).diagonal()) # TODO get rid of diagonal business

def beta_predictive(priorcounts,newcounts):
    prior_nsuc, prior_nfail = priorcounts
    nsuc, nfail = newcounts

    numer = scipy.special.gammaln(np.array([nsuc+prior_nsuc, nfail+prior_nfail, prior_nsuc+prior_nfail])).sum()
    denom = scipy.special.gammaln(np.array([prior_nsuc, prior_nfail, prior_nsuc+prior_nfail+nsuc+nfail])).sum()
    return numer - denom

<|endoftext|>


<commit_before>############################################################################
# Copyright 2016 Albin Severinson                                          #
#                                                                          #
# Licensed under the Apache License, Version 2.0 (the "License");          #
# you may not use this file except in compliance with the License.         #
# You may obtain a copy of the License at                                  #
#                                                                          #
#     http://www.apache.org/licenses/LICENSE-2.0                           #
#                                                                          #
# Unless required by applicable law or agreed to in writing, software      #
# distributed under the License is distributed on an "AS IS" BASIS,        #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and      #
# limitations under the License.                                           #
############################################################################

'''Project statistics module.'''

import math
import random
import functools
import statistics
import numpy as np
import scipy as sp
import scipy.stats

def order_sample(icdf, total, order):
    '''Sample the order statistic.

    Args:

    icdf: Distribution ICDF function.

    total: Total number of realizations.

    order: Order of the statistic.

    Returns: A sample of the order statistic.

    '''
    assert isinstance(total, int)
    assert isinstance(order, int)
    assert total >= order, 'order must be less or equal to total.'
    samples = [icdf(random.random()) for _ in range(total)]
    partition = np.partition(np.asarray(samples), order-1)
    # assert partition[order-1] <= partition[order]
    # assert partition[order] <= partition[order+1]
    return partition[order-1]

def order_mean_empiric(icdf, total, order, samples=1000):
    '''Compute the order static mean numerically.

    Args:

    icdf: Distribution ICDF function.

    total: Total number of realizations.

    order: Order of the statistic.

    samples: Samples to compute the mean from.

    Returns: The mean of the order statistic with variables drawn from
    the given distribution.

    '''
    samples = [order_sample(icdf, total, order) for _ in range(samples)]
    mean = statistics.mean(samples)
    variance = statistics.variance(samples, xbar=mean)
    return mean, variance

@functools.lru_cache(maxsize=1024)
def order_mean_shiftexp(total, order, parameter=1):
    '''Compute the mean of the shifted exponential order statistic.

    Args:

    total: Total number of variables.

    order: Statistic order.

    parameter: Distribution parameter.

    Returns: The mean of the order statistic with variables drawn from
    the shifted exponential distribution.

    '''
    mean = 1
    for i in range(total-order+1, total+1):
        mean += 1 / i

    mean *= parameter
    return mean

@functools.lru_cache(maxsize=1024)
def order_variance_shiftexp(total, order, parameter):
    '''Compute the variance of the shifted exponential order statistic.

    Args:

    total: Total number of variables.

    order: Statistic order.

    parameter: Distribution parameter.

    Returns: The variance of the order statistic with variables drawn from
    the shifted exponential distribution.

    '''
    variance = 0
    for i in range(total-order+1, total+1):
        variance += 1 / math.pow(i, 2)

    variance *= math.pow(parameter, 2)
    return variance

class ShiftexpOrder(object):
    '''Shifted exponential order statistic random variable.'''

    def __init__(self, parameter, total, order):
        '''Initialize the object.

        Args:

        parameter: Shifted exponential parameter.

        '''
        assert 0 < parameter < math.inf
        assert 0 < total < math.inf and total % 1 == 0
        assert 0 < order <= total and order % 1 == 0
        self.parameter = parameter
        self.total = total
        self.order = order
        self.beta = self.mean() / self.variance()
        self.alpha = self.mean() * self.beta
        return

    def pdf(self, value):
        '''Probability density function.'''
        assert 0 <= value <= math.inf
        return scipy.stats.gamma.pdf(value, self.alpha, scale=1/self.beta,
                                     loc=self.parameter)

    def mean(self):
        '''Expected value.'''
        return order_mean_shiftexp(self.total, self.order,
                                   parameter=self.parameter,
                                   exact=True)

    def variance(self):
        '''Random variable variance.'''
        return order_variance_shiftexp(self.total, self.order,
                                       self.parameter)

class Shiftexp(object):
    '''Shifted exponential distributed random variable.'''

    def __init__(self, parameter):
        '''Initialize the object.

        Args:

        parameter: Shifted exponential parameter.

        '''
        assert 0 < parameter < math.inf
        self.parameter = parameter
        return

    def pdf(self, value):
        '''Shifted exponential distribution PDF.'''
        assert 0 <= value <= math.inf
        if value < self.parameter:
            return 0
        else:
            return 1 / self.parameter * math.exp(-(value / self.parameter - 1))

    def cdf(self, value):
        '''Shifted exponential distribution CDF.'''
        assert 0 <= value <= math.inf
        if value < self.parameter:
            return 0
        else:
            return 1 - math.exp(-(value / self.parameter - 1))

    def icdf(self, value):
        '''Shifted exponential distribution inverse CDF.'''
        assert 0 <= value < 1, 'Value must be <= 0 and < 1.'
        return self.parameter * (1 - math.log(1 - value))

    def mean(self):
        '''Expected value.'''
        return self.parameter

def validate():
    '''Validate the analytic computation of the order stats.'''
    total = 1000
    order = 900
    mu = 2
    icdf = lambda x: icdf_shiftexp(x, parameter=mu)
    mean, variance = order_mean_empiric(icdf, total, order, samples=10000)

    # samples = [order_sample(icdf, total, order) for _ in range(100000)]
    # mean = sum(samples) / len(samples)
    analytic_mean = order_mean_shiftexp(total, order, parameter=mu)
    analytic_variance = order_variance_shiftexp(total, order, parameter=mu)

    try:
        fast_analytic = order_mean_shiftexp(total, order, parameter=mu, exact=False)
    except (ValueError, NotImplementedError):
        fast_analytic = math.inf
    print(mean, analytic_mean)
    print(variance, analytic_variance)
    return

if __name__ == '__main__':
    validate()
<commit_msg>shifted exponential order statistic CDF and addgregated CDF<commit_after>############################################################################
# Copyright 2016 Albin Severinson                                          #
#                                                                          #
# Licensed under the Apache License, Version 2.0 (the "License");          #
# you may not use this file except in compliance with the License.         #
# You may obtain a copy of the License at                                  #
#                                                                          #
#     http://www.apache.org/licenses/LICENSE-2.0                           #
#                                                                          #
# Unless required by applicable law or agreed to in writing, software      #
# distributed under the License is distributed on an "AS IS" BASIS,        #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and      #
# limitations under the License.                                           #
############################################################################

'''Project statistics module.'''

import math
import random
import functools
import statistics
import numpy as np
import scipy as sp
import scipy.stats
import matplotlib.pyplot as plt

from scipy.special import gamma, gammainc

def order_sample(icdf=None, total=None, order=None):
    '''Sample the order statistic.

    Args:

    icdf: Distribution ICDF function.

    total: Total number of realizations.

    order: Order of the statistic.

    Returns: A sample of the order statistic.

    '''
    assert isinstance(total, int)
    assert isinstance(order, int)
    assert total >= order, 'order must be less or equal to total.'
    samples = [icdf(random.random()) for _ in range(total)]
    partition = np.partition(np.asarray(samples), order-1)
    return partition[order-1]

def order_samples(icdf=None, total=None, order=None, samples=None):
    '''Sample the order statistic of the distribution given by the icdf'''
    return [order_sample(icdf, total, order) for _ in range(samples)]

def order_mean_empiric(icdf, total, order, samples=1000):
    '''Compute the order static mean numerically.

    Args:

    icdf: Distribution ICDF function.

    total: Total number of realizations.

    order: Order of the statistic.

    samples: Samples to compute the mean from.

    Returns: The mean of the order statistic with variables drawn from
    the given distribution.

    '''
    samples = [order_sample(icdf, total, order) for _ in range(samples)]
    mean = statistics.mean(samples)
    variance = statistics.variance(samples, xbar=mean)
    return mean, variance

@functools.lru_cache(maxsize=1024)
def order_mean_shiftexp(total, order, parameter=1):
    '''Compute the mean of the shifted exponential order statistic.

    Args:

    total: Total number of variables.

    order: Statistic order.

    parameter: Distribution parameter.

    Returns: The mean of the order statistic with variables drawn from
    the shifted exponential distribution.

    '''
    mean = 1
    for i in range(total-order+1, total+1):
        mean += 1 / i

    mean *= parameter
    return mean

@functools.lru_cache(maxsize=1024)
def order_variance_shiftexp(total, order, parameter):
    '''Compute the variance of the shifted exponential order statistic.

    Args:

    total: Total number of variables.

    order: Statistic order.

    parameter: Distribution parameter.

    Returns: The variance of the order statistic with variables drawn from
    the shifted exponential distribution.

    '''
    variance = 0
    for i in range(total-order+1, total+1):
        variance += 1 / math.pow(i, 2)

    variance *= math.pow(parameter, 2)
    return variance

@functools.lru_cache(maxsize=1024)
def order_cdf_shiftexp(value, total=None, order=None, parameter=None):
    '''CDF of the shifted exponential order statistic.

    args:

    value: value to evaluate the CDF at.

    total: Total number of variables.

    order: Statistic order.

    parameter: Distribution parameter.

    returns: the CDF of the shifted exponential order statistic evaluated at
    value.

    '''
    assert 0 <= value <= math.inf
    if value < parameter:
        return 0
    a = order_mean_shiftexp(total, order, parameter) - parameter
    a /= order_variance_shiftexp(total, order, parameter)
    b = pow(order_mean_shiftexp(total, order, parameter) - parameter, 2)
    b /= order_variance_shiftexp(total, order, parameter)

    # the CDF is given by the regularized lower incomplete gamma function.
    return gammainc(b, a*(value-parameter))

def order_aggregate_cdf_shiftexp(value, parameter=None, total=None,
                                 orders=None, order_probabilities=None):
    '''aggregate CDF of the shifted exponential order statistic.

    this method aggregates the order statistic CDF when the order is a random
    variable.

    args:

    value: value to evaluate the aggregate CDF at.

    parameter: shifted exponential distribution parameter.

    total: total number of variables.

    orders: numpy array-like of orders to aggregate the CDF from.

    order_probabilities: numpy array-like with probabilities for each order
    being active.

    returns: the aggregate CDF evaluated at value.

    '''
    assert len(orders) == len(order_probabilities)
    assert np.allclose(order_probabilities.sum(), 1)

    # evaluate the shifted exponential order statistic CDF at each order
    cdf_values = np.fromiter(
        (order_cdf_shiftexp(value, total=total, order=i, parameter=parameter) for i in orders),
        dtype=float,
    )

    # weigh the results by the probability of the respective order
    cdf_values *= order_probabilities

    # summing the array gives the weighted average, which is the value of the
    # aggregate CDF.
    return cdf_values.sum()

class ShiftexpOrder(object):
    '''Shifted exponential order statistic random variable.'''

    def __init__(self, parameter, total, order):
        '''Initialize the object.

        Args:

        parameter: Shifted exponential parameter.

        '''
        assert 0 < parameter < math.inf
        assert 0 < total < math.inf and total % 1 == 0
        assert 0 < order <= total and order % 1 == 0
        self.parameter = parameter
        self.total = total
        self.order = order

        # this variable is gamma distributed. compute the distribution
        # parameters from its mean and variance.
        self.a = (self.mean()-self.parameter) / self.variance()
        self.b = pow(self.mean()-self.parameter, 2) / self.variance()
        return

    def pdf(self, value):
        '''Probability density function.'''
        assert 0 <= value <= math.inf
        return scipy.stats.gamma.pdf(
            value,
            self.b,
            scale=1/self.a,
            loc=self.parameter,
        )

    def cdf(self, value):
        '''cumulative distribution function'''
        return order_cdf_shiftexp(
            value=value,
            total=self.total,
            order=self.order,
            parameter=self.parameter,
        )

    def mean(self):
        '''Expected value.'''
        return order_mean_shiftexp(
            self.total,
            self.order,
            parameter=self.parameter,
        )

    def variance(self):
        '''Random variable variance.'''
        return order_variance_shiftexp(
            self.total,
            self.order,
            self.parameter,
        )

class Shiftexp(object):
    '''Shifted exponential distributed random variable.'''

    def __init__(self, parameter):
        '''Initialize the object.

        Args:

        parameter: Shifted exponential parameter.

        '''
        assert 0 < parameter < math.inf
        self.parameter = parameter
        return

    def pdf(self, value):
        '''Shifted exponential distribution PDF.'''
        assert 0 <= value <= math.inf
        if value < self.parameter:
            return 0
        else:
            return 1 / self.parameter * math.exp(-(value / self.parameter - 1))

    def cdf(self, value):
        '''Shifted exponential distribution CDF.'''
        assert 0 <= value <= math.inf
        if value < self.parameter:
            return 0
        else:
            return 1 - math.exp(-(value / self.parameter - 1))

    def icdf(self, value):
        '''Shifted exponential distribution inverse CDF.'''
        assert 0 <= value < 1, 'Value must be <= 0 and < 1.'
        return self.parameter * (1 - math.log(1 - value))

    def mean(self):
        '''Expected value.'''
        return self.parameter

def validate():
    '''Validate the analytic computation of the order stats.'''
    total = 9
    order = 6
    samples = 100000
    mu = 2
    x = Shiftexp(mu)
    x_order = ShiftexpOrder(mu, total, order)
    samples = order_samples(icdf=x.icdf, total=total, order=order, samples=samples)
    t = np.linspace(min(samples), max(samples), 100)

    print('min:', min(samples), 'max:', max(samples))

    # pdf = [x_order.pdf(i) for i in t]
    # print(pdf)
    # return

    plt.figure()
    plt.hist(samples, bins=200, histtype='stepfilled', density=True)
    pdf = [x_order.pdf(i) for i in t]
    plt.plot(t, pdf)
    plt.grid()

    plt.figure()
    plt.hist(samples, bins=100, cumulative=True, density=True)
    cdf = [x_order.cdf(i) for i in t]
    plt.plot(t, cdf)
    plt.grid()

    plt.figure()
    plt.hist(samples, bins=100, cumulative=True, density=True)
    cdf = [order_aggregate_cdf_shiftexp(
        i,
        parameter=mu,
        total=total,
        orders=np.array([order, order+1]),
        order_probabilities=np.array([0.75, 0.25])
    ) for i in t]
    plt.plot(t, cdf)
    plt.grid()
    plt.show()
    return

if __name__ == '__main__':
    validate()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

###############################################################################
# Module Imports
###############################################################################

#import csv
#import matplotlib
#import re
import arrow
import bs4
import collections
import crawler
import logging
import re
#import numpy as np
import pandas as pd

###############################################################################
# Global Constants And Variables
###############################################################################

logger = logging.getLogger('scp.stats')
logger.setLevel(logging.DEBUG)

###############################################################################
# Classes
###############################################################################

class StatGenerator:

    def __init__(self):
        self.datadir = crawler.Snapshot.database_directory + 'stats/'

    ###########################################################################
    # Page Processors
    ###########################################################################

    ###########################################################################
    # Generator
    ###########################################################################

    def _generate_pages(self, frames, page):
        basic = ('url', 'title', 'rating', 'author', 'created', 'wordcount')
        for x in basic:
            frames['pages'][x][page._pageid] = getattr(page, x)
        lists = ('images', 'comments', 'history')
        for x in lists:
            frames['pages'][x][page._pageid] = len(getattr(page, x))
        for i in page.tags:
            index = len(frames['tags']['page'])
            frames['tags']['page'][index] = page._pageid
            frames['tags']['tag'][index] = i

    def _generate_votes(self, frames, page):
        for i in page.votes:
            index = len(frames['votes']['page'])
            for x in ('user', 'value'):
                frames['votes'][x][index] = getattr(i, x)
            frames['votes']['page'][index] = page._pageid

    def _generate_revisions(self, frames, page):
        for i in page.history:
            index = len(frames['revisions']['page'])
            for x in ('user', 'time'):
                frames['revisions'][x][index] = getattr(i, x)
            frames['revisions']['page'][index] = page._pageid

    def _generate_comments(self, frames, page):
        for i in page.comments:
            for x in ('user', 'time', 'title'):
                frames['comments'][x][i.post_id] = getattr(i, x)
            wc = len(re.findall(r"[\w'█_-]+",
                                bs4.BeautifulSoup(i.content).text))
            frames['comments']['wordcount'][i.post_id] = wc
            frames['comments']['page'][i.post_id] = page._pageid

    def generate(self):
        frames = collections.defaultdict(lambda: collections.defaultdict(dict))
        with crawler.Page.from_snapshot():
            logger.info('Generating DataFrame objects')
            for n, url in enumerate(crawler.Page.sn.list_all_pages()):
                logger.info('Processing page #{}'.format(n))
                p = crawler.Page(url)
                self._generate_pages(frames, p)
                self._generate_votes(frames, p)
                self._generate_revisions(frames, p)
                self._generate_comments(frames, p)
        for i in frames:
            frame = pd.DataFrame(frames[i])
            frame.to_csv(
                '{}_{}.csv'.format(self.datadir, i),
                index_label='index')
            setattr(self, i, frame)

    def load(self):
        for i in ('pages', 'votes', 'tags', 'revisions', 'comments'):
            frame = pd.read_csv(
                '{}_{}.csv'.format(self.datadir, i),
                index_col='index')
            setattr(self, i, frame)

    ###########################################################################
    # Output Methods
    ###########################################################################

    def print_basic(self):
        msgs = (
            'Number of pages on the site:',
            'Number of unique authors:',
            'Total number of votes:',
            '    Upvotes:',
            '    Downvotes:',
            'Total Net Rating:',
            '    Average:',
            '    Mode:',
            '    Deviation:',
            'Total Wordcount:',
            '    Average:',
            'Total Comments:',
            'Total Revisions:',
            'Total Images:')
        funcs = (
            lambda x: x.index.size,
            lambda x: x.author.nunique(),
            lambda x: self.votes[self.votes.page.isin(x.index)].index.size,
            lambda x: self.votes[
                (self.votes.page.isin(x.index)) &
                (self.votes.value == 1)].index.size,
            lambda x: self.votes[
                (self.votes.page.isin(x.index)) &
                (self.votes.value == -1)].index.size,
            lambda x: x.rating.sum(),
            lambda x: x.rating.mean(),
            lambda x: x.rating.mode()[0],
            lambda x: x.rating.std(),
            lambda x: x.wordcount.sum(),
            lambda x: x.wordcount.mean(),
            lambda x: x.comments.sum(),
            lambda x: x.history.sum(),
            lambda x: x.images.sum(),
        )
        skips = self.tags[self.tags.tag == 'scp'].page
        tales = self.tags[self.tags.tag == 'tale'].page
        header = '| {:<33} | {:<8} | {:<8} | {:<8} |'
        header = header.format('Field', 'Total', 'Skips', 'Tales')
        print(header)
        row = '| {:<33} | {:<8.0f} | {:<8.0f} | {:<8.0f} |'
        for msg, func in zip(msgs, funcs):
            print(row.format(
                msg,
                func(self.pages),
                func(self.pages[self.pages.index.isin(skips)]),
                func(self.pages[self.pages.index.isin(tales)])))

    def create_table_authors(self):
        pf = self.pages
        rf = self.revisions
        cf = self.comments
        columns = {}
        aus = pf.author.dropna().unique()
        au = lambda x: pf[pf.author == x]
        columns['pages created'] = {i: au(i).index.size for i in aus}
        columns['net rating'] = {i: au(i).rating.sum() for i in aus}
        columns['average rating'] = {i: au(i).rating.mean().round(2) for i in aus}
        columns['rating per day'] = {}
        for i in aus:
            first_comment = cf[cf.user == i].time.dropna().min()
            first_revision = rf[rf.user == i].time.dropna().min()
            if pd.isnull(first_comment) and pd.isnull(first_revision):
                continue
            elif pd.isnull(first_comment):
                first_activity = arrow.get(first_revision)
            elif pd.isnull(first_revision):
                first_activity = arrow.get(first_comment)
            else:
                first_activity = min(
                    arrow.get(first_comment), arrow.get(first_revision))
            days_on_site = (arrow.now() - first_activity).days
            rating = au(i).rating.sum() / days_on_site
            columns['rating per day'][i] = rating
        columns['wordcount'] = {i: au(i).wordcount.sum() for i in aus}
        columns['average wordcount'] = {i: au(i).wordcount.mean() for i in aus}
        columns['image count'] = {i: au(i).images.sum() for i in aus}
        table_authors = pd.DataFrame(columns)
        table_authors.to_csv(self.dir + 'table_authors.csv', index_col='user')

###############################################################################

def main():
    gen = StatGenerator()
    gen.load()
    gen.print_basic()
    #print(gen.basic_lists['unique authors'])
    #gen.print_basic()
    #print(gen.basic_lists['ratings'])
    pass

if __name__ == "__main__":
    crawler.enable_logging(logger)
    main()
<commit_msg>add rating statistics<commit_after>#!/usr/bin/env python3

###############################################################################
# Module Imports
###############################################################################

#import csv
#import matplotlib
#import re
import arrow
import bs4
import collections
import crawler
import logging
import re
#import numpy as np
import pandas as pd

###############################################################################
# Global Constants And Variables
###############################################################################

logger = logging.getLogger('scp.stats')
logger.setLevel(logging.DEBUG)

###############################################################################
# Classes
###############################################################################

class StatGenerator:

    def __init__(self):
        self.datadir = crawler.Snapshot.database_directory + 'stats/'

    ###########################################################################
    # Page Processors
    ###########################################################################

    ###########################################################################
    # Generator
    ###########################################################################

    def _generate_pages(self, frames, page):
        basic = ('url', 'title', 'rating', 'author', 'created', 'wordcount')
        for x in basic:
            frames['pages'][x][page._pageid] = getattr(page, x)
        lists = ('images', 'comments', 'history')
        for x in lists:
            frames['pages'][x][page._pageid] = len(getattr(page, x))
        for i in page.tags:
            index = len(frames['tags']['page'])
            frames['tags']['page'][index] = page._pageid
            frames['tags']['tag'][index] = i

    def _generate_votes(self, frames, page):
        for i in page.votes:
            index = len(frames['votes']['page'])
            for x in ('user', 'value'):
                frames['votes'][x][index] = getattr(i, x)
            frames['votes']['page'][index] = page._pageid

    def _generate_revisions(self, frames, page):
        for i in page.history:
            index = len(frames['revisions']['page'])
            for x in ('user', 'time'):
                frames['revisions'][x][index] = getattr(i, x)
            frames['revisions']['page'][index] = page._pageid

    def _generate_comments(self, frames, page):
        for i in page.comments:
            for x in ('user', 'time', 'title'):
                frames['comments'][x][i.post_id] = getattr(i, x)
            wc = len(re.findall(r"[\w'█_-]+",
                                bs4.BeautifulSoup(i.content).text))
            frames['comments']['wordcount'][i.post_id] = wc
            frames['comments']['page'][i.post_id] = page._pageid

    def generate(self):
        frames = collections.defaultdict(lambda: collections.defaultdict(dict))
        with crawler.Page.from_snapshot():
            logger.info('Generating DataFrame objects')
            for n, url in enumerate(crawler.Page.sn.list_all_pages()):
                logger.info('Processing page #{}'.format(n))
                p = crawler.Page(url)
                self._generate_pages(frames, p)
                self._generate_votes(frames, p)
                self._generate_revisions(frames, p)
                self._generate_comments(frames, p)
        for i in frames:
            frame = pd.DataFrame(frames[i])
            frame.to_csv(
                '{}_{}.csv'.format(self.datadir, i),
                index_label='index')
            setattr(self, i, frame)

    def load(self):
        for i in ('pages', 'votes', 'tags', 'revisions', 'comments'):
            frame = pd.read_csv(
                '{}_{}.csv'.format(self.datadir, i),
                index_col='index')
            setattr(self, i, frame)

    ###########################################################################
    # Output Methods
    ###########################################################################

    def print_basic(self):
        msgs = (
            '~ Pages',
            '~ Users',
            'Authors',
            'Voters',
            'Editors',
            '~ Votes',
            'Upvotes',
            'Downvotes',
            '~ Rating',
            'Average',
            'Mode',
            'Deviation',
            '~ Wordcount',
            'Average',
            '~ Comments',
            '~ Revisions',
            '~ Images')
        funcs = (
            lambda x: x.index.size,
            lambda x: len(
                set(x.author.unique()) |
                set(self.votes[self.votes.page.isin(x.index)].user.unique()) |
                set(self.revisions[
                    self.revisions.page.isin(x.index)].user.unique())),
            lambda x: x.author.nunique(),
            lambda x: self.votes[self.votes.page.isin(x.index)].user.nunique(),
            lambda x: self.revisions[
                self.revisions.page.isin(x.index)].user.nunique(),
            lambda x: self.votes[self.votes.page.isin(x.index)].index.size,
            lambda x: self.votes[
                (self.votes.page.isin(x.index)) &
                (self.votes.value == 1)].index.size,
            lambda x: self.votes[
                (self.votes.page.isin(x.index)) &
                (self.votes.value == -1)].index.size,
            lambda x: x.rating.sum(),
            lambda x: x.rating.mean(),
            lambda x: x.rating.mode()[0],
            lambda x: x.rating.std(),
            lambda x: x.wordcount.sum(),
            lambda x: x.wordcount.mean(),
            lambda x: x.comments.sum(),
            lambda x: x.history.sum(),
            lambda x: x.images.sum())
        skips = self.tags[self.tags.tag == 'scp'].page
        tales = self.tags[self.tags.tag == 'tale'].page
        header = '||~ {} ||~ {} ||~ {} ||~ {} ||'
        header = header.format('Field', 'Total', 'Skips', 'Tales')
        print(header)
        row = '||{} || {:.0f} || {:.0f} || {:.0f} ||'
        for msg, func in zip(msgs, funcs):
            print(row.format(
                msg,
                func(self.pages),
                func(self.pages[self.pages.index.isin(skips)]),
                func(self.pages[self.pages.index.isin(tales)])))

    def _days_on_the_site(self, user):
        print(user)
        com = self.comments[self.comments.user == user].time.dropna().min()
        rev = self.revisions[self.revisions.user == user].time.dropna().min()
        if pd.isnull(com) and pd.isnull(rev):
            return 999999 # this is a dirty hack :(
        elif pd.isnull(com) or pd.isnull(rev):
            act = com if pd.notnull(com) else rev
        else:
            act = min(com, rev)
        return (arrow.now() - arrow.get(act)).days

    def create_table_authors(self):
        authors = pf.author.dropna().unique()
        author_data = lambda x: self.pages[self.pages.author == x]
        columns['pages created'] = {i: au(i).index.size for i in aus}
        columns['net rating'] = {i: au(i).rating.sum() for i in aus}
        columns['average rating'] = {i: au(i).rating.mean().round(2) for i in aus}
        columns['rating per day'] = {}
        for i in aus:
            first_comment = cf[cf.user == i].time.dropna().min()
            first_revision = rf[rf.user == i].time.dropna().min()
            if pd.isnull(first_comment) and pd.isnull(first_revision):
                continue
            elif pd.isnull(first_comment):
                first_activity = arrow.get(first_revision)
            elif pd.isnull(first_revision):
                first_activity = arrow.get(first_comment)
            else:
                first_activity = min(
                    arrow.get(first_comment), arrow.get(first_revision))
            days_on_site = (arrow.now() - first_activity).days
            rating = au(i).rating.sum() / days_on_site
            columns['rating per day'][i] = rating
        columns['wordcount'] = {i: au(i).wordcount.sum() for i in aus}
        columns['average wordcount'] = {i: au(i).wordcount.mean() for i in aus}
        columns['image count'] = {i: au(i).images.sum() for i in aus}
        table_authors = pd.DataFrame(columns)
        table_authors.to_csv(self.dir + 'table_authors.csv', index_col='user')

    def create_table_ratings(self):
        ratings = collections.defaultdict(dict)
        contributors = self.pages.author.unique()
        newbies = [
            i for i in self.votes.user.unique()
            if self._days_on_the_site(i) < 180]
        staff = [
            'DrEverettMann', 'DrBright', 'DrClef', 'Drewbear',
            'Photosynthetic', 'Sorts', 'thedeadlymoose', 'TroyL', 'Crayne',
            'Dexanote', 'Eskobar', 'Gaffney', 'Pig_catapult', 'Roget',
            'Silberescher', 'Sophia', 'SoullessSingularity', 'Vivax', 'Zyn',
            'Accelerando', 'Doctor', 'anqxyr', 'pxdnbluesoul', 'Bouncl',
            'Faminepulse', 'FlameShirt', 'FortuneFavorsBold', 'Kalinin',
            'MisterFlames', 'murphy_slaw', 'Nioki', 'ProcyonLotor', 'Reject',
            'Riemann', 'Rumetzen', 'spikebrennan', 'thattallfellow', 'Tuomey',
            'Vincent_Redgrave', 'weizhong', 'Wogglebug', 'Blaroth', 'Chubert',
            'Devereaux', 'djkaktus', 'Fantem', 'Kate', 'LurkD', 'Pixeltasim']
        for n, (index, row) in enumerate(self.pages.iterrows()):
            print(n)
            ratings['full rating'][row['url']] = row['rating']
            ratings['contributor rating'][row['url']] = self.votes[
                (self.votes.user.isin(contributors)) &
                (self.votes.page == row.name)].value.sum()
            ratings['newbie rating'][row['url']] = self.votes[
                (self.votes.user.isin(newbies)) &
                (self.votes.page == row.name)].value.sum()
            ratings['staff rating'][row['url']] = self.votes[
                (self.votes.user.isin(staff)) &
                (self.votes.page == row.name)].value.sum()
        ratings = pd.DataFrame(ratings)
        ratings.to_csv(
            '{}table_ratings.csv'.format(self.datadir),
            index_label='page')

###############################################################################

def main():
    gen = StatGenerator()
    gen.load()
    gen.create_table_ratings()
    #gen.print_basic()
    #print(gen.basic_lists['unique authors'])
    #gen.print_basic()
    #print(gen.basic_lists['ratings'])
    pass

if __name__ == "__main__":
    crawler.enable_logging(logger)
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

import sys
import random
import logging
import math

from collections import namedtuple
from functools import partial
from itertools import takewhile, repeat, islice

from imread import imread, imwrite
import numpy as np
from moviepy.editor import VideoClip
import nphusl

logging.basicConfig(level=logging.INFO)

###################
# Filtering pixels

_props = namedtuple("channel", "h s l")

def hsl_filter(h=(), s=(), l=()):
    """Returns a namedtuple of properties that are used to find
    'channels' of a certain hue, saturation, and/or lightness in an image.
    If the user wants to filter for light rows/columns of pixels, they might
    use `hsl_filter(l=[90.5, 99.0])."""
    h, s, l = (tuple(p) for p in [h, s, l])  # let's make them all tuples

    # let's make sure the user has specified (min, max) for HSL
    if any(len(p) not in (0, 2) for p in (h, s, l)):
        raise ValueError("HSL filters can only be tuples like (min, max)")

    # let's make sure min, max values fit within their appropriate range
    if h:
        h = _valid_h(*h)
    if s:
        s = _valid_s(*s)
    if l:
        l = _valid_l(*l)

    return _props(h, s, l)

def _valid_h(hmin, hmax):
    return (max(hmin, 0), max(hmax, 0))

def _valid_s(smin, smax):
    return (min(max(smin, 0), 100),
            min(max(smax, 0), 100))

_valid_l = _valid_s  # saturation has the same range as lightness

def get_channel(img, filter_hsl, avg_husl):
    """Returns row indices for which the HSL filter constraints are met"""
    idx_select = np.ones(img.shape[0], dtype=bool)  # no "rows" selected initially
    for prop_idx, prop in enumerate(filter_hsl):
        if not prop:
            continue
        pmin, pmax = prop
        avg = avg_husl[:, prop_idx]
        # add/subtract 0.1 for convenient comparison of numpy arrays of floats
        # i.e. we want 100 to be not < 100.0000000000023
        idx_select[avg < (pmin - 0.1)] = 0
        idx_select[avg > (pmax + 0.1)] = 0
    return idx_select

################
# Moving pixels

@profile
def move(img, travel):
    """Shift `img` a distance of `travel` in the positive direction.
    The pixel array wraps around, so the last pixels will end up being
    the first pixels."""
    # NOTE: This copy is needed. Doing a backwards slice assignment is a good
    # workaround for an in place shift (numpy.roll creates a copy!), but it
    # is a real hack. Backwards slice assignment will work with c array
    # ordering, for example, but will break for Fortran style arrays.
    if len(img.shape) > 1:
        img = img.swapaxes(0, 1)
    if travel < 0:
        tail = img[:-travel].copy()
        img[:travel] = img[-travel:]
        img[travel:] = tail
    else:
        tail = img[-travel:].copy()  # pixel array wraps around
        img[travel:] = img[:-travel]  # move bulk of pixels
        img[:travel] = tail  # move the saved `tail` into vacated space
    return img

def flipped(fn):
    def wrapper(img, select, *args, **kwargs):
        _img = np.rot90(img)
        _select = np.rot90(select)
        for _ in fn(_img, _select, *args, **kwargs):
            yield img
    return wrapper

def run_while_changed(fn, img, *args, **kwargs):
    changed = True
    while changed:
        changed = fn(img, *args, **kwargs)
        yield img

@profile
def clump_rows(img, select, moves):
    rwhere, cwhere = np.nonzero(select)
    rows = np.unique(rwhere)  # row indices involved
    if len(moves) == 1:
        travels = np.zeros((len(rows),))
        travels[:] = moves[0]
    else:
        travels = np.random.choice(moves, len(cols))
    nz_travels = np.nonzero(travels)[0]
    rwhere = rwhere[nz_travels]
    cwhere = cwhere[nz_travels]
    rows = rows[nz_travels]

    s = select[(rows,)]
    col_indices = np.arange(0, img.shape[1])
    col_matrix = np.zeros(s.shape, dtype=np.float)
    col_matrix[:, None] = col_indices
    col_matrix[~s] = np.nan
    row_means = np.nanmean(col_matrix, axis=1).astype(np.int)

    total_mean = np.mean(cwhere)
    diff = total_mean - row_means
#    img[:] = 255
#    for row, mean in zip(rows, row_means):
#        if mean < total_mean:
#            img[row, mean-3: mean+3] = 200, 0, 200
#        else:
#            img[row, mean-3: mean+3] = 100, 0, 100
#    img[:, total_mean] = 255, 0, 255
#    img[select] = 0, 200, 200
    if max(moves) > 1:
        abs_diff = np.abs(diff)
        travels[abs_diff < travels] = 1
    travels[diff < 0] *= -1  # reverse row travel dir if row avg < total avg
    all_travels = tuple(-m for m in moves) + moves
    for travel in all_travels:
        rows_to_move = rows[travels == travel]
        for start, stop in _chunk_select(rows_to_move):
            move(img[start: stop], travel)
            move(select[start: stop], travel)
    #return False
    return True

def _chunk_select(rows):
    """Generate contiguous chunks of indices in tuples of
    (start_index, stop_index) where stop_index is not inclusive"""
    contiguous = np.diff(rows) == 1
    start, stop = 0, 1
    while stop < (rows.size - 1):
        if not contiguous[start]:
            stop = start + 1
            yield rows[start], rows[start] + 1
        else:
            remaining = contiguous[start:]
            stop = np.argmin(remaining) + start
            if stop == start:
                yield rows[start], rows[-1]
                break
            else:
                yield rows[start], rows[stop] + 1  # INCLUDE argmin
        start = stop

clump_horz = partial(run_while_changed, clump_rows)
clump_vert = flipped(clump_horz)

def disperse_cols(img, select, moves):
    rwhere, cwhere = np.nonzero(select)
    total_mean = np.mean(rwhere)
    max_dist = img.shape[1] // 2
    cols = np.unique(cwhere)
    if len(moves) == 1:
        travels = np.zeros((len(cols),))
        travels[:] = moves[0]
    else:
        travels = np.random.choice(moves, len(cols))
    n_changes = 0
    nz = np.nonzero(travels)
    cols, travels = cols[nz], travels[nz]
    for col, travel in zip(cols, travels):
        heights = rwhere[cwhere == col]
        col_avg = np.mean(heights)
        diff = col_avg - total_mean
        abs_diff = abs(diff)
        if abs_diff < travel:
            travel = 1
        if col_avg < total_mean:
            travel = -travel
        n_changes += 1
#        move(img[:, col], travel)
#        move(select[:, col], travel)
    return n_changes

disperse_vert = partial(run_while_changed, disperse_cols)
disperse_horz = flipped(disperse_vert)

def fuzz_horz(img, select, moves=(0, 1)):
    moves = tuple(moves)
    fuzz_moves = tuple(-m for m in moves if m) + moves
    while True:
        rows = np.nonzero(np.any(select, axis=1))[0]
        fuzz_rows(img, select, rows, fuzz_moves)
        yield img

fuzz_vert = flipped(fuzz_horz)

def fuzz_rows(img, select, rows, moves):
    travels = np.random.choice(moves, len(rows))
    for row, travel in zip(rows, travels):
        if not travel:
            continue
        move(img[row, :], travel)
        move(select[row, :], travel)

######################
# Creating animations

def read_img(path):
    img = np.squeeze(imread(path))
    if len(img.shape) == 2:
        _img = np.ndarray(img.shape + (3,), dtype=img.dtype)
        _img[:] = img[..., None]
        img = _img
    logging.info("Initial image shape: {}".format(img.shape))
    logging.info("Working image shape: {}".format(img.shape))
    return img

def dark_clump_turns(img, select, moves):
    yield img
    while True:
        for _ in clump_vert(img, select, moves):
            yield img
        for _ in clump_horz(img, select, moves):
            yield img

def zip_effects(img, *effects):
    yield img
    for _ in zip(*effects):
        yield img

def frame_maker(effects):
    def make(_):
        return next(effects)
    return make

#################
# Whole programs

def clump_dark(filename, percentile=4.0):
    img = read_img(filename)
    hsl = nphusl.to_husl(img)
    _, _, L = (hsl[..., n] for n in range(3))
    dark = L < np.percentile(L, 4.0)
    logging.info("Selection ratio: {:1.1f}%".format(
                 100 * np.count_nonzero(dark) / dark.size))
    travel = (1,)
   # img[dark] = 255, 0, 255
    vert = clump_vert(img, dark, travel)
    horz = clump_horz(img, dark, travel)
    return zip_effects(img, horz)

if __name__ == "__main__":
    infile, outfile = sys.argv[1: 3]
    frames = clump_dark(infile, 4.0)
    make_frame = frame_maker(frames)
    animation = VideoClip(make_frame, duration=10)
    animation.write_videofile(outfile, fps=24, audio=False, threads=2)

<commit_msg>major performance improvements<commit_after>#!/usr/bin/env python
# coding: utf-8

import sys
import random
import logging
import math

from collections import namedtuple
from functools import partial
from itertools import takewhile, repeat, islice, zip_longest

from imread import imread, imwrite
import numpy as np
from moviepy.editor import VideoClip
import nphusl

logging.basicConfig(level=logging.INFO)

###################
# Filtering pixels

_props = namedtuple("channel", "h s l")

def hsl_filter(h=(), s=(), l=()):
    """Returns a namedtuple of properties that are used to find
    'channels' of a certain hue, saturation, and/or lightness in an image.
    If the user wants to filter for light rows/columns of pixels, they might
    use `hsl_filter(l=[90.5, 99.0])."""
    h, s, l = (tuple(p) for p in [h, s, l])  # let's make them all tuples

    # let's make sure the user has specified (min, max) for HSL
    if any(len(p) not in (0, 2) for p in (h, s, l)):
        raise ValueError("HSL filters can only be tuples like (min, max)")

    # let's make sure min, max values fit within their appropriate range
    if h:
        h = _valid_h(*h)
    if s:
        s = _valid_s(*s)
    if l:
        l = _valid_l(*l)

    return _props(h, s, l)

def _valid_h(hmin, hmax):
    return (max(hmin, 0), max(hmax, 0))

def _valid_s(smin, smax):
    return (min(max(smin, 0), 100),
            min(max(smax, 0), 100))

_valid_l = _valid_s  # saturation has the same range as lightness

def get_channel(img, filter_hsl, avg_husl):
    """Returns row indices for which the HSL filter constraints are met"""
    idx_select = np.ones(img.shape[0], dtype=bool)  # no "rows" selected initially
    for prop_idx, prop in enumerate(filter_hsl):
        if not prop:
            continue
        pmin, pmax = prop
        avg = avg_husl[:, prop_idx]
        # add/subtract 0.1 for convenient comparison of numpy arrays of floats
        # i.e. we want 100 to be not < 100.0000000000023
        idx_select[avg < (pmin - 0.1)] = 0
        idx_select[avg > (pmax + 0.1)] = 0
    return idx_select

################
# Moving pixels

@profile
def move(i, travel):
    """Shift `img` a distance of `travel` in the positive direction.
    The pixel array wraps around, so the last pixels will end up being
    the first pixels."""
    # NOTE: This copy is needed. Doing a backwards slice assignment is a good
    # workaround for an in place shift (numpy.roll creates a copy!), but it
    # is a real hack. Backwards slice assignment will work with c array
    # ordering, for example, but will break for Fortran style arrays.
 #   if len(img.shape) > 2:
 #       img = img.swapaxes(0, 1)
    #i = img.squeeze()
    if travel < 0:
        tail = i[:-travel].copy()
        i[:travel] = i[-travel:]
        i[travel:] = tail
    else:
        tail = i[-travel:].copy()  # pixel array wraps around
        i[travel:] = i[:-travel]  # move bulk of pixels
        i[:travel] = tail  # move the saved `tail` into vacated space
    return i

def mass_move(img, travel):
    if travel < 0:
        tail = img[:-travel].copy()

def flipped(fn):
    def wrapper(img, select, *args, **kwargs):
        _img = np.rot90(img)
        _select = np.rot90(select)
        for _ in fn(_img, _select, *args, **kwargs):
            yield img
    return wrapper

def run_while_changed(fn, img, *args, **kwargs):
    changed = True
    while changed:
        changed = fn(img, *args, **kwargs)
        yield img

@profile
def clump_cols(img, select, moves):
    rwhere, cwhere = np.nonzero(select)
    total_avg = np.mean(rwhere)
 #   img[:] = 255
 #   img[total_avg, :] = 255, 0, 255
    cols = np.unique(cwhere)
    if len(moves) == 1:
        travels = np.zeros((cols.size,))
        travels[:] = moves[0]
    else:
        travels = np.random.choice(moves, cols.size)
    nz = np.nonzero(travels)
    cols, travels = cols[nz], travels[nz]

    s = select[:, cols]
    row_indices = np.arange(0, img.shape[0])
    row_matrix = np.zeros(select.shape, dtype=np.float)
    rm = row_matrix.swapaxes(0, 1)
    rm[:, None] = row_indices
    row_matrix = rm.swapaxes(0, 1)
    row_matrix[~select] = np.nan
    col_avgs = np.nanmean(row_matrix, axis=0)

    diff = col_avgs - total_avg
    abs_diff = np.abs(diff)

    # when columns are as close the median as they can be, we want them to
    # gradually move with a random choice between moving 0 or 1 pixels
    stuck = abs_diff < travels
    travels[stuck] = np.random.choice((0, 1), np.count_nonzero(stuck))
    travels[diff > 0] *= -1  # reverse row travel dir if row avg < total avg
    nz = np.nonzero(travels)
    travels = travels[nz]
    cols = cols[nz]

#    for col, travel in zip(cols, travels):
        #heights = rwhere[cwhere == col]
        #col_avg = np.mean(heights)
        #abs_diff = abs(col_avg - total_avg)
        #if abs_diff < travel:
        #    travel = 1
        #if col_avg > total_avg:
        #    travel = -travel
#        move(img[:, col], travel)
#        move(select[:, col], travel)

#    return True

    #img[:] = 255
    #img[select] = 255, 0, 255

    all_travels = tuple(-m for m in moves) + moves
#    for travel in all_travels:
#        cols_to_move = cols[travels == travel]
#        if not cols_to_move.size:
#            continue
#        for buf in _chunk_select_2(cols_to_move):
#            for col in buf:
#                move(img[:, col], travel)
#                move(select[:, col], travel)
#    return True

    moved = 0
    for travel in all_travels:
        cols_to_move = cols[travels == travel]
        moved += cols_to_move.size
        if not cols_to_move.size:
            continue
        for start, stop in _chunk_select(cols_to_move):
#            for col in range(start, stop):
#                move(img[:, col], travel)
#                move(select[:, col], travel)
            move(img[:, start: stop], travel)
            move(select[:, start: stop], travel)
    #return False
    return True

@profile
def _chunk_select(rows):
    """Generate contiguous chunks of indices in tuples of
    (start_index, stop_index) where stop_index is not inclusive"""
    contiguous = np.diff(rows) == 1
    buf = []
    for r, n in zip_longest(rows, contiguous):
        buf.append(r)
        if not n:
            yield buf[0], buf[-1] + 1
            buf = []
    if buf:
        yield buf[0], buf[-1] + 1

clump_vert = partial(run_while_changed, clump_cols)
clump_horz = flipped(clump_vert)

def clump_cols2(img, select, moves):
    rwhere, cwhere = np.nonzero(select)
    total_avg = np.mean(rwhere)
#    img[:] = 255
#    img[total_avg, :] = 255, 0, 255
    cols = np.unique(cwhere)
    if len(moves) == 1:
        travels = np.zeros((len(cols),))
        travels[:] = moves[0]
    else:
        travels = np.random.choice(moves, len(cols))
    nz = np.nonzero(travels)
    cols, travels = cols[nz], travels[nz]
    for col, travel in zip(cols, travels):
        heights = rwhere[cwhere == col]
        col_avg = np.mean(heights)
#        img[col_avg, col] = 255, 0, 100
        abs_diff = abs(col_avg - total_avg)
        if abs_diff < travel:
            travel = 1
        if col_avg > total_avg:
            travel = -travel
        move(img[:, col], travel)
        move(select[:, col], travel)
    return True
    return False

clump_vert2 = partial(run_while_changed, clump_cols2)
clump_horz2 = flipped(clump_vert2)

#disperse_vert = partial(run_while_changed, disperse_cols)
#disperse_horz = flipped(disperse_vert)

def fuzz_horz(img, select, moves=(0, 1)):
    moves = tuple(moves)
    fuzz_moves = tuple(-m for m in moves if m) + moves
    while True:
        rows = np.nonzero(np.any(select, axis=1))[0]
        fuzz_rows(img, select, rows, fuzz_moves)
        yield img

fuzz_vert = flipped(fuzz_horz)

def fuzz_rows(img, select, rows, moves):
    travels = np.random.choice(moves, len(rows))
    for row, travel in zip(rows, travels):
        if not travel:
            continue
        move(img[row, :], travel)
        move(select[row, :], travel)

######################
# Creating animations

def read_img(path):
    img = np.squeeze(imread(path))
    if len(img.shape) == 2:
        _img = np.ndarray(img.shape + (3,), dtype=img.dtype)
        _img[:] = img[..., None]
        img = _img
    logging.info("Initial image shape: {}".format(img.shape))
    logging.info("Working image shape: {}".format(img.shape))
    return img

def dark_clump_turns(img, select, moves):
    yield img
    while True:
        for _ in clump_vert(img, select, moves):
            yield img
        for _ in clump_horz(img, select, moves):
            yield img

def zip_effects(img, *effects):
    yield img
    for _ in zip(*effects):
        yield img

def frame_maker(effects):
    def make(_):
        return next(effects)
    return make

#################
# Whole programs

def clump_dark(filename, percentile=4.0):
    img = read_img(filename)
    hsl = nphusl.to_husl(img)
    _, _, L = (hsl[..., n] for n in range(3))
    dark = L < np.percentile(L, 4.0)
    logging.info("Selection ratio: {:1.1f}%".format(
                 100 * np.count_nonzero(dark) / dark.size))
    travel = (1,)
   # img[dark] = 255, 0, 255
    vert = clump_vert(img, dark, travel)
    horz = clump_horz(img, dark, travel)
    return zip_effects(img, vert)

if __name__ == "__main__":
    infile, outfile = sys.argv[1: 3]
    frames = clump_dark(infile, 4.0)
    make_frame = frame_maker(frames)
    animation = VideoClip(make_frame, duration=10)
    animation.write_videofile(outfile, fps=24, audio=False, threads=2)

<|endoftext|>


<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This script is used to synthesize generated parts of this library."""

import synthtool.gcp as gcp
import synthtool.languages.java as java

gapic = gcp.GAPICBazel()

service = 'secretmanager'
versions = ['v1beta1']

java.bazel_library(
    service=service,
    version='v1',
    package_pattern='com.google.cloud.{service}.{version}',
    gapic=gapic,
)

for version in versions:
  library = java.bazel_library(
    service=service,
    version=version,
    package_pattern='com.google.cloud.{service}.{version}',
    gapic=gapic,
    proto_path=f'google/cloud/secrets/{version}',
    bazel_target=f'//google/cloud/secrets/{version}:google-cloud-{service}-{version}-java',
  )

java.common_templates()
<commit_msg>chore: enable context aware commits (#75)<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This script is used to synthesize generated parts of this library."""

import synthtool.gcp as gcp
import synthtool.languages.java as java

AUTOSYNTH_MULTIPLE_COMMITS = True

gapic = gcp.GAPICBazel()

service = 'secretmanager'
versions = ['v1beta1']

java.bazel_library(
    service=service,
    version='v1',
    package_pattern='com.google.cloud.{service}.{version}',
    gapic=gapic,
)

for version in versions:
  library = java.bazel_library(
    service=service,
    version=version,
    package_pattern='com.google.cloud.{service}.{version}',
    gapic=gapic,
    proto_path=f'google/cloud/secrets/{version}',
    bazel_target=f'//google/cloud/secrets/{version}:google-cloud-{service}-{version}-java',
  )

java.common_templates()
<|endoftext|>


<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This script is used to synthesize generated parts of this library."""

import os
import synthtool as s
import synthtool.gcp as gcp
import logging

logging.basicConfig(level=logging.DEBUG)

gapic = gcp.GAPICGenerator()

for version in ['V2', 'V2beta2', 'V2beta3']:
    lower_version = version.lower()
    library = gapic.php_library(
        service='tasks',
        version=lower_version,
        config_path=f'artman_cloudtasks_{lower_version}.yaml',
        artman_output_name=f'google-cloud-tasks-{lower_version}')

    # copy all src including partial veneer classes
    s.move(library / 'src')

    # copy proto files to src also
    s.move(library / f'proto/src/Google/Cloud/Tasks', f'src/')
    s.move(library / f'tests/')

    # copy GPBMetadata file to metadata
    s.move(library / f'proto/src/GPBMetadata/Google/Cloud/Tasks', f'metadata/')

# document and utilize apiEndpoint instead of serviceAddress
s.replace(
    "**/Gapic/*GapicClient.php",
    r"'serviceAddress' =>",
    r"'apiEndpoint' =>")
s.replace(
    "**/Gapic/*GapicClient.php",
    r"@type string \$serviceAddress\n\s+\*\s+The address",
    r"""@type string $serviceAddress
     *           **Deprecated**. This option will be removed in a future major release. Please
     *           utilize the `$apiEndpoint` option instead.
     *     @type string $apiEndpoint
     *           The address""")
s.replace(
    "**/Gapic/*GapicClient.php",
    r"\$transportConfig, and any \$serviceAddress",
    r"$transportConfig, and any `$apiEndpoint`")

# fix year
s.replace(
    'src/V2beta*/**/*.php',
    r'Copyright \d{4}',
    r'Copyright 2018')
s.replace(
    'tests/*/V2beta*/*Test.php',
    r'Copyright \d{4}',
    r'Copyright 2018')
s.replace(
    'src/V2/**/*.php',
    r'Copyright \d{4}',
    r'Copyright 2019')
s.replace(
    'tests/*/V2/*Test.php',
    r'Copyright \d{4}',
    r'Copyright 2019')

# Use new namespace in the doc sample. See
# https://github.com/googleapis/gapic-generator/issues/2141
s.replace(
    'src/*/Gapic/CloudTasksGapicClient.php',
    r'Task_View',
    r'Task\\View')

# Change the wording for the deprecation warning.
s.replace(
    'src/*/*_*.php',
    r'will be removed in the next major release',
    'will be removed in a future release')

# V2 is GA, so remove @experimental tags
s.replace(
    'src/V2/**/*Client.php',
    r'^(\s+\*\n)?\s+\*\s@experimental\n',
    '')

### [START] protoc backwards compatibility fixes

# roll back to private properties.
s.replace(
    "src/**/V*/**/*.php",
    r"Generated from protobuf field ([^\n]{0,})\n\s{5}\*/\n\s{4}protected \$",
    r"""Generated from protobuf field \1
     */
    private $""")

# prevent proto messages from being marked final
s.replace(
    "src/**/V*/**/*.php",
    r"final class",
    r"class")

# Replace "Unwrapped" with "Value" for method names.
s.replace(
    "src/**/V*/**/*.php",
    r"public function ([s|g]\w{3,})Unwrapped",
    r"public function \1Value"
)

### [END] protoc backwards compatibility fixes
<commit_msg>chore: fix backwards compatibility via synth (#2664)<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This script is used to synthesize generated parts of this library."""

import os
import synthtool as s
import synthtool.gcp as gcp
import logging

logging.basicConfig(level=logging.DEBUG)

gapic = gcp.GAPICGenerator()

for version in ['V2', 'V2beta2', 'V2beta3']:
    lower_version = version.lower()
    library = gapic.php_library(
        service='tasks',
        version=lower_version,
        config_path=f'artman_cloudtasks_{lower_version}.yaml',
        artman_output_name=f'google-cloud-tasks-{lower_version}')

    # copy all src including partial veneer classes
    s.move(library / 'src')

    # copy proto files to src also
    s.move(library / f'proto/src/Google/Cloud/Tasks', f'src/')
    s.move(library / f'tests/')

    # copy GPBMetadata file to metadata
    s.move(library / f'proto/src/GPBMetadata/Google/Cloud/Tasks', f'metadata/')

# document and utilize apiEndpoint instead of serviceAddress
s.replace(
    "**/Gapic/*GapicClient.php",
    r"'serviceAddress' =>",
    r"'apiEndpoint' =>")
s.replace(
    "**/Gapic/*GapicClient.php",
    r"@type string \$serviceAddress\n\s+\*\s+The address",
    r"""@type string $serviceAddress
     *           **Deprecated**. This option will be removed in a future major release. Please
     *           utilize the `$apiEndpoint` option instead.
     *     @type string $apiEndpoint
     *           The address""")
s.replace(
    "**/Gapic/*GapicClient.php",
    r"\$transportConfig, and any \$serviceAddress",
    r"$transportConfig, and any `$apiEndpoint`")

# fix year
s.replace(
    'src/V2beta*/**/*.php',
    r'Copyright \d{4}',
    r'Copyright 2018')
s.replace(
    'tests/*/V2beta*/*Test.php',
    r'Copyright \d{4}',
    r'Copyright 2018')
s.replace(
    'src/V2/**/*.php',
    r'Copyright \d{4}',
    r'Copyright 2019')
s.replace(
    'tests/*/V2/*Test.php',
    r'Copyright \d{4}',
    r'Copyright 2019')

# Use new namespace in the doc sample. See
# https://github.com/googleapis/gapic-generator/issues/2141
s.replace(
    'src/*/Gapic/CloudTasksGapicClient.php',
    r'Task_View',
    r'Task\\View')

# Change the wording for the deprecation warning.
s.replace(
    'src/*/*_*.php',
    r'will be removed in the next major release',
    'will be removed in a future release')

# V2 is GA, so remove @experimental tags
s.replace(
    'src/V2/**/*Client.php',
    r'^(\s+\*\n)?\s+\*\s@experimental\n',
    '')

# Fix missing formatting method
tasksBrokenGapics = [
    'src/V2beta3/Gapic/CloudTasksGapicClient.php',
    'src/V2beta2/Gapic/CloudTasksGapicClient.php',
    'src/V2/Gapic/CloudTasksGapicClient.php',
]
s.replace(
    tasksBrokenGapics,
    r"private static \$locationNameTemplate;\n\s{4}private static \$queueNameTemplate;",
    """private static $locationNameTemplate;
    private static $projectNameTemplate;
    private static $queueNameTemplate;"""
)
s.replace(
    tasksBrokenGapics,
    "private static function getQueueNameTemplate",
    """private static function getProjectNameTemplate()
    {
        if (null == self::$projectNameTemplate) {
            self::$projectNameTemplate = new PathTemplate('projects/{project}');
        }

        return self::$projectNameTemplate;
    }

    private static function getQueueNameTemplate"""
)
s.replace(
    tasksBrokenGapics,
    r"'location' => self::getLocationNameTemplate\(\),\n\s{0,}'queue' => self::getQueueNameTemplate\(\),",
    """'location' => self::getLocationNameTemplate(),
                'project' => self::getProjectNameTemplate(),
                'queue' => self::getQueueNameTemplate(),"""
)
s.replace(
    tasksBrokenGapics,
    r"\/\*\*\n\s{5}\* Parses a formatted name string and returns an",
    """/**
     * Formats a string containing the fully-qualified path to represent
     * a project resource.
     *
     * @param string $project
     *
     * @return string The formatted project resource.
     */
    public static function projectName($project)
    {
        return self::getProjectNameTemplate()->render([
            'project' => $project,
        ]);
    }

    /**
     * Parses a formatted name string and returns an"""
)

### [START] protoc backwards compatibility fixes

# roll back to private properties.
s.replace(
    "src/**/V*/**/*.php",
    r"Generated from protobuf field ([^\n]{0,})\n\s{5}\*/\n\s{4}protected \$",
    r"""Generated from protobuf field \1
     */
    private $""")

# prevent proto messages from being marked final
s.replace(
    "src/**/V*/**/*.php",
    r"final class",
    r"class")

# Replace "Unwrapped" with "Value" for method names.
s.replace(
    "src/**/V*/**/*.php",
    r"public function ([s|g]\w{3,})Unwrapped",
    r"public function \1Value"
)

### [END] protoc backwards compatibility fixes
<|endoftext|>


<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This script is used to synthesize generated parts of this library."""

import synthtool.gcp as gcp
import synthtool.languages.java as java

gapic = gcp.GAPICGenerator()

service = 'billing-budgets'
versions = ['v1beta1']

for version in versions:
    java.bazel_library(
        service=service,
        version=version,
        proto_path=f'google/cloud/billing/budgets/{version}',
        bazel_target=f'//google/cloud/billing/budgets/{version}:google-cloud-{service}-{version}-java',
        destination_name='billingbudgets',
    )

java.common_templates()
<commit_msg>chore: enable context aware commits (#102)<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This script is used to synthesize generated parts of this library."""

import synthtool.gcp as gcp
import synthtool.languages.java as java

AUTOSYNTH_MULTIPLE_COMMITS = True

gapic = gcp.GAPICGenerator()

service = 'billing-budgets'
versions = ['v1beta1']

for version in versions:
    java.bazel_library(
        service=service,
        version=version,
        proto_path=f'google/cloud/billing/budgets/{version}',
        bazel_target=f'//google/cloud/billing/budgets/{version}:google-cloud-{service}-{version}-java',
        destination_name='billingbudgets',
    )

java.common_templates()
<|endoftext|>


<commit_before>import synthtool as s
import synthtool.gcp as gcp
import logging
from pathlib import Path
import subprocess

logging.basicConfig(level=logging.DEBUG)

gapic = gcp.GAPICGenerator()

# tasks has two product names, and a poorly named artman yaml
v2beta2_library = gapic._generate_code(
    'spanner', 'v1', 'nodejs',
    config_path='/google/spanner/artman_spanner.yaml')

# Copy all files except for 'README.md' and 'package.json'
s.copy(v2beta2_library / 'src/v1', excludes="src/v1/index.js")

# '''
# Node.js specific cleanup
# '''
# # prettify and lint
subprocess.run(['npm', 'run', 'prettier'])
subprocess.run(['npm', 'run', 'lint'])
<commit_msg>use node_library not not internal generate method (#247)<commit_after>import synthtool as s
import synthtool.gcp as gcp
import logging
from pathlib import Path
import subprocess

logging.basicConfig(level=logging.DEBUG)

gapic = gcp.GAPICGenerator()

# tasks has two product names, and a poorly named artman yaml
v2beta2_library = gapic.node_library(
    'spanner', 'v1', config_path='/google/spanner/artman_spanner.yaml')

# Copy all files except for 'README.md' and 'package.json'
s.copy(v2beta2_library / 'src/v1', excludes="src/v1/index.js")

# '''
# Node.js specific cleanup
# '''
# # prettify and lint
subprocess.run(['npm', 'run', 'prettier'])
subprocess.run(['npm', 'run', 'lint'])
<|endoftext|>