<commit_before>import json
import warnings

from cached_property import cached_property
from frozendict import frozendict

import fabricio

from . import Image

class Container(object):

    image = None  # type: Image

    cmd = None

    stop_timeout = 10

    # default options
    user = None
    ports = None
    env = None
    volumes = None
    links = None
    hosts = None
    network = None
    restart_policy = None
    stop_signal = None

    _options = {}

    def __init__(self, name, options=None, **kwargs):
        self.name = name
        if options:
            warnings.warn(
                '`options` deprecated and will be removed in v0.4',
                category=RuntimeWarning, stacklevel=2,
            )
        is_default_option = self.default_options.__contains__
        self.options = options = options or {}
        self.overridden_options = set()
        for option, value in kwargs.items():
            if is_default_option(option):
                setattr(self, option, value)
            else:
                options[option] = value

    def __setattr__(self, attr, value):
        if attr in self.default_options:
            self.overridden_options.add(attr)
        super(Container, self).__setattr__(attr, value)

    @cached_property
    def default_options(self):
        return self._get_options(all_options=False)

    def fork(self, name=None, options=None, **kwargs):
        if options:
            warnings.warn(
                '`options` deprecated and will be removed in v0.4',
                category=RuntimeWarning, stacklevel=2,
            )
        if name is None:
            name = self.name
        deprecated_options = options or {}
        options = self._options.copy()
        options.update(deprecated_options)
        for option in self.overridden_options:
            options[option] = getattr(self, option)
        options.update(kwargs)
        return self.__class__(name, **options)

    def _get_options(self, all_options=True):
        initial = self._options if all_options else ()
        return frozendict(
            initial,
            user=self.user,
            ports=self.ports,
            env=self.env,
            volumes=self.volumes,
            links=self.links,
            hosts=self.hosts,
            network=self.network,
            restart_policy=self.restart_policy,
            stop_signal=self.stop_signal,
        )

    def _set_options(self, options):
        self._options = options

    options = property(_get_options, _set_options)

    def __str__(self):
        return str(self.name)

    def __copy__(self):
        return self.fork()

    @property
    def info(self):
        command = 'docker inspect --type container {container}'
        info = fabricio.run(command.format(container=self))
        return json.loads(info)[0]

    def delete(self, force=False, ignore_errors=False):
        command = 'docker rm {force}{container}'
        force = force and '--force ' or ''
        fabricio.run(
            command.format(container=self, force=force),
            ignore_errors=ignore_errors,
        )

    def run(self, tag=None, registry=None):
        self.__class__.image[registry:tag].run(
            cmd=self.cmd,
            temporary=False,
            name=self.name,
            options=self.options,
        )

    def execute(self, cmd, ignore_errors=False, quiet=True, use_cache=False):
        command = 'docker exec --tty --interactive {container} {cmd}'
        return fabricio.run(
            command.format(container=self, cmd=cmd),
            ignore_errors=ignore_errors,
            quiet=quiet,
            use_cache=use_cache,
        )

    def start(self):
        command = 'docker start {container}'
        fabricio.run(command.format(container=self))

    def stop(self, timeout=None):
        if timeout is None:
            timeout = self.stop_timeout
        command = 'docker stop --time {timeout} {container}'
        fabricio.run(command.format(container=self, timeout=timeout))

    def restart(self, timeout=None):
        if timeout is None:
            timeout = self.stop_timeout
        command = 'docker restart --time {timeout} {container}'
        fabricio.run(command.format(container=self, timeout=timeout))

    def rename(self, new_name):
        command = 'docker rename {container} {new_name}'
        fabricio.run(command.format(container=self, new_name=new_name))
        self.name = new_name

    def signal(self, signal):
        command = 'docker kill --signal {signal} {container}'
        fabricio.run(command.format(container=self, signal=signal))

    def update(self, tag=None, registry=None, force=False):
        if not force:
            try:
                current_image_id = self.image.id
            except RuntimeError:  # current container not found
                pass
            else:
                new_image = self.__class__.image[registry:tag]
                if current_image_id == new_image.id:
                    fabricio.log('No change detected, update skipped.')
                    self.start()  # force starting container
                    return False
        new_container = self.fork(name=self.name)
        obsolete_container = self.get_backup_container()
        try:
            obsolete_image = obsolete_container.image
        except RuntimeError:
            pass
        else:
            obsolete_container.delete()
            obsolete_image.delete(ignore_errors=True)
        try:
            backup_container = self.fork()
            backup_container.rename(obsolete_container.name)
        except RuntimeError:
            pass
        else:
            backup_container.stop()
        new_container.run(tag=tag, registry=registry)
        return True

    def revert(self):
        failed_image = self.image
        self.stop()
        self.delete()
        failed_image.delete(ignore_errors=True)
        backup_container = self.get_backup_container()
        backup_container.start()
        backup_container.rename(self.name)

    def get_backup_container(self):
        return self.fork(name='{container}_backup'.format(container=self))

    def migrate(self, tag=None, registry=None):
        pass

    def migrate_back(self):
        pass

    def backup(self):
        pass

    def restore(self, backup_name=None):
        pass
<commit_msg>fixed name property assignment<commit_after>import json
import warnings

from cached_property import cached_property
from frozendict import frozendict

import fabricio

from . import Image

class Container(object):

    image = None  # type: Image

    cmd = None

    stop_timeout = 10

    # default options
    user = None
    ports = None
    env = None
    volumes = None
    links = None
    hosts = None
    network = None
    restart_policy = None
    stop_signal = None

    _options = {}
    name = None

    def __init__(self, name, options=None, **kwargs):
        self.name = name
        if options:
            warnings.warn(
                '`options` deprecated and will be removed in v0.4',
                category=RuntimeWarning, stacklevel=2,
            )
        is_default_option = self.default_options.__contains__
        self.options = options = options or {}
        self.overridden_options = set()
        for option, value in kwargs.items():
            if is_default_option(option):
                setattr(self, option, value)
            else:
                options[option] = value

    def __setattr__(self, attr, value):
        if attr in self.default_options:
            self.overridden_options.add(attr)
        super(Container, self).__setattr__(attr, value)

    @cached_property
    def default_options(self):
        return self._get_options(all_options=False)

    def fork(self, name=None, options=None, **kwargs):
        if options:
            warnings.warn(
                '`options` deprecated and will be removed in v0.4',
                category=RuntimeWarning, stacklevel=2,
            )
        if name is None:
            name = self.name
        deprecated_options = options or {}
        options = self._options.copy()
        options.update(deprecated_options)
        for option in self.overridden_options:
            options[option] = getattr(self, option)
        options.update(kwargs)
        return self.__class__(name, **options)

    def _get_options(self, all_options=True):
        initial = self._options if all_options else ()
        return frozendict(
            initial,
            user=self.user,
            ports=self.ports,
            env=self.env,
            volumes=self.volumes,
            links=self.links,
            hosts=self.hosts,
            network=self.network,
            restart_policy=self.restart_policy,
            stop_signal=self.stop_signal,
        )

    def _set_options(self, options):
        self._options = options

    options = property(_get_options, _set_options)

    def __str__(self):
        return str(self.name)

    def __copy__(self):
        return self.fork()

    @property
    def info(self):
        command = 'docker inspect --type container {container}'
        info = fabricio.run(command.format(container=self))
        return json.loads(info)[0]

    def delete(self, force=False, ignore_errors=False):
        command = 'docker rm {force}{container}'
        force = force and '--force ' or ''
        fabricio.run(
            command.format(container=self, force=force),
            ignore_errors=ignore_errors,
        )

    def run(self, tag=None, registry=None):
        self.__class__.image[registry:tag].run(
            cmd=self.cmd,
            temporary=False,
            name=self.name,
            options=self.options,
        )

    def execute(self, cmd, ignore_errors=False, quiet=True, use_cache=False):
        command = 'docker exec --tty --interactive {container} {cmd}'
        return fabricio.run(
            command.format(container=self, cmd=cmd),
            ignore_errors=ignore_errors,
            quiet=quiet,
            use_cache=use_cache,
        )

    def start(self):
        command = 'docker start {container}'
        fabricio.run(command.format(container=self))

    def stop(self, timeout=None):
        if timeout is None:
            timeout = self.stop_timeout
        command = 'docker stop --time {timeout} {container}'
        fabricio.run(command.format(container=self, timeout=timeout))

    def restart(self, timeout=None):
        if timeout is None:
            timeout = self.stop_timeout
        command = 'docker restart --time {timeout} {container}'
        fabricio.run(command.format(container=self, timeout=timeout))

    def rename(self, new_name):
        command = 'docker rename {container} {new_name}'
        fabricio.run(command.format(container=self, new_name=new_name))
        self.name = new_name

    def signal(self, signal):
        command = 'docker kill --signal {signal} {container}'
        fabricio.run(command.format(container=self, signal=signal))

    def update(self, tag=None, registry=None, force=False):
        if not force:
            try:
                current_image_id = self.image.id
            except RuntimeError:  # current container not found
                pass
            else:
                new_image = self.__class__.image[registry:tag]
                if current_image_id == new_image.id:
                    fabricio.log('No change detected, update skipped.')
                    self.start()  # force starting container
                    return False
        new_container = self.fork(name=self.name)
        obsolete_container = self.get_backup_container()
        try:
            obsolete_image = obsolete_container.image
        except RuntimeError:
            pass
        else:
            obsolete_container.delete()
            obsolete_image.delete(ignore_errors=True)
        try:
            backup_container = self.fork()
            backup_container.rename(obsolete_container.name)
        except RuntimeError:
            pass
        else:
            backup_container.stop()
        new_container.run(tag=tag, registry=registry)
        return True

    def revert(self):
        failed_image = self.image
        self.stop()
        self.delete()
        failed_image.delete(ignore_errors=True)
        backup_container = self.get_backup_container()
        backup_container.start()
        backup_container.rename(self.name)

    def get_backup_container(self):
        return self.fork(name='{container}_backup'.format(container=self))

    def migrate(self, tag=None, registry=None):
        pass

    def migrate_back(self):
        pass

    def backup(self):
        pass

    def restore(self, backup_name=None):
        pass
<|endoftext|>


<commit_before>"""
Idempotent API for managing PostgreSQL users and databases
"""
from fabtools.postgres import *

def user(name, passwd):
    """
    I can haz PostgreSQL user
    """
    if not user_exists(name):
        create_user(name, passwd)

def database(name, owner):
    """
    I can haz PostgreSQL database
    """
    if not database_exists(name):
        create_database(name, owner)
<commit_msg>Added requirement for a PostgreSQL server.<commit_after>"""
Idempotent API for managing PostgreSQL users and databases
"""
from fabtools.postgres import *
from fabtools.icanhaz.deb import package
from fabtools.icanhaz.service import started

def server(version='8.4'):
    """
    I can haz PostgreSQL server
    """
    package('postgresql-%s' % version)
    started('postgresql-%s' % version)

def user(name, passwd):
    """
    I can haz PostgreSQL user
    """
    if not user_exists(name):
        create_user(name, passwd)

def database(name, owner):
    """
    I can haz PostgreSQL database
    """
    if not database_exists(name):
        create_database(name, owner)
<|endoftext|>


<commit_before>"""Provide access to Python's configuration information.  The specific names
defined in the module depend heavily on the platform and configuration.

Written by:   Fred L. Drake, Jr.
Email:        <fdrake@acm.org>
Initial date: 17-Dec-1998
"""

__revision__ = "$Id$"

import os
import re
import string
import sys

from errors import DistutilsPlatformError

# These are needed in a couple of spots, so just compute them once.
PREFIX = os.path.normpath(sys.prefix)
EXEC_PREFIX = os.path.normpath(sys.exec_prefix)

# Boolean; if it's true, we're still building Python, so
# we use different (hard-wired) directories.

python_build = 0

def set_python_build():
    """Set the python_build flag to true; this means that we're
    building Python itself.  Only called from the setup.py script
    shipped with Python.
    """

    global python_build
    python_build = 1

def get_python_inc(plat_specific=0, prefix=None):
    """Return the directory containing installed Python header files.

    If 'plat_specific' is false (the default), this is the path to the
    non-platform-specific header files, i.e. Python.h and so on;
    otherwise, this is the path to platform-specific header files
    (namely config.h).

    If 'prefix' is supplied, use it instead of sys.prefix or
    sys.exec_prefix -- i.e., ignore 'plat_specific'.
    """
    if prefix is None:
        prefix = (plat_specific and EXEC_PREFIX or PREFIX)
    if os.name == "posix":
        if python_build:
            return "Include/"
        return os.path.join(prefix, "include", "python" + sys.version[:3])
    elif os.name == "nt":
        return os.path.join(prefix, "Include") # include or Include?
    elif os.name == "mac":
        return os.path.join(prefix, "Include")
    else:
        raise DistutilsPlatformError, \
              ("I don't know where Python installs its C header files " +
               "on platform '%s'") % os.name

def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
    """Return the directory containing the Python library (standard or
    site additions).

    If 'plat_specific' is true, return the directory containing
    platform-specific modules, i.e. any module from a non-pure-Python
    module distribution; otherwise, return the platform-shared library
    directory.  If 'standard_lib' is true, return the directory
    containing standard Python library modules; otherwise, return the
    directory for site-specific modules.

    If 'prefix' is supplied, use it instead of sys.prefix or
    sys.exec_prefix -- i.e., ignore 'plat_specific'.
    """
    if prefix is None:
        prefix = (plat_specific and EXEC_PREFIX or PREFIX)

    if os.name == "posix":
        libpython = os.path.join(prefix,
                                 "lib", "python" + sys.version[:3])
        if standard_lib:
            return libpython
        else:
            return os.path.join(libpython, "site-packages")

    elif os.name == "nt":
        if standard_lib:
            return os.path.join(PREFIX, "Lib")
        else:
            return prefix

    elif os.name == "mac":
        if plat_specific:
            if standard_lib:
                return os.path.join(EXEC_PREFIX, "Mac", "Plugins")
            else:
                raise DistutilsPlatformError, \
                      "OK, where DO site-specific extensions go on the Mac?"
        else:
            if standard_lib:
                return os.path.join(PREFIX, "Lib")
            else:
                raise DistutilsPlatformError, \
                      "OK, where DO site-specific modules go on the Mac?"
    else:
        raise DistutilsPlatformError, \
              ("I don't know where Python installs its library " +
               "on platform '%s'") % os.name

# get_python_lib()

def customize_compiler (compiler):
    """Do any platform-specific customization of the CCompiler instance
    'compiler'.  Mainly needed on Unix, so we can plug in the information
    that varies across Unices and is stored in Python's Makefile.
    """
    if compiler.compiler_type == "unix":
        (cc, opt, ccshared, ldshared, so_ext) = \
            get_config_vars('CC', 'OPT', 'CCSHARED', 'LDSHARED', 'SO')

        cc_cmd = cc + ' ' + opt
        compiler.set_executables(
            preprocessor=cc + " -E",    # not always!
            compiler=cc_cmd,
            compiler_so=cc_cmd + ' ' + ccshared,
            linker_so=ldshared,
            linker_exe=cc)

        compiler.shared_lib_extension = so_ext

def get_config_h_filename():
    """Return full pathname of installed config.h file."""
    if python_build: inc_dir = '.'
    else:            inc_dir = get_python_inc(plat_specific=1)
    return os.path.join(inc_dir, "config.h")

def get_makefile_filename():
    """Return full pathname of installed Makefile from the Python build."""
    if python_build:
        return './Makefile'
    lib_dir = get_python_lib(plat_specific=1, standard_lib=1)
    return os.path.join(lib_dir, "config", "Makefile")

def parse_config_h(fp, g=None):
    """Parse a config.h-style file.

    A dictionary containing name/value pairs is returned.  If an
    optional dictionary is passed in as the second argument, it is
    used instead of a new dictionary.
    """
    if g is None:
        g = {}
    define_rx = re.compile("#define ([A-Z][A-Z0-9_]+) (.*)\n")
    undef_rx = re.compile("/[*] #undef ([A-Z][A-Z0-9_]+) [*]/\n")
    #
    while 1:
        line = fp.readline()
        if not line:
            break
        m = define_rx.match(line)
        if m:
            n, v = m.group(1, 2)
            try: v = string.atoi(v)
            except ValueError: pass
            g[n] = v
        else:
            m = undef_rx.match(line)
            if m:
                g[m.group(1)] = 0
    return g

# Regexes needed for parsing Makefile (and similar syntaxes,
# like old-style Setup files).
_variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")

def parse_makefile(fn, g=None):
    """Parse a Makefile-style file.

    A dictionary containing name/value pairs is returned.  If an
    optional dictionary is passed in as the second argument, it is
    used instead of a new dictionary.

    """
    from distutils.text_file import TextFile
    fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1)

    if g is None:
        g = {}
    done = {}
    notdone = {}

    while 1:
        line = fp.readline()
        if line is None:                # eof
            break
        m = _variable_rx.match(line)
        if m:
            n, v = m.group(1, 2)
            v = string.strip(v)
            if "$" in v:
                notdone[n] = v
            else:
                try: v = string.atoi(v)
                except ValueError: pass
                done[n] = v

    # do variable interpolation here
    while notdone:
        for name in notdone.keys():
            value = notdone[name]
            m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
            if m:
                n = m.group(1)
                if done.has_key(n):
                    after = value[m.end():]
                    value = value[:m.start()] + str(done[n]) + after
                    if "$" in after:
                        notdone[name] = value
                    else:
                        try: value = string.atoi(value)
                        except ValueError:
                            done[name] = string.strip(value)
                        else:
                            done[name] = value
                        del notdone[name]
                elif notdone.has_key(n):
                    # get it on a subsequent round
                    pass
                else:
                    done[n] = ""
                    after = value[m.end():]
                    value = value[:m.start()] + after
                    if "$" in after:
                        notdone[name] = value
                    else:
                        try: value = string.atoi(value)
                        except ValueError:
                            done[name] = string.strip(value)
                        else:
                            done[name] = value
                        del notdone[name]
            else:
                # bogus variable reference; just drop it since we can't deal
                del notdone[name]

    fp.close()

    # save the results in the global dictionary
    g.update(done)
    return g

def expand_makefile_vars(s, vars):
    """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
    'string' according to 'vars' (a dictionary mapping variable names to
    values).  Variables not present in 'vars' are silently expanded to the
    empty string.  The variable values in 'vars' should not contain further
    variable expansions; if 'vars' is the output of 'parse_makefile()',
    you're fine.  Returns a variable-expanded version of 's'.
    """

    # This algorithm does multiple expansion, so if vars['foo'] contains
    # "${bar}", it will expand ${foo} to ${bar}, and then expand
    # ${bar}... and so forth.  This is fine as long as 'vars' comes from
    # 'parse_makefile()', which takes care of such expansions eagerly,
    # according to make's variable expansion semantics.

    while 1:
        m = _findvar1_rx.search(s) or _findvar2_rx.search(s)
        if m:
            name = m.group(1)
            (beg, end) = m.span()
            s = s[0:beg] + vars.get(m.group(1)) + s[end:]
        else:
            break
    return s

_config_vars = None

def _init_posix():
    """Initialize the module as appropriate for POSIX systems."""
    g = {}
    # load the installed Makefile:
    try:
        filename = get_makefile_filename()
        parse_makefile(filename, g)
    except IOError, msg:
        my_msg = "invalid Python installation: unable to open %s" % filename
        if hasattr(msg, "strerror"):
            my_msg = my_msg + " (%s)" % msg.strerror

        raise DistutilsPlatformError, my_msg

    # On AIX, there are wrong paths to the linker scripts in the Makefile
    # -- these paths are relative to the Python source, but when installed
    # the scripts are in another directory.
    if sys.platform == 'aix4':          # what about AIX 3.x ?
        # Linker script is in the config directory, not in Modules as the
        # Makefile says.
        python_lib = get_python_lib(standard_lib=1)
        ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix')
        python_exp = os.path.join(python_lib, 'config', 'python.exp')

        g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp)

    elif sys.platform == 'beos':

        # Linker script is in the config directory.  In the Makefile it is
        # relative to the srcdir, which after installation no longer makes
        # sense.
        python_lib = get_python_lib(standard_lib=1)
        linkerscript_name = os.path.basename(string.split(g['LDSHARED'])[0])
        linkerscript = os.path.join(python_lib, 'config', linkerscript_name)

        # XXX this isn't the right place to do this: adding the Python
        # library to the link, if needed, should be in the "build_ext"
        # command.  (It's also needed for non-MS compilers on Windows, and
        # it's taken care of for them by the 'build_ext.get_libraries()'
        # method.)
        g['LDSHARED'] = ("%s -L%s/lib -lpython%s" %
                         (linkerscript, PREFIX, sys.version[0:3]))

    global _config_vars
    _config_vars = g

def _init_nt():
    """Initialize the module as appropriate for NT"""
    g = {}
    # set basic install directories
    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)

    # XXX hmmm.. a normal install puts include files here
    g['INCLUDEPY'] = get_python_inc(plat_specific=0)

    g['SO'] = '.pyd'
    g['EXE'] = ".exe"

    global _config_vars
    _config_vars = g

def _init_mac():
    """Initialize the module as appropriate for Macintosh systems"""
    g = {}
    # set basic install directories
    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)

    # XXX hmmm.. a normal install puts include files here
    g['INCLUDEPY'] = get_python_inc(plat_specific=0)

    g['SO'] = '.ppc.slb'

    # XXX are these used anywhere?
    g['install_lib'] = os.path.join(EXEC_PREFIX, "Lib")
    g['install_platlib'] = os.path.join(EXEC_PREFIX, "Mac", "Lib")

    global _config_vars
    _config_vars = g

def get_config_vars(*args):
    """With no arguments, return a dictionary of all configuration
    variables relevant for the current platform.  Generally this includes
    everything needed to build extensions and install both pure modules and
    extensions.  On Unix, this means every variable defined in Python's
    installed Makefile; on Windows and Mac OS it's a much smaller set.

    With arguments, return a list of values that result from looking up
    each argument in the configuration variable dictionary.
    """
    global _config_vars
    if _config_vars is None:
        func = globals().get("_init_" + os.name)
        if func:
            func()
        else:
            _config_vars = {}

        # Normalized versions of prefix and exec_prefix are handy to have;
        # in fact, these are the standard versions used most places in the
        # Distutils.
        _config_vars['prefix'] = PREFIX
        _config_vars['exec_prefix'] = EXEC_PREFIX

    if args:
        vals = []
        for name in args:
            vals.append(_config_vars.get(name))
        return vals
    else:
        return _config_vars

def get_config_var(name):
    """Return the value of a single variable using the dictionary
    returned by 'get_config_vars()'.  Equivalent to
      get_config_vars().get(name)
    """
    return get_config_vars().get(name)
<commit_msg>Linking just got simpiler on AIX and BeOS (closes SF patch #103679).<commit_after>"""Provide access to Python's configuration information.  The specific names
defined in the module depend heavily on the platform and configuration.

Written by:   Fred L. Drake, Jr.
Email:        <fdrake@acm.org>
Initial date: 17-Dec-1998
"""

__revision__ = "$Id$"

import os
import re
import string
import sys

from errors import DistutilsPlatformError

# These are needed in a couple of spots, so just compute them once.
PREFIX = os.path.normpath(sys.prefix)
EXEC_PREFIX = os.path.normpath(sys.exec_prefix)

# Boolean; if it's true, we're still building Python, so
# we use different (hard-wired) directories.

python_build = 0

def set_python_build():
    """Set the python_build flag to true; this means that we're
    building Python itself.  Only called from the setup.py script
    shipped with Python.
    """

    global python_build
    python_build = 1

def get_python_inc(plat_specific=0, prefix=None):
    """Return the directory containing installed Python header files.

    If 'plat_specific' is false (the default), this is the path to the
    non-platform-specific header files, i.e. Python.h and so on;
    otherwise, this is the path to platform-specific header files
    (namely config.h).

    If 'prefix' is supplied, use it instead of sys.prefix or
    sys.exec_prefix -- i.e., ignore 'plat_specific'.
    """
    if prefix is None:
        prefix = (plat_specific and EXEC_PREFIX or PREFIX)
    if os.name == "posix":
        if python_build:
            return "Include/"
        return os.path.join(prefix, "include", "python" + sys.version[:3])
    elif os.name == "nt":
        return os.path.join(prefix, "Include") # include or Include?
    elif os.name == "mac":
        return os.path.join(prefix, "Include")
    else:
        raise DistutilsPlatformError, \
              ("I don't know where Python installs its C header files " +
               "on platform '%s'") % os.name

def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
    """Return the directory containing the Python library (standard or
    site additions).

    If 'plat_specific' is true, return the directory containing
    platform-specific modules, i.e. any module from a non-pure-Python
    module distribution; otherwise, return the platform-shared library
    directory.  If 'standard_lib' is true, return the directory
    containing standard Python library modules; otherwise, return the
    directory for site-specific modules.

    If 'prefix' is supplied, use it instead of sys.prefix or
    sys.exec_prefix -- i.e., ignore 'plat_specific'.
    """
    if prefix is None:
        prefix = (plat_specific and EXEC_PREFIX or PREFIX)

    if os.name == "posix":
        libpython = os.path.join(prefix,
                                 "lib", "python" + sys.version[:3])
        if standard_lib:
            return libpython
        else:
            return os.path.join(libpython, "site-packages")

    elif os.name == "nt":
        if standard_lib:
            return os.path.join(PREFIX, "Lib")
        else:
            return prefix

    elif os.name == "mac":
        if plat_specific:
            if standard_lib:
                return os.path.join(EXEC_PREFIX, "Mac", "Plugins")
            else:
                raise DistutilsPlatformError, \
                      "OK, where DO site-specific extensions go on the Mac?"
        else:
            if standard_lib:
                return os.path.join(PREFIX, "Lib")
            else:
                raise DistutilsPlatformError, \
                      "OK, where DO site-specific modules go on the Mac?"
    else:
        raise DistutilsPlatformError, \
              ("I don't know where Python installs its library " +
               "on platform '%s'") % os.name

# get_python_lib()

def customize_compiler (compiler):
    """Do any platform-specific customization of the CCompiler instance
    'compiler'.  Mainly needed on Unix, so we can plug in the information
    that varies across Unices and is stored in Python's Makefile.
    """
    if compiler.compiler_type == "unix":
        (cc, opt, ccshared, ldshared, so_ext) = \
            get_config_vars('CC', 'OPT', 'CCSHARED', 'LDSHARED', 'SO')

        cc_cmd = cc + ' ' + opt
        compiler.set_executables(
            preprocessor=cc + " -E",    # not always!
            compiler=cc_cmd,
            compiler_so=cc_cmd + ' ' + ccshared,
            linker_so=ldshared,
            linker_exe=cc)

        compiler.shared_lib_extension = so_ext

def get_config_h_filename():
    """Return full pathname of installed config.h file."""
    if python_build: inc_dir = '.'
    else:            inc_dir = get_python_inc(plat_specific=1)
    return os.path.join(inc_dir, "config.h")

def get_makefile_filename():
    """Return full pathname of installed Makefile from the Python build."""
    if python_build:
        return './Makefile'
    lib_dir = get_python_lib(plat_specific=1, standard_lib=1)
    return os.path.join(lib_dir, "config", "Makefile")

def parse_config_h(fp, g=None):
    """Parse a config.h-style file.

    A dictionary containing name/value pairs is returned.  If an
    optional dictionary is passed in as the second argument, it is
    used instead of a new dictionary.
    """
    if g is None:
        g = {}
    define_rx = re.compile("#define ([A-Z][A-Z0-9_]+) (.*)\n")
    undef_rx = re.compile("/[*] #undef ([A-Z][A-Z0-9_]+) [*]/\n")
    #
    while 1:
        line = fp.readline()
        if not line:
            break
        m = define_rx.match(line)
        if m:
            n, v = m.group(1, 2)
            try: v = string.atoi(v)
            except ValueError: pass
            g[n] = v
        else:
            m = undef_rx.match(line)
            if m:
                g[m.group(1)] = 0
    return g

# Regexes needed for parsing Makefile (and similar syntaxes,
# like old-style Setup files).
_variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")

def parse_makefile(fn, g=None):
    """Parse a Makefile-style file.

    A dictionary containing name/value pairs is returned.  If an
    optional dictionary is passed in as the second argument, it is
    used instead of a new dictionary.

    """
    from distutils.text_file import TextFile
    fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1)

    if g is None:
        g = {}
    done = {}
    notdone = {}

    while 1:
        line = fp.readline()
        if line is None:                # eof
            break
        m = _variable_rx.match(line)
        if m:
            n, v = m.group(1, 2)
            v = string.strip(v)
            if "$" in v:
                notdone[n] = v
            else:
                try: v = string.atoi(v)
                except ValueError: pass
                done[n] = v

    # do variable interpolation here
    while notdone:
        for name in notdone.keys():
            value = notdone[name]
            m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
            if m:
                n = m.group(1)
                if done.has_key(n):
                    after = value[m.end():]
                    value = value[:m.start()] + str(done[n]) + after
                    if "$" in after:
                        notdone[name] = value
                    else:
                        try: value = string.atoi(value)
                        except ValueError:
                            done[name] = string.strip(value)
                        else:
                            done[name] = value
                        del notdone[name]
                elif notdone.has_key(n):
                    # get it on a subsequent round
                    pass
                else:
                    done[n] = ""
                    after = value[m.end():]
                    value = value[:m.start()] + after
                    if "$" in after:
                        notdone[name] = value
                    else:
                        try: value = string.atoi(value)
                        except ValueError:
                            done[name] = string.strip(value)
                        else:
                            done[name] = value
                        del notdone[name]
            else:
                # bogus variable reference; just drop it since we can't deal
                del notdone[name]

    fp.close()

    # save the results in the global dictionary
    g.update(done)
    return g

def expand_makefile_vars(s, vars):
    """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
    'string' according to 'vars' (a dictionary mapping variable names to
    values).  Variables not present in 'vars' are silently expanded to the
    empty string.  The variable values in 'vars' should not contain further
    variable expansions; if 'vars' is the output of 'parse_makefile()',
    you're fine.  Returns a variable-expanded version of 's'.
    """

    # This algorithm does multiple expansion, so if vars['foo'] contains
    # "${bar}", it will expand ${foo} to ${bar}, and then expand
    # ${bar}... and so forth.  This is fine as long as 'vars' comes from
    # 'parse_makefile()', which takes care of such expansions eagerly,
    # according to make's variable expansion semantics.

    while 1:
        m = _findvar1_rx.search(s) or _findvar2_rx.search(s)
        if m:
            name = m.group(1)
            (beg, end) = m.span()
            s = s[0:beg] + vars.get(m.group(1)) + s[end:]
        else:
            break
    return s

_config_vars = None

def _init_posix():
    """Initialize the module as appropriate for POSIX systems."""
    g = {}
    # load the installed Makefile:
    try:
        filename = get_makefile_filename()
        parse_makefile(filename, g)
    except IOError, msg:
        my_msg = "invalid Python installation: unable to open %s" % filename
        if hasattr(msg, "strerror"):
            my_msg = my_msg + " (%s)" % msg.strerror

        raise DistutilsPlatformError, my_msg

    # On AIX, there are wrong paths to the linker scripts in the Makefile
    # -- these paths are relative to the Python source, but when installed
    # the scripts are in another directory.
    if python_build:
	g['LDSHARED'] = g['BLDSHARED']

    global _config_vars
    _config_vars = g

def _init_nt():
    """Initialize the module as appropriate for NT"""
    g = {}
    # set basic install directories
    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)

    # XXX hmmm.. a normal install puts include files here
    g['INCLUDEPY'] = get_python_inc(plat_specific=0)

    g['SO'] = '.pyd'
    g['EXE'] = ".exe"

    global _config_vars
    _config_vars = g

def _init_mac():
    """Initialize the module as appropriate for Macintosh systems"""
    g = {}
    # set basic install directories
    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)

    # XXX hmmm.. a normal install puts include files here
    g['INCLUDEPY'] = get_python_inc(plat_specific=0)

    g['SO'] = '.ppc.slb'

    # XXX are these used anywhere?
    g['install_lib'] = os.path.join(EXEC_PREFIX, "Lib")
    g['install_platlib'] = os.path.join(EXEC_PREFIX, "Mac", "Lib")

    global _config_vars
    _config_vars = g

def get_config_vars(*args):
    """With no arguments, return a dictionary of all configuration
    variables relevant for the current platform.  Generally this includes
    everything needed to build extensions and install both pure modules and
    extensions.  On Unix, this means every variable defined in Python's
    installed Makefile; on Windows and Mac OS it's a much smaller set.

    With arguments, return a list of values that result from looking up
    each argument in the configuration variable dictionary.
    """
    global _config_vars
    if _config_vars is None:
        func = globals().get("_init_" + os.name)
        if func:
            func()
        else:
            _config_vars = {}

        # Normalized versions of prefix and exec_prefix are handy to have;
        # in fact, these are the standard versions used most places in the
        # Distutils.
        _config_vars['prefix'] = PREFIX
        _config_vars['exec_prefix'] = EXEC_PREFIX

    if args:
        vals = []
        for name in args:
            vals.append(_config_vars.get(name))
        return vals
    else:
        return _config_vars

def get_config_var(name):
    """Return the value of a single variable using the dictionary
    returned by 'get_config_vars()'.  Equivalent to
      get_config_vars().get(name)
    """
    return get_config_vars().get(name)
<|endoftext|>


<commit_before>from .utils import get_test_data_path
from featurex.stimuli.video import VideoStim, VideoFrameStim, DerivedVideoStim
from featurex.stimuli.text import ComplexTextStim
from featurex.stimuli.audio import AudioStim
from featurex.stimuli.image import ImageStim
from featurex.extractors import Extractor
from featurex.stimuli import Stim
from featurex.core import Value, Event, Timeline
from featurex.support.download import download_nltk_data
import numpy as np
from os.path import join
import pandas as pd
import pytest

class DummyExtractor(Extractor):

    target = Stim

    def _extract(self, stim):
        return Value(stim, self, {'constant': 1})

class DummyIterableExtractor(Extractor):

    target = Stim

    def _extract(self, stim):

        events = []
        time_bins = np.arange(0., stim.duration, 1.)
        for i, tb in enumerate(time_bins):
            ev = Event(onset=tb, duration=1000)
            ev.add_value(Value(stim, self, {'second': i}))
        return events

@pytest.fixture(scope='module')
def get_nltk():
    download_nltk_data()

@pytest.fixture(scope='module')
def dummy_extractor():
    return DummyExtractor()

@pytest.fixture(scope='module')
def dummy_iter_extractor():
    return DummyIterableExtractor()

def test_image_stim(dummy_iter_extractor):
    filename = join(get_test_data_path(), 'image', 'apple.jpg')
    stim = ImageStim(filename)
    assert stim.data.shape == (288, 420, 3)
    values = stim.extract([dummy_iter_extractor])
    assert isinstance(values, Value)

def test_video_stim():
    ''' Test VideoStim functionality. '''
    filename = join(get_test_data_path(), 'video', 'small.mp4')
    video = VideoStim(filename)
    assert video.fps == 30
    assert video.n_frames == 168
    assert video.width == 560

    # Test frame iterator
    frames = [f for f in video]
    assert len(frames) == 168
    f = frames[100]
    assert isinstance(f, VideoFrameStim)
    assert isinstance(f.onset, float)
    f.data.shape == (320, 560, 3)

def test_derived_video_stim():
    ''' Test DerivedVideoStim functionality. '''
    filename = join(get_test_data_path(), 'video', 'small.mp4')
    video = DerivedVideoStim(filename)
    assert video.fps == 30
    assert video.n_frames == 168
    assert video.width == 560

    # Test frame filters
    video.filter(every=3)
    assert len(video.elements) == video.n_frames / 3
    assert type(next(f for f in video)) == VideoFrameStim
    assert next(f for f in video).duration == 3 * (1 / 30.0)

    # Should refilter from original frames
    video.filter(hertz=15)
    assert len(video.elements) == video.n_frames / 2
    assert type(next(f for f in video)) == VideoFrameStim
    assert next(f for f in video).duration == 1 * (1 / 15.0)

    # Should refilter from original frames
    video.filter(num_frames=5)
    assert len(video.elements) == 5
    assert type(next(f for f in video)) == VideoFrameStim

    # Test filter history
    assert video.history.shape == (4, 3)
    assert np.array_equal(video.history['filter'], ['None', 'every', 'hertz', 'num_frames'])

def test_audio_stim(dummy_iter_extractor):
    audio_dir = join(get_test_data_path(), 'audio')
    stim = AudioStim(join(audio_dir, 'barber.wav'))
    assert round(stim.duration) == 57
    assert stim.sampling_rate == 11025
    tl = stim.extract([dummy_iter_extractor])
    assert isinstance(tl, Timeline)

def test_complex_text_stim():
    text_dir = join(get_test_data_path(), 'text')
    stim = ComplexTextStim(join(text_dir, 'complex_stim_no_header.txt'),
                           columns='ot', default_duration=0.2)
    assert len(stim.elements) == 4
    assert stim.elements[2].onset == 34
    assert stim.elements[2].duration == 0.2
    stim = ComplexTextStim(join(text_dir, 'complex_stim_with_header.txt'))
    assert len(stim.elements) == 4
    assert stim.elements[2].duration == 0.1

def test_complex_stim_from_text():
    textfile = join(get_test_data_path(), 'text', 'scandal.txt')
    text = open(textfile).read().strip()
    stim = ComplexTextStim.from_text(text)
    target = ['To', 'Sherlock', 'Holmes']
    assert [w.text for w in stim.elements[:3]] == target
    assert len(stim.elements) == 231
    stim = ComplexTextStim.from_text(text, unit='sent')
    # Custom tokenizer
    stim = ComplexTextStim.from_text(text, tokenizer='(\w+)')
    assert len(stim.elements) == 209

def test_complex_stim_from_srt():
    srtfile = join(get_test_data_path(), 'text', 'wonderful.srt')
    textfile = join(get_test_data_path(), 'text', 'wonderful.txt')
    df = pd.read_csv(textfile, sep='\t')
    target = df["text"].tolist()
    srt_stim = ComplexTextStim(srtfile)
    texts = [sent.text for sent in srt_stim.elements]
    assert texts == target
<commit_msg>modified test to get around lack of cv2<commit_after>from .utils import get_test_data_path
from featurex.stimuli.video import VideoStim, VideoFrameStim, DerivedVideoStim
from featurex.stimuli.text import ComplexTextStim
from featurex.stimuli.audio import AudioStim
from featurex.stimuli.image import ImageStim
from featurex.extractors import Extractor
from featurex.stimuli import Stim
from featurex.core import Value, Event, Timeline
from featurex.support.download import download_nltk_data
import numpy as np
from os.path import join
import pandas as pd
import pytest

class DummyExtractor(Extractor):

    target = Stim

    def _extract(self, stim):
        return Value(stim, self, {'constant': 1})

class DummyIterableExtractor(Extractor):

    target = Stim

    def _extract(self, stim):

        events = []
        time_bins = np.arange(0., stim.duration, 1.)
        for i, tb in enumerate(time_bins):
            ev = Event(onset=tb, duration=1000)
            ev.add_value(Value(stim, self, {'second': i}))
        return events

@pytest.fixture(scope='module')
def get_nltk():
    download_nltk_data()

@pytest.fixture(scope='module')
def dummy_extractor():
    return DummyExtractor()

@pytest.fixture(scope='module')
def dummy_iter_extractor():
    return DummyIterableExtractor()

def test_image_stim(dummy_iter_extractor):
    filename = join(get_test_data_path(), 'image', 'apple.jpg')
    stim = ImageStim(filename)
    assert stim.data.shape == (288, 420, 3)
    values = stim.extract([dummy_iter_extractor])
    assert isinstance(values, Value)

def test_video_stim():
    ''' Test VideoStim functionality. '''
    filename = join(get_test_data_path(), 'video', 'small.mp4')
    video = VideoStim(filename)
    assert video.fps == 30
    assert video.n_frames == 168
    assert video.width == 560

    # Test frame iterator
    frames = [f for f in video]
    assert len(frames) == 168
    f = frames[100]
    assert isinstance(f, VideoFrameStim)
    assert isinstance(f.onset, float)
    f.data.shape == (320, 560, 3)

def test_derived_video_stim():
    ''' Test DerivedVideoStim functionality. '''
    filename = join(get_test_data_path(), 'video', 'small.mp4')
    video = DerivedVideoStim(filename)
    assert video.fps == 30
    assert video.n_frames == 168
    assert video.width == 560

    # Test frame filters
    video.filter(every=3)
    assert len(video.elements) == video.n_frames / 3
    assert type(next(f for f in video)) == VideoFrameStim
    assert next(f for f in video).duration == 3 * (1 / 30.0)

    # Should refilter from original frames
    video.filter(hertz=15)
    assert len(video.elements) == video.n_frames / 2
    assert type(next(f for f in video)) == VideoFrameStim
    assert next(f for f in video).duration == 1 * (1 / 15.0)

    # Test filter history
    assert video.history.shape == (3, 3)
    assert np.array_equal(video.history['filter'], ['None', 'every', 'hertz'])

def test_derived_video_stim_cv2():
    pytest.importorskip('cv2')
    filename = join(get_test_data_path(), 'video', 'small.mp4')
    video = DerivedVideoStim(filename)

    video.filter(num_frames=5)
    assert len(video.elements) == 5
    assert type(next(f for f in video)) == VideoFrameStim

def test_audio_stim(dummy_iter_extractor):
    audio_dir = join(get_test_data_path(), 'audio')
    stim = AudioStim(join(audio_dir, 'barber.wav'))
    assert round(stim.duration) == 57
    assert stim.sampling_rate == 11025
    tl = stim.extract([dummy_iter_extractor])
    assert isinstance(tl, Timeline)

def test_complex_text_stim():
    text_dir = join(get_test_data_path(), 'text')
    stim = ComplexTextStim(join(text_dir, 'complex_stim_no_header.txt'),
                           columns='ot', default_duration=0.2)
    assert len(stim.elements) == 4
    assert stim.elements[2].onset == 34
    assert stim.elements[2].duration == 0.2
    stim = ComplexTextStim(join(text_dir, 'complex_stim_with_header.txt'))
    assert len(stim.elements) == 4
    assert stim.elements[2].duration == 0.1

def test_complex_stim_from_text():
    textfile = join(get_test_data_path(), 'text', 'scandal.txt')
    text = open(textfile).read().strip()
    stim = ComplexTextStim.from_text(text)
    target = ['To', 'Sherlock', 'Holmes']
    assert [w.text for w in stim.elements[:3]] == target
    assert len(stim.elements) == 231
    stim = ComplexTextStim.from_text(text, unit='sent')
    # Custom tokenizer
    stim = ComplexTextStim.from_text(text, tokenizer='(\w+)')
    assert len(stim.elements) == 209

def test_complex_stim_from_srt():
    srtfile = join(get_test_data_path(), 'text', 'wonderful.srt')
    textfile = join(get_test_data_path(), 'text', 'wonderful.txt')
    df = pd.read_csv(textfile, sep='\t')
    target = df["text"].tolist()
    srt_stim = ComplexTextStim(srtfile)
    texts = [sent.text for sent in srt_stim.elements]
    assert texts == target
<|endoftext|>


<commit_before>from feedly.feeds.base import UserBaseFeed
from feedly.tasks import fanout_operation, follow_many, unfollow_many
from feedly.utils import chunks
from feedly.utils.timing import timer
import logging

logger = logging.getLogger(__name__)

def add_operation(feed, activities, trim=True, batch_interface=None):
    '''
    Add the activities to the feed
    functions used in tasks need to be at the main level of the module
    '''
    t = timer()
    msg_format = 'running %s.add_many operation for %s activities batch interface %s and trim %s'
    logger.debug(msg_format, feed, len(activities), batch_interface, trim)
    feed.add_many(activities, batch_interface=batch_interface, trim=trim)
    logger.debug('add many operation took %s seconds', t.next())

def remove_operation(feed, activities, trim=True, batch_interface=None):
    '''
    Remove the activities from the feed
    functions used in tasks need to be at the main level of the module
    '''
    t = timer()
    msg_format = 'running %s.remove_many operation for %s activities batch interface %s'
    logger.debug(msg_format, feed, len(activities), batch_interface)
    feed.remove_many(activities, trim=trim, batch_interface=batch_interface)
    logger.debug('remove many operation took %s seconds', t.next())

class BaseFeedly(object):
    pass

class Feedly(BaseFeedly):

    '''
    The Feedly class handles the fanout from a user's activity
    to all their follower's feeds

    .. note::

        Fanout is the process by which you get all the users which follow the user
        and spawn many asynchronous tasks which all push a bit of data
        to the feeds of these followers.

    See the :class:`.PinFeedly` class for an example implementation.

    You will definitely need to implement:

    - feed_classes
    - user_feed_class
    - get_user_follower_ids

    '''
    #: a dictionary with the feeds to fanout to
    #: for example feed_classes = dict(normal=PinFeed, aggregated=AggregatedPinFeed)
    feed_classes = {}
    #: the user feed class (it stores the latest activity by one user)
    user_feed_class = UserBaseFeed

    #: the number of activities which enter your feed when you follow someone
    follow_activity_limit = 5000
    #: the number of users which are handled in one asynchronous task
    #: when doing the fanout
    fanout_chunk_size = 100

    def __init__(self):
        '''
        This manager is built specifically for the love feed

        :feed_class the feed
        :user_feed_class where user activity gets stored (defaults to same as :feed_class param)

        '''
        pass

    def get_user_follower_ids(self, user_id):
        '''
        returns the list of ids of user_id followers
        this depends on how you store followers/friends etc
        and is not implemented in feedly
        '''
        raise NotImplementedError()

    def get_feeds(self, user_id):
        '''
        get the feed that contains the sum of all activity
        from feeds :user_id is subscribed to

        :returns dict: a dictionary with the feeds we're pushing to
        '''
        return dict([(k, feed(user_id)) for k, feed in self.feed_classes.items()])

    def get_user_feed(self, user_id):
        '''
        feed where activity from :user_id is saved

        :param user_id: the id of the user
        '''
        return self.user_feed_class(user_id)

    def add_user_activity(self, user_id, activity):
        '''
        Store the new activity and then fanout to user followers

        This function will
        - store the activity in the activity storage
        - store it in the user feed (list of activities for one user)
        - fanout for all feed_classes

        :param user_id: the id of the user
        :param activity: the activity which to add
        '''
        self.user_feed_class.insert_activity(activity)
        user_feed = self.get_user_feed(user_id)
        user_feed.add(activity)
        self._start_fanout(
            self.feed_classes,
            user_id,
            add_operation,
            activities=[activity],
            # Enable trimming to prevent infinite data storage :)
            trim=True
        )
        return

    def update_user_activities(self, activities):
        self.user_feed_class.insert_activities(activities)

    def update_user_activity(self, activity):
        self.user_feed_class.insert_activities([activity])

    def remove_user_activity(self, user_id, activity):
        '''
        Remove the activity and then fanout to user followers

        :param user_id: the id of the user
        :param activity: the activity which to add
        '''
        # The we can only remove this after the other tasks completed
        # TODO: clean up the activity after the fanout
        # self.user_feed_class.remove_activity(activity)

        user_feed = self.get_user_feed(user_id)
        user_feed.remove(activity)
        self._start_fanout(
            self.feed_classes,
            user_id,
            remove_operation,
            activities=[activity],
            # Enable trimming to prevent infinite data storage :)
            trim=True
        )
        return

    def follow_feed(self, feed, activities):
        '''
        copies source_feed entries into feed
        it will only copy follow_activity_limit activities

        :param feed: the feed to copy to
        :param activities: the activities to copy into the feed
        '''
        if activities:
            return feed.add_many(activities)

    def unfollow_feed(self, feed, source_feed):
        '''
        removes entries originating from the source feed form the feed class
        this will remove all activities, so this could take a wh
        :param feed: the feed to copy to
        :param source_feed: the feed with a list of activities to remove
        '''
        activities = source_feed[:]  # need to slice
        if activities:
            return feed.remove_many(activities)

    def follow_user(self, user_id, target_user_id, async=True):
        '''
        user_id starts following target_user_id

        :param user_id: the user which is doing the following/unfollowing
        :target_user_id: the user which is being unfollowed
        '''
        source_feed = self.get_user_feed(target_user_id)
        # fetch the activities only once
        activities = source_feed[:self.follow_activity_limit]
        for user_feed in self.get_feeds(user_id).values():
            self.follow_feed(user_feed, activities)

    def unfollow_user(self, user_id, target_user_id, async=True):
        '''
        unfollows the user

        :param user_id: the user which is doing the following/unfollowing
        :target_user_id: the user which is being unfollowed
        '''
        if async:
            unfollow_many_fn = unfollow_many.delay
        else:
            unfollow_many_fn = unfollow_many

        unfollow_many_fn(self, user_id, [target_user_id])

    def follow_many_users(self, user_id, target_ids, async=True):
        '''
        copies feeds for target_ids in user_id

        :param user_id: the user which is doing the following/unfollowing
        :param target_ids: the user to follow
        :param async: controls if the operation should be done via celery
        '''
        if async:
            follow_many_fn = follow_many.delay
        else:
            follow_many_fn = follow_many

        follow_many_fn(
            self,
            user_id,
            target_ids,
            self.follow_activity_limit
        )

    def _start_fanout(self, feed_classes, user_id, operation, follower_ids=None, *args, **kwargs):
        '''
        Start fanout applies the given operation to the feeds of the users
        followers

        It takes the following ids and distributes them per fanout_chunk_size
        into smaller tasks

        :param feed_classes: the feed classes to run the operation on
        :param user_id: the user id to run the operation for
        :param operation: the operation function applied to all follower feeds
        :param follower_ids: (optionally) specify the list of followers
        :param args: args passed to the operation
        :param kwargs: kwargs passed to the operation
        '''
        user_ids = follower_ids or self.get_user_follower_ids(user_id=user_id)
        user_ids_chunks = list(chunks(user_ids, self.fanout_chunk_size))
        msg_format = 'spawning %s subtasks for %s user ids in chunks of %s users'
        logger.info(msg_format, len(user_ids_chunks),
                    len(user_ids), self.fanout_chunk_size)

        # now actually create the tasks
        subs = []
        for ids_chunk in user_ids_chunks:
            for name, feed_class in feed_classes.items():
                feed_class_dict = dict()
                feed_class_dict[name] = feed_class
                task_args = [
                    self, feed_class_dict, ids_chunk, operation] + list(args)
                sub = fanout_operation.apply_async(
                    args=task_args,
                    kwargs=kwargs
                )
                subs.append(sub)
        return subs

    def _fanout_task(self, user_ids, feed_classes, operation, *args, **kwargs):
        '''
        This functionality is called from within feedly.tasks.fanout_operation

        :param user_ids: the list of user ids which feeds we should apply the
        operation against
        :param feed_classes: the feed classes to change
        :param operation: the function to run on all the feeds
        :param args: args to pass to the operation
        :param kwargs: kwargs to pass to the operation
        '''
        separator = '===' * 10
        logger.info('%s starting fanout %s', separator, separator)
        for name, feed_class in feed_classes.items():
            batch_context_manager = feed_class.get_timeline_batch_interface()
            msg_format = 'starting batch interface for feed %s, fanning out to %s users'
            with batch_context_manager as batch_interface:
                logger.info(msg_format, name, len(user_ids))
                kwargs['batch_interface'] = batch_interface
                for user_id in user_ids:
                    logger.debug('now handling fanout to user %s', user_id)
                    feed = feed_class(user_id)
                    operation(feed, *args, **kwargs)
            logger.info('finished fanout for feed %s', name)

    def batch_import(self, user_id, activities, chunk_size=500):
        '''
        Batch import all of the users activities and distributes
        them to the users followers

        **Example**::

            activities = [long list of activities]
            feedly.batch_import(13, activities, 500)

        :param user_id: the user who created the activities
        :param activities: a list of activities from this user
        :param chunk_size: per how many activities to run the batch operations

        '''
        activities = list(activities)
        # skip empty lists
        if not activities:
            return
        logger.info('running batch import for user %s', user_id)
        follower_ids = self.get_user_follower_ids(user_id=user_id)
        logger.info('retrieved %s follower ids', len(follower_ids))
        user_feed = self.get_user_feed(user_id)
        if activities[0].actor_id != user_id:
            raise ValueError('Send activities for only one user please')

        activity_chunks = list(chunks(activities, chunk_size))
        logger.info('processing %s items in %s chunks of %s',
                    len(activities), len(activity_chunks), chunk_size)

        for index, activity_chunk in enumerate(activity_chunks):
            # first insert into the global activity storage
            self.user_feed_class.insert_activities(activity_chunk)
            logger.info(
                'inserted chunk %s (length %s) into the global activity store', index, len(activity_chunk))
            # next add the activities to the users personal timeline
            user_feed.add_many(activity_chunk)
            logger.info(
                'inserted chunk %s (length %s) into the user feed', index, len(activity_chunk))
            # now start a big fanout task
            logger.info('starting task fanout for chunk %s', index)
            self._start_fanout(
                self.feed_classes,
                user_id,
                add_operation,
                follower_ids=follower_ids,
                activities=activity_chunk,
                # disable trimming during the import as its really really slow
                trim=False
            )

    def flush(self):
        '''
        Flushes all the feeds
        '''
        # TODO why do we have this method?
        for name, feed_class in self.feed_classes.items():
            feed_class.flush()
        self.user_feed_class.flush()
<commit_msg>ability to turn of the fanout for batch import<commit_after>from feedly.feeds.base import UserBaseFeed
from feedly.tasks import fanout_operation, follow_many, unfollow_many
from feedly.utils import chunks
from feedly.utils.timing import timer
import logging

logger = logging.getLogger(__name__)

def add_operation(feed, activities, trim=True, batch_interface=None):
    '''
    Add the activities to the feed
    functions used in tasks need to be at the main level of the module
    '''
    t = timer()
    msg_format = 'running %s.add_many operation for %s activities batch interface %s and trim %s'
    logger.debug(msg_format, feed, len(activities), batch_interface, trim)
    feed.add_many(activities, batch_interface=batch_interface, trim=trim)
    logger.debug('add many operation took %s seconds', t.next())

def remove_operation(feed, activities, trim=True, batch_interface=None):
    '''
    Remove the activities from the feed
    functions used in tasks need to be at the main level of the module
    '''
    t = timer()
    msg_format = 'running %s.remove_many operation for %s activities batch interface %s'
    logger.debug(msg_format, feed, len(activities), batch_interface)
    feed.remove_many(activities, trim=trim, batch_interface=batch_interface)
    logger.debug('remove many operation took %s seconds', t.next())

class BaseFeedly(object):
    pass

class Feedly(BaseFeedly):

    '''
    The Feedly class handles the fanout from a user's activity
    to all their follower's feeds

    .. note::

        Fanout is the process by which you get all the users which follow the user
        and spawn many asynchronous tasks which all push a bit of data
        to the feeds of these followers.

    See the :class:`.PinFeedly` class for an example implementation.

    You will definitely need to implement:

    - feed_classes
    - user_feed_class
    - get_user_follower_ids

    '''
    #: a dictionary with the feeds to fanout to
    #: for example feed_classes = dict(normal=PinFeed, aggregated=AggregatedPinFeed)
    feed_classes = {}
    #: the user feed class (it stores the latest activity by one user)
    user_feed_class = UserBaseFeed

    #: the number of activities which enter your feed when you follow someone
    follow_activity_limit = 5000
    #: the number of users which are handled in one asynchronous task
    #: when doing the fanout
    fanout_chunk_size = 100

    def __init__(self):
        '''
        This manager is built specifically for the love feed

        :feed_class the feed
        :user_feed_class where user activity gets stored (defaults to same as :feed_class param)

        '''
        pass

    def get_user_follower_ids(self, user_id):
        '''
        returns the list of ids of user_id followers
        this depends on how you store followers/friends etc
        and is not implemented in feedly
        '''
        raise NotImplementedError()

    def get_feeds(self, user_id):
        '''
        get the feed that contains the sum of all activity
        from feeds :user_id is subscribed to

        :returns dict: a dictionary with the feeds we're pushing to
        '''
        return dict([(k, feed(user_id)) for k, feed in self.feed_classes.items()])

    def get_user_feed(self, user_id):
        '''
        feed where activity from :user_id is saved

        :param user_id: the id of the user
        '''
        return self.user_feed_class(user_id)

    def add_user_activity(self, user_id, activity):
        '''
        Store the new activity and then fanout to user followers

        This function will
        - store the activity in the activity storage
        - store it in the user feed (list of activities for one user)
        - fanout for all feed_classes

        :param user_id: the id of the user
        :param activity: the activity which to add
        '''
        self.user_feed_class.insert_activity(activity)
        user_feed = self.get_user_feed(user_id)
        user_feed.add(activity)
        self._start_fanout(
            self.feed_classes,
            user_id,
            add_operation,
            activities=[activity],
            # Enable trimming to prevent infinite data storage :)
            trim=True
        )
        return

    def update_user_activities(self, activities):
        self.user_feed_class.insert_activities(activities)

    def update_user_activity(self, activity):
        self.user_feed_class.insert_activities([activity])

    def remove_user_activity(self, user_id, activity):
        '''
        Remove the activity and then fanout to user followers

        :param user_id: the id of the user
        :param activity: the activity which to add
        '''
        # The we can only remove this after the other tasks completed
        # TODO: clean up the activity after the fanout
        # self.user_feed_class.remove_activity(activity)

        user_feed = self.get_user_feed(user_id)
        user_feed.remove(activity)
        self._start_fanout(
            self.feed_classes,
            user_id,
            remove_operation,
            activities=[activity],
            # Enable trimming to prevent infinite data storage :)
            trim=True
        )
        return

    def follow_feed(self, feed, activities):
        '''
        copies source_feed entries into feed
        it will only copy follow_activity_limit activities

        :param feed: the feed to copy to
        :param activities: the activities to copy into the feed
        '''
        if activities:
            return feed.add_many(activities)

    def unfollow_feed(self, feed, source_feed):
        '''
        removes entries originating from the source feed form the feed class
        this will remove all activities, so this could take a wh
        :param feed: the feed to copy to
        :param source_feed: the feed with a list of activities to remove
        '''
        activities = source_feed[:]  # need to slice
        if activities:
            return feed.remove_many(activities)

    def follow_user(self, user_id, target_user_id, async=True):
        '''
        user_id starts following target_user_id

        :param user_id: the user which is doing the following/unfollowing
        :target_user_id: the user which is being unfollowed
        '''
        source_feed = self.get_user_feed(target_user_id)
        # fetch the activities only once
        activities = source_feed[:self.follow_activity_limit]
        for user_feed in self.get_feeds(user_id).values():
            self.follow_feed(user_feed, activities)

    def unfollow_user(self, user_id, target_user_id, async=True):
        '''
        unfollows the user

        :param user_id: the user which is doing the following/unfollowing
        :target_user_id: the user which is being unfollowed
        '''
        if async:
            unfollow_many_fn = unfollow_many.delay
        else:
            unfollow_many_fn = unfollow_many

        unfollow_many_fn(self, user_id, [target_user_id])

    def follow_many_users(self, user_id, target_ids, async=True):
        '''
        copies feeds for target_ids in user_id

        :param user_id: the user which is doing the following/unfollowing
        :param target_ids: the user to follow
        :param async: controls if the operation should be done via celery
        '''
        if async:
            follow_many_fn = follow_many.delay
        else:
            follow_many_fn = follow_many

        follow_many_fn(
            self,
            user_id,
            target_ids,
            self.follow_activity_limit
        )

    def _start_fanout(self, feed_classes, user_id, operation, follower_ids=None, *args, **kwargs):
        '''
        Start fanout applies the given operation to the feeds of the users
        followers

        It takes the following ids and distributes them per fanout_chunk_size
        into smaller tasks

        :param feed_classes: the feed classes to run the operation on
        :param user_id: the user id to run the operation for
        :param operation: the operation function applied to all follower feeds
        :param follower_ids: (optionally) specify the list of followers
        :param args: args passed to the operation
        :param kwargs: kwargs passed to the operation
        '''
        user_ids = follower_ids or self.get_user_follower_ids(user_id=user_id)
        user_ids_chunks = list(chunks(user_ids, self.fanout_chunk_size))
        msg_format = 'spawning %s subtasks for %s user ids in chunks of %s users'
        logger.info(msg_format, len(user_ids_chunks),
                    len(user_ids), self.fanout_chunk_size)

        # now actually create the tasks
        subs = []
        for ids_chunk in user_ids_chunks:
            for name, feed_class in feed_classes.items():
                feed_class_dict = dict()
                feed_class_dict[name] = feed_class
                task_args = [
                    self, feed_class_dict, ids_chunk, operation] + list(args)
                sub = fanout_operation.apply_async(
                    args=task_args,
                    kwargs=kwargs
                )
                subs.append(sub)
        return subs

    def _fanout_task(self, user_ids, feed_classes, operation, *args, **kwargs):
        '''
        This functionality is called from within feedly.tasks.fanout_operation

        :param user_ids: the list of user ids which feeds we should apply the
        operation against
        :param feed_classes: the feed classes to change
        :param operation: the function to run on all the feeds
        :param args: args to pass to the operation
        :param kwargs: kwargs to pass to the operation
        '''
        separator = '===' * 10
        logger.info('%s starting fanout %s', separator, separator)
        for name, feed_class in feed_classes.items():
            batch_context_manager = feed_class.get_timeline_batch_interface()
            msg_format = 'starting batch interface for feed %s, fanning out to %s users'
            with batch_context_manager as batch_interface:
                logger.info(msg_format, name, len(user_ids))
                kwargs['batch_interface'] = batch_interface
                for user_id in user_ids:
                    logger.debug('now handling fanout to user %s', user_id)
                    feed = feed_class(user_id)
                    operation(feed, *args, **kwargs)
            logger.info('finished fanout for feed %s', name)

    def batch_import(self, user_id, activities, fanout=True, chunk_size=500):
        '''
        Batch import all of the users activities and distributes
        them to the users followers

        **Example**::

            activities = [long list of activities]
            feedly.batch_import(13, activities, 500)

        :param user_id: the user who created the activities
        :param activities: a list of activities from this user
        :param chunk_size: per how many activities to run the batch operations

        '''
        activities = list(activities)
        # skip empty lists
        if not activities:
            return
        logger.info('running batch import for user %s', user_id)
        follower_ids = self.get_user_follower_ids(user_id=user_id)
        logger.info('retrieved %s follower ids', len(follower_ids))
        user_feed = self.get_user_feed(user_id)
        if activities[0].actor_id != user_id:
            raise ValueError('Send activities for only one user please')

        activity_chunks = list(chunks(activities, chunk_size))
        logger.info('processing %s items in %s chunks of %s',
                    len(activities), len(activity_chunks), chunk_size)

        for index, activity_chunk in enumerate(activity_chunks):
            # first insert into the global activity storage
            self.user_feed_class.insert_activities(activity_chunk)
            logger.info(
                'inserted chunk %s (length %s) into the global activity store', index, len(activity_chunk))
            # next add the activities to the users personal timeline
            user_feed.add_many(activity_chunk)
            logger.info(
                'inserted chunk %s (length %s) into the user feed', index, len(activity_chunk))
            # now start a big fanout task
            if fanout:
                logger.info('starting task fanout for chunk %s', index)
                self._start_fanout(
                    self.feed_classes,
                    user_id,
                    add_operation,
                    follower_ids=follower_ids,
                    activities=activity_chunk,
                    # disable trimming during the import as its really really slow
                    trim=False
                )

    def flush(self):
        '''
        Flushes all the feeds
        '''
        # TODO why do we have this method?
        for name, feed_class in self.feed_classes.items():
            feed_class.flush()
        self.user_feed_class.flush()
<|endoftext|>


<commit_before>from feedly.utils import chunks
from feedly.tasks import fanout_operation
from feedly.tasks import follow_many
from feedly.feeds.base import UserBaseFeed

def add_operation(feed, activities, batch_interface):
    '''
    Add the activities to the feed
    functions used in tasks need to be at the main level of the module
    '''
    feed.add_many(activities, batch_interface=batch_interface)

def remove_operation(feed, activities, batch_interface):
    '''
    Remove the activities from the feed
    functions used in tasks need to be at the main level of the module
    '''
    feed.remove_many(activities, batch_interface=batch_interface)

class BaseFeedly(object):
    pass

class Feedly(BaseFeedly):
    '''
    The Feedly class handles the fanout from a user's activity
    to all their follower's feeds

    .. note::

        Fanout is the process by which you get all the users which follow the user
        and spawn many asynchronous tasks which all push a bit of data
        to the feeds of these followers.

    See the :class:`.PinFeedly` class for an example implementation.

    You will definitely need to implement:

    - feed_classes
    - user_feed_class
    - get_user_follower_ids

    '''
    #: a dictionary with the feeds to fanout to
    #: for example feed_classes = dict(normal=PinFeed, aggregated=AggregatedPinFeed)
    feed_classes = {}
    #: the user feed class (it stores the latest activity by one user)
    user_feed_class = UserBaseFeed

    #: the number of activities which enter your feed when you follow someone
    follow_activity_limit = 5000
    #: the number of users which are handled in one asynchronous task
    #: when doing the fanout
    fanout_chunk_size = 1000

    def __init__(self):
        '''
        This manager is built specifically for the love feed

        :feed_class the feed
        :user_feed_class where user activity gets stored (defaults to same as :feed_class param)

        '''
        pass

    def get_user_follower_ids(self, user_id):
        '''
        returns the list of ids of user_id followers
        this depends on how you store followers/friends etc
        and is not implemented in feedly
        '''
        raise NotImplementedError()

    def get_feeds(self, user_id):
        '''
        get the feed that contains the sum of all activity
        from feeds :user_id is subscribed to

        :returns dict: a dictionary with the feeds we're pushing to
        '''
        return dict([(k, feed(user_id)) for k, feed in self.feed_classes.items()])

    def get_user_feed(self, user_id):
        '''
        feed where activity from :user_id is saved

        :param user_id: the id of the user
        '''
        return self.user_feed_class(user_id)

    def add_user_activity(self, user_id, activity):
        '''
        Store the new activity and then fanout to user followers

        This function will
        - store the activity in the activity storage
        - store it in the user feed (list of activities for one user)
        - fanout for all feed_classes

        :param user_id: the id of the user
        :param activity: the activity which to add
        '''
        self.user_feed_class.insert_activity(activity)
        user_feed = self.get_user_feed(user_id)
        user_feed.add(activity)
        self._fanout(
            self.feed_classes,
            user_id,
            add_operation,
            activities=[activity]
        )
        return

    def remove_user_activity(self, user_id, activity):
        '''
        Remove the activity and then fanout to user followers

        :param user_id: the id of the user
        :param activity: the activity which to add
        '''
        self.user_feed_class.remove_activity(activity)

        user_feed = self.get_user_feed(user_id)
        user_feed.remove(activity)
        self._fanout(
            self.feed_classes,
            user_id,
            remove_operation,
            activities=[activity]
        )
        return

    def follow_feed(self, feed, source_feed):
        '''
        copies source_feed entries into feed
        it will only copy follow_activity_limit activities

        :param feed: the feed to copy to
        :param source_feed: the feed to copy from
        '''
        activities = source_feed[:self.follow_activity_limit]
        return feed.add_many(activities)

    def unfollow_feed(self, feed, source_feed):
        '''
        removes entries originating from the source feed form the feed class
        this will remove all activities, so this could take a while

        :param feed: the feed to copy to
        :param source_feed: the feed with a list of activities to remove
        '''
        activities = source_feed[:]  # need to slice
        return feed.remove_many(activities)

    def follow_user(self, user_id, target_user_id):
        '''
        user_id starts following target_user_id

        :param user_id: the user which is doing the following/unfollowing
        :target_user_id: the user which is being unfollowed
        '''
        source_feed = self.get_user_feed(target_user_id)
        for feed_name, user_feed in self.get_feeds(user_id).items():
            self.follow_feed(user_feed, source_feed)

    def unfollow_user(self, user_id, target_user_id):
        '''
        unfollows the user

        :param user_id: the user which is doing the following/unfollowing
        :target_user_id: the user which is being unfollowed
        '''
        source_feed = self.get_user_feed(target_user_id)
        for feed in self.get_feeds(user_id).values():
            self.unfollow_feed(feed, source_feed)

    def follow_many_users(self, user_id, target_ids, async=True):
        '''
        copies feeds for target_ids in user_id

        :param user_id: the user which is doing the following/unfollowing
        :param target_ids: the user to follow
        :param async: controls if the operation should be done via celery
        '''
        if async:
            follow_many_fn = follow_many.delay
        else:
            follow_many_fn = follow_many

        follow_many_fn(
            feeds=self.get_feeds(user_id).values(),
            target_feeds=map(self.get_user_feed, target_ids),
            follow_limit=self.follow_activity_limit
        )

    def _fanout(self, feed_classes, user_id, operation, *args, **kwargs):
        '''
        Generic functionality for running an operation on all of your
        follower's feeds

        It takes the following ids and distributes them per fanout_chunk_size
        '''
        user_ids = self.get_user_follower_ids(user_id)
        user_ids_chunks = chunks(user_ids, self.fanout_chunk_size)
        for ids_chunk in user_ids_chunks:
            fanout_operation(
                self, feed_classes, ids_chunk, operation, *args, **kwargs
            )

    def _fanout_task(self, user_ids, feed_classes, operation, *args, **kwargs):
        '''
        This bit of the fan-out is normally called via an Async task
        this shouldnt do any db queries whatsoever
        '''
        for name, feed_class in feed_classes.items():
            with feed_class.get_timeline_batch_interface() as batch_interface:
                kwargs['batch_interface'] = batch_interface
                for user_id in user_ids:
                    feed = feed_class(user_id)
                    operation(feed, *args, **kwargs)

    def flush(self):
        '''
        Flushes all the feeds
        '''
        #TODO why do we have this method?
        for name, feed_class in self.feed_classes.items():
            feed_class.flush()
        self.user_feed_class.flush()
<commit_msg>better signature<commit_after>from feedly.utils import chunks
from feedly.tasks import fanout_operation
from feedly.tasks import follow_many
from feedly.feeds.base import UserBaseFeed

def add_operation(feed, activities, batch_interface):
    '''
    Add the activities to the feed
    functions used in tasks need to be at the main level of the module
    '''
    feed.add_many(activities, batch_interface=batch_interface)

def remove_operation(feed, activities, batch_interface):
    '''
    Remove the activities from the feed
    functions used in tasks need to be at the main level of the module
    '''
    feed.remove_many(activities, batch_interface=batch_interface)

class BaseFeedly(object):
    pass

class Feedly(BaseFeedly):
    '''
    The Feedly class handles the fanout from a user's activity
    to all their follower's feeds

    .. note::

        Fanout is the process by which you get all the users which follow the user
        and spawn many asynchronous tasks which all push a bit of data
        to the feeds of these followers.

    See the :class:`.PinFeedly` class for an example implementation.

    You will definitely need to implement:

    - feed_classes
    - user_feed_class
    - get_user_follower_ids

    '''
    #: a dictionary with the feeds to fanout to
    #: for example feed_classes = dict(normal=PinFeed, aggregated=AggregatedPinFeed)
    feed_classes = {}
    #: the user feed class (it stores the latest activity by one user)
    user_feed_class = UserBaseFeed

    #: the number of activities which enter your feed when you follow someone
    follow_activity_limit = 5000
    #: the number of users which are handled in one asynchronous task
    #: when doing the fanout
    fanout_chunk_size = 1000

    def __init__(self):
        '''
        This manager is built specifically for the love feed

        :feed_class the feed
        :user_feed_class where user activity gets stored (defaults to same as :feed_class param)

        '''
        pass

    def get_user_follower_ids(self, user_id):
        '''
        returns the list of ids of user_id followers
        this depends on how you store followers/friends etc
        and is not implemented in feedly
        '''
        raise NotImplementedError()

    def get_feeds(self, user_id):
        '''
        get the feed that contains the sum of all activity
        from feeds :user_id is subscribed to

        :returns dict: a dictionary with the feeds we're pushing to
        '''
        return dict([(k, feed(user_id)) for k, feed in self.feed_classes.items()])

    def get_user_feed(self, user_id):
        '''
        feed where activity from :user_id is saved

        :param user_id: the id of the user
        '''
        return self.user_feed_class(user_id)

    def add_user_activity(self, user_id, activity):
        '''
        Store the new activity and then fanout to user followers

        This function will
        - store the activity in the activity storage
        - store it in the user feed (list of activities for one user)
        - fanout for all feed_classes

        :param user_id: the id of the user
        :param activity: the activity which to add
        '''
        self.user_feed_class.insert_activity(activity)
        user_feed = self.get_user_feed(user_id)
        user_feed.add(activity)
        self._fanout(
            self.feed_classes,
            user_id,
            add_operation,
            activities=[activity]
        )
        return

    def remove_user_activity(self, user_id, activity):
        '''
        Remove the activity and then fanout to user followers

        :param user_id: the id of the user
        :param activity: the activity which to add
        '''
        self.user_feed_class.remove_activity(activity)

        user_feed = self.get_user_feed(user_id)
        user_feed.remove(activity)
        self._fanout(
            self.feed_classes,
            user_id,
            remove_operation,
            activities=[activity]
        )
        return

    def follow_feed(self, feed, source_feed):
        '''
        copies source_feed entries into feed
        it will only copy follow_activity_limit activities

        :param feed: the feed to copy to
        :param source_feed: the feed to copy from
        '''
        activities = source_feed[:self.follow_activity_limit]
        return feed.add_many(activities)

    def unfollow_feed(self, feed, source_feed):
        '''
        removes entries originating from the source feed form the feed class
        this will remove all activities, so this could take a while

        :param feed: the feed to copy to
        :param source_feed: the feed with a list of activities to remove
        '''
        activities = source_feed[:]  # need to slice
        return feed.remove_many(activities)

    def follow_user(self, user_id, target_user_id):
        '''
        user_id starts following target_user_id

        :param user_id: the user which is doing the following/unfollowing
        :target_user_id: the user which is being unfollowed
        '''
        source_feed = self.get_user_feed(target_user_id)
        for feed_name, user_feed in self.get_feeds(user_id).items():
            self.follow_feed(user_feed, source_feed)

    def unfollow_user(self, user_id, target_user_id):
        '''
        unfollows the user

        :param user_id: the user which is doing the following/unfollowing
        :target_user_id: the user which is being unfollowed
        '''
        source_feed = self.get_user_feed(target_user_id)
        for feed in self.get_feeds(user_id).values():
            self.unfollow_feed(feed, source_feed)

    def follow_many_users(self, user_id, target_ids, async=True):
        '''
        copies feeds for target_ids in user_id

        :param user_id: the user which is doing the following/unfollowing
        :param target_ids: the user to follow
        :param async: controls if the operation should be done via celery
        '''
        if async:
            follow_many_fn = follow_many.delay
        else:
            follow_many_fn = follow_many

        follow_many_fn(
            feeds=self.get_feeds(user_id).values(),
            target_feeds=map(self.get_user_feed, target_ids),
            follow_limit=self.follow_activity_limit
        )

    def _fanout(self, feed_classes, user_id, operation, *args, **kwargs):
        '''
        Generic functionality for running an operation on all of your
        follower's feeds

        It takes the following ids and distributes them per fanout_chunk_size
        '''
        user_ids = self.get_user_follower_ids(user_id=user_id)
        user_ids_chunks = chunks(user_ids, self.fanout_chunk_size)
        for ids_chunk in user_ids_chunks:
            fanout_operation(
                self, feed_classes, ids_chunk, operation, *args, **kwargs
            )

    def _fanout_task(self, user_ids, feed_classes, operation, *args, **kwargs):
        '''
        This bit of the fan-out is normally called via an Async task
        this shouldnt do any db queries whatsoever
        '''
        for name, feed_class in feed_classes.items():
            with feed_class.get_timeline_batch_interface() as batch_interface:
                kwargs['batch_interface'] = batch_interface
                for user_id in user_ids:
                    feed = feed_class(user_id)
                    operation(feed, *args, **kwargs)

    def flush(self):
        '''
        Flushes all the feeds
        '''
        #TODO why do we have this method?
        for name, feed_class in self.feed_classes.items():
            feed_class.flush()
        self.user_feed_class.flush()
<|endoftext|>


<commit_before>__version__ = "0.1"
<commit_msg>bump version<commit_after>__version__ = "0.1.1"
<|endoftext|>