<commit_before># -*- coding: utf-8 -*-
from redis.exceptions import WatchError

from cacheops.conf import redis_client
from cacheops.utils import get_model_name

__all__ = ('invalidate_obj', 'invalidate_model', 'invalidate_all')

def serialize_scheme(scheme):
    return ','.join(scheme)

def deserialize_scheme(scheme):
    return tuple(scheme.split(','))

def conj_cache_key(model, conj):
    return 'conj:%s:' % get_model_name(model) + '&'.join('%s=%s' % t for t in sorted(conj))

def conj_cache_key_from_scheme(model, scheme, values):
    return 'conj:%s:' % get_model_name(model) + '&'.join('%s=%s' % (f, values[f]) for f in scheme)

class ConjSchemes(object):
    """
    A container for managing models scheme collections.
    Schemes are stored in redis and cached locally.
    """
    def __init__(self):
        self.local = {}
        self.versions = {}

    def get_lookup_key(self, model_or_name):
        if not isinstance(model_or_name, basestring):
            model_or_name = get_model_name(model_or_name)
        return 'schemes:%s' % model_or_name

    def get_version_key(self, model_or_name):
        if not isinstance(model_or_name, basestring):
            model_or_name = get_model_name(model_or_name)
        return 'schemes:%s:version' % model_or_name

    def load_schemes(self, model):
        model_name = get_model_name(model)

        txn = redis_client.pipeline()
        txn.get(self.get_version_key(model))
        txn.smembers(self.get_lookup_key(model_name))
        version, members = txn.execute()

        self.local[model_name] = set(map(deserialize_scheme, members))
        self.local[model_name].add(()) # Всегда добавляем пустую схему
        self.versions[model_name] = int(version or 0)
        return self.local[model_name]

    def schemes(self, model):
        model_name = get_model_name(model)
        try:
            return self.local[model_name]
        except KeyError:
            return self.load_schemes(model)

    def version(self, model):
        try:
            return self.versions[get_model_name(model)]
        except KeyError:
            return 0

    def ensure_known(self, model, new_schemes):
        """
        Ensure that `new_schemes` are known or know them
        """
        new_schemes = set(new_schemes)
        model_name = get_model_name(model)
        loaded = False

        if model_name not in self.local:
            self.load_schemes(model)
            loaded = True
        schemes = self.local[model_name]

        if new_schemes - schemes:
            if not loaded:
                schemes = self.load_schemes(model)
            if new_schemes - schemes:
                # Write new schemes to redis
                txn = redis_client.pipeline()
                txn.incr(self.get_version_key(model_name)) # Увеличиваем версию схем

                lookup_key = self.get_lookup_key(model_name)
                for scheme in new_schemes - schemes:
                    txn.sadd(lookup_key, serialize_scheme(scheme))
                txn.execute()

                # Updating local version
                self.local[model_name].update(new_schemes)
                # We increment here instead of using incr result from redis,
                # because even our updated collection could be already obsolete
                self.versions[model_name] += 1

    def clear(self, model):
        """
        Clears schemes for models
        """
        redis_client.delete(self.get_lookup_key(model))
        redis_client.incr(self.get_version_key(model))

    def clear_all(self):
        self.local = {}
        for model_name in self.versions:
            self.versions[model_name] += 1

cache_schemes = ConjSchemes()

def invalidate_from_dict(model, values):
    """
    Invalidates caches that can possibly be influenced by object
    """

    schemes = cache_schemes.schemes(model)
    # Create a list of invalidators from list of schemes and values of object fields
    conjs_keys = [conj_cache_key_from_scheme(model, scheme, values) for scheme in schemes]

    # Optimistic locking: we hope schemes and invalidators won't change while we remove them
    # Ignoring this could lead to cache key hanging with it's invalidator removed
    version_key = cache_schemes.get_version_key(model)

    def _invalidate_conjs(pipe):

        # Starting MULTI block

        pipe.multi()
        # Check if our version of schemes for model is obsolete, update them and redo if needed
        # This shouldn't be happen too often once schemes are filled a bit
        pipe.get(version_key)
        # Get a union of all cache keys registered in invalidators
        pipe.sunion(conjs_keys)
        # `conjs_keys` are keys of sets containing `cache_keys` we are going to delete,
        # so we'll remove them too.
        # NOTE: There could be some other invalidators not matched with current object,
        #       which reference cache keys we delete, they will be hanging out for a while.
        # A transation will fail if schemes or any of invalidator sets is changed in between,
        # in that case we redo the whole thing
        pipe.delete(*conjs_keys)

    try_count = 3
    while try_count > 0:
        try_count -= 1
        version, cache_keys, _ = redis_client.transaction(_invalidate_conjs)
        if cache_keys:
            redis_client.delete(*cache_keys)
        if int(version or 0) != cache_schemes.version(model):
            cache_schemes.load_schemes(model)

def invalidate_obj(obj):
    """
    Invalidates caches that can possibly be influenced by object
    """
    invalidate_from_dict(obj.__class__, obj.__dict__)

def invalidate_model(model):
    """
    Invalidates all caches for given model.
    NOTE: This is a heavy artilery which uses redis KEYS request,
          which could be relatively slow on large datasets.
    """
    conjs_keys = redis_client.keys('conj:%s:*' % get_model_name(model))
    if isinstance(conjs_keys, str):
        conjs_keys = conjs_keys.split()

    if conjs_keys:
        cache_keys = redis_client.sunion(conjs_keys)
        redis_client.delete(*(list(cache_keys) + conjs_keys))

    # BUG: a race bug here, ignoring since invalidate_model() is not for hot production use
    cache_schemes.clear(model)

def invalidate_all():
    redis_client.flushdb()
    cache_schemes.clear_all()
<commit_msg>fix extra loop<commit_after># -*- coding: utf-8 -*-
from redis.exceptions import WatchError

from cacheops.conf import redis_client
from cacheops.utils import get_model_name

__all__ = ('invalidate_obj', 'invalidate_model', 'invalidate_all')

def serialize_scheme(scheme):
    return ','.join(scheme)

def deserialize_scheme(scheme):
    return tuple(scheme.split(','))

def conj_cache_key(model, conj):
    return 'conj:%s:' % get_model_name(model) + '&'.join('%s=%s' % t for t in sorted(conj))

def conj_cache_key_from_scheme(model, scheme, values):
    return 'conj:%s:' % get_model_name(model) + '&'.join('%s=%s' % (f, values[f]) for f in scheme)

class ConjSchemes(object):
    """
    A container for managing models scheme collections.
    Schemes are stored in redis and cached locally.
    """
    def __init__(self):
        self.local = {}
        self.versions = {}

    def get_lookup_key(self, model_or_name):
        if not isinstance(model_or_name, basestring):
            model_or_name = get_model_name(model_or_name)
        return 'schemes:%s' % model_or_name

    def get_version_key(self, model_or_name):
        if not isinstance(model_or_name, basestring):
            model_or_name = get_model_name(model_or_name)
        return 'schemes:%s:version' % model_or_name

    def load_schemes(self, model):
        model_name = get_model_name(model)

        txn = redis_client.pipeline()
        txn.get(self.get_version_key(model))
        txn.smembers(self.get_lookup_key(model_name))
        version, members = txn.execute()

        self.local[model_name] = set(map(deserialize_scheme, members))
        self.local[model_name].add(()) # Всегда добавляем пустую схему
        self.versions[model_name] = int(version or 0)
        return self.local[model_name]

    def schemes(self, model):
        model_name = get_model_name(model)
        try:
            return self.local[model_name]
        except KeyError:
            return self.load_schemes(model)

    def version(self, model):
        try:
            return self.versions[get_model_name(model)]
        except KeyError:
            return 0

    def ensure_known(self, model, new_schemes):
        """
        Ensure that `new_schemes` are known or know them
        """
        new_schemes = set(new_schemes)
        model_name = get_model_name(model)
        loaded = False

        if model_name not in self.local:
            self.load_schemes(model)
            loaded = True
        schemes = self.local[model_name]

        if new_schemes - schemes:
            if not loaded:
                schemes = self.load_schemes(model)
            if new_schemes - schemes:
                # Write new schemes to redis
                txn = redis_client.pipeline()
                txn.incr(self.get_version_key(model_name)) # Увеличиваем версию схем

                lookup_key = self.get_lookup_key(model_name)
                for scheme in new_schemes - schemes:
                    txn.sadd(lookup_key, serialize_scheme(scheme))
                txn.execute()

                # Updating local version
                self.local[model_name].update(new_schemes)
                # We increment here instead of using incr result from redis,
                # because even our updated collection could be already obsolete
                self.versions[model_name] += 1

    def clear(self, model):
        """
        Clears schemes for models
        """
        redis_client.delete(self.get_lookup_key(model))
        redis_client.incr(self.get_version_key(model))

    def clear_all(self):
        self.local = {}
        for model_name in self.versions:
            self.versions[model_name] += 1

cache_schemes = ConjSchemes()

def invalidate_from_dict(model, values):
    """
    Invalidates caches that can possibly be influenced by object
    """

    schemes = cache_schemes.schemes(model)
    # Create a list of invalidators from list of schemes and values of object fields
    conjs_keys = [conj_cache_key_from_scheme(model, scheme, values) for scheme in schemes]

    # Optimistic locking: we hope schemes and invalidators won't change while we remove them
    # Ignoring this could lead to cache key hanging with it's invalidator removed
    version_key = cache_schemes.get_version_key(model)

    def _invalidate_conjs(pipe):

        # Starting MULTI block

        pipe.multi()
        # Check if our version of schemes for model is obsolete, update them and redo if needed
        # This shouldn't be happen too often once schemes are filled a bit
        pipe.get(version_key)
        # Get a union of all cache keys registered in invalidators
        pipe.sunion(conjs_keys)
        # `conjs_keys` are keys of sets containing `cache_keys` we are going to delete,
        # so we'll remove them too.
        # NOTE: There could be some other invalidators not matched with current object,
        #       which reference cache keys we delete, they will be hanging out for a while.
        # A transation will fail if schemes or any of invalidator sets is changed in between,
        # in that case we redo the whole thing
        pipe.delete(*conjs_keys)

    try_count = 3
    while try_count > 0:
        try_count -= 1
        version, cache_keys, _ = redis_client.transaction(_invalidate_conjs)
        if cache_keys:
            redis_client.delete(*cache_keys)
        if int(version or 0) != cache_schemes.version(model):
            cache_schemes.load_schemes(model)
        else:
            break

def invalidate_obj(obj):
    """
    Invalidates caches that can possibly be influenced by object
    """
    invalidate_from_dict(obj.__class__, obj.__dict__)

def invalidate_model(model):
    """
    Invalidates all caches for given model.
    NOTE: This is a heavy artilery which uses redis KEYS request,
          which could be relatively slow on large datasets.
    """
    conjs_keys = redis_client.keys('conj:%s:*' % get_model_name(model))
    if isinstance(conjs_keys, str):
        conjs_keys = conjs_keys.split()

    if conjs_keys:
        cache_keys = redis_client.sunion(conjs_keys)
        redis_client.delete(*(list(cache_keys) + conjs_keys))

    # BUG: a race bug here, ignoring since invalidate_model() is not for hot production use
    cache_schemes.clear(model)

def invalidate_all():
    redis_client.flushdb()
    cache_schemes.clear_all()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import collectd
import requests

class Plugin(object):
    DOCKER_URL_PREFIX = "http://docker.lain:2375"
    READ_INTERVAL = 600  # 600 seconds
    TIMEOUT = 6  # 6 seconds
    NAME = "lain.cluster.node"

    def init(self):
        collectd.info("node_monitor plugin has been initialized.")

    def read_docker_used_cpu_cores(self):
        '''
        Unit: CPU Cores
        '''
        try:
            containers = requests.get(
                "{}/containers/json".format(self.DOCKER_URL_PREFIX),
                timeout=self.TIMEOUT).json()
            docekr_used_cpu_cores = 0
            for container in containers:
                stats = requests.get(
                    "{}/containers/{}/stats?stream=false".format(
                        self.DOCKER_URL_PREFIX, container["Id"]),
                    timeout=self.TIMEOUT).json()
                docekr_used_cpu_cores += self.__calculate_cpu_cores(stats)
            metric = collectd.Values()
            metric.plugin = self.NAME
            metric.plugin_instance = "docker_used_cpu_cores"
            metric.type = "val"
            metric.values = [docekr_used_cpu_cores]
            metric.dispatch()
        except Exception as e:
            collectd.error(
                "read_docker_used_cpu_cores() failed, exception: {}".format(e))

    def read(self):
        self.read_docker_used_cpu_cores()

    def shutdown(self):
        collectd.info("node_monitor plugin has been shutdown.")

    def __calculate_cpu_cores(self, stats):
        '''
        Unit: CPU Cores
        '''
        cpu_delta = stats["cpu_stats"]["cpu_usage"]["total_usage"] - stats[
            "precpu_stats"]["cpu_usage"]["total_usage"]
        system_delta = stats["cpu_stats"]["system_cpu_usage"] - stats[
            "precpu_stats"]["system_cpu_usage"]
        if cpu_delta > 0 and system_delta > 0:
            return (float(cpu_delta) / system_delta
                    ) * len(stats["cpu_stats"]["cpu_usage"]["percpu_usage"])

        return 0

if __name__ != "__main__":
    node_monitor = Plugin()
    collectd.register_init(node_monitor.init)
    collectd.register_read(node_monitor.read, node_monitor.READ_INTERVAL)
    collectd.register_shutdown(node_monitor.shutdown)
<commit_msg>monitor docker reserved cpu<commit_after># -*- coding: utf-8 -*-

import collectd
import requests
import time

class Plugin(object):
    DOCKER_URL_PREFIX = "http://docker.lain:2375"
    READ_INTERVAL = 600  # 600 seconds
    TIMEOUT = 6  # 6 seconds
    NAME = "lain.cluster.node"

    def init(self):
        collectd.info("node_monitor plugin has been initialized.")

    def __get(self, location, params=None, **kwargs):
        url = "{}{}".format(self.DOCKER_URL_PREFIX, location)
        return requests.get(url, params, timeout=self.TIMEOUT, **kwargs).json()

    def __get_all_container_ids(self):
        containers = self.__get("/containers/json")
        return [c["Id"] for c in containers]

    def read_docker_reserved_cpu_cores(self, container_ids):
        '''
        Unit: CPU Cores
        '''
        docker_reserved_cpu_cores = 0
        for container_id in container_ids:
            inspect_info = self.__get("/containers/{}/json".format(
                container_id))
            docker_reserved_cpu_cores += self.__calculate_docker_reserved_cpu_cores(
                inspect_info)

        metric = collectd.Values()
        metric.plugin = self.NAME
        metric.plugin_instance = "docker_reserved_cpu_cores"
        metric.type = "val"
        metric.values = [docker_reserved_cpu_cores]
        metric.dispatch()

    def __calculate_docker_reserved_cpu_cores(self, inspect_info):
        cpu_quota = inspect_info["HostConfig"]["CpuQuota"]
        cpu_period = inspect_info["HostConfig"]["CpuPeriod"]
        if cpu_quota > 0 and cpu_period > 0:
            return (float(cpu_quota) / cpu_period)

        return 0

    def read_docker_used_cpu_cores(self, container_ids):
        '''
        Unit: CPU Cores
        '''
        docekr_used_cpu_cores = 0
        for container_id in container_ids:
            stats = self.__get("/containers/{}/stats?stream=false".format(
                container_id))
            docekr_used_cpu_cores += self.__calculate_docker_used_cpu_cores(
                stats)
        metric = collectd.Values()
        metric.plugin = self.NAME
        metric.plugin_instance = "docker_used_cpu_cores"
        metric.type = "val"
        metric.values = [docekr_used_cpu_cores]
        metric.dispatch()

    def __calculate_docker_used_cpu_cores(self, stats):
        '''
        Unit: CPU Cores
        '''
        cpu_delta = stats["cpu_stats"]["cpu_usage"]["total_usage"] - stats[
            "precpu_stats"]["cpu_usage"]["total_usage"]
        system_delta = stats["cpu_stats"]["system_cpu_usage"] - stats[
            "precpu_stats"]["system_cpu_usage"]
        if cpu_delta > 0 and system_delta > 0:
            return (float(cpu_delta) / system_delta
                    ) * len(stats["cpu_stats"]["cpu_usage"]["percpu_usage"])

        return 0

    def read(self):
        try:
            container_ids = self.__get_all_container_ids()
            self.read_docker_used_cpu_cores(container_ids)
            time.sleep(10)
            self.read_docker_reserved_cpu_cores(container_ids)
        except Exception as e:
            collectd.error("read() failed, exception: {}".format(e))

    def shutdown(self):
        collectd.info("node_monitor plugin has been shutdown.")

if __name__ != "__main__":
    node_monitor = Plugin()
    collectd.register_init(node_monitor.init)
    collectd.register_read(node_monitor.read, node_monitor.READ_INTERVAL)
    collectd.register_shutdown(node_monitor.shutdown)
<|endoftext|>


<commit_before>from data_collection.management.commands import BaseScotlandSpatialHubImporter

"""
Note:
This importer provides coverage for 87/107 districts
due to incomplete/poor quality data

Data at

http://gis-pkc.opendata.arcgis.com/datasets?q=polling&sort_by=-updatedAt
https://github.com/wdiv-scrapers/data/tree/master/S12000024

is also be worth a look, but at last check there were
some issues with overlapping polygons etc so could also be problematic
"""

class Command(BaseScotlandSpatialHubImporter):
    council_id = "S12000048"
    council_name = "Perth and Kinross"
    elections = ["parl.2017-06-08"]
<commit_msg>Import script for Perth and Kinross.<commit_after>from data_collection.management.commands import BaseScotlandSpatialHubImporter
from data_collection.slugger import Slugger

"""
Note:
This importer provides coverage for 103/107 districts
due to incomplete/poor quality data

Station RAILWAY STAFF CLUB, PERTH has no districts assigned to it.
Districts Wellshill and Fairfield are assinged to
Fairfield Neighbourhood Centre, Perth. But this station is  not in the polling
stations source file. The RAILWAY STAFF CLUB could be correct for these.
District Hoole has polling place 'Inchture Village Hall, Kinrossie' aassigned
which is ambiguous becasue Kinrossie and Inchture are not the same place and
District Inchture has 'Inchture Village Hall, Inchture' which looks correct.
District East Ruthenfield has no polling station due to an abscence of electors.

Data at
http://gis-pkc.opendata.arcgis.com/datasets?q=polling&sort_by=-updatedAt
https://github.com/wdiv-scrapers/data/tree/master/S12000024

is also be worth a look, but at last check there were
some issues with overlapping polygons etc so could also be problematic
"""

class Command(BaseScotlandSpatialHubImporter):
    council_id = "S12000048"
    council_name = "Perth and Kinross"
    elections = ["europarl.2019-05-23"]
    station_map = {}

    def district_record_to_dict(self, record):
        if record[2] == self.council_name:
            station_slug = Slugger.slugify(record[3])
            self.station_map.setdefault(station_slug, []).append(record[0])
            return super().district_record_to_dict(record)

    def station_record_to_dict(self, record):
        if record[2] == self.council_name:
            try:
                station_slug = Slugger.slugify(record[3])
                if station_slug == "pitcairngreen-village-hall-pitcairngreen":
                    station_slug = "pitcairngreen-village-hall-almondbank"

                codes = self.station_map[station_slug]

            except KeyError:
                return None

            stations = []
            for code in codes:
                rec = {
                    "internal_council_id": code,
                    "postcode": "",
                    "address": record[3],
                }
                stations.append(rec)
            return stations
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function
from builtins import str
from copy import deepcopy

from panoptes_client.exportable import Exportable
from panoptes_client.panoptes import PanoptesObject, LinkResolver
from panoptes_client.subject import Subject
from panoptes_client.subject_set import SubjectSet
from panoptes_client.utils import batchable

class Workflow(PanoptesObject, Exportable):
    _api_slug = 'workflows'
    _link_slug = 'workflows'
    _edit_attributes = (
        'active',
        'configuration',
        'display_name',
        'first_task',
        'mobile_friendly',
        'primary_language',
        'retirement',
        'tasks',
        {
            'links': (
                    'project',
            )
        },
    )

    def __init__(self, raw={}, etag=None):
        super(Workflow, self).__init__(raw, etag)
        if not self.configuration:
            self.configuration = {}
            self._original_configuration = {}
        if not self.retirement:
            self.retirement = {}
            self._original_retirement = {}
        if not self.tasks:
            self.tasks = {}
            self._original_tasks = {}

    def set_raw(self, raw, etag=None, loaded=True):
        super(Workflow, self).set_raw(raw, etag, loaded)
        if loaded:
            if self.configuration:
                self._original_configuration = deepcopy(self.configuration)
            if self.retirement:
                self._original_retirement = deepcopy(self.retirement)
            if self.tasks:
                self._original_tasks = deepcopy(self.tasks)
        elif loaded:
            self._original_configuration = None
            self._original_retirement = None
            self._original_tasks = None

    def save(self):
        """
        Adds workflow configuration, retirement, and tasks dicts to the list of
        savable attributes if it has changed.
        """
        if not self.configuration == self._original_configuration:
            self.modified_attributes.add('configuration')
        if not self.retirement == self._original_retirement:
            self.modified_attributes.add('retirement')
        if not self.tasks == self._original_tasks:
            self.modified_attributes.add('tasks')

        super(Workflow, self).save()

    @batchable
    def retire_subjects(self, subjects, reason='other'):
        """
        Retires subjects in this workflow.

        - **subjects** can be a list of :py:class:`Subject` instances, a list
          of subject IDs, a single :py:class:`Subject` instance, or a single
          subject ID.
        - **reason** gives the reason the :py:class:`Subject` has been retired.
          Defaults to **other**.

        Examples::

            workflow.retire_subjects(1234)
            workflow.retire_subjects([1,2,3,4])
            workflow.retire_subjects(Subject(1234))
            workflow.retire_subjects([Subject(12), Subject(34)])
        """

        subjects = [ s.id if isinstance(s, Subject) else s for s in subjects ]

        return Workflow.http_post(
            '{}/retired_subjects'.format(self.id),
            json={
                'subject_ids': subjects,
                'retirement_reason': reason
            },
        )

    @batchable
    def unretire_subjects(self, subjects, subject_sets):
        """
        some notes here
        """

        subjects = [ s.id if isinstance(s, Subject) else s for s in subjects ]
        subject_sets = [ ss.id if isinstance(ss, SubjectSet) else ss for ss in subject_sets ]
        return Workflow.http_post(
            '{}/unretire_subjects'.format(self.id),
            json={
                'subject_ids': subjects,
                'subject_set_ids': subject_sets
            },
        )

    def add_subject_sets(self, subject_sets):
        """
        A wrapper around :py:meth:`.LinkCollection.add`. Equivalent to::

            workflow.links.subject_sets.add(subject_sets)
        """

        return self.links.subject_sets.add(subject_sets)

    def remove_subject_sets(self, subject_sets):
        """
        A wrapper around :py:meth:`.LinkCollection.remove`. Equivalent to::

            workflow.links.subject_sets.remove(subject_sets)
        """

        return self.links.subject_sets.remove(subject_sets)

    @property
    def versions(self):
        """
        A generator which yields all :py:class:`.WorkflowVersion` instances for
        this workflow.
        """

        return WorkflowVersion.where(workflow=self)

LinkResolver.register(Workflow)
LinkResolver.register(Workflow, 'active_workflows', readonly=True)

from panoptes_client.workflow_version import WorkflowVersion
<commit_msg>Update workflow.py<commit_after>from __future__ import absolute_import, division, print_function
from builtins import str
from copy import deepcopy

from panoptes_client.exportable import Exportable
from panoptes_client.panoptes import PanoptesObject, LinkResolver
from panoptes_client.subject import Subject
from panoptes_client.subject_set import SubjectSet
from panoptes_client.utils import batchable

class Workflow(PanoptesObject, Exportable):
    _api_slug = 'workflows'
    _link_slug = 'workflows'
    _edit_attributes = (
        'active',
        'configuration',
        'display_name',
        'first_task',
        'mobile_friendly',
        'primary_language',
        'retirement',
        'tasks',
        {
            'links': (
                    'project',
            )
        },
    )

    def __init__(self, raw={}, etag=None):
        super(Workflow, self).__init__(raw, etag)
        if not self.configuration:
            self.configuration = {}
            self._original_configuration = {}
        if not self.retirement:
            self.retirement = {}
            self._original_retirement = {}
        if not self.tasks:
            self.tasks = {}
            self._original_tasks = {}

    def set_raw(self, raw, etag=None, loaded=True):
        super(Workflow, self).set_raw(raw, etag, loaded)
        if loaded:
            if self.configuration:
                self._original_configuration = deepcopy(self.configuration)
            if self.retirement:
                self._original_retirement = deepcopy(self.retirement)
            if self.tasks:
                self._original_tasks = deepcopy(self.tasks)
        elif loaded:
            self._original_configuration = None
            self._original_retirement = None
            self._original_tasks = None

    def save(self):
        """
        Adds workflow configuration, retirement, and tasks dicts to the list of
        savable attributes if it has changed.
        """
        if not self.configuration == self._original_configuration:
            self.modified_attributes.add('configuration')
        if not self.retirement == self._original_retirement:
            self.modified_attributes.add('retirement')
        if not self.tasks == self._original_tasks:
            self.modified_attributes.add('tasks')

        super(Workflow, self).save()

    @batchable
    def retire_subjects(self, subjects, reason='other'):
        """
        Retires subjects in this workflow.

        - **subjects** can be a list of :py:class:`Subject` instances, a list
          of subject IDs, a single :py:class:`Subject` instance, or a single
          subject ID.
        - **reason** gives the reason the :py:class:`Subject` has been retired.
          Defaults to **other**.

        Examples::

            workflow.retire_subjects(1234)
            workflow.retire_subjects([1,2,3,4])
            workflow.retire_subjects(Subject(1234))
            workflow.retire_subjects([Subject(12), Subject(34)])
        """

        subjects = [ s.id if isinstance(s, Subject) else s for s in subjects ]

        return Workflow.http_post(
            '{}/retired_subjects'.format(self.id),
            json={
                'subject_ids': subjects,
                'retirement_reason': reason
            },
        )

    @batchable
    def unretire_subjects(self, subjects, subject_sets):
        """
        Un-retires subjects in this workflow.

        - **subjects** can be a list of :py:class:`Subject` instances, a list
          of subject IDs, a single :py:class:`Subject` instance, or a single
          subject ID.

        - **subjects_sets* can be a list of :py:class:`SubjectSet` instances, a list
          of subject_set IDs, a single :py:class:`SubjectSet` instance, or a single
          subject_set ID.
        """

        subjects = [ s.id if isinstance(s, Subject) else s for s in subjects ]
        subject_sets = [ ss.id if isinstance(ss, SubjectSet) else ss for ss in subject_sets ]
        return Workflow.http_post(
            '{}/unretire_subjects'.format(self.id),
            json={
                'subject_ids': subjects,
                'subject_set_ids': subject_sets
            },
        )

    def add_subject_sets(self, subject_sets):
        """
        A wrapper around :py:meth:`.LinkCollection.add`. Equivalent to::

            workflow.links.subject_sets.add(subject_sets)
        """

        return self.links.subject_sets.add(subject_sets)

    def remove_subject_sets(self, subject_sets):
        """
        A wrapper around :py:meth:`.LinkCollection.remove`. Equivalent to::

            workflow.links.subject_sets.remove(subject_sets)
        """

        return self.links.subject_sets.remove(subject_sets)

    @property
    def versions(self):
        """
        A generator which yields all :py:class:`.WorkflowVersion` instances for
        this workflow.
        """

        return WorkflowVersion.where(workflow=self)

LinkResolver.register(Workflow)
LinkResolver.register(Workflow, 'active_workflows', readonly=True)

from panoptes_client.workflow_version import WorkflowVersion
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import csv
import os

import pandas as pd
from nose.tools import (assert_equal,
                        assert_false,
                        assert_list_equal,
                        assert_true,
                        raises)

from .. import main

try:
    data_directory = os.environ['FILE_VALIDATOR_DATA_DIRECTORY']
except KeyError:
    message = """
The File Validator could not find the data directory in the shell
environment.

To set the data directory for the current shell session, from the
terminal run
    export FILE_VALIDATOR_DATA_DIRECTORY='/home/foo/bar/'

To set the data directory for the current and all future shell
sessions, from the terminal run
    echo 'export FILE_VALIDATOR_DATA_DIRECTORY='/home/foo/bar/' >> ~/.bashrc
"""
    raise(EnvironmentError(message))

def assert_data_equal(left, right):

    """
    Extends nose.tools.assert_list_equal.

    xlrd does not differentiate between integers and floats; both are
    stored as floats. Floats must be conditionally type casted into
    integer string representations. xlrd pads records with empty
    values. Trailing missing data is dropped to achieve "fuzzy"
    comparison.
    """

    processed_left = _drop_trailing_missing_data(
        _smart_float_coerce(data=left))
    processed_right = _drop_trailing_missing_data(
        _smart_float_coerce(data=right))
    assert_list_equal(processed_left, processed_right)

def _drop_trailing_missing_data(data):

    """
    Returns List.

    Drop trailing missing data for each record.

    Parameters
    ----------
    data : Iterable
    """

    processed_data = list()

    for record in data:
        for value in reversed(record):
            if value == '':
                record.pop()
            else:
                break
        processed_data.append(record)

    return processed_data

def _smart_float_coerce(data):

    """
    Returns List.

    Conditionally type cast values into their float representation if
    possible.

    Parameters
    ---------
    data : Iterable
    """

    processed_data = list()

    for record in data:
        processed_record = list()
        # This does not support handling empty records.
        for value in record:
            try:
                processed_value = float(value)
            except ValueError:
                processed_value = value
            processed_record.append(processed_value)
        processed_data.append(processed_record)

    return processed_data

def test_drop_trailing_missing_data():

    input_data = [['foo', 'bar', ''], ['eggs', 'ham', '']]
    expected_data = [['foo', 'bar'], ['eggs', 'ham']]
    output_data = _drop_trailing_missing_data(data=input_data)

    assert_equal(output_data, expected_data)

def test_smart_float_coerce():

    input_data = [['foo', '0'], ['', '1']]
    expected_data = [['foo', 0.0], ['', 1.0]]
    output_data = _smart_float_coerce(data=input_data)

    assert_equal(output_data, expected_data)

def test_data_table_from_data_frame():

    data_frame = pd.DataFrame.from_items([('foo', ('eggs', 'ham')),
                                          ('bar', ('0', '1'))])
    expected_data_table = main.DataTable([['foo', 'bar'],
                                          ['eggs', '0'],
                                          ['ham', '1']])
    output_data_table = main.DataTable.from_data_frame(data_frame)

    assert_list_equal(output_data_table, expected_data_table)

@raises(AssertionError)
def test_validation_result_unset():

    validation_results = main.ValidationResults()
    validation_results.validate()

def test_csv():

    validation_results = setup_test_csv()
    assert_false(validation_results.is_skewed)

def setup_test_csv():

    file_path = data_directory + '/' + 'students.csv'
    is_excel = False
    raw_delimiter = '1'
    has_header = True

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   raw_delimiter=raw_delimiter,
                                   has_header=has_header)

    return validation_results

def test_csv_missing_header():

    file_path = data_directory + '/' + 'students-missing-header.csv'
    is_excel = False
    raw_delimiter = '1'
    has_header = False
    header_file_path = data_directory + '/' + 'head.csv'

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   raw_delimiter=raw_delimiter,
                                   has_header=has_header,
                                   header_file_path=header_file_path)

    assert_false(validation_results.is_skewed)

    expected_data_frame = setup_test_csv().source_data_table
    assert_data_equal(validation_results.processed_data_table,
                      expected_data_frame)

def test_csv_skewed():

    file_path = data_directory + '/' + 'students-skewed.csv'
    is_excel = False
    raw_delimiter = '1'
    has_header = True

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   raw_delimiter=raw_delimiter,
                                   has_header=has_header)

    assert_true(validation_results.is_skewed)

def test_csv_missing_header_skewed():

    file_path = data_directory + '/' + 'students-missing-header-skewed.csv'
    is_excel = False
    raw_delimiter = '1'
    has_header = False
    header_file_path = data_directory + '/' + 'head.csv'

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   raw_delimiter=raw_delimiter,
                                   has_header=has_header,
                                   header_file_path=header_file_path)

    assert_true(validation_results.is_skewed)

def test_tab_delimited():

    validation_results = setup_test_tab_delimited()
    assert_false(validation_results.is_skewed)

def setup_test_tab_delimited():

    file_path = data_directory + '/' + 'students.txt'
    is_excel = False
    raw_delimiter = '2'
    has_header = True

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   raw_delimiter=raw_delimiter,
                                   has_header=has_header)

    return validation_results

def test_tab_delimited_missing_header():

    file_path = data_directory + '/' + 'students-missing-header.txt'
    is_excel = False
    raw_delimiter = '2'
    has_header = False
    header_file_path = data_directory + '/' + 'head.txt'

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   raw_delimiter=raw_delimiter,
                                   has_header=has_header,
                                   header_file_path=header_file_path)

    assert_false(validation_results.is_skewed)

    expected_data_frame = setup_test_tab_delimited().source_data_table
    assert_data_equal(validation_results.processed_data_table,
                      expected_data_frame)

def test_excel():

    validation_results = setup_test_excel()
    assert_false(validation_results.is_skewed)

def setup_test_excel():

    file_path = data_directory + '/' + 'students.xlsx'
    is_excel = True
    has_header = True

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   has_header=has_header)

    return validation_results

def test_excel_missing_header():

    file_path = data_directory + '/' + 'students-missing-header.xlsx'
    is_excel = True
    has_header = False
    header_file_path = data_directory + '/' + 'head.csv'

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   has_header=has_header,
                                   header_file_path=header_file_path)

    assert_false(validation_results.is_skewed)

    expected_data_frame = setup_test_excel().source_data_table
    assert_data_equal(validation_results.processed_data_table,
                      expected_data_frame)

def test_excel_skewed():

    file_path = data_directory + '/' + 'students-skewed.xlsx'
    is_excel = True
    has_header = True

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   has_header=has_header)

    assert_true(validation_results.is_skewed)

def test_excel_missing_header_skewed():

    file_path = data_directory + '/' + 'students-missing-header-skewed.xlsx'
    is_excel = True
    has_header = False
    header_file_path = data_directory + '/' + 'head.csv'

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   has_header=has_header,
                                   header_file_path=header_file_path)

    assert_true(validation_results.is_skewed)

def test_excel_missing_data():

    validation_results = setup_test_excel()
    assert_false(validation_results.is_skewed)

def setup_test_excel_missing_data():

    file_path = data_directory + '/' + 'students-missing-data.xlsx'
    is_excel = True
    has_header = True

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   has_header=has_header)

    return validation_results

def test_excel_missing_data_missing_header():

    file_path = data_directory + '/' + 'students-missing-data-missing-header.xlsx'
    is_excel = True
    has_header = False
    header_file_path = data_directory + '/' + 'head.csv'

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   has_header=has_header,
                                   header_file_path=header_file_path)

    assert_false(validation_results.is_skewed)

    expected_data_frame = setup_test_excel_missing_data().source_data_table
    assert_data_equal(validation_results.processed_data_table,
                      expected_data_frame)

def test_excel_missing_data_skewed():

    file_path = data_directory + '/' + 'students-missing-data-skewed.xlsx'
    is_excel = True
    has_header = True

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   has_header=has_header)

    assert_true(validation_results.is_skewed)

def test_excel_missing_data_missing_header_skewed():

    file_path = data_directory + '/' + 'students-missing-data-missing-header-skewed.xlsx'
    is_excel = True
    has_header = False
    header_file_path = data_directory + '/' + 'head.csv'

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   has_header=has_header,
                                   header_file_path=header_file_path)

    assert_true(validation_results.is_skewed)

def test_primitive_read_excel():

    _test_primitive_read_excel_helper(left='students.xlsx',
                                      right='students.csv')

def test_primitive_read_excel_skewed():

    _test_primitive_read_excel_helper(left='students-skewed.xlsx',
                                      right='students-skewed.csv')

def _test_primitive_read_excel_helper(left, right):

    """
    Returns None.

    Parameters
    ----------
    left : String
        Left file name or path.
    right : String
        Right file name or path.
    """

    xlsx_file_path = data_directory + '/' + left
    csv_file_path = data_directory + '/' + right
    csv_representation = list()

    with open(csv_file_path, 'r') as file:
        for line in csv.reader(file):
            csv_representation.append([unicode(value) for value in line])

    xlsx_representation = main._primitive_read_excel(
        file_path=xlsx_file_path)

    assert_data_equal(xlsx_representation, csv_representation)

<commit_msg>Refactored _drop_trailing_missing_data()<commit_after># -*- coding: utf-8 -*-

import copy
import csv
import os

import pandas as pd
from nose.tools import (assert_equal,
                        assert_false,
                        assert_list_equal,
                        assert_true,
                        raises)

from .. import main

try:
    data_directory = os.environ['FILE_VALIDATOR_DATA_DIRECTORY']
except KeyError:
    message = """
The File Validator could not find the data directory in the shell
environment.

To set the data directory for the current shell session, from the
terminal run
    export FILE_VALIDATOR_DATA_DIRECTORY='/home/foo/bar/'

To set the data directory for the current and all future shell
sessions, from the terminal run
    echo 'export FILE_VALIDATOR_DATA_DIRECTORY='/home/foo/bar/' >> ~/.bashrc
"""
    raise(EnvironmentError(message))

def assert_data_equal(left, right):

    """
    Extends nose.tools.assert_list_equal.

    xlrd does not differentiate between integers and floats; both are
    stored as floats. Floats must be conditionally type casted into
    integer string representations. xlrd pads records with empty
    values. Trailing missing data is dropped to achieve "fuzzy"
    comparison.
    """

    processed_left = _drop_trailing_missing_data(
        _smart_float_coerce(data=left))
    processed_right = _drop_trailing_missing_data(
        _smart_float_coerce(data=right))
    assert_list_equal(processed_left, processed_right)

def _drop_trailing_missing_data(data):

    """
    Returns List.

    Drop trailing missing data for each record.

    Parameters
    ----------
    data : Iterable
    """

    processed_data = copy.deepcopy(data)

    for record in processed_data:
        while record[-1] == '':
            record.pop()

    return processed_data

def _smart_float_coerce(data):

    """
    Returns List.

    Conditionally type cast values into their float representation if
    possible.

    Parameters
    ---------
    data : Iterable
    """

    processed_data = list()

    for record in data:
        processed_record = list()
        # This does not support handling empty records.
        for value in record:
            try:
                processed_value = float(value)
            except ValueError:
                processed_value = value
            processed_record.append(processed_value)
        processed_data.append(processed_record)

    return processed_data

def test_drop_trailing_missing_data():

    input_data = [['foo', 'bar', ''], ['eggs', 'ham', '']]
    expected_data = [['foo', 'bar'], ['eggs', 'ham']]
    output_data = _drop_trailing_missing_data(data=input_data)

    assert_equal(output_data, expected_data)

def test_smart_float_coerce():

    input_data = [['foo', '0'], ['', '1']]
    expected_data = [['foo', 0.0], ['', 1.0]]
    output_data = _smart_float_coerce(data=input_data)

    assert_equal(output_data, expected_data)

def test_data_table_from_data_frame():

    data_frame = pd.DataFrame.from_items([('foo', ('eggs', 'ham')),
                                          ('bar', ('0', '1'))])
    expected_data_table = main.DataTable([['foo', 'bar'],
                                          ['eggs', '0'],
                                          ['ham', '1']])
    output_data_table = main.DataTable.from_data_frame(data_frame)

    assert_list_equal(output_data_table, expected_data_table)

@raises(AssertionError)
def test_validation_result_unset():

    validation_results = main.ValidationResults()
    validation_results.validate()

def test_csv():

    validation_results = setup_test_csv()
    assert_false(validation_results.is_skewed)

def setup_test_csv():

    file_path = data_directory + '/' + 'students.csv'
    is_excel = False
    raw_delimiter = '1'
    has_header = True

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   raw_delimiter=raw_delimiter,
                                   has_header=has_header)

    return validation_results

def test_csv_missing_header():

    file_path = data_directory + '/' + 'students-missing-header.csv'
    is_excel = False
    raw_delimiter = '1'
    has_header = False
    header_file_path = data_directory + '/' + 'head.csv'

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   raw_delimiter=raw_delimiter,
                                   has_header=has_header,
                                   header_file_path=header_file_path)

    assert_false(validation_results.is_skewed)

    expected_data_frame = setup_test_csv().source_data_table
    assert_data_equal(validation_results.processed_data_table,
                      expected_data_frame)

def test_csv_skewed():

    file_path = data_directory + '/' + 'students-skewed.csv'
    is_excel = False
    raw_delimiter = '1'
    has_header = True

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   raw_delimiter=raw_delimiter,
                                   has_header=has_header)

    assert_true(validation_results.is_skewed)

def test_csv_missing_header_skewed():

    file_path = data_directory + '/' + 'students-missing-header-skewed.csv'
    is_excel = False
    raw_delimiter = '1'
    has_header = False
    header_file_path = data_directory + '/' + 'head.csv'

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   raw_delimiter=raw_delimiter,
                                   has_header=has_header,
                                   header_file_path=header_file_path)

    assert_true(validation_results.is_skewed)

def test_tab_delimited():

    validation_results = setup_test_tab_delimited()
    assert_false(validation_results.is_skewed)

def setup_test_tab_delimited():

    file_path = data_directory + '/' + 'students.txt'
    is_excel = False
    raw_delimiter = '2'
    has_header = True

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   raw_delimiter=raw_delimiter,
                                   has_header=has_header)

    return validation_results

def test_tab_delimited_missing_header():

    file_path = data_directory + '/' + 'students-missing-header.txt'
    is_excel = False
    raw_delimiter = '2'
    has_header = False
    header_file_path = data_directory + '/' + 'head.txt'

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   raw_delimiter=raw_delimiter,
                                   has_header=has_header,
                                   header_file_path=header_file_path)

    assert_false(validation_results.is_skewed)

    expected_data_frame = setup_test_tab_delimited().source_data_table
    assert_data_equal(validation_results.processed_data_table,
                      expected_data_frame)

def test_excel():

    validation_results = setup_test_excel()
    assert_false(validation_results.is_skewed)

def setup_test_excel():

    file_path = data_directory + '/' + 'students.xlsx'
    is_excel = True
    has_header = True

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   has_header=has_header)

    return validation_results

def test_excel_missing_header():

    file_path = data_directory + '/' + 'students-missing-header.xlsx'
    is_excel = True
    has_header = False
    header_file_path = data_directory + '/' + 'head.csv'

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   has_header=has_header,
                                   header_file_path=header_file_path)

    assert_false(validation_results.is_skewed)

    expected_data_frame = setup_test_excel().source_data_table
    assert_data_equal(validation_results.processed_data_table,
                      expected_data_frame)

def test_excel_skewed():

    file_path = data_directory + '/' + 'students-skewed.xlsx'
    is_excel = True
    has_header = True

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   has_header=has_header)

    assert_true(validation_results.is_skewed)

def test_excel_missing_header_skewed():

    file_path = data_directory + '/' + 'students-missing-header-skewed.xlsx'
    is_excel = True
    has_header = False
    header_file_path = data_directory + '/' + 'head.csv'

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   has_header=has_header,
                                   header_file_path=header_file_path)

    assert_true(validation_results.is_skewed)

def test_excel_missing_data():

    validation_results = setup_test_excel()
    assert_false(validation_results.is_skewed)

def setup_test_excel_missing_data():

    file_path = data_directory + '/' + 'students-missing-data.xlsx'
    is_excel = True
    has_header = True

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   has_header=has_header)

    return validation_results

def test_excel_missing_data_missing_header():

    file_path = data_directory + '/' + 'students-missing-data-missing-header.xlsx'
    is_excel = True
    has_header = False
    header_file_path = data_directory + '/' + 'head.csv'

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   has_header=has_header,
                                   header_file_path=header_file_path)

    assert_false(validation_results.is_skewed)

    expected_data_frame = setup_test_excel_missing_data().source_data_table
    assert_data_equal(validation_results.processed_data_table,
                      expected_data_frame)

def test_excel_missing_data_skewed():

    file_path = data_directory + '/' + 'students-missing-data-skewed.xlsx'
    is_excel = True
    has_header = True

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   has_header=has_header)

    assert_true(validation_results.is_skewed)

def test_excel_missing_data_missing_header_skewed():

    file_path = data_directory + '/' + 'students-missing-data-missing-header-skewed.xlsx'
    is_excel = True
    has_header = False
    header_file_path = data_directory + '/' + 'head.csv'

    validation_results = main.main(file_path=file_path,
                                   is_excel=is_excel,
                                   has_header=has_header,
                                   header_file_path=header_file_path)

    assert_true(validation_results.is_skewed)

def test_primitive_read_excel():

    _test_primitive_read_excel_helper(left='students.xlsx',
                                      right='students.csv')

def test_primitive_read_excel_skewed():

    _test_primitive_read_excel_helper(left='students-skewed.xlsx',
                                      right='students-skewed.csv')

def _test_primitive_read_excel_helper(left, right):

    """
    Returns None.

    Parameters
    ----------
    left : String
        Left file name or path.
    right : String
        Right file name or path.
    """

    xlsx_file_path = data_directory + '/' + left
    csv_file_path = data_directory + '/' + right
    csv_representation = list()

    with open(csv_file_path, 'r') as file:
        for line in csv.reader(file):
            csv_representation.append([unicode(value) for value in line])

    xlsx_representation = main._primitive_read_excel(
        file_path=xlsx_file_path)

    assert_data_equal(xlsx_representation, csv_representation)

<|endoftext|>


<commit_before>#
# Copyright 2012 New Dream Network, LLC (DreamHost)
#
# Author: Doug Hellmann <doug.hellmann@dreamhost.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Publish a sample using the preferred RPC mechanism.
"""

import itertools
import operator
import six.moves.urllib.parse as urlparse

from oslo.config import cfg
import oslo.messaging
import oslo.messaging._drivers.common

from ceilometer import messaging
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log
from ceilometer import publisher
from ceilometer.publisher import utils

LOG = log.getLogger(__name__)

METER_PUBLISH_OPTS = [
    cfg.StrOpt('metering_topic',
               default='metering',
               help='The topic that ceilometer uses for metering messages.',
               deprecated_group="DEFAULT",
               ),
]

def register_opts(config):
    """Register the options for publishing metering messages.
    """
    config.register_opts(METER_PUBLISH_OPTS, group="publisher_rpc")

register_opts(cfg.CONF)

def oslo_messaging_is_rabbit():
    kombu = ['ceilometer.openstack.common.rpc.impl_kombu',
             'oslo.messaging._drivers.impl_rabbit:RabbitDriver'
             'rabbit']
    return cfg.CONF.rpc_backend in kombu or (
        cfg.CONF.transport_url and
        cfg.CONF.transport_url.startswith('rabbit://'))

def override_backend_retry_config(value):
    """Override the retry config option native to the configured
       rpc backend (if such a native config option exists).

       :param value: the value to override
    """
    # TODO(sileht): ultimately we should add to olso a more generic concept
    # of retry config (i.e. not specific to an individual AMQP provider)
    # see: https://bugs.launchpad.net/ceilometer/+bug/1244698
    # and: https://bugs.launchpad.net/oslo.messaging/+bug/1282639
    if oslo_messaging_is_rabbit():
        if 'rabbit_max_retries' in cfg.CONF:
            cfg.CONF.set_override('rabbit_max_retries', value)

class RPCPublisher(publisher.PublisherBase):

    def __init__(self, parsed_url):
        options = urlparse.parse_qs(parsed_url.query)
        # the values of the option is a list of url params values
        # only take care of the latest one if the option
        # is provided more than once
        self.per_meter_topic = bool(int(
            options.get('per_meter_topic', [0])[-1]))

        self.target = options.get('target', ['record_metering_data'])[0]

        self.policy = options.get('policy', ['default'])[-1]
        self.max_queue_length = int(options.get(
            'max_queue_length', [1024])[-1])

        self.local_queue = []

        if self.policy in ['queue', 'drop']:
            LOG.info(_('Publishing policy set to %s, '
                       'override backend retry config to 1') % self.policy)
            override_backend_retry_config(1)
        elif self.policy == 'default':
            LOG.info(_('Publishing policy set to %s') % self.policy)
        else:
            LOG.warn(_('Publishing policy is unknown (%s) force to default')
                     % self.policy)
            self.policy = 'default'

        self.rpc_client = messaging.get_rpc_client(version='1.0')

    def publish_samples(self, context, samples):
        """Publish samples on RPC.

        :param context: Execution context from the service or RPC call.
        :param samples: Samples from pipeline after transformation.

        """

        meters = [
            utils.meter_message_from_counter(
                sample,
                cfg.CONF.publisher.metering_secret)
            for sample in samples
        ]

        topic = cfg.CONF.publisher_rpc.metering_topic
        self.local_queue.append((context, topic, meters))

        if self.per_meter_topic:
            for meter_name, meter_list in itertools.groupby(
                    sorted(meters, key=operator.itemgetter('counter_name')),
                    operator.itemgetter('counter_name')):
                meter_list = list(meter_list)
                topic_name = topic + '.' + meter_name
                LOG.audit(_('Publishing %(m)d samples on %(n)s') % (
                          {'m': len(meter_list), 'n': topic_name}))
                self.local_queue.append((context, topic_name, meter_list))

        self.flush()

    def flush(self):
        # NOTE(sileht):
        # IO of the rpc stuff in handled by eventlet,
        # this is why the self.local_queue, is emptied before processing the
        # queue and the remaining messages in the queue are added to
        # self.local_queue after in case of a other call have already added
        # something in the self.local_queue
        queue = self.local_queue
        self.local_queue = []
        self.local_queue = self._process_queue(queue, self.policy) + \
            self.local_queue
        if self.policy == 'queue':
            self._check_queue_length()

    def _check_queue_length(self):
        queue_length = len(self.local_queue)
        if queue_length > self.max_queue_length > 0:
            count = queue_length - self.max_queue_length
            self.local_queue = self.local_queue[count:]
            LOG.warn(_("Publisher max local_queue length is exceeded, "
                     "dropping %d oldest samples") % count)

    def _process_queue(self, queue, policy):
        # NOTE(sileht):
        # the behavior of rpc.cast call depends of rabbit_max_retries
        # if rabbit_max_retries <= 0:
        #   it returns only if the msg has been sent on the amqp queue
        # if rabbit_max_retries > 0:
        #   it raises a exception if rabbitmq is unreachable
        #
        # the default policy just respect the rabbitmq configuration
        # nothing special is done if rabbit_max_retries <= 0
        # and exception is reraised if rabbit_max_retries > 0
        while queue:
            context, topic, meters = queue[0]
            try:
                self.rpc_client.prepare(topic=topic).cast(
                    context, self.target, data=meters)
            except oslo.messaging._drivers.common.RPCException:
                samples = sum([len(m) for __, __, m in queue])
                if policy == 'queue':
                    LOG.warn(_("Failed to publish %d samples, queue them"),
                             samples)
                    return queue
                elif policy == 'drop':
                    LOG.warn(_("Failed to publish %d samples, dropping them"),
                             samples)
                    return []
                # default, occur only if rabbit_max_retries > 0
                raise
            else:
                queue.pop(0)
        return []
<commit_msg>Downgrade publisher logging to debug level again<commit_after>#
# Copyright 2012 New Dream Network, LLC (DreamHost)
#
# Author: Doug Hellmann <doug.hellmann@dreamhost.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Publish a sample using the preferred RPC mechanism.
"""

import itertools
import operator
import six.moves.urllib.parse as urlparse

from oslo.config import cfg
import oslo.messaging
import oslo.messaging._drivers.common

from ceilometer import messaging
from ceilometer.openstack.common.gettextutils import _
from ceilometer.openstack.common import log
from ceilometer import publisher
from ceilometer.publisher import utils

LOG = log.getLogger(__name__)

METER_PUBLISH_OPTS = [
    cfg.StrOpt('metering_topic',
               default='metering',
               help='The topic that ceilometer uses for metering messages.',
               deprecated_group="DEFAULT",
               ),
]

def register_opts(config):
    """Register the options for publishing metering messages.
    """
    config.register_opts(METER_PUBLISH_OPTS, group="publisher_rpc")

register_opts(cfg.CONF)

def oslo_messaging_is_rabbit():
    kombu = ['ceilometer.openstack.common.rpc.impl_kombu',
             'oslo.messaging._drivers.impl_rabbit:RabbitDriver'
             'rabbit']
    return cfg.CONF.rpc_backend in kombu or (
        cfg.CONF.transport_url and
        cfg.CONF.transport_url.startswith('rabbit://'))

def override_backend_retry_config(value):
    """Override the retry config option native to the configured
       rpc backend (if such a native config option exists).

       :param value: the value to override
    """
    # TODO(sileht): ultimately we should add to olso a more generic concept
    # of retry config (i.e. not specific to an individual AMQP provider)
    # see: https://bugs.launchpad.net/ceilometer/+bug/1244698
    # and: https://bugs.launchpad.net/oslo.messaging/+bug/1282639
    if oslo_messaging_is_rabbit():
        if 'rabbit_max_retries' in cfg.CONF:
            cfg.CONF.set_override('rabbit_max_retries', value)

class RPCPublisher(publisher.PublisherBase):

    def __init__(self, parsed_url):
        options = urlparse.parse_qs(parsed_url.query)
        # the values of the option is a list of url params values
        # only take care of the latest one if the option
        # is provided more than once
        self.per_meter_topic = bool(int(
            options.get('per_meter_topic', [0])[-1]))

        self.target = options.get('target', ['record_metering_data'])[0]

        self.policy = options.get('policy', ['default'])[-1]
        self.max_queue_length = int(options.get(
            'max_queue_length', [1024])[-1])

        self.local_queue = []

        if self.policy in ['queue', 'drop']:
            LOG.info(_('Publishing policy set to %s, '
                       'override backend retry config to 1') % self.policy)
            override_backend_retry_config(1)
        elif self.policy == 'default':
            LOG.info(_('Publishing policy set to %s') % self.policy)
        else:
            LOG.warn(_('Publishing policy is unknown (%s) force to default')
                     % self.policy)
            self.policy = 'default'

        self.rpc_client = messaging.get_rpc_client(version='1.0')

    def publish_samples(self, context, samples):
        """Publish samples on RPC.

        :param context: Execution context from the service or RPC call.
        :param samples: Samples from pipeline after transformation.

        """

        meters = [
            utils.meter_message_from_counter(
                sample,
                cfg.CONF.publisher.metering_secret)
            for sample in samples
        ]

        topic = cfg.CONF.publisher_rpc.metering_topic
        self.local_queue.append((context, topic, meters))

        if self.per_meter_topic:
            for meter_name, meter_list in itertools.groupby(
                    sorted(meters, key=operator.itemgetter('counter_name')),
                    operator.itemgetter('counter_name')):
                meter_list = list(meter_list)
                topic_name = topic + '.' + meter_name
                LOG.debug('Publishing %(m)d samples on %(n)s',
                          {'m': len(meter_list), 'n': topic_name})
                self.local_queue.append((context, topic_name, meter_list))

        self.flush()

    def flush(self):
        # NOTE(sileht):
        # IO of the rpc stuff in handled by eventlet,
        # this is why the self.local_queue, is emptied before processing the
        # queue and the remaining messages in the queue are added to
        # self.local_queue after in case of a other call have already added
        # something in the self.local_queue
        queue = self.local_queue
        self.local_queue = []
        self.local_queue = self._process_queue(queue, self.policy) + \
            self.local_queue
        if self.policy == 'queue':
            self._check_queue_length()

    def _check_queue_length(self):
        queue_length = len(self.local_queue)
        if queue_length > self.max_queue_length > 0:
            count = queue_length - self.max_queue_length
            self.local_queue = self.local_queue[count:]
            LOG.warn(_("Publisher max local_queue length is exceeded, "
                     "dropping %d oldest samples") % count)

    def _process_queue(self, queue, policy):
        # NOTE(sileht):
        # the behavior of rpc.cast call depends of rabbit_max_retries
        # if rabbit_max_retries <= 0:
        #   it returns only if the msg has been sent on the amqp queue
        # if rabbit_max_retries > 0:
        #   it raises a exception if rabbitmq is unreachable
        #
        # the default policy just respect the rabbitmq configuration
        # nothing special is done if rabbit_max_retries <= 0
        # and exception is reraised if rabbit_max_retries > 0
        while queue:
            context, topic, meters = queue[0]
            try:
                self.rpc_client.prepare(topic=topic).cast(
                    context, self.target, data=meters)
            except oslo.messaging._drivers.common.RPCException:
                samples = sum([len(m) for __, __, m in queue])
                if policy == 'queue':
                    LOG.warn(_("Failed to publish %d samples, queue them"),
                             samples)
                    return queue
                elif policy == 'drop':
                    LOG.warn(_("Failed to publish %d samples, dropping them"),
                             samples)
                    return []
                # default, occur only if rabbit_max_retries > 0
                raise
            else:
                queue.pop(0)
        return []
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from redis.exceptions import WatchError

from cacheops.conf import redis_client
from cacheops.utils import get_model_name

__all__ = ('invalidate_obj', 'invalidate_model', 'invalidate_all')

def serialize_scheme(scheme):
    return ','.join(scheme)

def deserialize_scheme(scheme):
    return tuple(scheme.split(','))

def conj_cache_key(model, conj):
    return 'conj:%s:' % get_model_name(model) + '&'.join('%s=%s' % t for t in sorted(conj))

def conj_cache_key_from_scheme(model, scheme, values):
    return 'conj:%s:' % get_model_name(model) + '&'.join('%s=%s' % (f, values[f]) for f in scheme)

class ConjSchemes(object):
    """
    A container for managing models scheme collections.
    Schemes are stored in redis and cached locally.
    """
    def __init__(self):
        self.local = {}
        self.versions = {}

    def get_lookup_key(self, model_or_name):
        if not isinstance(model_or_name, str):
            model_or_name = get_model_name(model_or_name)
        return 'schemes:%s' % model_or_name

    def get_version_key(self, model_or_name):
        if not isinstance(model_or_name, str):
            model_or_name = get_model_name(model_or_name)
        return 'schemes:%s:version' % model_or_name

    def load_schemes(self, model):
        model_name = get_model_name(model)

        txn = redis_client.pipeline()
        txn.get(self.get_version_key(model))
        txn.smembers(self.get_lookup_key(model_name))
        version, members = txn.execute()

        self.local[model_name] = set(map(deserialize_scheme, members))
        self.local[model_name].add(()) # Всегда добавляем пустую схему
        self.versions[model_name] = int(version or 0)
        return self.local[model_name]

    def schemes(self, model):
        model_name = get_model_name(model)
        try:
            return self.local[model_name]
        except KeyError:
            return self.load_schemes(model)

    def version(self, model):
        try:
            return self.versions[get_model_name(model)]
        except KeyError:
            return 0

    def ensure_known(self, model, new_schemes):
        """
        Ensure that `new_schemes` are known or know them
        """
        new_schemes = set(new_schemes)
        model_name = get_model_name(model)
        loaded = False

        if model_name not in self.local:
            self.load_schemes(model)
            loaded = True
        schemes = self.local[model_name]

        if new_schemes - schemes:
            if not loaded:
                schemes = self.load_schemes(model)
            if new_schemes - schemes:
                # Write new schemes to redis
                txn = redis_client.pipeline()
                txn.incr(self.get_version_key(model_name)) # Увеличиваем версию схем

                lookup_key = self.get_lookup_key(model_name)
                for scheme in new_schemes - schemes:
                    txn.sadd(lookup_key, serialize_scheme(scheme))
                txn.execute()

                # Updating local version
                self.local[model_name].update(new_schemes)
                # We increment here instead of using incr result from redis,
                # because even our updated collection could be already obsolete
                self.versions[model_name] += 1

    def clear(self, model):
        """
        Clears schemes for models
        """
        redis_client.delete(self.get_lookup_key(model))
        redis_client.incr(self.get_version_key(model))

cache_schemes = ConjSchemes()

def invalidate_from_dict(model, values):
    """
    Invalidates caches that can possibly be influenced by object
    """
    # Create a list of invalidators from list of schemes and values of object fields
    schemes = cache_schemes.schemes(model)
    conjs_keys = [conj_cache_key_from_scheme(model, scheme, values) for scheme in schemes]

    # Get a union of all cache keys registered in invalidators
    # Get schemes version at the same time, hoping it's unchanged
    version_key = cache_schemes.get_version_key(model)
    pipe = redis_client.pipeline(transaction=False)
    # Optimistic locking: we hope schemes and invalidators won't change while we remove them
    # Ignoring this could lead to cache key hanging with it's invalidator removed
    # HACK: fix for strange WATCH handling in redis-py 2.4.6+
    if hasattr(pipe, 'pipeline_execute_command'):
        pipe.pipeline_execute_command('watch', version_key, *conjs_keys)
    else:
        pipe.watch(version_key, *conjs_keys)
    pipe.get(version_key)
    pipe.sunion(conjs_keys)
    _, version, cache_keys = pipe.execute()

    # Check if our version if schemes for model is obsolete, update them and redo if needed
    # This shouldn't be happen too often once schemes are filled a bit
    if version is None:
        version = 0
    if int(version) != cache_schemes.version(model):
        redis_client.unwatch()
        cache_schemes.load_schemes(model)
        invalidate_from_dict(model, values)

    elif cache_keys or conjs_keys:
        # `conjs_keys` are keys of sets containing `cache_keys` we are going to delete,
        # so we'll remove them too.
        # NOTE: There could be some other invalidators not matched with current object,
        #       which reference cache keys we delete, they will be hanging out for a while.
        try:
            txn = redis_client.pipeline()
            txn.delete(*(list(cache_keys) + conjs_keys))
            txn.execute()
        except WatchError:
            # Optimistic locking failed: schemes or one of invalidator sets changed.
            # Just redo everything.
            # NOTE: Scipting will be a cool alternative to optimistic locking
            invalidate_from_dict(model, values)

    else:
        redis_client.unwatch()

def invalidate_obj(obj):
    """
    Invalidates caches that can possibly be influenced by object
    """
    invalidate_from_dict(obj.__class__, obj.__dict__)

def invalidate_model(model):
    """
    Invalidates all caches for given model.
    NOTE: This is a heavy artilery which uses redis KEYS request,
          which could be relatively slow on large datasets.
    """
    conjs_keys = redis_client.keys('conj:%s:*' % get_model_name(model))
    if isinstance(conjs_keys, str):
        conjs_keys = conjs_keys.split()

    if conjs_keys:
        cache_keys = redis_client.sunion(conjs_keys)
        redis_client.delete(*(list(cache_keys) + conjs_keys))

    # BUG: a race bug here, ignoring since invalidate_model() is not for hot production use
    cache_schemes.clear(model)

def invalidate_all():
    redis_client.flushdb()
<commit_msg>Fixed a bug in invalidate_all<commit_after># -*- coding: utf-8 -*-
from redis.exceptions import WatchError

from cacheops.conf import redis_client
from cacheops.utils import get_model_name

__all__ = ('invalidate_obj', 'invalidate_model', 'invalidate_all')

def serialize_scheme(scheme):
    return ','.join(scheme)

def deserialize_scheme(scheme):
    return tuple(scheme.split(','))

def conj_cache_key(model, conj):
    return 'conj:%s:' % get_model_name(model) + '&'.join('%s=%s' % t for t in sorted(conj))

def conj_cache_key_from_scheme(model, scheme, values):
    return 'conj:%s:' % get_model_name(model) + '&'.join('%s=%s' % (f, values[f]) for f in scheme)

class ConjSchemes(object):
    """
    A container for managing models scheme collections.
    Schemes are stored in redis and cached locally.
    """
    def __init__(self):
        self.local = {}
        self.versions = {}

    def get_lookup_key(self, model_or_name):
        if not isinstance(model_or_name, str):
            model_or_name = get_model_name(model_or_name)
        return 'schemes:%s' % model_or_name

    def get_version_key(self, model_or_name):
        if not isinstance(model_or_name, str):
            model_or_name = get_model_name(model_or_name)
        return 'schemes:%s:version' % model_or_name

    def load_schemes(self, model):
        model_name = get_model_name(model)

        txn = redis_client.pipeline()
        txn.get(self.get_version_key(model))
        txn.smembers(self.get_lookup_key(model_name))
        version, members = txn.execute()

        self.local[model_name] = set(map(deserialize_scheme, members))
        self.local[model_name].add(()) # Всегда добавляем пустую схему
        self.versions[model_name] = int(version or 0)
        return self.local[model_name]

    def schemes(self, model):
        model_name = get_model_name(model)
        try:
            return self.local[model_name]
        except KeyError:
            return self.load_schemes(model)

    def version(self, model):
        try:
            return self.versions[get_model_name(model)]
        except KeyError:
            return 0

    def ensure_known(self, model, new_schemes):
        """
        Ensure that `new_schemes` are known or know them
        """
        new_schemes = set(new_schemes)
        model_name = get_model_name(model)
        loaded = False

        if model_name not in self.local:
            self.load_schemes(model)
            loaded = True
        schemes = self.local[model_name]

        if new_schemes - schemes:
            if not loaded:
                schemes = self.load_schemes(model)
            if new_schemes - schemes:
                # Write new schemes to redis
                txn = redis_client.pipeline()
                txn.incr(self.get_version_key(model_name)) # Увеличиваем версию схем

                lookup_key = self.get_lookup_key(model_name)
                for scheme in new_schemes - schemes:
                    txn.sadd(lookup_key, serialize_scheme(scheme))
                txn.execute()

                # Updating local version
                self.local[model_name].update(new_schemes)
                # We increment here instead of using incr result from redis,
                # because even our updated collection could be already obsolete
                self.versions[model_name] += 1

    def clear(self, model):
        """
        Clears schemes for models
        """
        redis_client.delete(self.get_lookup_key(model))
        redis_client.incr(self.get_version_key(model))

    def clear_all(self):
        self.local = {}
        for model_name in self.versions:
            self.versions[model_name] += 1

cache_schemes = ConjSchemes()

def invalidate_from_dict(model, values):
    """
    Invalidates caches that can possibly be influenced by object
    """
    # Create a list of invalidators from list of schemes and values of object fields
    schemes = cache_schemes.schemes(model)
    conjs_keys = [conj_cache_key_from_scheme(model, scheme, values) for scheme in schemes]

    # Get a union of all cache keys registered in invalidators
    # Get schemes version at the same time, hoping it's unchanged
    version_key = cache_schemes.get_version_key(model)
    pipe = redis_client.pipeline(transaction=False)
    # Optimistic locking: we hope schemes and invalidators won't change while we remove them
    # Ignoring this could lead to cache key hanging with it's invalidator removed
    # HACK: fix for strange WATCH handling in redis-py 2.4.6+
    if hasattr(pipe, 'pipeline_execute_command'):
        pipe.pipeline_execute_command('watch', version_key, *conjs_keys)
    else:
        pipe.watch(version_key, *conjs_keys)
    pipe.get(version_key)
    pipe.sunion(conjs_keys)
    _, version, cache_keys = pipe.execute()

    # Check if our version if schemes for model is obsolete, update them and redo if needed
    # This shouldn't be happen too often once schemes are filled a bit
    if version is None:
        version = 0
    if int(version) != cache_schemes.version(model):
        redis_client.unwatch()
        cache_schemes.load_schemes(model)
        invalidate_from_dict(model, values)

    elif cache_keys or conjs_keys:
        # `conjs_keys` are keys of sets containing `cache_keys` we are going to delete,
        # so we'll remove them too.
        # NOTE: There could be some other invalidators not matched with current object,
        #       which reference cache keys we delete, they will be hanging out for a while.
        try:
            txn = redis_client.pipeline()
            txn.delete(*(list(cache_keys) + conjs_keys))
            txn.execute()
        except WatchError:
            # Optimistic locking failed: schemes or one of invalidator sets changed.
            # Just redo everything.
            # NOTE: Scipting will be a cool alternative to optimistic locking
            invalidate_from_dict(model, values)

    else:
        redis_client.unwatch()

def invalidate_obj(obj):
    """
    Invalidates caches that can possibly be influenced by object
    """
    invalidate_from_dict(obj.__class__, obj.__dict__)

def invalidate_model(model):
    """
    Invalidates all caches for given model.
    NOTE: This is a heavy artilery which uses redis KEYS request,
          which could be relatively slow on large datasets.
    """
    conjs_keys = redis_client.keys('conj:%s:*' % get_model_name(model))
    if isinstance(conjs_keys, str):
        conjs_keys = conjs_keys.split()

    if conjs_keys:
        cache_keys = redis_client.sunion(conjs_keys)
        redis_client.delete(*(list(cache_keys) + conjs_keys))

    # BUG: a race bug here, ignoring since invalidate_model() is not for hot production use
    cache_schemes.clear(model)

def invalidate_all():
    redis_client.flushdb()
    cache_schemes.clear_all()
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
from easy_thumbnails import fields as easy_thumbnails_fields, \
    files as easy_thumbnails_files

from filer import settings as filer_settings
from filer.utils.filer_easy_thumbnails import ThumbnailerNameMixin
from filer.utils.cdn import get_cdn_url

STORAGES = {
    'public': filer_settings.FILER_PUBLICMEDIA_STORAGE,
    'private': filer_settings.FILER_PRIVATEMEDIA_STORAGE,
}
THUMBNAIL_STORAGES = {
    'public': filer_settings.FILER_PUBLICMEDIA_THUMBNAIL_STORAGE,
    'private': filer_settings.FILER_PRIVATEMEDIA_THUMBNAIL_STORAGE,
}
THUMBNAIL_OPTIONS = {
    'public': filer_settings.FILER_PUBLICMEDIA_THUMBNAIL_OPTIONS,
    'private': filer_settings.FILER_PRIVATEMEDIA_THUMBNAIL_OPTIONS,
}

def generate_filename_multistorage(instance, filename):
    if instance.is_public:
        upload_to = filer_settings.FILER_PUBLICMEDIA_UPLOAD_TO
    else:
        upload_to = filer_settings.FILER_PRIVATEMEDIA_UPLOAD_TO

    if callable(upload_to):
        return upload_to(instance, filename)
    else:
        return upload_to

class CdnAwareThumbnailFile(object):

    def __init__(self, thumbnail_file, filer_file):
        self._thumbnail_file = thumbnail_file
        self._filer_file = filer_file

    def __getattr__(self, attr_name):
        return getattr(self._thumbnail_file, attr_name)

    @property
    def url(self):
        return get_cdn_url(self._filer_file, self._thumbnail_file.url)

class MultiStorageFieldFile(ThumbnailerNameMixin,
                            easy_thumbnails_files.ThumbnailerFieldFile):
    def __init__(self, instance, field, name):
        """
        This is a little weird, but I couldn't find a better solution.
        Thumbnailer.__init__ is called first for proper object inizialization.
        Then we override some attributes defined at runtime with properties.
        We cannot simply call super().__init__ because filer Field objects
        doesn't have a storage attribute.
        """
        easy_thumbnails_files.Thumbnailer.__init__(self, None, name)
        self.instance = instance
        self.field = field
        self._committed = True
        self.storages = self.field.storages
        self.thumbnail_storages = self.field.thumbnail_storages
        self.thumbnail_options = self.field.thumbnail_options
        self.storage = self._storage
        self.source_storage = self._source_storage
        self.thumbnail_storage = self._thumbnail_storage
        self.thumbnail_basedir = self._thumbnail_base_dir

    @property
    def _storage(self):
        if self.instance.is_public:
            return self.storages['public']
        else:
            return self.storages['private']

    @property
    def _source_storage(self):
        if self.instance.is_public:
            return self.storages['public']
        else:
            return self.storages['private']

    @property
    def _thumbnail_storage(self):
        if self.instance.is_public:
            return self.thumbnail_storages['public']
        else:
            return self.thumbnail_storages['private']

    @property
    def _thumbnail_base_dir(self):
        if self.instance.is_public:
            return self.thumbnail_options['public'].get('base_dir', '')
        else:
            return self.thumbnail_options['private'].get('base_dir', '')

    def _get_url(self):
        url = super(MultiStorageFieldFile, self)._get_url()
        return get_cdn_url(self.instance, url)
    url = property(_get_url)

    def save(self, name, content, save=True):
        content.seek(0) # Ensure we upload the whole file
        super(MultiStorageFieldFile, self).save(name, content, save)

    def get_thumbnail(self, opts, save=True, generate=None):
        thumbnail = super(MultiStorageFieldFile, self).get_thumbnail(opts, save, generate)
        return CdnAwareThumbnailFile(thumbnail, self.instance)

class MultiStorageFileField(easy_thumbnails_fields.ThumbnailerField):
    attr_class = MultiStorageFieldFile

    def __init__(self, verbose_name=None, name=None,
                 storages=None, thumbnail_storages=None, thumbnail_options=None, **kwargs):
        self.storages = storages or STORAGES
        self.thumbnail_storages = thumbnail_storages or THUMBNAIL_STORAGES
        self.thumbnail_options = thumbnail_options or THUMBNAIL_OPTIONS
        super(easy_thumbnails_fields.ThumbnailerField, self).__init__(
                                      verbose_name=verbose_name, name=name,
                                      upload_to=generate_filename_multistorage,
                                      storage=None, **kwargs)
<commit_msg>Removed extra blank line (PEP-8)<commit_after>#-*- coding: utf-8 -*-
from easy_thumbnails import fields as easy_thumbnails_fields, \
    files as easy_thumbnails_files

from filer import settings as filer_settings
from filer.utils.filer_easy_thumbnails import ThumbnailerNameMixin
from filer.utils.cdn import get_cdn_url

STORAGES = {
    'public': filer_settings.FILER_PUBLICMEDIA_STORAGE,
    'private': filer_settings.FILER_PRIVATEMEDIA_STORAGE,
}
THUMBNAIL_STORAGES = {
    'public': filer_settings.FILER_PUBLICMEDIA_THUMBNAIL_STORAGE,
    'private': filer_settings.FILER_PRIVATEMEDIA_THUMBNAIL_STORAGE,
}
THUMBNAIL_OPTIONS = {
    'public': filer_settings.FILER_PUBLICMEDIA_THUMBNAIL_OPTIONS,
    'private': filer_settings.FILER_PRIVATEMEDIA_THUMBNAIL_OPTIONS,
}

def generate_filename_multistorage(instance, filename):
    if instance.is_public:
        upload_to = filer_settings.FILER_PUBLICMEDIA_UPLOAD_TO
    else:
        upload_to = filer_settings.FILER_PRIVATEMEDIA_UPLOAD_TO

    if callable(upload_to):
        return upload_to(instance, filename)
    else:
        return upload_to

class CdnAwareThumbnailFile(object):

    def __init__(self, thumbnail_file, filer_file):
        self._thumbnail_file = thumbnail_file
        self._filer_file = filer_file

    def __getattr__(self, attr_name):
        return getattr(self._thumbnail_file, attr_name)

    @property
    def url(self):
        return get_cdn_url(self._filer_file, self._thumbnail_file.url)

class MultiStorageFieldFile(ThumbnailerNameMixin,
                            easy_thumbnails_files.ThumbnailerFieldFile):
    def __init__(self, instance, field, name):
        """
        This is a little weird, but I couldn't find a better solution.
        Thumbnailer.__init__ is called first for proper object inizialization.
        Then we override some attributes defined at runtime with properties.
        We cannot simply call super().__init__ because filer Field objects
        doesn't have a storage attribute.
        """
        easy_thumbnails_files.Thumbnailer.__init__(self, None, name)
        self.instance = instance
        self.field = field
        self._committed = True
        self.storages = self.field.storages
        self.thumbnail_storages = self.field.thumbnail_storages
        self.thumbnail_options = self.field.thumbnail_options
        self.storage = self._storage
        self.source_storage = self._source_storage
        self.thumbnail_storage = self._thumbnail_storage
        self.thumbnail_basedir = self._thumbnail_base_dir

    @property
    def _storage(self):
        if self.instance.is_public:
            return self.storages['public']
        else:
            return self.storages['private']

    @property
    def _source_storage(self):
        if self.instance.is_public:
            return self.storages['public']
        else:
            return self.storages['private']

    @property
    def _thumbnail_storage(self):
        if self.instance.is_public:
            return self.thumbnail_storages['public']
        else:
            return self.thumbnail_storages['private']

    @property
    def _thumbnail_base_dir(self):
        if self.instance.is_public:
            return self.thumbnail_options['public'].get('base_dir', '')
        else:
            return self.thumbnail_options['private'].get('base_dir', '')

    def _get_url(self):
        url = super(MultiStorageFieldFile, self)._get_url()
        return get_cdn_url(self.instance, url)
    url = property(_get_url)

    def save(self, name, content, save=True):
        content.seek(0) # Ensure we upload the whole file
        super(MultiStorageFieldFile, self).save(name, content, save)

    def get_thumbnail(self, opts, save=True, generate=None):
        thumbnail = super(MultiStorageFieldFile, self).get_thumbnail(opts, save, generate)
        return CdnAwareThumbnailFile(thumbnail, self.instance)

class MultiStorageFileField(easy_thumbnails_fields.ThumbnailerField):
    attr_class = MultiStorageFieldFile

    def __init__(self, verbose_name=None, name=None,
                 storages=None, thumbnail_storages=None, thumbnail_options=None, **kwargs):
        self.storages = storages or STORAGES
        self.thumbnail_storages = thumbnail_storages or THUMBNAIL_STORAGES
        self.thumbnail_options = thumbnail_options or THUMBNAIL_OPTIONS
        super(easy_thumbnails_fields.ThumbnailerField, self).__init__(
                                      verbose_name=verbose_name, name=name,
                                      upload_to=generate_filename_multistorage,
                                      storage=None, **kwargs)
<|endoftext|>