<commit_before>from __future__ import print_function
import copy
import datetime
import json
import logging
import os
import six
import subprocess

DATE_FORMAT = '%Y%m%dT%H%M%SZ'
REPR_OUTPUT_SIZE = 10
PENDING = 'pending'
COMPLETED = 'completed'

logger = logging.getLogger(__name__)

class TaskWarriorException(Exception):
    pass

class TaskResource(object):
    read_only_fields = []

    def _load_data(self, data):
        self._data = data

    def __getitem__(self, key):
        hydrate_func = getattr(self, 'deserialize_{0}'.format(key),
                               lambda x: x)
        return hydrate_func(self._data.get(key))

    def __setitem__(self, key, value):
        if key in self.read_only_fields:
            raise RuntimeError('Field \'%s\' is read-only' % key)
        dehydrate_func = getattr(self, 'serialize_{0}'.format(key),
                                 lambda x: x)
        self._data[key] = dehydrate_func(value)
        self._modified_fields.add(key)

    def __str__(self):
        s = six.text_type(self.__unicode__())
        if not six.PY3:
            s = s.encode('utf-8')
        return s

    def __repr__(self):
        return str(self)

class TaskAnnotation(TaskResource):
    read_only_fields = ['entry', 'description']

    def __init__(self, task, data={}):
        self.task = task
        self._load_data(data)

    def deserialize_entry(self, data):
        return datetime.datetime.strptime(data, DATE_FORMAT) if data else None

    def serialize_entry(self, date):
        return date.strftime(DATE_FORMAT) if date else ''

    def remove(self):
        self.task.remove_annotation(self)

    def __unicode__(self):
        return self['description']

    __repr__ = __unicode__

class Task(TaskResource):
    read_only_fields = ['id', 'entry', 'urgency']

    class DoesNotExist(Exception):
        pass

    def __init__(self, warrior, data={}):
        self.warrior = warrior
        self._load_data(data)
        self._modified_fields = set()

    def __unicode__(self):
        return self['description']

    def serialize_due(self, date):
        return date.strftime(DATE_FORMAT)

    def deserialize_due(self, date_str):
        if not date_str:
            return None
        return datetime.datetime.strptime(date_str, DATE_FORMAT)

    def deserialize_annotations(self, data):
        return [TaskAnnotation(self, d) for d in data] if data else []

    def deserialize_tags(self, tags):
        if isinstance(tags, basestring):
            return tags.split(',') if tags else []
        return tags

    def serialize_tags(self, tags):
        return ','.join(tags) if tags else ''

    def delete(self):
        self.warrior.execute_command([self['id'], 'delete'], config_override={
            'confirmation': 'no',
        })

    def done(self):
        self.warrior.execute_command([self['id'], 'done'])

    def save(self):
        args = [self['id'], 'modify'] if self['id'] else ['add']
        args.extend(self._get_modified_fields_as_args())
        self.warrior.execute_command(args)
        self._modified_fields.clear()

    def add_annotation(self, annotation):
        args = [self['id'], 'annotate', annotation]
        self.warrior.execute_command(args)
        self.refresh(only_fields=['annotations'])

    def remove_annotation(self, annotation):
        if isinstance(annotation, TaskAnnotation):
            annotation = annotation['description']
        args = [self['id'], 'denotate', annotation]
        self.warrior.execute_command(args)
        self.refresh(only_fields=['annotations'])

    def _get_modified_fields_as_args(self):
        args = []
        for field in self._modified_fields:
            args.append('{}:{}'.format(field, self._data[field]))
        return args

    def refresh(self, only_fields=[]):
        args = [self['uuid'], 'export']
        new_data = json.loads(self.warrior.execute_command(args)[0])
        if only_fields:
            to_update = dict(
                [(k, new_data.get(k)) for k in only_fields])
            self._data.update(to_update)
        else:
            self._data = new_data

class TaskFilter(object):
    """
    A set of parameters to filter the task list with.
    """

    def __init__(self, filter_params=[]):
        self.filter_params = filter_params

    def add_filter(self, filter_str):
        self.filter_params.append(filter_str)

    def add_filter_param(self, key, value):
        key = key.replace('__', '.')

        # Replace the value with empty string, since that is the
        # convention in TW for empty values
        value = value if value is not None else ''
        self.filter_params.append('{0}:{1}'.format(key, value))

    def get_filter_params(self):
        return [f for f in self.filter_params if f]

    def clone(self):
        c = self.__class__()
        c.filter_params = list(self.filter_params)
        return c

class TaskQuerySet(object):
    """
    Represents a lazy lookup for a task objects.
    """

    def __init__(self, warrior=None, filter_obj=None):
        self.warrior = warrior
        self._result_cache = None
        self.filter_obj = filter_obj or TaskFilter()

    def __deepcopy__(self, memo):
        """
        Deep copy of a QuerySet doesn't populate the cache
        """
        obj = self.__class__()
        for k, v in self.__dict__.items():
            if k in ('_iter', '_result_cache'):
                obj.__dict__[k] = None
            else:
                obj.__dict__[k] = copy.deepcopy(v, memo)
        return obj

    def __repr__(self):
        data = list(self[:REPR_OUTPUT_SIZE + 1])
        if len(data) > REPR_OUTPUT_SIZE:
            data[-1] = "...(remaining elements truncated)..."
        return repr(data)

    def __len__(self):
        if self._result_cache is None:
            self._result_cache = list(self)
        return len(self._result_cache)

    def __iter__(self):
        if self._result_cache is None:
            self._result_cache = self._execute()
        return iter(self._result_cache)

    def __getitem__(self, k):
        if self._result_cache is None:
            self._result_cache = list(self)
        return self._result_cache.__getitem__(k)

    def __bool__(self):
        if self._result_cache is not None:
            return bool(self._result_cache)
        try:
            next(iter(self))
        except StopIteration:
            return False
        return True

    def __nonzero__(self):
        return type(self).__bool__(self)

    def _clone(self, klass=None, **kwargs):
        if klass is None:
            klass = self.__class__
        filter_obj = self.filter_obj.clone()
        c = klass(warrior=self.warrior, filter_obj=filter_obj)
        c.__dict__.update(kwargs)
        return c

    def _execute(self):
        """
        Fetch the tasks which match the current filters.
        """
        return self.warrior.filter_tasks(self.filter_obj)

    def all(self):
        """
        Returns a new TaskQuerySet that is a copy of the current one.
        """
        return self._clone()

    def pending(self):
        return self.filter(status=PENDING)

    def completed(self):
        return self.filter(status=COMPLETED)

    def filter(self, *args, **kwargs):
        """
        Returns a new TaskQuerySet with the given filters added.
        """
        clone = self._clone()
        for f in args:
            clone.filter_obj.add_filter(f)
        for key, value in kwargs.items():
            clone.filter_obj.add_filter_param(key, value)
        return clone

    def get(self, **kwargs):
        """
        Performs the query and returns a single object matching the given
        keyword arguments.
        """
        clone = self.filter(**kwargs)
        num = len(clone)
        if num == 1:
            return clone._result_cache[0]
        if not num:
            raise Task.DoesNotExist(
                'Task matching query does not exist. '
                'Lookup parameters were {0}'.format(kwargs))
        raise ValueError(
            'get() returned more than one Task -- it returned {0}! '
            'Lookup parameters were {1}'.format(num, kwargs))

class TaskWarrior(object):
    def __init__(self, data_location='~/.task', create=True):
        data_location = os.path.expanduser(data_location)
        if create and not os.path.exists(data_location):
            os.makedirs(data_location)
        self.config = {
            'data.location': os.path.expanduser(data_location),
        }
        self.tasks = TaskQuerySet(self)

    def _get_command_args(self, args, config_override={}):
        command_args = ['task', 'rc:/']
        config = self.config.copy()
        config.update(config_override)
        for item in config.items():
            command_args.append('rc.{0}={1}'.format(*item))
        command_args.extend(map(str, args))
        return command_args

    def execute_command(self, args, config_override={}):
        command_args = self._get_command_args(
            args, config_override=config_override)
        logger.debug(' '.join(command_args))
        p = subprocess.Popen(command_args, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        stdout, stderr = [x.decode('utf-8') for x in p.communicate()]
        if p.returncode:
            if stderr.strip():
                error_msg = stderr.strip().splitlines()[-1]
            else:
                error_msg = stdout.strip()
            raise TaskWarriorException(error_msg)
        return stdout.strip().split('\n')

    def filter_tasks(self, filter_obj):
        args = ['export', '--'] + filter_obj.get_filter_params()
        tasks = []
        for line in self.execute_command(args):
            if line:
                data = line.strip(',')
                try:
                    tasks.append(Task(self, json.loads(data)))
                except ValueError:
                    raise TaskWarriorException('Invalid JSON: %s' % data)
        return tasks

    def merge_with(self, path, push=False):
        path = path.rstrip('/') + '/'
        self.execute_command(['merge', path], config_override={
            'merge.autopush': 'yes' if push else 'no',
        })

    def undo(self):
        self.execute_command(['undo'], config_override={
            'confirmation': 'no',
        })
<commit_msg>Task: Perform done and deleted operations only when necessary<commit_after>from __future__ import print_function
import copy
import datetime
import json
import logging
import os
import six
import subprocess

DATE_FORMAT = '%Y%m%dT%H%M%SZ'
REPR_OUTPUT_SIZE = 10
PENDING = 'pending'
COMPLETED = 'completed'

logger = logging.getLogger(__name__)

class TaskWarriorException(Exception):
    pass

class TaskResource(object):
    read_only_fields = []

    def _load_data(self, data):
        self._data = data

    def __getitem__(self, key):
        hydrate_func = getattr(self, 'deserialize_{0}'.format(key),
                               lambda x: x)
        return hydrate_func(self._data.get(key))

    def __setitem__(self, key, value):
        if key in self.read_only_fields:
            raise RuntimeError('Field \'%s\' is read-only' % key)
        dehydrate_func = getattr(self, 'serialize_{0}'.format(key),
                                 lambda x: x)
        self._data[key] = dehydrate_func(value)
        self._modified_fields.add(key)

    def __str__(self):
        s = six.text_type(self.__unicode__())
        if not six.PY3:
            s = s.encode('utf-8')
        return s

    def __repr__(self):
        return str(self)

class TaskAnnotation(TaskResource):
    read_only_fields = ['entry', 'description']

    def __init__(self, task, data={}):
        self.task = task
        self._load_data(data)

    def deserialize_entry(self, data):
        return datetime.datetime.strptime(data, DATE_FORMAT) if data else None

    def serialize_entry(self, date):
        return date.strftime(DATE_FORMAT) if date else ''

    def remove(self):
        self.task.remove_annotation(self)

    def __unicode__(self):
        return self['description']

    __repr__ = __unicode__

class Task(TaskResource):
    read_only_fields = ['id', 'entry', 'urgency']

    class DoesNotExist(Exception):
        pass

    class CompletedTask(Exception):
        """
        Raised when the operation cannot be performed on the completed task.
        """
        pass

    class DeletedTask(Exception):
        """
        Raised when the operation cannot be performed on the deleted task.
        """
        pass

    def __init__(self, warrior, data={}):
        self.warrior = warrior
        self._load_data(data)
        self._modified_fields = set()

    def __unicode__(self):
        return self['description']

    @property
    def completed(self):
        return self['status'] == six.text_type('completed')

    @property
    def deleted(self):
        return self['status'] == six.text_type('deleted')

    @property
    def waiting(self):
        return self['status'] == six.text_type('waiting')

    @property
    def pending(self):
        return self['status'] == six.text_type('pending')

    def serialize_due(self, date):
        return date.strftime(DATE_FORMAT)

    def deserialize_due(self, date_str):
        if not date_str:
            return None
        return datetime.datetime.strptime(date_str, DATE_FORMAT)

    def deserialize_annotations(self, data):
        return [TaskAnnotation(self, d) for d in data] if data else []

    def deserialize_tags(self, tags):
        if isinstance(tags, basestring):
            return tags.split(',') if tags else []
        return tags

    def serialize_tags(self, tags):
        return ','.join(tags) if tags else ''

    def delete(self):
        # Refresh the status, and raise exception if the task is deleted
        self.refresh(only_fields=['status'])

        if self.deleted:
            raise self.DeletedTask("Task was already deleted")

        self.warrior.execute_command([self['id'], 'delete'], config_override={
            'confirmation': 'no',
        })

        # Refresh the status again, so that we have updated info stored
        self.refresh(only_fields=['status'])

    def done(self):
        # Refresh, and raise exception if task is already completed/deleted
        self.refresh(only_fields=['status'])

        if self.completed:
            raise self.CompletedTask("Cannot complete a completed task")
        elif self.deleted:
            raise self.DeletedTask("Deleted task cannot be completed")

        self.warrior.execute_command([self['id'], 'done'])

        # Refresh the status again, so that we have updated info stored
        self.refresh(only_fields=['status'])

    def save(self):
        args = [self['id'], 'modify'] if self['id'] else ['add']
        args.extend(self._get_modified_fields_as_args())
        self.warrior.execute_command(args)
        self._modified_fields.clear()

    def add_annotation(self, annotation):
        args = [self['id'], 'annotate', annotation]
        self.warrior.execute_command(args)
        self.refresh(only_fields=['annotations'])

    def remove_annotation(self, annotation):
        if isinstance(annotation, TaskAnnotation):
            annotation = annotation['description']
        args = [self['id'], 'denotate', annotation]
        self.warrior.execute_command(args)
        self.refresh(only_fields=['annotations'])

    def _get_modified_fields_as_args(self):
        args = []
        for field in self._modified_fields:
            args.append('{}:{}'.format(field, self._data[field]))
        return args

    def refresh(self, only_fields=[]):
        args = [self['uuid'], 'export']
        new_data = json.loads(self.warrior.execute_command(args)[0])
        if only_fields:
            to_update = dict(
                [(k, new_data.get(k)) for k in only_fields])
            self._data.update(to_update)
        else:
            self._data = new_data

class TaskFilter(object):
    """
    A set of parameters to filter the task list with.
    """

    def __init__(self, filter_params=[]):
        self.filter_params = filter_params

    def add_filter(self, filter_str):
        self.filter_params.append(filter_str)

    def add_filter_param(self, key, value):
        key = key.replace('__', '.')

        # Replace the value with empty string, since that is the
        # convention in TW for empty values
        value = value if value is not None else ''
        self.filter_params.append('{0}:{1}'.format(key, value))

    def get_filter_params(self):
        return [f for f in self.filter_params if f]

    def clone(self):
        c = self.__class__()
        c.filter_params = list(self.filter_params)
        return c

class TaskQuerySet(object):
    """
    Represents a lazy lookup for a task objects.
    """

    def __init__(self, warrior=None, filter_obj=None):
        self.warrior = warrior
        self._result_cache = None
        self.filter_obj = filter_obj or TaskFilter()

    def __deepcopy__(self, memo):
        """
        Deep copy of a QuerySet doesn't populate the cache
        """
        obj = self.__class__()
        for k, v in self.__dict__.items():
            if k in ('_iter', '_result_cache'):
                obj.__dict__[k] = None
            else:
                obj.__dict__[k] = copy.deepcopy(v, memo)
        return obj

    def __repr__(self):
        data = list(self[:REPR_OUTPUT_SIZE + 1])
        if len(data) > REPR_OUTPUT_SIZE:
            data[-1] = "...(remaining elements truncated)..."
        return repr(data)

    def __len__(self):
        if self._result_cache is None:
            self._result_cache = list(self)
        return len(self._result_cache)

    def __iter__(self):
        if self._result_cache is None:
            self._result_cache = self._execute()
        return iter(self._result_cache)

    def __getitem__(self, k):
        if self._result_cache is None:
            self._result_cache = list(self)
        return self._result_cache.__getitem__(k)

    def __bool__(self):
        if self._result_cache is not None:
            return bool(self._result_cache)
        try:
            next(iter(self))
        except StopIteration:
            return False
        return True

    def __nonzero__(self):
        return type(self).__bool__(self)

    def _clone(self, klass=None, **kwargs):
        if klass is None:
            klass = self.__class__
        filter_obj = self.filter_obj.clone()
        c = klass(warrior=self.warrior, filter_obj=filter_obj)
        c.__dict__.update(kwargs)
        return c

    def _execute(self):
        """
        Fetch the tasks which match the current filters.
        """
        return self.warrior.filter_tasks(self.filter_obj)

    def all(self):
        """
        Returns a new TaskQuerySet that is a copy of the current one.
        """
        return self._clone()

    def pending(self):
        return self.filter(status=PENDING)

    def completed(self):
        return self.filter(status=COMPLETED)

    def filter(self, *args, **kwargs):
        """
        Returns a new TaskQuerySet with the given filters added.
        """
        clone = self._clone()
        for f in args:
            clone.filter_obj.add_filter(f)
        for key, value in kwargs.items():
            clone.filter_obj.add_filter_param(key, value)
        return clone

    def get(self, **kwargs):
        """
        Performs the query and returns a single object matching the given
        keyword arguments.
        """
        clone = self.filter(**kwargs)
        num = len(clone)
        if num == 1:
            return clone._result_cache[0]
        if not num:
            raise Task.DoesNotExist(
                'Task matching query does not exist. '
                'Lookup parameters were {0}'.format(kwargs))
        raise ValueError(
            'get() returned more than one Task -- it returned {0}! '
            'Lookup parameters were {1}'.format(num, kwargs))

class TaskWarrior(object):
    def __init__(self, data_location='~/.task', create=True):
        data_location = os.path.expanduser(data_location)
        if create and not os.path.exists(data_location):
            os.makedirs(data_location)
        self.config = {
            'data.location': os.path.expanduser(data_location),
        }
        self.tasks = TaskQuerySet(self)

    def _get_command_args(self, args, config_override={}):
        command_args = ['task', 'rc:/']
        config = self.config.copy()
        config.update(config_override)
        for item in config.items():
            command_args.append('rc.{0}={1}'.format(*item))
        command_args.extend(map(str, args))
        return command_args

    def execute_command(self, args, config_override={}):
        command_args = self._get_command_args(
            args, config_override=config_override)
        logger.debug(' '.join(command_args))
        p = subprocess.Popen(command_args, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        stdout, stderr = [x.decode('utf-8') for x in p.communicate()]
        if p.returncode:
            if stderr.strip():
                error_msg = stderr.strip().splitlines()[-1]
            else:
                error_msg = stdout.strip()
            raise TaskWarriorException(error_msg)
        return stdout.strip().split('\n')

    def filter_tasks(self, filter_obj):
        args = ['export', '--'] + filter_obj.get_filter_params()
        tasks = []
        for line in self.execute_command(args):
            if line:
                data = line.strip(',')
                try:
                    tasks.append(Task(self, json.loads(data)))
                except ValueError:
                    raise TaskWarriorException('Invalid JSON: %s' % data)
        return tasks

    def merge_with(self, path, push=False):
        path = path.rstrip('/') + '/'
        self.execute_command(['merge', path], config_override={
            'merge.autopush': 'yes' if push else 'no',
        })

    def undo(self):
        self.execute_command(['undo'], config_override={
            'confirmation': 'no',
        })
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from contextlib import contextmanager
from hashlib import md5
from random import random
import datetime
import logging
import logging.config
import os
import traceback

from slugify import slugify as do_slug
import dateutil.parser
import mutagen

import shiva
from shiva.exceptions import MetadataManagerReadError

def get_shiva_path():
    return os.path.dirname(os.path.abspath(shiva.__file__))

def get_logger():
    logging_conf = os.path.join(get_shiva_path(), 'logging.conf')
    logging.config.fileConfig(logging_conf)

    return logging.getLogger('shiva')

def randstr(length=None):
    """ Generates a random string of the given length. """

    if length < 1:
        return ''

    digest = ''
    while len(digest) < length:
        digest += md5(str(random())).hexdigest()

    if length:
        return digest[:length]

    return digest

def slugify(text):
    """
    Generates an alphanumeric slug. If the resulting slug is numeric-only a
    hyphen and a random string is appended to it.

    """

    if not text:
        return ''

    slug = do_slug(text)
    if not slug:
        slug = randstr(length=6)

    try:
        is_int = isinstance(int(slug), int)
    except ValueError:
        is_int = False

    if is_int:
        slug += u'-%s' % randstr(6)

    return slug

def _import(class_path):
    """ Imports a module or class from a string in dot notation. """

    bits = class_path.split('.')
    mod_name = '.'.join(bits[:-1])
    cls_name = bits[-1]

    mod = __import__(mod_name, None, None, cls_name)

    return getattr(mod, cls_name)

@contextmanager
def ignored(*exceptions, **kwargs):
    """Context manager that ignores all of the specified exceptions. This will
    be in the standard library starting with Python 3.4."""
    try:
        yield
    except exceptions:
        if kwargs.get('print_traceback'):
            print(traceback.format_exc())

class MetadataManager(object):
    """A format-agnostic metadata wrapper around Mutagen.

    This makes reading/writing audio metadata easy across all possible audio
    formats by using properties for the different keys.

    In order to persist changes to the metadata, the ``save()`` method needs to
    be called.

    """

    def __init__(self, filepath):
        self._original_path = filepath
        try:
            self.reader = mutagen.File(filepath, easy=True)
        except Exception, e:
            raise MetadataManagerReadError(e.message)

    @property
    def title(self):
        return self._getter('title')

    @property
    def artist(self):
        """The artist name."""
        return self._getter('artist')

    @artist.setter
    def artist(self, value):
        self.reader['artist'] = value

    @property
    def album(self):
        """The album name."""
        return self._getter('album')

    @album.setter
    def album(self, value):
        self.reader['album'] = value

    @property
    def release_year(self):
        """The album release year."""
        default_date = datetime.datetime(datetime.MINYEAR, 1, 1)
        default_date = default_date.replace(tzinfo=None)
        date = self._getter('date', '')
        try:
            parsed_date = dateutil.parser.parse(date, default=default_date)
        except ValueError:
            return None

        parsed_date = parsed_date.replace(tzinfo=None)
        if parsed_date != default_date:
            return parsed_date.year

        return None

    @release_year.setter
    def release_year(self, value):
        self.reader['year'] = value

    @property
    def track_number(self):
        """The track number."""

        try:
            _number = int(self._getter('tracknumber'))
        except (TypeError, ValueError):
            _number = None

        return _number

    @track_number.setter
    def track_number(self, value):
        self.reader['tracknumber'] = value

    @property
    def genre(self):
        """The music genre."""
        return self._getter('genre')

    @genre.setter
    def genre(self, value):
        self.genre = value

    @property
    def length(self):
        """The length of the song in seconds."""
        return int(round(self.reader.info.length))

    @property
    def bitrate(self):
        """The audio bitrate."""
        if hasattr(self.reader.info, 'bitrate'):
            return self.reader.info.bitrate

    @property
    def sample_rate(self):
        """The audio sample rate."""
        return self.reader.info.sample_rate

    @property
    def filename(self):
        """The file name of this audio file."""
        return os.path.basename(self.reader.filename)

    @property
    def filepath(self):
        """The absolute path to this audio file."""
        return os.path.abspath(self.reader.filename)

    @property
    def origpath(self):
        """The original path with which this class was instantiated. This
        function avoids a call to ``os.path``.  Usually you'll want to use
        either :meth:`.filename` or :meth:`.filepath` instead."""
        return self._original_path

    @property
    def filesize(self):
        """The size of this audio file in the filesystem."""
        return os.stat(self.reader.filename).st_size

    # Helper functions

    def _getter(self, attr, fallback=None):
        """Return the first list item of the specified attribute or fall back
        to a default value if attribute is not available."""
        return self.reader[attr][0] if attr in self.reader else fallback

    def save(self):
        """Save changes to file metadata."""
        self.reader.save()
<commit_msg>utils.ignored uses the logger<commit_after># -*- coding: utf-8 -*-
from contextlib import contextmanager
from hashlib import md5
from random import random
import datetime
import logging
import logging.config
import os
import traceback

from slugify import slugify as do_slug
import dateutil.parser
import mutagen

import shiva
from shiva.exceptions import MetadataManagerReadError

def get_shiva_path():
    return os.path.dirname(os.path.abspath(shiva.__file__))

def get_logger():
    logging_conf = os.path.join(get_shiva_path(), 'logging.conf')
    logging.config.fileConfig(logging_conf)

    return logging.getLogger('shiva')

def randstr(length=None):
    """ Generates a random string of the given length. """

    if length < 1:
        return ''

    digest = ''
    while len(digest) < length:
        digest += md5(str(random())).hexdigest()

    if length:
        return digest[:length]

    return digest

def slugify(text):
    """
    Generates an alphanumeric slug. If the resulting slug is numeric-only a
    hyphen and a random string is appended to it.

    """

    if not text:
        return ''

    slug = do_slug(text)
    if not slug:
        slug = randstr(length=6)

    try:
        is_int = isinstance(int(slug), int)
    except ValueError:
        is_int = False

    if is_int:
        slug += u'-%s' % randstr(6)

    return slug

def _import(class_path):
    """ Imports a module or class from a string in dot notation. """

    bits = class_path.split('.')
    mod_name = '.'.join(bits[:-1])
    cls_name = bits[-1]

    mod = __import__(mod_name, None, None, cls_name)

    return getattr(mod, cls_name)

@contextmanager
def ignored(*exceptions, **kwargs):
    """Context manager that ignores all of the specified exceptions. This will
    be in the standard library starting with Python 3.4."""

    log = get_logger()
    try:
        yield
    except exceptions:
        if kwargs.get('print_traceback'):
            log.debug(traceback.format_exc())

class MetadataManager(object):
    """A format-agnostic metadata wrapper around Mutagen.

    This makes reading/writing audio metadata easy across all possible audio
    formats by using properties for the different keys.

    In order to persist changes to the metadata, the ``save()`` method needs to
    be called.

    """

    def __init__(self, filepath):
        self._original_path = filepath
        try:
            self.reader = mutagen.File(filepath, easy=True)
        except Exception, e:
            raise MetadataManagerReadError(e.message)

    @property
    def title(self):
        return self._getter('title')

    @property
    def artist(self):
        """The artist name."""
        return self._getter('artist')

    @artist.setter
    def artist(self, value):
        self.reader['artist'] = value

    @property
    def album(self):
        """The album name."""
        return self._getter('album')

    @album.setter
    def album(self, value):
        self.reader['album'] = value

    @property
    def release_year(self):
        """The album release year."""
        default_date = datetime.datetime(datetime.MINYEAR, 1, 1)
        default_date = default_date.replace(tzinfo=None)
        date = self._getter('date', '')
        try:
            parsed_date = dateutil.parser.parse(date, default=default_date)
        except ValueError:
            return None

        parsed_date = parsed_date.replace(tzinfo=None)
        if parsed_date != default_date:
            return parsed_date.year

        return None

    @release_year.setter
    def release_year(self, value):
        self.reader['year'] = value

    @property
    def track_number(self):
        """The track number."""

        try:
            _number = int(self._getter('tracknumber'))
        except (TypeError, ValueError):
            _number = None

        return _number

    @track_number.setter
    def track_number(self, value):
        self.reader['tracknumber'] = value

    @property
    def genre(self):
        """The music genre."""
        return self._getter('genre')

    @genre.setter
    def genre(self, value):
        self.genre = value

    @property
    def length(self):
        """The length of the song in seconds."""
        return int(round(self.reader.info.length))

    @property
    def bitrate(self):
        """The audio bitrate."""
        if hasattr(self.reader.info, 'bitrate'):
            return self.reader.info.bitrate

    @property
    def sample_rate(self):
        """The audio sample rate."""
        return self.reader.info.sample_rate

    @property
    def filename(self):
        """The file name of this audio file."""
        return os.path.basename(self.reader.filename)

    @property
    def filepath(self):
        """The absolute path to this audio file."""
        return os.path.abspath(self.reader.filename)

    @property
    def origpath(self):
        """The original path with which this class was instantiated. This
        function avoids a call to ``os.path``.  Usually you'll want to use
        either :meth:`.filename` or :meth:`.filepath` instead."""
        return self._original_path

    @property
    def filesize(self):
        """The size of this audio file in the filesystem."""
        return os.stat(self.reader.filename).st_size

    # Helper functions

    def _getter(self, attr, fallback=None):
        """Return the first list item of the specified attribute or fall back
        to a default value if attribute is not available."""
        return self.reader[attr][0] if attr in self.reader else fallback

    def save(self):
        """Save changes to file metadata."""
        self.reader.save()
<|endoftext|>


<commit_before>import ujson as json

class ModelFormat(object):
    def __init__(self):
        pass
    def __call__(self, model):
        raise NotImplemented

    @classmethod
    def iter_options(cls):
        yield JsonFormat
        yield RedisFormat

class TsvFormat(ModelFormat):
    """ Format model output as tab separated values """
    @staticmethod
    def items_to_tsv(items):
        key_order = None
        for item in items:
            if key_order == None:
                key_order = []
                if '_id' in item:
                    key_order.append('_id')
                key_order += sorted(k for k in item.iterkeys() if k != '_id')
            yield '\t'.join(str(item[k]) for k in key_order)

    def __call__(self, model):
        return model.mapPartitions(self.items_to_tsv)

    @classmethod
    def add_arguments(cls, p):
        p.set_defaults(fmtcls=cls)
        return p

class JsonFormat(ModelFormat):
    """ Format model output as json """
    def __call__(self, model):
        return model.map(json.dumps)

    @classmethod
    def add_arguments(cls, p):
        p.set_defaults(fmtcls=cls)
        return p

class RedisFormat(ModelFormat):
    """ Format model output as redis protocol SET commands """
    def __init__(self, prefix):
        self.prefix = prefix

    def __call__(self, model):
        cmd = '\r\n'.join(["*3", "$3", "SET", "${}", "{}", "${}", "{}"])+'\r'
        return model\
            .map(lambda i: ((self.prefix+i['_id'].replace('"','\\"')).encode('utf-8'), json.dumps(i)))\
            .map(lambda (t, c): cmd.format(len(t), t, len(c), c))

    @classmethod
    def add_arguments(cls, p):
        p.add_argument('--prefix', required=False, default='', metavar='PREFIX')
        p.set_defaults(fmtcls=cls)
        return p
<commit_msg>Add tsv to model format options and encode output<commit_after>import ujson as json

class ModelFormat(object):
    def __init__(self):
        pass
    def __call__(self, model):
        raise NotImplemented

    @classmethod
    def iter_options(cls):
        yield JsonFormat
        yield RedisFormat
        yield TsvFormat

class TsvFormat(ModelFormat):
    """ Format model output as tab separated values """
    @staticmethod
    def items_to_tsv(items):
        key_order = None
        for item in items:
            if key_order == None:
                key_order = []
                if '_id' in item:
                    key_order.append('_id')
                key_order += sorted(k for k in item.iterkeys() if k != '_id')

            # todo: proper field serialization and escapes
            yield u'\t'.join(unicode(item[k]) for k in key_order).encode('utf-8')

    def __call__(self, model):
        return model.mapPartitions(self.items_to_tsv)

    @classmethod
    def add_arguments(cls, p):
        p.set_defaults(fmtcls=cls)
        return p

class JsonFormat(ModelFormat):
    """ Format model output as json """
    def __call__(self, model):
        return model.map(json.dumps)

    @classmethod
    def add_arguments(cls, p):
        p.set_defaults(fmtcls=cls)
        return p

class RedisFormat(ModelFormat):
    """ Format model output as redis protocol SET commands """
    def __init__(self, prefix):
        self.prefix = prefix

    def __call__(self, model):
        cmd = '\r\n'.join(["*3", "$3", "SET", "${}", "{}", "${}", "{}"])+'\r'
        return model\
            .map(lambda i: ((self.prefix+i['_id'].replace('"','\\"')).encode('utf-8'), json.dumps(i)))\
            .map(lambda (t, c): cmd.format(len(t), t, len(c), c))

    @classmethod
    def add_arguments(cls, p):
        p.add_argument('--prefix', required=False, default='', metavar='PREFIX')
        p.set_defaults(fmtcls=cls)
        return p
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import logging
import re
from codecs import open
from collections import namedtuple
from datetime import datetime
from glob import glob
from os.path import basename

from tsstats.client import Client, Clients

re_log_filename = re.compile(r'ts3server_(?P<date>\d{4}-\d\d-\d\d)'
                             '__(?P<time>\d\d_\d\d_\d\d.\d+)_(?P<sid>\d).log')
re_log_entry = re.compile('(?P<timestamp>\d{4}-\d\d-\d\d\ \d\d:\d\d:\d\d.\d+)'
                          '\|\ *(?P<level>\w+)\ *\|\ *(?P<component>\w+)\ *'
                          '\|\ *(?P<sid>\d+)\ *\|\ *(?P<message>.*)')
re_dis_connect = re.compile(
    r"client (?P<action>(dis)?connected) '(?P<nick>.*)'\(id:(?P<clid>\d+)\)")
re_disconnect_invoker = re.compile(
    r'invokername=(.*)\ invokeruid=(.*)\ reasonmsg'
)

log_timestamp_format = '%Y-%m-%d %H:%M:%S.%f'

TimedLog = namedtuple('TimedLog', ['path', 'timestamp'])

logger = logging.getLogger('tsstats')

def parse_logs(log_glob, ident_map=None, online_dc=True, *args, **kwargs):
    '''
    parse logs from `log_glob`

    :param log_glob: path to server-logs (supports globbing)
    :param ident_map: identmap used for Client-initializations

    :type log_glob: str
    :type ident_map: dict

    :return: clients bundled by virtual-server
    :rtype: dict
    '''
    vserver_clients = {}
    for virtualserver_id, logs in\
            _bundle_logs(log_file for log_file in glob(log_glob)).items():
        clients = Clients(ident_map)
        # keep last log out of the iteration for now
        for log in logs[:-1]:
            # don't reconnect connected clients for all logs except last one
            # because that would lead to insane onlinetimes
            _parse_details(log.path, clients=clients, online_dc=False,
                           *args, **kwargs)
        # now parse details of last log with correct online_dc set
        _parse_details(logs[-1].path, clients=clients, online_dc=online_dc,
                       *args, **kwargs)
        if len(clients) >= 1:
            vserver_clients[virtualserver_id] = clients
    return vserver_clients

def _bundle_logs(logs):
    '''
    bundle `logs` by virtualserver-id
    and sort by timestamp from filename (if exists)

    :param logs: list of paths to logfiles

    :type logs: list

    :return: `logs` bundled by virtualserver-id and sorted by timestamp
    :rtype: dict{str: [TimedLog]}
    '''
    vserver_logfiles = {}  # sid: [/path/to/log1, ..., /path/to/logn]
    for log in logs:
        # try to get date and sid from filename
        match = re_log_filename.match(basename(log))
        if match:
            match = match.groupdict()
            timestamp = datetime.strptime('{0} {1}'.format(
                match['date'], match['time'].replace('_', ':')),
                log_timestamp_format)
            tl = TimedLog(log, timestamp)
            sid = match['sid']
            if sid in vserver_logfiles:
                # if already exists, keep list sorted by timestamp
                vserver_logfiles[sid].append(tl)
                vserver_logfiles[sid] =\
                    sorted(vserver_logfiles[sid],
                           key=lambda tl: tl.timestamp)
            else:
                # if not exists, just create a list
                vserver_logfiles[match['sid']] = [tl]
        else:
            # fallback to plain sorting
            vserver_logfiles.setdefault('', [])\
                .append(TimedLog(log, None))
            vserver_logfiles[''] =\
                sorted(vserver_logfiles[''],
                       key=lambda tl: tl.path)
    return vserver_logfiles

def _parse_details(log_path, ident_map=None, clients=None, online_dc=True):
    '''
    extract details from log-files

    detailed parsing is done here: onlinetime, kicks, pkicks, bans, pbans

    :param log_path: path to log-file
    :param ident_map: :doc:`identmap`
    :param clients: clients-object to add parsing-results to
    :param online_cd: disconnect online clients after parsing

    :type log_path: str
    :type ident_map: dict
    :type clients: tsstats.client.Clients
    :type online_cd: bool

    :return: parsed clients
    :rtype: tsstats.client.Clients
    '''
    if clients is None:
        clients = Clients(ident_map)
    log_file = open(log_path, encoding='utf-8')
    # process lines
    logger.debug('Started parsing of %s', log_file.name)
    for line in log_file:
        match = re_log_entry.match(line)
        if not match:
            logger.debug('No match: "%s"', line)
            continue
        match = match.groupdict()
        logdatetime = datetime.strptime(match['timestamp'],
                                        log_timestamp_format)
        message = match['message']
        if message.startswith('client'):
            match = re_dis_connect.match(message)
            if not match:
                logger.debug('Not supported client action: "%s"', message)
                continue
            nick, clid = match.group('nick'), match.group('clid')
            client = clients.setdefault(clid, Client(clid, nick))
            client.nick = nick  # set nick to display changes
            action = match.group('action')
            if action == 'connected':
                client.connect(logdatetime)
            elif action == 'disconnected':
                client.disconnect(logdatetime)
                if 'invokeruid' in message:
                    re_disconnect_data = re_disconnect_invoker.findall(
                        message)
                    invokernick, invokeruid = re_disconnect_data[0]
                    invoker = clients.setdefault(invokeruid,
                                                 Client(invokeruid))
                    invoker.nick = invokernick
                    if 'bantime' in message:
                        invoker.ban(client)
                    else:
                        invoker.kick(client)
    if online_dc:
        def _reconnect(client):
            client.disconnect(datetime.utcnow())
            client.connected += 1
        [_reconnect(client) for client in clients if client.connected]
    logger.debug('Finished parsing of %s', log_file.name)
    return clients
<commit_msg>Remove double-iteratation over logfiles<commit_after># -*- coding: utf-8 -*-

import logging
import re
from codecs import open
from collections import namedtuple
from datetime import datetime
from glob import glob
from os.path import basename

from tsstats.client import Client, Clients

re_log_filename = re.compile(r'ts3server_(?P<date>\d{4}-\d\d-\d\d)'
                             '__(?P<time>\d\d_\d\d_\d\d.\d+)_(?P<sid>\d).log')
re_log_entry = re.compile('(?P<timestamp>\d{4}-\d\d-\d\d\ \d\d:\d\d:\d\d.\d+)'
                          '\|\ *(?P<level>\w+)\ *\|\ *(?P<component>\w+)\ *'
                          '\|\ *(?P<sid>\d+)\ *\|\ *(?P<message>.*)')
re_dis_connect = re.compile(
    r"client (?P<action>(dis)?connected) '(?P<nick>.*)'\(id:(?P<clid>\d+)\)")
re_disconnect_invoker = re.compile(
    r'invokername=(.*)\ invokeruid=(.*)\ reasonmsg'
)

log_timestamp_format = '%Y-%m-%d %H:%M:%S.%f'

TimedLog = namedtuple('TimedLog', ['path', 'timestamp'])

logger = logging.getLogger('tsstats')

def parse_logs(log_glob, ident_map=None, online_dc=True, *args, **kwargs):
    '''
    parse logs from `log_glob`

    :param log_glob: path to server-logs (supports globbing)
    :param ident_map: identmap used for Client-initializations

    :type log_glob: str
    :type ident_map: dict

    :return: clients bundled by virtual-server
    :rtype: dict
    '''
    vserver_clients = {}
    for virtualserver_id, logs in\
            _bundle_logs(glob(log_glob)).items():
        clients = Clients(ident_map)
        # keep last log out of the iteration for now
        for log in logs[:-1]:
            # don't reconnect connected clients for all logs except last one
            # because that would lead to insane onlinetimes
            _parse_details(log.path, clients=clients, online_dc=False,
                           *args, **kwargs)
        # now parse details of last log with correct online_dc set
        _parse_details(logs[-1].path, clients=clients, online_dc=online_dc,
                       *args, **kwargs)
        if len(clients) >= 1:
            vserver_clients[virtualserver_id] = clients
    return vserver_clients

def _bundle_logs(logs):
    '''
    bundle `logs` by virtualserver-id
    and sort by timestamp from filename (if exists)

    :param logs: list of paths to logfiles

    :type logs: list

    :return: `logs` bundled by virtualserver-id and sorted by timestamp
    :rtype: dict{str: [TimedLog]}
    '''
    vserver_logfiles = {}  # sid: [/path/to/log1, ..., /path/to/logn]
    for log in logs:
        # try to get date and sid from filename
        match = re_log_filename.match(basename(log))
        if match:
            match = match.groupdict()
            timestamp = datetime.strptime('{0} {1}'.format(
                match['date'], match['time'].replace('_', ':')),
                log_timestamp_format)
            tl = TimedLog(log, timestamp)
            sid = match['sid']
            if sid in vserver_logfiles:
                # if already exists, keep list sorted by timestamp
                vserver_logfiles[sid].append(tl)
                vserver_logfiles[sid] =\
                    sorted(vserver_logfiles[sid],
                           key=lambda tl: tl.timestamp)
            else:
                # if not exists, just create a list
                vserver_logfiles[match['sid']] = [tl]
        else:
            # fallback to plain sorting
            vserver_logfiles.setdefault('', [])\
                .append(TimedLog(log, None))
            vserver_logfiles[''] =\
                sorted(vserver_logfiles[''],
                       key=lambda tl: tl.path)
    return vserver_logfiles

def _parse_details(log_path, ident_map=None, clients=None, online_dc=True):
    '''
    extract details from log-files

    detailed parsing is done here: onlinetime, kicks, pkicks, bans, pbans

    :param log_path: path to log-file
    :param ident_map: :doc:`identmap`
    :param clients: clients-object to add parsing-results to
    :param online_cd: disconnect online clients after parsing

    :type log_path: str
    :type ident_map: dict
    :type clients: tsstats.client.Clients
    :type online_cd: bool

    :return: parsed clients
    :rtype: tsstats.client.Clients
    '''
    if clients is None:
        clients = Clients(ident_map)
    log_file = open(log_path, encoding='utf-8')
    # process lines
    logger.debug('Started parsing of %s', log_file.name)
    for line in log_file:
        match = re_log_entry.match(line)
        if not match:
            logger.debug('No match: "%s"', line)
            continue
        match = match.groupdict()
        logdatetime = datetime.strptime(match['timestamp'],
                                        log_timestamp_format)
        message = match['message']
        if message.startswith('client'):
            match = re_dis_connect.match(message)
            if not match:
                logger.debug('Not supported client action: "%s"', message)
                continue
            nick, clid = match.group('nick'), match.group('clid')
            client = clients.setdefault(clid, Client(clid, nick))
            client.nick = nick  # set nick to display changes
            action = match.group('action')
            if action == 'connected':
                client.connect(logdatetime)
            elif action == 'disconnected':
                client.disconnect(logdatetime)
                if 'invokeruid' in message:
                    re_disconnect_data = re_disconnect_invoker.findall(
                        message)
                    invokernick, invokeruid = re_disconnect_data[0]
                    invoker = clients.setdefault(invokeruid,
                                                 Client(invokeruid))
                    invoker.nick = invokernick
                    if 'bantime' in message:
                        invoker.ban(client)
                    else:
                        invoker.kick(client)
    if online_dc:
        def _reconnect(client):
            client.disconnect(datetime.utcnow())
            client.connected += 1
        [_reconnect(client) for client in clients if client.connected]
    logger.debug('Finished parsing of %s', log_file.name)
    return clients
<|endoftext|>


<commit_before>from chainer import cuda
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition
from chainer.utils import conv
from models import upsampling_2d

import chainer.functions as F
import numpy
import unittest

@testing.parameterize(
    {'in_shape': (4, 3, 6, 8)},
    {'in_shape': (4, 3, 5, 7)},
)
class TestUpsampling2D(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, self.in_shape).astype('f')
        self.p = F.MaxPooling2D(2, 2, use_cudnn=False)
        self.pooled_y = self.p(self.x)
        self.gy = numpy.random.uniform(
            -1, 1, self.in_shape).astype(numpy.float32)

    def check_forward(self, y):
        y = upsampling_2d.upsampling_2d(
            self.pooled_y, self.p.indexes, ksize=(self.p.kh, self.p.kw),
            stride=(self.p.sy, self.p.sx), pad=(self.p.ph, self.p.pw),
            outsize=self.in_shape[2:], cover_all=self.p.cover_all)
        if isinstance(y.data, numpy.ndarray):
            y = conv.im2col_cpu(y.data, self.p.kh, self.p.kw,
                                self.p.sy, self.p.sx, self.p.ph, self.p.pw)
        else:
            y = conv.im2col_gpu(y.data, self.p.kh, self.p.kw,
                                self.p.sy, self.p.sx, self.p.ph, self.p.pw)
        for i in numpy.ndindex(y.shape):
            n, c, sy, sx, ky, kx = i
            up_y = y[n, c, sy, sx, ky, kx]
            if sy * y.shape[3] + sx == self.p.indexes[n, c, ky, kx]:
                in_y = self.pooled_y.data[n, c, ky, kx]
                testing.assert_allclose(in_y, up_y)
            else:
                testing.assert_allclose(up_y, 0)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.pooled_y.to_cpu()
        self.check_forward(self.pooled_y)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.pooled_y.to_gpu()
        self.check_forward(self.pooled_y)

    def check_backward(self, x_data, y_grad):
        func = upsampling_2d.Upsampling2D(
            self.p.indexes, ksize=(self.p.kh, self.p.kw),
            stride=(self.p.sy, self.p.sx), pad=(self.p.ph, self.p.pw),
            outsize=self.in_shape[2:], cover_all=self.p.cover_all)
        gradient_check.check_backward(func, x_data, y_grad)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.pooled_y.data, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(
            self.pooled_y.data), cuda.to_gpu(self.gy))
<commit_msg>Fix test codes<commit_after>from chainer import cuda
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition
from chainer.utils import conv

import chainer.functions as F
import numpy
import unittest

@testing.parameterize(
    {'in_shape': (4, 3, 6, 8)},
    {'in_shape': (4, 3, 5, 7)},
)
class TestUpsampling2D(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, self.in_shape).astype('f')
        self.p = F.MaxPooling2D(2, 2, use_cudnn=False)
        self.pooled_y = self.p(self.x)
        self.gy = numpy.random.uniform(
            -1, 1, self.in_shape).astype(numpy.float32)

    def check_forward(self, y):
        y = F.upsampling_2d(
            self.pooled_y, self.p.indexes, ksize=(self.p.kh, self.p.kw),
            stride=(self.p.sy, self.p.sx), pad=(self.p.ph, self.p.pw),
            outsize=self.in_shape[2:], cover_all=self.p.cover_all)
        if isinstance(y.data, numpy.ndarray):
            y = conv.im2col_cpu(y.data, self.p.kh, self.p.kw,
                                self.p.sy, self.p.sx, self.p.ph, self.p.pw)
        else:
            y = conv.im2col_gpu(y.data, self.p.kh, self.p.kw,
                                self.p.sy, self.p.sx, self.p.ph, self.p.pw)
        for i in numpy.ndindex(y.shape):
            n, c, sy, sx, ky, kx = i
            up_y = y[n, c, sy, sx, ky, kx]
            if sy * y.shape[3] + sx == self.p.indexes[n, c, ky, kx]:
                in_y = self.pooled_y.data[n, c, ky, kx]
                testing.assert_allclose(in_y, up_y)
            else:
                testing.assert_allclose(up_y, 0)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.pooled_y.to_cpu()
        self.check_forward(self.pooled_y)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.pooled_y.to_gpu()
        self.check_forward(self.pooled_y)

    def check_backward(self, x_data, y_grad):
        func = F.Upsampling2D(
            self.p.indexes, ksize=(self.p.kh, self.p.kw),
            stride=(self.p.sy, self.p.sx), pad=(self.p.ph, self.p.pw),
            outsize=self.in_shape[2:], cover_all=self.p.cover_all)
        gradient_check.check_backward(func, x_data, y_grad)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.pooled_y.data, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(
            self.pooled_y.data), cuda.to_gpu(self.gy))
<|endoftext|>


<commit_before># coding=utf-8
import os
import shutil
from tempfile import mkdtemp

from django.core.management.base import BaseCommand

from realtime.tasks.test.test_realtime_tasks import flood_layer_uri
from realtime.tasks.realtime.flood import process_flood

class Command(BaseCommand):
    """Script to load flood test data for demo purpose only.

    """
    help = 'Script to load flood test data for demo purpose only.'

    def handle(self, *args, **options):
        # Copy file to hazard drop directory
        REALTIME_HAZARD_DROP = os.environ.get(
            'REALTIME_HAZARD_DROP',
            '/home/realtime/hazard-drop/')
        hazard_drop_path = mkdtemp(dir=REALTIME_HAZARD_DROP)
        hazard_drop_path = os.path.join(
            hazard_drop_path, os.path.basename(flood_layer_uri))
        print 'Copy flood data to %s' % hazard_drop_path
        shutil.copy(flood_layer_uri, hazard_drop_path)

        flood_id = '2018022511-6-rw'
        print 'Send flood data to InaSAFE Django with flood id = %s' % flood_id
        process_flood.delay(
            flood_id=flood_id,
            data_source='hazard_file',
            data_source_args={
                'filename': flood_layer_uri
            }
        )
<commit_msg>Fix wrong path to flood data to push.<commit_after># coding=utf-8
import os
import shutil
from tempfile import mkdtemp

from django.core.management.base import BaseCommand

from realtime.tasks.test.test_realtime_tasks import flood_layer_uri
from realtime.tasks.realtime.flood import process_flood

class Command(BaseCommand):
    """Script to load flood test data for demo purpose only.

    """
    help = 'Script to load flood test data for demo purpose only.'

    def handle(self, *args, **options):
        # Copy file to hazard drop directory
        REALTIME_HAZARD_DROP = os.environ.get(
            'REALTIME_HAZARD_DROP',
            '/home/realtime/hazard-drop/')
        hazard_drop_path = mkdtemp(dir=REALTIME_HAZARD_DROP)
        hazard_drop_path = os.path.join(
            hazard_drop_path, os.path.basename(flood_layer_uri))
        print 'Copy flood data to %s' % hazard_drop_path
        shutil.copy(flood_layer_uri, hazard_drop_path)

        flood_id = '2018022511-6-rw'
        print 'Send flood data to InaSAFE Django with flood id = %s' % flood_id
        process_flood.delay(
            flood_id=flood_id,
            data_source='hazard_file',
            data_source_args={
                'filename': hazard_drop_path
            }
        )
<|endoftext|>


<commit_before>import pytest

import copy
import subprocess
import time
import signal
import datetime
import requests
import boto3

from great_expectations.data_context.util import file_relative_path

USAGE_STATISTICS_QA_URL = "https://qa.stats.greatexpectations.io/great_expectations/v1/usage_statistics"

logGroupName = "/great_expectations/usage_statistics/qa"

@pytest.fixture(scope="session")
def valid_usage_statistics_message():
    return {
    'event_payload': {
        'platform.system': 'Darwin',
        'platform.release': '19.3.0',
        'version_info': "sys.version_info(major=3, minor=7, micro=4, releaselevel='final', serial=0)",
        'anonymized_datasources': [
            {
                'anonymized_name': 'f57d8a6edae4f321b833384801847498',
                'parent_class': 'SqlAlchemyDatasource',
                'sqlalchemy_dialect': 'postgresql'
            }
        ],
        'anonymized_stores': [
            {
                'anonymized_name': '078eceafc1051edf98ae2f911484c7f7',
                'parent_class': 'ExpectationsStore',
                'anonymized_store_backend': {
                    'parent_class': 'TupleFilesystemStoreBackend'
                }
            },
            {
                'anonymized_name': '313cbd9858dd92f3fc2ef1c10ab9c7c8',
                'parent_class': 'ValidationsStore',
                'anonymized_store_backend': {
                    'parent_class': 'TupleFilesystemStoreBackend'
                }
            },
            {
                'anonymized_name': '2d487386aa7b39e00ed672739421473f',
                'parent_class': 'EvaluationParameterStore',
                'anonymized_store_backend': {
                    'parent_class': 'InMemoryStoreBackend'
                }
            }
        ],
        'anonymized_validation_operators': [
            {
                'anonymized_name': '99d14cc00b69317551690fb8a61aca94',
                'parent_class': 'ActionListValidationOperator',
                'anonymized_action_list': [
                    {
                        'anonymized_name': '5a170e5b77c092cc6c9f5cf2b639459a',
                        'parent_class': 'StoreValidationResultAction'
                    },
                    {
                        'anonymized_name': '0fffe1906a8f2a5625a5659a848c25a3',
                        'parent_class': 'StoreEvaluationParametersAction'
                    },
                    {
                        'anonymized_name': '101c746ab7597e22b94d6e5f10b75916',
                        'parent_class': 'UpdateDataDocsAction'
                    }
                ]
            }
        ],
        'anonymized_data_docs_sites': [
            {
                'parent_class': 'SiteBuilder',
                'anonymized_name': 'eaf0cf17ad63abf1477f7c37ad192700',
                'anonymized_store_backend': {'parent_class': 'TupleFilesystemStoreBackend'},
                'anonymized_site_index_builder': {
                    'parent_class': 'DefaultSiteIndexBuilder',
                    'show_cta_footer': True
                }
            }
        ],
        'anonymized_expectation_suites': [
            {
                'anonymized_name': '238e99998c7674e4ff26a9c529d43da4',
                'expectation_count': 8,
                'anonymized_expectation_type_counts': {
                    'expect_column_value_lengths_to_be_between': 1,
                    'expect_table_row_count_to_be_between': 1,
                    'expect_column_values_to_not_be_null': 2,
                    'expect_column_distinct_values_to_be_in_set': 1,
                    'expect_column_kl_divergence_to_be_less_than': 1,
                    'expect_table_column_count_to_equal': 1,
                    'expect_table_columns_to_match_ordered_list': 1
                }
            }
        ]
    },
    'event': 'data_context.__init__',
    'success': True,
    'version': '1.0.0',
    'event_time': '2020-03-28T01:14:21.155Z',
    'data_context_id': '96c547fe-e809-4f2e-b122-0dc91bb7b3ad',
    'data_context_instance_id': '445a8ad1-2bd0-45ce-bb6b-d066afe996dd',
    'ge_version': '0.9.7+244.g56d67e51d.dirty'
}

@pytest.fixture(scope="session")
def logstream(valid_usage_statistics_message):
    client = boto3.client('logs', region_name='us-east-1')
    # Warm up a logstream
    logStreamName = None
    message = copy.deepcopy(valid_usage_statistics_message)
    message["data_context_id"] = "00000000-0000-0000-0000-000000000000"
    res = requests.post(USAGE_STATISTICS_QA_URL, json=message)
    assert res.status_code == 201
    attempts = 0
    while attempts < 3:
        attempts += 1
        time.sleep(2)
        logStreams = client.describe_log_streams(
            logGroupName=logGroupName,
            orderBy='LastEventTime',
            descending=True,
            limit=2,
        )
        lastEventTimestamp = logStreams["logStreams"][0].get("lastEventTimestamp")
        if lastEventTimestamp is not None:
            lastEvent = datetime.datetime.fromtimestamp(lastEventTimestamp / 1000)
            if (lastEvent - datetime.datetime.now()) < datetime.timedelta(seconds=30):
                logStreamName = logStreams["logStreams"][0]["logStreamName"]
                break
    if logStreamName is None:
        assert False, "Unable to warm up a log stream for integration testing."
    return client, logStreamName

def test_send_malformed_data(valid_usage_statistics_message):
    # We should be able to successfully send a valid message, but find that
    # a malformed message is not accepted
    res = requests.post(USAGE_STATISTICS_QA_URL, json=valid_usage_statistics_message)
    assert res.status_code == 201
    invalid_usage_statistics_message = copy.deepcopy(valid_usage_statistics_message)
    del invalid_usage_statistics_message["data_context_id"]
    res = requests.post(USAGE_STATISTICS_QA_URL, json=invalid_usage_statistics_message)
    assert res.status_code == 400

def test_usage_statistics_transmission(logstream):
    client, logStreamName = logstream
    pre_events = client.get_log_events(
        logGroupName=logGroupName,
        logStreamName=logStreamName,
        limit=100,
    )
    assert len(pre_events) < 100, "This test assumed small logstream sizes in the qa stream. Consider changing " \
                                  "fetch limit."

    p = subprocess.Popen(
        ["python", file_relative_path(__file__, "./instantiate_context_with_usage_statistics.py"), "0"],
        stdout=subprocess.PIPE, stderr=subprocess.PIPE
    )
    outs, errs = p.communicate()
    outs = str(outs)
    errs = str(errs)
    assert "INFO" not in outs
    assert "Done constructing a DataContext" in outs
    assert "Ending a long nap" in outs
    assert "KeyboardInterrupt" not in errs

    # Wait a bit for the log events to post
    time.sleep(5)
    post_events = client.get_log_events(
        logGroupName=logGroupName,
        logStreamName=logStreamName,
        limit=100,
    )
    assert len(pre_events["events"]) + 2 == len(post_events["events"])

def test_send_completes_on_kill(logstream):
    client, logStreamName = logstream
    pre_events = client.get_log_events(
        logGroupName=logGroupName,
        logStreamName=logStreamName,
        limit=100,
    )
    """Test that having usage statistics enabled does not negatively impact kill signals or cause loss of queued usage statistics. """
    # Execute process that initializes data context
    acceptable_startup_time = 6
    acceptable_shutdown_time = 1
    nap_time = 30
    start = datetime.datetime.now()
    # Instruct the process to wait for 30 seconds after initializing before completing.
    p = subprocess.Popen(
        ["python", file_relative_path(__file__, "./instantiate_context_with_usage_statistics.py"),
         str(nap_time), "False", "True"],
        stdout=subprocess.PIPE, stderr=subprocess.PIPE
    )

    time.sleep(acceptable_startup_time)

    # Send a signal to kill
    p.send_signal(signal.SIGINT)
    outs, errs = p.communicate()
    end = datetime.datetime.now()

    # Ensure that the process shut down earlier than it would have
    assert datetime.timedelta(
        seconds=acceptable_startup_time
    ) < (end - start) < datetime.timedelta(
        seconds=acceptable_startup_time + acceptable_shutdown_time)

    outs = str(outs)
    errs = str(errs)
    assert "INFO" not in outs
    assert "Done constructing a DataContext" in outs
    assert "Ending a long nap" not in outs
    assert "KeyboardInterrupt" in errs
    time.sleep(5)
    post_events = client.get_log_events(
        logGroupName=logGroupName,
        logStreamName=logStreamName,
        limit=100,
    )
    assert len(pre_events["events"]) + 2 == len(post_events["events"])

def test_graceful_failure_with_no_internet():
    """Test that having usage statistics enabled does not negatively impact kill signals or cause loss of queued usage statistics. """

    # Execute process that initializes data context
    # NOTE - JPC - 20200227 - this is crazy long (not because of logging I think, but worth revisiting)
    acceptable_startup_time = 6
    acceptable_shutdown_time = 1
    nap_time = 0
    start = datetime.datetime.now()
    # Instruct the process to wait for 30 seconds after initializing before completing.
    p = subprocess.Popen(
        ["python", file_relative_path(__file__, "./instantiate_context_with_usage_statistics.py"),
         str(nap_time), "True", "True"],
        stdout=subprocess.PIPE, stderr=subprocess.PIPE
    )
    outs, errs = p.communicate()
    end = datetime.datetime.now()
    # We didn't wait or send a signal, so just check that times were reasonable
    assert (end - start) < datetime.timedelta(
        seconds=acceptable_startup_time + acceptable_shutdown_time)
    outs = str(outs)
    errs = str(errs)
    assert "INFO" not in outs
    assert "Done constructing a DataContext" in outs
    assert "Ending a long nap" in outs
<commit_msg>do not reshare boto3 client<commit_after>import copy
import datetime
import signal
import subprocess
import time

import boto3
import pytest
import requests

from great_expectations.data_context.util import file_relative_path

USAGE_STATISTICS_QA_URL = "https://qa.stats.greatexpectations.io/great_expectations/v1/usage_statistics"

logGroupName = "/great_expectations/usage_statistics/qa"

@pytest.fixture(scope="session")
def valid_usage_statistics_message():
    return {
    'event_payload': {
        'platform.system': 'Darwin',
        'platform.release': '19.3.0',
        'version_info': "sys.version_info(major=3, minor=7, micro=4, releaselevel='final', serial=0)",
        'anonymized_datasources': [
            {
                'anonymized_name': 'f57d8a6edae4f321b833384801847498',
                'parent_class': 'SqlAlchemyDatasource',
                'sqlalchemy_dialect': 'postgresql'
            }
        ],
        'anonymized_stores': [
            {
                'anonymized_name': '078eceafc1051edf98ae2f911484c7f7',
                'parent_class': 'ExpectationsStore',
                'anonymized_store_backend': {
                    'parent_class': 'TupleFilesystemStoreBackend'
                }
            },
            {
                'anonymized_name': '313cbd9858dd92f3fc2ef1c10ab9c7c8',
                'parent_class': 'ValidationsStore',
                'anonymized_store_backend': {
                    'parent_class': 'TupleFilesystemStoreBackend'
                }
            },
            {
                'anonymized_name': '2d487386aa7b39e00ed672739421473f',
                'parent_class': 'EvaluationParameterStore',
                'anonymized_store_backend': {
                    'parent_class': 'InMemoryStoreBackend'
                }
            }
        ],
        'anonymized_validation_operators': [
            {
                'anonymized_name': '99d14cc00b69317551690fb8a61aca94',
                'parent_class': 'ActionListValidationOperator',
                'anonymized_action_list': [
                    {
                        'anonymized_name': '5a170e5b77c092cc6c9f5cf2b639459a',
                        'parent_class': 'StoreValidationResultAction'
                    },
                    {
                        'anonymized_name': '0fffe1906a8f2a5625a5659a848c25a3',
                        'parent_class': 'StoreEvaluationParametersAction'
                    },
                    {
                        'anonymized_name': '101c746ab7597e22b94d6e5f10b75916',
                        'parent_class': 'UpdateDataDocsAction'
                    }
                ]
            }
        ],
        'anonymized_data_docs_sites': [
            {
                'parent_class': 'SiteBuilder',
                'anonymized_name': 'eaf0cf17ad63abf1477f7c37ad192700',
                'anonymized_store_backend': {'parent_class': 'TupleFilesystemStoreBackend'},
                'anonymized_site_index_builder': {
                    'parent_class': 'DefaultSiteIndexBuilder',
                    'show_cta_footer': True
                }
            }
        ],
        'anonymized_expectation_suites': [
            {
                'anonymized_name': '238e99998c7674e4ff26a9c529d43da4',
                'expectation_count': 8,
                'anonymized_expectation_type_counts': {
                    'expect_column_value_lengths_to_be_between': 1,
                    'expect_table_row_count_to_be_between': 1,
                    'expect_column_values_to_not_be_null': 2,
                    'expect_column_distinct_values_to_be_in_set': 1,
                    'expect_column_kl_divergence_to_be_less_than': 1,
                    'expect_table_column_count_to_equal': 1,
                    'expect_table_columns_to_match_ordered_list': 1
                }
            }
        ]
    },
    'event': 'data_context.__init__',
    'success': True,
    'version': '1.0.0',
    'event_time': '2020-03-28T01:14:21.155Z',
    'data_context_id': '96c547fe-e809-4f2e-b122-0dc91bb7b3ad',
    'data_context_instance_id': '445a8ad1-2bd0-45ce-bb6b-d066afe996dd',
    'ge_version': '0.9.7+244.g56d67e51d.dirty'
}

@pytest.fixture(scope="session")
def logstream(valid_usage_statistics_message):
    client = boto3.client('logs', region_name='us-east-1')
    # Warm up a logstream
    logStreamName = None
    message = copy.deepcopy(valid_usage_statistics_message)
    message["data_context_id"] = "00000000-0000-0000-0000-000000000000"
    res = requests.post(USAGE_STATISTICS_QA_URL, json=message)
    assert res.status_code == 201
    attempts = 0
    while attempts < 3:
        attempts += 1
        time.sleep(2)
        logStreams = client.describe_log_streams(
            logGroupName=logGroupName,
            orderBy='LastEventTime',
            descending=True,
            limit=2,
        )
        lastEventTimestamp = logStreams["logStreams"][0].get("lastEventTimestamp")
        if lastEventTimestamp is not None:
            lastEvent = datetime.datetime.fromtimestamp(lastEventTimestamp / 1000)
            if (lastEvent - datetime.datetime.now()) < datetime.timedelta(seconds=30):
                logStreamName = logStreams["logStreams"][0]["logStreamName"]
                break
    if logStreamName is None:
        assert False, "Unable to warm up a log stream for integration testing."
    return logStreamName

def test_send_malformed_data(valid_usage_statistics_message):
    # We should be able to successfully send a valid message, but find that
    # a malformed message is not accepted
    res = requests.post(USAGE_STATISTICS_QA_URL, json=valid_usage_statistics_message)
    assert res.status_code == 201
    invalid_usage_statistics_message = copy.deepcopy(valid_usage_statistics_message)
    del invalid_usage_statistics_message["data_context_id"]
    res = requests.post(USAGE_STATISTICS_QA_URL, json=invalid_usage_statistics_message)
    assert res.status_code == 400

def test_usage_statistics_transmission(logstream):
    logStreamName = logstream
    client = boto3.client('logs', region_name='us-east-1')
    pre_events = client.get_log_events(
        logGroupName=logGroupName,
        logStreamName=logStreamName,
        limit=100,
    )
    assert len(pre_events) < 100, "This test assumed small logstream sizes in the qa stream. Consider changing " \
                                  "fetch limit."

    p = subprocess.Popen(
        ["python", file_relative_path(__file__, "./instantiate_context_with_usage_statistics.py"), "0"],
        stdout=subprocess.PIPE, stderr=subprocess.PIPE
    )
    outs, errs = p.communicate()
    outs = str(outs)
    errs = str(errs)
    assert "INFO" not in outs
    assert "Done constructing a DataContext" in outs
    assert "Ending a long nap" in outs
    assert "KeyboardInterrupt" not in errs

    # Wait a bit for the log events to post
    time.sleep(5)
    post_events = client.get_log_events(
        logGroupName=logGroupName,
        logStreamName=logStreamName,
        limit=100,
    )
    assert len(pre_events["events"]) + 2 == len(post_events["events"])

def test_send_completes_on_kill(logstream):
    logStreamName = logstream
    client = boto3.client('logs', region_name='us-east-1')
    pre_events = client.get_log_events(
        logGroupName=logGroupName,
        logStreamName=logStreamName,
        limit=100,
    )
    """Test that having usage statistics enabled does not negatively impact kill signals or cause loss of queued usage statistics. """
    # Execute process that initializes data context
    acceptable_startup_time = 6
    acceptable_shutdown_time = 1
    nap_time = 30
    start = datetime.datetime.now()
    # Instruct the process to wait for 30 seconds after initializing before completing.
    p = subprocess.Popen(
        ["python", file_relative_path(__file__, "./instantiate_context_with_usage_statistics.py"),
         str(nap_time), "False", "True"],
        stdout=subprocess.PIPE, stderr=subprocess.PIPE
    )

    time.sleep(acceptable_startup_time)

    # Send a signal to kill
    p.send_signal(signal.SIGINT)
    outs, errs = p.communicate()
    end = datetime.datetime.now()

    # Ensure that the process shut down earlier than it would have
    assert datetime.timedelta(
        seconds=acceptable_startup_time
    ) < (end - start) < datetime.timedelta(
        seconds=acceptable_startup_time + acceptable_shutdown_time)

    outs = str(outs)
    errs = str(errs)
    assert "INFO" not in outs
    assert "Done constructing a DataContext" in outs
    assert "Ending a long nap" not in outs
    assert "KeyboardInterrupt" in errs
    time.sleep(5)
    post_events = client.get_log_events(
        logGroupName=logGroupName,
        logStreamName=logStreamName,
        limit=100,
    )
    assert len(pre_events["events"]) + 2 == len(post_events["events"])

def test_graceful_failure_with_no_internet():
    """Test that having usage statistics enabled does not negatively impact kill signals or cause loss of queued usage statistics. """

    # Execute process that initializes data context
    # NOTE - JPC - 20200227 - this is crazy long (not because of logging I think, but worth revisiting)
    acceptable_startup_time = 6
    acceptable_shutdown_time = 1
    nap_time = 0
    start = datetime.datetime.now()
    # Instruct the process to wait for 30 seconds after initializing before completing.
    p = subprocess.Popen(
        ["python", file_relative_path(__file__, "./instantiate_context_with_usage_statistics.py"),
         str(nap_time), "True", "True"],
        stdout=subprocess.PIPE, stderr=subprocess.PIPE
    )
    outs, errs = p.communicate()
    end = datetime.datetime.now()
    # We didn't wait or send a signal, so just check that times were reasonable
    assert (end - start) < datetime.timedelta(
        seconds=acceptable_startup_time + acceptable_shutdown_time)
    outs = str(outs)
    errs = str(errs)
    assert "INFO" not in outs
    assert "Done constructing a DataContext" in outs
    assert "Ending a long nap" in outs
<|endoftext|>


<commit_before># disable missing docstring
# pylint: disable=C0111

from lettuce import world
from nose.tools import assert_equal, assert_in  # pylint: disable=E0611
from terrain.steps import reload_the_page
from common import type_in_codemirror
from selenium.webdriver.common.keys import Keys
from cms.envs.common import FEATURES

@world.absorb
def create_component_instance(step, category, component_type=None, is_advanced=False):
    """
    Create a new component in a Unit.

    Parameters
    ----------
    category: component type (discussion, html, problem, video)
    component_type: for components with multiple templates, the link text in the menu
    is_advanced: for problems, is the desired component under the advanced menu?
    """
    assert_in(category, ['problem', 'html', 'video', 'discussion'])

    component_button_css = 'span.large-{}-icon'.format(category.lower())
    if category == 'problem':
        module_css = 'div.xmodule_CapaModule'
    else:
        module_css = 'div.xmodule_{}Module'.format(category.title())

    # Count how many of that module is on the page. Later we will
    # assert that one more was added.
    # We need to use world.browser.find_by_css instead of world.css_find
    # because it's ok if there are currently zero of them.
    module_count_before =  len(world.browser.find_by_css(module_css))

    # Disable the jquery animation for the transition to the menus.
    world.disable_jquery_animations()
    world.css_click(component_button_css)

    if category in ('problem', 'html'):
        world.wait_for_invisible(component_button_css)
        click_component_from_menu(category, component_type, is_advanced)

    expected_count = module_count_before + 1
    world.wait_for(
        lambda _: len(world.css_find(module_css)) == expected_count,
        timeout=20
    )

@world.absorb
def click_new_component_button(step, component_button_css):
    step.given('I have clicked the new unit button')

    world.css_click(component_button_css)

def _click_advanced():
    css = 'ul.problem-type-tabs a[href="#tab2"]'
    world.css_click(css)

    # Wait for the advanced tab items to be displayed
    tab2_css = 'div.ui-tabs-panel#tab2'
    world.wait_for_visible(tab2_css)

def _find_matching_link(category, component_type):
    """
    Find the link with the specified text. There should be one and only one.
    """

    # The tab shows links for the given category
    links = world.css_find('div.new-component-{} a'.format(category))

    # Find the link whose text matches what you're looking for
    matched_links = [link for link in links if link.text == component_type]

    # There should be one and only one
    assert_equal(len(matched_links), 1)
    return matched_links[0]

def click_component_from_menu(category, component_type, is_advanced):
    """
    Creates a component for a category with more
    than one template, i.e. HTML and Problem.
    For some problem types, it is necessary to click to
    the Advanced tab.
    The component_type is the link text, e.g. "Blank Common Problem"
    """
    if is_advanced:
        # Sometimes this click does not work if you go too fast.
        world.retry_on_exception(_click_advanced,
            ignored_exceptions=AssertionError)

    # Retry this in case the list is empty because you tried too fast.
    link = world.retry_on_exception(
        lambda: _find_matching_link(category, component_type),
        ignored_exceptions=AssertionError
    )

    # Wait for the link to be clickable. If you go too fast it is not.
    world.retry_on_exception(lambda: link.click())

@world.absorb
def edit_component_and_select_settings():
    world.edit_component()
    world.ensure_settings_visible()

@world.absorb
def ensure_settings_visible():
    # Select the 'settings' tab if there is one (it isn't displayed if it is the only option)
    settings_button = world.browser.find_by_css('.settings-button')
    if len(settings_button) > 0:
        world.css_click('.settings-button')

@world.absorb
def edit_component():
    world.wait_for(lambda _driver: world.css_visible('a.edit-button'))
    world.css_click('a.edit-button')

@world.absorb
def select_editor_tab(tab_name):
    editor_tabs = world.browser.find_by_css('.editor-tabs a')
    expected_tab_text = tab_name.strip().upper()
    matching_tabs = [tab for tab in editor_tabs if tab.text.upper() == expected_tab_text]
    assert len(matching_tabs) == 1
    tab = matching_tabs[0]
    tab.click()
    world.wait_for_ajax_complete()

def enter_xml_in_advanced_problem(step, text):
    """
    Edits an advanced problem (assumes only on page),
    types the provided XML, and saves the component.
    """
    world.edit_component()
    type_in_codemirror(0, text)
    world.save_component()

@world.absorb
def verify_setting_entry(setting, display_name, value, explicitly_set):
    """
    Verify the capa module fields are set as expected in the
    Advanced Settings editor.

    Parameters
    ----------
    setting: the WebDriverElement object found in the browser
    display_name: the string expected as the label
    html: the expected field value
    explicitly_set: True if the value is expected to have been explicitly set
        for the problem, rather than derived from the defaults. This is verified
        by the existence of a "Clear" button next to the field value.
    """
    assert_equal(display_name, setting.find_by_css('.setting-label')[0].html.strip())

    # Check if the web object is a list type
    # If so, we use a slightly different mechanism for determining its value
    if setting.has_class('metadata-list-enum') or setting.has_class('metadata-dict') or setting.has_class('metadata-video-translations'):
        list_value = ', '.join(ele.value for ele in setting.find_by_css('.list-settings-item'))
        assert_equal(value, list_value)
    elif setting.has_class('metadata-videolist-enum'):
        list_value = ', '.join(ele.find_by_css('input')[0].value for ele in setting.find_by_css('.videolist-settings-item'))
        assert_equal(value, list_value)
    else:
        assert_equal(value, setting.find_by_css('.setting-input')[0].value)

    # VideoList doesn't have clear button
    if not setting.has_class('metadata-videolist-enum'):
        settingClearButton = setting.find_by_css('.setting-clear')[0]
        assert_equal(explicitly_set, settingClearButton.has_class('active'))
        assert_equal(not explicitly_set, settingClearButton.has_class('inactive'))

@world.absorb
def verify_all_setting_entries(expected_entries):
    settings = world.browser.find_by_css('.wrapper-comp-setting')
    assert_equal(len(expected_entries), len(settings))
    for (counter, setting) in enumerate(settings):
        world.verify_setting_entry(
            setting, expected_entries[counter][0],
            expected_entries[counter][1], expected_entries[counter][2]
        )

@world.absorb
def save_component():
    world.css_click("a.action-save")
    world.wait_for_ajax_complete()

@world.absorb
def save_component_and_reopen(step):
    save_component()
    # We have a known issue that modifications are still shown within the edit window after cancel (though)
    # they are not persisted. Refresh the browser to make sure the changes WERE persisted after Save.
    reload_the_page(step)
    edit_component_and_select_settings()

@world.absorb
def cancel_component(step):
    world.css_click("a.action-cancel")
    # We have a known issue that modifications are still shown within the edit window after cancel (though)
    # they are not persisted. Refresh the browser to make sure the changes were not persisted.
    reload_the_page(step)

@world.absorb
def revert_setting_entry(label):
    get_setting_entry(label).find_by_css('.setting-clear')[0].click()

@world.absorb
def get_setting_entry(label):
    def get_setting():
        settings = world.css_find('.wrapper-comp-setting')
        for setting in settings:
            if setting.find_by_css('.setting-label')[0].value == label:
                return setting
        return None
    return world.retry_on_exception(get_setting)

@world.absorb
def get_setting_entry_index(label):
    def get_index():
        settings = world.css_find('.metadata_edit .wrapper-comp-setting')
        for index, setting in enumerate(settings):
            if setting.find_by_css('.setting-label')[0].value == label:
                return index
        return None
    return world.retry_on_exception(get_index)

@world.absorb
def set_field_value(index, value):
    """
    Set the field to the specified value.

    Note: we cannot use css_fill here because the value is not set
    until after you move away from that field.
    Instead we will find the element, set its value, then hit the Tab key
    to get to the next field.
    """
    elem = world.css_find('.metadata_edit div.wrapper-comp-setting input.setting-input')[index]
    elem.value = value
    elem.type(Keys.TAB)
<commit_msg>Wait for ajax to complete when editing components in Studio acceptance tests<commit_after># disable missing docstring
# pylint: disable=C0111

from lettuce import world
from nose.tools import assert_equal, assert_in  # pylint: disable=E0611
from terrain.steps import reload_the_page
from common import type_in_codemirror
from selenium.webdriver.common.keys import Keys
from cms.envs.common import FEATURES

@world.absorb
def create_component_instance(step, category, component_type=None, is_advanced=False):
    """
    Create a new component in a Unit.

    Parameters
    ----------
    category: component type (discussion, html, problem, video)
    component_type: for components with multiple templates, the link text in the menu
    is_advanced: for problems, is the desired component under the advanced menu?
    """
    assert_in(category, ['problem', 'html', 'video', 'discussion'])

    component_button_css = 'span.large-{}-icon'.format(category.lower())
    if category == 'problem':
        module_css = 'div.xmodule_CapaModule'
    else:
        module_css = 'div.xmodule_{}Module'.format(category.title())

    # Count how many of that module is on the page. Later we will
    # assert that one more was added.
    # We need to use world.browser.find_by_css instead of world.css_find
    # because it's ok if there are currently zero of them.
    module_count_before =  len(world.browser.find_by_css(module_css))

    # Disable the jquery animation for the transition to the menus.
    world.disable_jquery_animations()
    world.css_click(component_button_css)

    if category in ('problem', 'html'):
        world.wait_for_invisible(component_button_css)
        click_component_from_menu(category, component_type, is_advanced)

    expected_count = module_count_before + 1
    world.wait_for(
        lambda _: len(world.css_find(module_css)) == expected_count,
        timeout=20
    )

@world.absorb
def click_new_component_button(step, component_button_css):
    step.given('I have clicked the new unit button')

    world.css_click(component_button_css)

def _click_advanced():
    css = 'ul.problem-type-tabs a[href="#tab2"]'
    world.css_click(css)

    # Wait for the advanced tab items to be displayed
    tab2_css = 'div.ui-tabs-panel#tab2'
    world.wait_for_visible(tab2_css)

def _find_matching_link(category, component_type):
    """
    Find the link with the specified text. There should be one and only one.
    """

    # The tab shows links for the given category
    links = world.css_find('div.new-component-{} a'.format(category))

    # Find the link whose text matches what you're looking for
    matched_links = [link for link in links if link.text == component_type]

    # There should be one and only one
    assert_equal(len(matched_links), 1)
    return matched_links[0]

def click_component_from_menu(category, component_type, is_advanced):
    """
    Creates a component for a category with more
    than one template, i.e. HTML and Problem.
    For some problem types, it is necessary to click to
    the Advanced tab.
    The component_type is the link text, e.g. "Blank Common Problem"
    """
    if is_advanced:
        # Sometimes this click does not work if you go too fast.
        world.retry_on_exception(_click_advanced,
            ignored_exceptions=AssertionError)

    # Retry this in case the list is empty because you tried too fast.
    link = world.retry_on_exception(
        lambda: _find_matching_link(category, component_type),
        ignored_exceptions=AssertionError
    )

    # Wait for the link to be clickable. If you go too fast it is not.
    world.retry_on_exception(lambda: link.click())

@world.absorb
def edit_component_and_select_settings():
    world.edit_component()
    world.ensure_settings_visible()

@world.absorb
def ensure_settings_visible():
    # Select the 'settings' tab if there is one (it isn't displayed if it is the only option)
    settings_button = world.browser.find_by_css('.settings-button')
    if len(settings_button) > 0:
        world.css_click('.settings-button')

@world.absorb
def edit_component():
    world.wait_for(lambda _driver: world.css_visible('a.edit-button'))
    world.css_click('a.edit-button')
    world.wait_for_ajax_complete()

@world.absorb
def select_editor_tab(tab_name):
    editor_tabs = world.browser.find_by_css('.editor-tabs a')
    expected_tab_text = tab_name.strip().upper()
    matching_tabs = [tab for tab in editor_tabs if tab.text.upper() == expected_tab_text]
    assert len(matching_tabs) == 1
    tab = matching_tabs[0]
    tab.click()
    world.wait_for_ajax_complete()

def enter_xml_in_advanced_problem(step, text):
    """
    Edits an advanced problem (assumes only on page),
    types the provided XML, and saves the component.
    """
    world.edit_component()
    type_in_codemirror(0, text)
    world.save_component()

@world.absorb
def verify_setting_entry(setting, display_name, value, explicitly_set):
    """
    Verify the capa module fields are set as expected in the
    Advanced Settings editor.

    Parameters
    ----------
    setting: the WebDriverElement object found in the browser
    display_name: the string expected as the label
    html: the expected field value
    explicitly_set: True if the value is expected to have been explicitly set
        for the problem, rather than derived from the defaults. This is verified
        by the existence of a "Clear" button next to the field value.
    """
    assert_equal(display_name, setting.find_by_css('.setting-label')[0].html.strip())

    # Check if the web object is a list type
    # If so, we use a slightly different mechanism for determining its value
    if setting.has_class('metadata-list-enum') or setting.has_class('metadata-dict') or setting.has_class('metadata-video-translations'):
        list_value = ', '.join(ele.value for ele in setting.find_by_css('.list-settings-item'))
        assert_equal(value, list_value)
    elif setting.has_class('metadata-videolist-enum'):
        list_value = ', '.join(ele.find_by_css('input')[0].value for ele in setting.find_by_css('.videolist-settings-item'))
        assert_equal(value, list_value)
    else:
        assert_equal(value, setting.find_by_css('.setting-input')[0].value)

    # VideoList doesn't have clear button
    if not setting.has_class('metadata-videolist-enum'):
        settingClearButton = setting.find_by_css('.setting-clear')[0]
        assert_equal(explicitly_set, settingClearButton.has_class('active'))
        assert_equal(not explicitly_set, settingClearButton.has_class('inactive'))

@world.absorb
def verify_all_setting_entries(expected_entries):
    settings = world.browser.find_by_css('.wrapper-comp-setting')
    assert_equal(len(expected_entries), len(settings))
    for (counter, setting) in enumerate(settings):
        world.verify_setting_entry(
            setting, expected_entries[counter][0],
            expected_entries[counter][1], expected_entries[counter][2]
        )

@world.absorb
def save_component():
    world.css_click("a.action-save")
    world.wait_for_ajax_complete()

@world.absorb
def save_component_and_reopen(step):
    save_component()
    # We have a known issue that modifications are still shown within the edit window after cancel (though)
    # they are not persisted. Refresh the browser to make sure the changes WERE persisted after Save.
    reload_the_page(step)
    edit_component_and_select_settings()

@world.absorb
def cancel_component(step):
    world.css_click("a.action-cancel")
    # We have a known issue that modifications are still shown within the edit window after cancel (though)
    # they are not persisted. Refresh the browser to make sure the changes were not persisted.
    reload_the_page(step)

@world.absorb
def revert_setting_entry(label):
    get_setting_entry(label).find_by_css('.setting-clear')[0].click()

@world.absorb
def get_setting_entry(label):
    def get_setting():
        settings = world.css_find('.wrapper-comp-setting')
        for setting in settings:
            if setting.find_by_css('.setting-label')[0].value == label:
                return setting
        return None
    return world.retry_on_exception(get_setting)

@world.absorb
def get_setting_entry_index(label):
    def get_index():
        settings = world.css_find('.metadata_edit .wrapper-comp-setting')
        for index, setting in enumerate(settings):
            if setting.find_by_css('.setting-label')[0].value == label:
                return index
        return None
    return world.retry_on_exception(get_index)

@world.absorb
def set_field_value(index, value):
    """
    Set the field to the specified value.

    Note: we cannot use css_fill here because the value is not set
    until after you move away from that field.
    Instead we will find the element, set its value, then hit the Tab key
    to get to the next field.
    """
    elem = world.css_find('.metadata_edit div.wrapper-comp-setting input.setting-input')[index]
    elem.value = value
    elem.type(Keys.TAB)
<|endoftext|>