<commit_before># -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

from __future__ import division, absolute_import, print_function
import six

"""Gets genres for imported music based on Last.fm tags.

Uses a provided whitelist file to determine which tags are valid genres.
The included (default) genre list was originally produced by scraping Wikipedia
and has been edited to remove some questionable entries.
The scraper script used is available here:
https://gist.github.com/1241307
"""
import pylast
import codecs
import os
import yaml
import traceback

from beets import plugins
from beets import ui
from beets import config
from beets.util import normpath, plurality
from beets import library

LASTFM = pylast.LastFMNetwork(api_key=plugins.LASTFM_KEY)

PYLAST_EXCEPTIONS = (
    pylast.WSError,
    pylast.MalformedResponseError,
    pylast.NetworkError,
)

REPLACE = {
    u'\u2010': '-',
}

def deduplicate(seq):
    """Remove duplicates from sequence wile preserving order.
    """
    seen = set()
    return [x for x in seq if x not in seen and not seen.add(x)]

# Canonicalization tree processing.

def flatten_tree(elem, path, branches):
    """Flatten nested lists/dictionaries into lists of strings
    (branches).
    """
    if not path:
        path = []

    if isinstance(elem, dict):
        for (k, v) in elem.items():
            flatten_tree(v, path + [k], branches)
    elif isinstance(elem, list):
        for sub in elem:
            flatten_tree(sub, path, branches)
    else:
        branches.append(path + [six.text_type(elem)])

def find_parents(candidate, branches):
    """Find parents genre of a given genre, ordered from the closest to
    the further parent.
    """
    for branch in branches:
        try:
            idx = branch.index(candidate.lower())
            return list(reversed(branch[:idx + 1]))
        except ValueError:
            continue
    return [candidate]

# Main plugin logic.

WHITELIST = os.path.join(os.path.dirname(__file__), 'genres.txt')
C14N_TREE = os.path.join(os.path.dirname(__file__), 'genres-tree.yaml')

class LastGenrePlugin(plugins.BeetsPlugin):
    def __init__(self):
        super(LastGenrePlugin, self).__init__()

        self.config.add({
            'whitelist': True,
            'min_weight': 10,
            'count': 1,
            'fallback': None,
            'canonical': False,
            'source': 'album',
            'force': True,
            'auto': True,
            'separator': u', ',
            'prefer_specific': False,
        })

        self.setup()

    def setup(self):
        """Setup plugin from config options
        """
        if self.config['auto']:
            self.import_stages = [self.imported]

        self._genre_cache = {}

        # Read the whitelist file if enabled.
        self.whitelist = set()
        wl_filename = self.config['whitelist'].get()
        if wl_filename in (True, ''):  # Indicates the default whitelist.
            wl_filename = WHITELIST
        if wl_filename:
            wl_filename = normpath(wl_filename)
            with open(wl_filename, 'rb') as f:
                for line in f:
                    line = line.decode('utf-8').strip().lower()
                    if line and not line.startswith(u'#'):
                        self.whitelist.add(line)

        # Read the genres tree for canonicalization if enabled.
        self.c14n_branches = []
        c14n_filename = self.config['canonical'].get()
        self.canonicalize = c14n_filename is not False

        # Default tree
        if c14n_filename in (True, ''):
            c14n_filename = C14N_TREE
        elif not self.canonicalize and self.config['prefer_specific'].get():
            # prefer_specific requires a tree, load default tree
            c14n_filename = C14N_TREE

        # Read the tree
        if c14n_filename:
            self._log.debug('Loading canonicalization tree {0}', c14n_filename)
            c14n_filename = normpath(c14n_filename)
            with codecs.open(c14n_filename, 'r', encoding='utf-8') as f:
                genres_tree = yaml.load(f)
            flatten_tree(genres_tree, [], self.c14n_branches)

    @property
    def sources(self):
        """A tuple of allowed genre sources. May contain 'track',
        'album', or 'artist.'
        """
        source = self.config['source'].as_choice(('track', 'album', 'artist'))
        if source == 'track':
            return 'track', 'album', 'artist'
        elif source == 'album':
            return 'album', 'artist'
        elif source == 'artist':
            return 'artist',

    def _get_depth(self, tag):
        """Find the depth of a tag in the genres tree.
        """
        depth = None
        for key, value in enumerate(self.c14n_branches):
            if tag in value:
                depth = value.index(tag)
                break
        return depth

    def _sort_by_depth(self, tags):
        """Given a list of tags, sort the tags by their depths in the
        genre tree.
        """
        depth_tag_pairs = [(self._get_depth(t), t) for t in tags]
        depth_tag_pairs = [e for e in depth_tag_pairs if e[0] is not None]
        depth_tag_pairs.sort(reverse=True)
        return [p[1] for p in depth_tag_pairs]

    def _resolve_genres(self, tags):
        """Given a list of strings, return a genre by joining them into a
        single string and (optionally) canonicalizing each.
        """
        if not tags:
            return None

        count = self.config['count'].get(int)
        if self.canonicalize:
            # Extend the list to consider tags parents in the c14n tree
            tags_all = []
            for tag in tags:
                # Add parents that are in the whitelist, or add the oldest
                # ancestor if no whitelist
                if self.whitelist:
                    parents = [x for x in find_parents(tag, self.c14n_branches)
                               if self._is_allowed(x)]
                else:
                    parents = [find_parents(tag, self.c14n_branches)[-1]]

                tags_all += parents
                # Stop if we have enough tags already, unless we need to find
                # the most specific tag (instead of the most popular).
                if (not self.config['prefer_specific'] and
                        len(tags_all) >= count):
                    break
            tags = tags_all

        tags = deduplicate(tags)

        # Sort the tags by specificity.
        if self.config['prefer_specific']:
            tags = self._sort_by_depth(tags)

        # c14n only adds allowed genres but we may have had forbidden genres in
        # the original tags list
        tags = [x.title() for x in tags if self._is_allowed(x)]

        return self.config['separator'].as_str().join(
            tags[:self.config['count'].get(int)]
        )

    def fetch_genre(self, lastfm_obj):
        """Return the genre for a pylast entity or None if no suitable genre
        can be found. Ex. 'Electronic, House, Dance'
        """
        min_weight = self.config['min_weight'].get(int)
        return self._resolve_genres(self._tags_for(lastfm_obj, min_weight))

    def _is_allowed(self, genre):
        """Determine whether the genre is present in the whitelist,
        returning a boolean.
        """
        if genre is None:
            return False
        if not self.whitelist or genre in self.whitelist:
            return True
        return False

    # Cached entity lookups.

    def _last_lookup(self, entity, method, *args):
        """Get a genre based on the named entity using the callable `method`
        whose arguments are given in the sequence `args`. The genre lookup
        is cached based on the entity name and the arguments. Before the
        lookup, each argument is has some Unicode characters replaced with
        rough ASCII equivalents in order to return better results from the
        Last.fm database.
        """
        # Shortcut if we're missing metadata.
        if any(not s for s in args):
            return None

        key = u'{0}.{1}'.format(entity,
                                u'-'.join(six.text_type(a) for a in args))
        if key in self._genre_cache:
            return self._genre_cache[key]
        else:
            args_replaced = []
            for arg in args:
                for k, v in REPLACE.items():
                    arg = arg.replace(k, v)
                args_replaced.append(arg)

            genre = self.fetch_genre(method(*args_replaced))
            self._genre_cache[key] = genre
            return genre

    def fetch_album_genre(self, obj):
        """Return the album genre for this Item or Album.
        """
        return self._last_lookup(
            u'album', LASTFM.get_album, obj.albumartist, obj.album
        )

    def fetch_album_artist_genre(self, obj):
        """Return the album artist genre for this Item or Album.
        """
        return self._last_lookup(
            u'artist', LASTFM.get_artist, obj.albumartist
        )

    def fetch_artist_genre(self, item):
        """Returns the track artist genre for this Item.
        """
        return self._last_lookup(
            u'artist', LASTFM.get_artist, item.artist
        )

    def fetch_track_genre(self, obj):
        """Returns the track genre for this Item.
        """
        return self._last_lookup(
            u'track', LASTFM.get_track, obj.artist, obj.title
        )

    def _get_genre(self, obj):
        """Get the genre string for an Album or Item object based on
        self.sources. Return a `(genre, source)` pair. The
        prioritization order is:
            - track (for Items only)
            - album
            - artist
            - original
            - fallback
            - None
        """

        # Shortcut to existing genre if not forcing.
        if not self.config['force'] and self._is_allowed(obj.genre):
            return obj.genre, 'keep'

        # Track genre (for Items only).
        if isinstance(obj, library.Item):
            if 'track' in self.sources:
                result = self.fetch_track_genre(obj)
                if result:
                    return result, 'track'

        # Album genre.
        if 'album' in self.sources:
            result = self.fetch_album_genre(obj)
            if result:
                return result, 'album'

        # Artist (or album artist) genre.
        if 'artist' in self.sources:
            result = None
            if isinstance(obj, library.Item):
                result = self.fetch_artist_genre(obj)
            elif obj.albumartist != config['va_name'].as_str():
                result = self.fetch_album_artist_genre(obj)
            else:
                # For "Various Artists", pick the most popular track genre.
                item_genres = []
                for item in obj.items():
                    item_genre = None
                    if 'track' in self.sources:
                        item_genre = self.fetch_track_genre(item)
                    if not item_genre:
                        item_genre = self.fetch_artist_genre(item)
                    if item_genre:
                        item_genres.append(item_genre)
                if item_genres:
                    result, _ = plurality(item_genres)

            if result:
                return result, 'artist'

        # Filter the existing genre.
        if obj.genre:
            result = self._resolve_genres([obj.genre])
            if result:
                return result, 'original'

        # Fallback string.
        fallback = self.config['fallback'].get()
        if fallback:
            return fallback, 'fallback'

        return None, None

    def commands(self):
        lastgenre_cmd = ui.Subcommand('lastgenre', help=u'fetch genres')
        lastgenre_cmd.parser.add_option(
            u'-f', u'--force', dest='force',
            action='store_true', default=False,
            help=u're-download genre when already present'
        )
        lastgenre_cmd.parser.add_option(
            u'-s', u'--source', dest='source', type='string',
            help=u'genre source: artist, album, or track'
        )

        def lastgenre_func(lib, opts, args):
            write = ui.should_write()
            self.config.set_args(opts)

            for album in lib.albums(ui.decargs(args)):
                album.genre, src = self._get_genre(album)
                self._log.info(u'genre for album {0} ({1}): {0.genre}',
                               album, src)
                album.store()

                for item in album.items():
                    # If we're using track-level sources, also look up each
                    # track on the album.
                    if 'track' in self.sources:
                        item.genre, src = self._get_genre(item)
                        item.store()
                        self._log.info(u'genre for track {0} ({1}): {0.genre}',
                                       item, src)

                    if write:
                        item.try_write()

        lastgenre_cmd.func = lastgenre_func
        return [lastgenre_cmd]

    def imported(self, session, task):
        """Event hook called when an import task finishes."""
        if task.is_album:
            album = task.album
            album.genre, src = self._get_genre(album)
            self._log.debug(u'added last.fm album genre ({0}): {1}',
                            src, album.genre)
            album.store()

            if 'track' in self.sources:
                for item in album.items():
                    item.genre, src = self._get_genre(item)
                    self._log.debug(u'added last.fm item genre ({0}): {1}',
                                    src, item.genre)
                    item.store()

        else:
            item = task.item
            item.genre, src = self._get_genre(item)
            self._log.debug(u'added last.fm item genre ({0}): {1}',
                            src, item.genre)
            item.store()

    def _tags_for(self, obj, min_weight=None):
        """Core genre identification routine.

        Given a pylast entity (album or track), return a list of
        tag names for that entity. Return an empty list if the entity is
        not found or another error occurs.

        If `min_weight` is specified, tags are filtered by weight.
        """
        # Work around an inconsistency in pylast where
        # Album.get_top_tags() does not return TopItem instances.
        # https://github.com/pylast/pylast/issues/86
        if isinstance(obj, pylast.Album):
            obj = super(pylast.Album, obj)

        try:
            res = obj.get_top_tags()
        except PYLAST_EXCEPTIONS as exc:
            self._log.debug(u'last.fm error: {0}', exc)
            return []
        except Exception as exc:
            # Isolate bugs in pylast.
            self._log.debug(u'{}', traceback.format_exc())
            self._log.error(u'error in pylast library: {0}', exc)
            return []

        # Filter by weight (optionally).
        if min_weight:
            res = [el for el in res if (int(el.weight or 0)) >= min_weight]

        # Get strings from tags.
        res = [el.item.get_name().lower() for el in res]

        return res
<commit_msg>Remove `default` arg for `lastgenre --force` flag<commit_after># -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

from __future__ import division, absolute_import, print_function
import six

"""Gets genres for imported music based on Last.fm tags.

Uses a provided whitelist file to determine which tags are valid genres.
The included (default) genre list was originally produced by scraping Wikipedia
and has been edited to remove some questionable entries.
The scraper script used is available here:
https://gist.github.com/1241307
"""
import pylast
import codecs
import os
import yaml
import traceback

from beets import plugins
from beets import ui
from beets import config
from beets.util import normpath, plurality
from beets import library

LASTFM = pylast.LastFMNetwork(api_key=plugins.LASTFM_KEY)

PYLAST_EXCEPTIONS = (
    pylast.WSError,
    pylast.MalformedResponseError,
    pylast.NetworkError,
)

REPLACE = {
    u'\u2010': '-',
}

def deduplicate(seq):
    """Remove duplicates from sequence wile preserving order.
    """
    seen = set()
    return [x for x in seq if x not in seen and not seen.add(x)]

# Canonicalization tree processing.

def flatten_tree(elem, path, branches):
    """Flatten nested lists/dictionaries into lists of strings
    (branches).
    """
    if not path:
        path = []

    if isinstance(elem, dict):
        for (k, v) in elem.items():
            flatten_tree(v, path + [k], branches)
    elif isinstance(elem, list):
        for sub in elem:
            flatten_tree(sub, path, branches)
    else:
        branches.append(path + [six.text_type(elem)])

def find_parents(candidate, branches):
    """Find parents genre of a given genre, ordered from the closest to
    the further parent.
    """
    for branch in branches:
        try:
            idx = branch.index(candidate.lower())
            return list(reversed(branch[:idx + 1]))
        except ValueError:
            continue
    return [candidate]

# Main plugin logic.

WHITELIST = os.path.join(os.path.dirname(__file__), 'genres.txt')
C14N_TREE = os.path.join(os.path.dirname(__file__), 'genres-tree.yaml')

class LastGenrePlugin(plugins.BeetsPlugin):
    def __init__(self):
        super(LastGenrePlugin, self).__init__()

        self.config.add({
            'whitelist': True,
            'min_weight': 10,
            'count': 1,
            'fallback': None,
            'canonical': False,
            'source': 'album',
            'force': True,
            'auto': True,
            'separator': u', ',
            'prefer_specific': False,
        })

        self.setup()

    def setup(self):
        """Setup plugin from config options
        """
        if self.config['auto']:
            self.import_stages = [self.imported]

        self._genre_cache = {}

        # Read the whitelist file if enabled.
        self.whitelist = set()
        wl_filename = self.config['whitelist'].get()
        if wl_filename in (True, ''):  # Indicates the default whitelist.
            wl_filename = WHITELIST
        if wl_filename:
            wl_filename = normpath(wl_filename)
            with open(wl_filename, 'rb') as f:
                for line in f:
                    line = line.decode('utf-8').strip().lower()
                    if line and not line.startswith(u'#'):
                        self.whitelist.add(line)

        # Read the genres tree for canonicalization if enabled.
        self.c14n_branches = []
        c14n_filename = self.config['canonical'].get()
        self.canonicalize = c14n_filename is not False

        # Default tree
        if c14n_filename in (True, ''):
            c14n_filename = C14N_TREE
        elif not self.canonicalize and self.config['prefer_specific'].get():
            # prefer_specific requires a tree, load default tree
            c14n_filename = C14N_TREE

        # Read the tree
        if c14n_filename:
            self._log.debug('Loading canonicalization tree {0}', c14n_filename)
            c14n_filename = normpath(c14n_filename)
            with codecs.open(c14n_filename, 'r', encoding='utf-8') as f:
                genres_tree = yaml.load(f)
            flatten_tree(genres_tree, [], self.c14n_branches)

    @property
    def sources(self):
        """A tuple of allowed genre sources. May contain 'track',
        'album', or 'artist.'
        """
        source = self.config['source'].as_choice(('track', 'album', 'artist'))
        if source == 'track':
            return 'track', 'album', 'artist'
        elif source == 'album':
            return 'album', 'artist'
        elif source == 'artist':
            return 'artist',

    def _get_depth(self, tag):
        """Find the depth of a tag in the genres tree.
        """
        depth = None
        for key, value in enumerate(self.c14n_branches):
            if tag in value:
                depth = value.index(tag)
                break
        return depth

    def _sort_by_depth(self, tags):
        """Given a list of tags, sort the tags by their depths in the
        genre tree.
        """
        depth_tag_pairs = [(self._get_depth(t), t) for t in tags]
        depth_tag_pairs = [e for e in depth_tag_pairs if e[0] is not None]
        depth_tag_pairs.sort(reverse=True)
        return [p[1] for p in depth_tag_pairs]

    def _resolve_genres(self, tags):
        """Given a list of strings, return a genre by joining them into a
        single string and (optionally) canonicalizing each.
        """
        if not tags:
            return None

        count = self.config['count'].get(int)
        if self.canonicalize:
            # Extend the list to consider tags parents in the c14n tree
            tags_all = []
            for tag in tags:
                # Add parents that are in the whitelist, or add the oldest
                # ancestor if no whitelist
                if self.whitelist:
                    parents = [x for x in find_parents(tag, self.c14n_branches)
                               if self._is_allowed(x)]
                else:
                    parents = [find_parents(tag, self.c14n_branches)[-1]]

                tags_all += parents
                # Stop if we have enough tags already, unless we need to find
                # the most specific tag (instead of the most popular).
                if (not self.config['prefer_specific'] and
                        len(tags_all) >= count):
                    break
            tags = tags_all

        tags = deduplicate(tags)

        # Sort the tags by specificity.
        if self.config['prefer_specific']:
            tags = self._sort_by_depth(tags)

        # c14n only adds allowed genres but we may have had forbidden genres in
        # the original tags list
        tags = [x.title() for x in tags if self._is_allowed(x)]

        return self.config['separator'].as_str().join(
            tags[:self.config['count'].get(int)]
        )

    def fetch_genre(self, lastfm_obj):
        """Return the genre for a pylast entity or None if no suitable genre
        can be found. Ex. 'Electronic, House, Dance'
        """
        min_weight = self.config['min_weight'].get(int)
        return self._resolve_genres(self._tags_for(lastfm_obj, min_weight))

    def _is_allowed(self, genre):
        """Determine whether the genre is present in the whitelist,
        returning a boolean.
        """
        if genre is None:
            return False
        if not self.whitelist or genre in self.whitelist:
            return True
        return False

    # Cached entity lookups.

    def _last_lookup(self, entity, method, *args):
        """Get a genre based on the named entity using the callable `method`
        whose arguments are given in the sequence `args`. The genre lookup
        is cached based on the entity name and the arguments. Before the
        lookup, each argument is has some Unicode characters replaced with
        rough ASCII equivalents in order to return better results from the
        Last.fm database.
        """
        # Shortcut if we're missing metadata.
        if any(not s for s in args):
            return None

        key = u'{0}.{1}'.format(entity,
                                u'-'.join(six.text_type(a) for a in args))
        if key in self._genre_cache:
            return self._genre_cache[key]
        else:
            args_replaced = []
            for arg in args:
                for k, v in REPLACE.items():
                    arg = arg.replace(k, v)
                args_replaced.append(arg)

            genre = self.fetch_genre(method(*args_replaced))
            self._genre_cache[key] = genre
            return genre

    def fetch_album_genre(self, obj):
        """Return the album genre for this Item or Album.
        """
        return self._last_lookup(
            u'album', LASTFM.get_album, obj.albumartist, obj.album
        )

    def fetch_album_artist_genre(self, obj):
        """Return the album artist genre for this Item or Album.
        """
        return self._last_lookup(
            u'artist', LASTFM.get_artist, obj.albumartist
        )

    def fetch_artist_genre(self, item):
        """Returns the track artist genre for this Item.
        """
        return self._last_lookup(
            u'artist', LASTFM.get_artist, item.artist
        )

    def fetch_track_genre(self, obj):
        """Returns the track genre for this Item.
        """
        return self._last_lookup(
            u'track', LASTFM.get_track, obj.artist, obj.title
        )

    def _get_genre(self, obj):
        """Get the genre string for an Album or Item object based on
        self.sources. Return a `(genre, source)` pair. The
        prioritization order is:
            - track (for Items only)
            - album
            - artist
            - original
            - fallback
            - None
        """

        # Shortcut to existing genre if not forcing.
        if not self.config['force'] and self._is_allowed(obj.genre):
            return obj.genre, 'keep'

        # Track genre (for Items only).
        if isinstance(obj, library.Item):
            if 'track' in self.sources:
                result = self.fetch_track_genre(obj)
                if result:
                    return result, 'track'

        # Album genre.
        if 'album' in self.sources:
            result = self.fetch_album_genre(obj)
            if result:
                return result, 'album'

        # Artist (or album artist) genre.
        if 'artist' in self.sources:
            result = None
            if isinstance(obj, library.Item):
                result = self.fetch_artist_genre(obj)
            elif obj.albumartist != config['va_name'].as_str():
                result = self.fetch_album_artist_genre(obj)
            else:
                # For "Various Artists", pick the most popular track genre.
                item_genres = []
                for item in obj.items():
                    item_genre = None
                    if 'track' in self.sources:
                        item_genre = self.fetch_track_genre(item)
                    if not item_genre:
                        item_genre = self.fetch_artist_genre(item)
                    if item_genre:
                        item_genres.append(item_genre)
                if item_genres:
                    result, _ = plurality(item_genres)

            if result:
                return result, 'artist'

        # Filter the existing genre.
        if obj.genre:
            result = self._resolve_genres([obj.genre])
            if result:
                return result, 'original'

        # Fallback string.
        fallback = self.config['fallback'].get()
        if fallback:
            return fallback, 'fallback'

        return None, None

    def commands(self):
        lastgenre_cmd = ui.Subcommand('lastgenre', help=u'fetch genres')
        lastgenre_cmd.parser.add_option(
            u'-f', u'--force', dest='force',
            action='store_true',
            help=u're-download genre when already present'
        )
        lastgenre_cmd.parser.add_option(
            u'-s', u'--source', dest='source', type='string',
            help=u'genre source: artist, album, or track'
        )

        def lastgenre_func(lib, opts, args):
            write = ui.should_write()
            self.config.set_args(opts)

            for album in lib.albums(ui.decargs(args)):
                album.genre, src = self._get_genre(album)
                self._log.info(u'genre for album {0} ({1}): {0.genre}',
                               album, src)
                album.store()

                for item in album.items():
                    # If we're using track-level sources, also look up each
                    # track on the album.
                    if 'track' in self.sources:
                        item.genre, src = self._get_genre(item)
                        item.store()
                        self._log.info(u'genre for track {0} ({1}): {0.genre}',
                                       item, src)

                    if write:
                        item.try_write()

        lastgenre_cmd.func = lastgenre_func
        return [lastgenre_cmd]

    def imported(self, session, task):
        """Event hook called when an import task finishes."""
        if task.is_album:
            album = task.album
            album.genre, src = self._get_genre(album)
            self._log.debug(u'added last.fm album genre ({0}): {1}',
                            src, album.genre)
            album.store()

            if 'track' in self.sources:
                for item in album.items():
                    item.genre, src = self._get_genre(item)
                    self._log.debug(u'added last.fm item genre ({0}): {1}',
                                    src, item.genre)
                    item.store()

        else:
            item = task.item
            item.genre, src = self._get_genre(item)
            self._log.debug(u'added last.fm item genre ({0}): {1}',
                            src, item.genre)
            item.store()

    def _tags_for(self, obj, min_weight=None):
        """Core genre identification routine.

        Given a pylast entity (album or track), return a list of
        tag names for that entity. Return an empty list if the entity is
        not found or another error occurs.

        If `min_weight` is specified, tags are filtered by weight.
        """
        # Work around an inconsistency in pylast where
        # Album.get_top_tags() does not return TopItem instances.
        # https://github.com/pylast/pylast/issues/86
        if isinstance(obj, pylast.Album):
            obj = super(pylast.Album, obj)

        try:
            res = obj.get_top_tags()
        except PYLAST_EXCEPTIONS as exc:
            self._log.debug(u'last.fm error: {0}', exc)
            return []
        except Exception as exc:
            # Isolate bugs in pylast.
            self._log.debug(u'{}', traceback.format_exc())
            self._log.error(u'error in pylast library: {0}', exc)
            return []

        # Filter by weight (optionally).
        if min_weight:
            res = [el for el in res if (int(el.weight or 0)) >= min_weight]

        # Get strings from tags.
        res = [el.item.get_name().lower() for el in res]

        return res
<|endoftext|>


<commit_before>from bids.grabbids import BIDSLayout
from bids.variables import (SparseRunVariable, SimpleVariable,
                            DenseRunVariable, load_variables)
from bids.variables.entities import Node, RunNode, NodeIndex
import pytest
from os.path import join
from bids.tests import get_test_data_path
from bids.config import set_option

@pytest.fixture
def layout1():
    path = join(get_test_data_path(), 'ds005')
    layout = BIDSLayout(path, exclude='derivatives/')
    return layout

@pytest.fixture(scope="module", params=["events", "preproc"])
def synthetic(request):
    if request.param == "preproc":
        set_option('loop_preproc', True)
    path = join(get_test_data_path(), 'synthetic')
    layout = BIDSLayout(path)
    return request.param, load_variables(layout)

def test_load_events(layout1):
    index = load_variables(layout1, types='events', scan_length=480)
    runs = index.get_nodes(level='run', entities={'subject': '01'})
    assert len(runs) == 3
    assert isinstance(runs[0], RunNode)
    variables = runs[0].variables
    assert len(variables) == 8
    targ_cols = {'parametric gain', 'PTval', 'trial_type', 'respnum'}
    assert not (targ_cols - set(variables.keys()))
    assert isinstance(variables['parametric gain'], SparseRunVariable)
    assert variables['parametric gain'].index.shape == (86, 3)
    assert variables['parametric gain'].source == 'events'

def test_load_participants(layout1):
    index = load_variables(layout1, types='participants')
    assert isinstance(index, NodeIndex)
    dataset = index.get_nodes(level='dataset')[0]
    assert isinstance(dataset, Node)
    assert len(dataset.variables) == 2
    assert {'age', 'sex'} == set(dataset.variables.keys())
    age = dataset.variables['age']
    assert isinstance(age, SimpleVariable)
    assert age.index.shape == (16, 1)
    assert age.values.shape == (16,)

    index = load_variables(layout1, types='participants', subject=['^1.*'])
    age = index.get_nodes(level='dataset')[0].variables['age']
    assert age.index.shape == (7, 1)
    assert age.values.shape == (7,)

def test_load_synthetic_dataset(synthetic):
    param, index = synthetic
    # Runs
    runs = index.get_nodes('run')
    assert len(runs) == 5 * 2 * 3
    runs = index.get_nodes('run', {'task': 'nback'})
    assert len(runs) == 5 * 2 * 2
    variables = runs[0].variables

    if param == 'preproc':
        # non-exhaustive
        match = ['Cosine01', 'stdDVARS', 'RotZ', 'FramewiseDisplacement']
        sum_dense = 52
    else:
        match = ['trial_type', 'weight', 'respiratory', 'cardiac']
        sum_dense = 54

    for v in match:
        assert v in variables.keys()

    assert sum([isinstance(v, DenseRunVariable)
                for v in variables.values()]) == sum_dense
    assert all([len(r.variables['weight'].values) == 42 for r in runs])

    # Sessions
    sessions = index.get_nodes('session')
    assert len(sessions) == 5 * 2
    assert set(sessions[0].variables.keys()) == {'acq_time'}
    data = sessions[0].variables['acq_time'].filter({'task': 'nback'})
    assert len(data.values) == 2

    # Subjects
    subs = index.get_nodes('subject')
    assert len(subs) == 5
    assert set(subs[0].variables.keys()) == {'systolic_blood_pressure'}
<commit_msg>Update tests layout path<commit_after>from bids.grabbids import BIDSLayout
from bids.variables import (SparseRunVariable, SimpleVariable,
                            DenseRunVariable, load_variables)
from bids.variables.entities import Node, RunNode, NodeIndex
import pytest
from os.path import join
from bids.tests import get_test_data_path
from bids.config import set_option

@pytest.fixture
def layout1():
    path = join(get_test_data_path(), 'ds005')
    layout = BIDSLayout(path, exclude='derivatives/')
    return layout

@pytest.fixture(scope="module", params=["events", "preproc"])
def synthetic(request):
    path = join(get_test_data_path(), 'synthetic')
    if request.param == "preproc":
        set_option('loop_preproc', True)
        path = (path, ['bids', 'derivatives'])
    layout = BIDSLayout(path)
    return request.param, load_variables(layout)

def test_load_events(layout1):
    index = load_variables(layout1, types='events', scan_length=480)
    runs = index.get_nodes(level='run', entities={'subject': '01'})
    assert len(runs) == 3
    assert isinstance(runs[0], RunNode)
    variables = runs[0].variables
    assert len(variables) == 8
    targ_cols = {'parametric gain', 'PTval', 'trial_type', 'respnum'}
    assert not (targ_cols - set(variables.keys()))
    assert isinstance(variables['parametric gain'], SparseRunVariable)
    assert variables['parametric gain'].index.shape == (86, 3)
    assert variables['parametric gain'].source == 'events'

def test_load_participants(layout1):
    index = load_variables(layout1, types='participants')
    assert isinstance(index, NodeIndex)
    dataset = index.get_nodes(level='dataset')[0]
    assert isinstance(dataset, Node)
    assert len(dataset.variables) == 2
    assert {'age', 'sex'} == set(dataset.variables.keys())
    age = dataset.variables['age']
    assert isinstance(age, SimpleVariable)
    assert age.index.shape == (16, 1)
    assert age.values.shape == (16,)

    index = load_variables(layout1, types='participants', subject=['^1.*'])
    age = index.get_nodes(level='dataset')[0].variables['age']
    assert age.index.shape == (7, 1)
    assert age.values.shape == (7,)

def test_load_synthetic_dataset(synthetic):
    param, index = synthetic
    # Runs
    runs = index.get_nodes('run')
    assert len(runs) == 5 * 2 * 3
    runs = index.get_nodes('run', {'task': 'nback'})
    assert len(runs) == 5 * 2 * 2
    variables = runs[0].variables

    if param == 'preproc':
        # non-exhaustive
        match = ['Cosine01', 'stdDVARS', 'RotZ', 'FramewiseDisplacement']
        sum_dense = 52
    else:
        match = ['trial_type', 'weight', 'respiratory', 'cardiac']
        sum_dense = 54

    for v in match:
        assert v in variables.keys()

    assert sum([isinstance(v, DenseRunVariable)
                for v in variables.values()]) == sum_dense
    assert all([len(r.variables['weight'].values) == 42 for r in runs])

    # Sessions
    sessions = index.get_nodes('session')
    assert len(sessions) == 5 * 2
    assert set(sessions[0].variables.keys()) == {'acq_time'}
    data = sessions[0].variables['acq_time'].filter({'task': 'nback'})
    assert len(data.values) == 2

    # Subjects
    subs = index.get_nodes('subject')
    assert len(subs) == 5
    assert set(subs[0].variables.keys()) == {'systolic_blood_pressure'}
<|endoftext|>


<commit_before>from bika.lims import logger
from bika.lims.content.analysis import Analysis
from bika.lims.testing import BIKA_FUNCTIONAL_TESTING
from bika.lims.tests.base import BikaFunctionalTestCase
from bika.lims.utils.analysisrequest import create_analysisrequest
from plone.app.testing import login, logout
from plone.app.testing import TEST_USER_NAME
from Products.CMFCore.utils import getToolByName
from plone import api
import unittest

try:
    import unittest2 as unittest
except ImportError: # Python 2.7
    import unittest

class LIMS2242(BikaFunctionalTestCase):
    layer = BIKA_FUNCTIONAL_TESTING

    def setUp(self):
        super(TestSamples, self).setUp()
        login(self.portal, TEST_USER_NAME)
        servs = self.portal.bika_setup.bika_analysisservices
        self.services = [servs['analysisservice-3'],
                         servs['analysisservice-6'],
                         servs['analysisservice-7']]
        # Enabling the workflow
        self.portal.bika_setup.setSamplingWorkflowEnabled(True)
        self.portal.bika_setup.setScheduleSamplingEnabled(True)

    def tearDown(self):
        # Enabling the workflow
        self.portal.bika_setup.setSamplingWorkflowEnabled(False)
        self.portal.bika_setup.setScheduleSamplingEnabled(False)
        logout()
        super(TestSamples, self).tearDown()

    def test_sample_workflow_action_schedule_sampling(self):
        """
        This test validates the function
        bika/lims//browser/analysisrequest/workflow.py/workflow_action_schedule_sampling
        """
        from bika.lims.utils.workflow import schedulesampling
        workflow = getToolByName(self.portal, 'portal_workflow')
        pc = getToolByName(self.portal, 'portal_catalog')
        sampler = api.user.get(username='sampler1')
        coordinator = self.createUser('SamplingCoordinator', 'cord1')
        mtool = getToolByName(self.portal, 'portal_membership')
        # Getting the client
        client = self.portal.clients['client-1']
        # Getting a sample type
        sampletype = self.portal.bika_setup.bika_sampletypes['sampletype-1']
        # Creating an AR
        values = {'Client': client.UID(),
                  'Contact': client.getContacts()[0].UID(),
                  'SamplingDate': '2015-01-01',
                  'SampleType': sampletype.UID()}
        request = {}
        services = [s.UID() for s in self.services]
        # creating the analysisrequest
        ar = create_analysisrequest(client, request, values, services)
        self.assertEqual(
            workflow.getInfoFor(ar, 'review_state'), 'to_be_sampled')
        # Changing user to coordinator
        logout()
        login(self.portal, 'cord1')
        # If ScheduledSamplingSampler is empty and ScheduleSamplingEnabled,
        # no workflow_action_schedule_sampling can be done
        schedulesampling.doTransition(ar.getSample())
        self.assertEqual(
            workflow.getInfoFor(ar, 'review_state'), 'to_be_sampled')
        self.assertEqual(
            workflow.getInfoFor(ar.getSample(), 'review_state'),
            'to_be_sampled')
        # set a value in ScheduledSamplingSampler
        ar.setScheduledSamplingSampler(sampler)
        schedulesampling.doTransition(ar.getSample())
        self.assertEqual(
            workflow.getInfoFor(ar, 'review_state'), 'scheduled_sampling')
        self.assertEqual(
            workflow.getInfoFor(ar.getSample(), 'review_state'),
            'scheduled_sampling')

    def createUser(self, role, username='test_1'):
        #workflow.getTransitionsFor(ar.getSample())
        member = self.portal.portal_registration.addMember(
            username,
            username,
            properties={
                'username': 'username',
                'email': username + "@example.com",
                'fullname': username}
        )
        # Add user to all specified groups
        group_id = role + "s"
        # Add user to all specified roles
        self.portal.portal_groups.addPrincipalToGroup(
            member.getUserName(), group_id)
        return member

def test_suite():
    suite = unittest.TestSuite()
    suite.addTest(unittest.makeSuite(TestLimitDetections))
    suite.layer = BIKA_FUNCTIONAL_TESTING
    return suite
<commit_msg>test 2252<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2019-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.

from __future__ import absolute_import, division, print_function, unicode_literals

import argparse
import os
import subprocess
import sys

from getdeps.buildopts import setup_build_options
from getdeps.load import load_project, manifests_in_dependency_order
from getdeps.manifest import ManifestParser
from getdeps.platform import HostType, context_from_host_tuple
from getdeps.subcmd import SubCmd, add_subcommands, cmd

sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "getdeps"))

@cmd("validate-manifest", "parse a manifest and validate that it is correct")
class ValidateManifest(SubCmd):
    def run(self, args):
        try:
            ManifestParser(file_name=args.file_name)
            print("OK", file=sys.stderr)
            return 0
        except Exception as exc:
            print("ERROR: %s" % str(exc), file=sys.stderr)
            return 1

    def setup_parser(self, parser):
        parser.add_argument("file_name", help="path to the manifest file")

@cmd("show-host-type", "outputs the host type tuple for the host machine")
class ShowHostType(SubCmd):
    def run(self, args):
        host = HostType()
        print("%s" % host.as_tuple_string())
        return 0

@cmd("fetch", "fetch the code for a given project")
class FetchCmd(SubCmd):
    def setup_parser(self, parser):
        parser.add_argument(
            "project",
            help=(
                "name of the project or path to a manifest "
                "file describing the project"
            ),
        )
        parser.add_argument(
            "--recursive",
            help="fetch the transitive deps also",
            action="store_true",
            default=False,
        )
        parser.add_argument(
            "--host-type",
            help=(
                "When recursively fetching, fetch deps for "
                "this host type rather than the current system"
            ),
        )

    def run(self, args):
        opts = setup_build_options(args)
        manifest = load_project(opts, args.project)
        ctx = context_from_host_tuple(args.host_type)
        if args.recursive:
            projects = manifests_in_dependency_order(opts, manifest, ctx)
        else:
            projects = [manifest]
        for m in projects:
            fetcher = m.create_fetcher(opts, ctx)
            fetcher.update()

@cmd("list-deps", "lists the transitive deps for a given project")
class ListDepsCmd(SubCmd):
    def run(self, args):
        opts = setup_build_options(args)
        manifest = load_project(opts, args.project)
        ctx = context_from_host_tuple(args.host_type)
        for m in manifests_in_dependency_order(opts, manifest, ctx):
            print(m.name)
        return 0

    def setup_parser(self, parser):
        parser.add_argument(
            "--host-type",
            help=(
                "Produce the list for the specified host type, "
                "rather than that of the current system"
            ),
        )
        parser.add_argument(
            "project",
            help=(
                "name of the project or path to a manifest "
                "file describing the project"
            ),
        )

def build_argparser():
    common_args = argparse.ArgumentParser(add_help=False)
    common_args.add_argument(
        "--scratch-path", help="Where to maintain checkouts and build dirs"
    )
    common_args.add_argument(
        "--install-prefix",
        help=(
            "Where the final build products will be installed "
            "(default is [scratch-path]/installed)"
        ),
    )
    common_args.add_argument(
        "--num-jobs",
        type=int,
        help=(
            "Number of concurrent jobs to use while building. "
            "(default=number of cpu cores)"
        ),
    )

    ap = argparse.ArgumentParser(
        description="Get and build dependencies and projects", parents=[common_args]
    )
    sub = ap.add_subparsers(
        # metavar suppresses the long and ugly default list of subcommands on a
        # single line.  We still render the nicer list below where we would
        # have shown the nasty one.
        metavar="",
        title="Available commands",
        help="",
    )

    add_subcommands(sub, common_args)

    return ap

def main():
    ap = build_argparser()
    args = ap.parse_args()
    if getattr(args, "func", None) is None:
        ap.print_help()
        return 0
    try:
        return args.func(args)
    except subprocess.CalledProcessError:
        print("!! Failed", file=sys.stderr)
        return 1

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>getdeps: add build subcommand<commit_after>#!/usr/bin/env python
# Copyright (c) 2019-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.

from __future__ import absolute_import, division, print_function, unicode_literals

import argparse
import os
import shutil
import subprocess
import sys

from getdeps.buildopts import setup_build_options
from getdeps.load import load_project, manifests_in_dependency_order
from getdeps.manifest import ManifestParser
from getdeps.platform import HostType, context_from_host_tuple
from getdeps.subcmd import SubCmd, add_subcommands, cmd

sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "getdeps"))

@cmd("validate-manifest", "parse a manifest and validate that it is correct")
class ValidateManifest(SubCmd):
    def run(self, args):
        try:
            ManifestParser(file_name=args.file_name)
            print("OK", file=sys.stderr)
            return 0
        except Exception as exc:
            print("ERROR: %s" % str(exc), file=sys.stderr)
            return 1

    def setup_parser(self, parser):
        parser.add_argument("file_name", help="path to the manifest file")

@cmd("show-host-type", "outputs the host type tuple for the host machine")
class ShowHostType(SubCmd):
    def run(self, args):
        host = HostType()
        print("%s" % host.as_tuple_string())
        return 0

@cmd("fetch", "fetch the code for a given project")
class FetchCmd(SubCmd):
    def setup_parser(self, parser):
        parser.add_argument(
            "project",
            help=(
                "name of the project or path to a manifest "
                "file describing the project"
            ),
        )
        parser.add_argument(
            "--recursive",
            help="fetch the transitive deps also",
            action="store_true",
            default=False,
        )
        parser.add_argument(
            "--host-type",
            help=(
                "When recursively fetching, fetch deps for "
                "this host type rather than the current system"
            ),
        )

    def run(self, args):
        opts = setup_build_options(args)
        manifest = load_project(opts, args.project)
        ctx = context_from_host_tuple(args.host_type)
        if args.recursive:
            projects = manifests_in_dependency_order(opts, manifest, ctx)
        else:
            projects = [manifest]
        for m in projects:
            fetcher = m.create_fetcher(opts, ctx)
            fetcher.update()

@cmd("list-deps", "lists the transitive deps for a given project")
class ListDepsCmd(SubCmd):
    def run(self, args):
        opts = setup_build_options(args)
        manifest = load_project(opts, args.project)
        ctx = context_from_host_tuple(args.host_type)
        for m in manifests_in_dependency_order(opts, manifest, ctx):
            print(m.name)
        return 0

    def setup_parser(self, parser):
        parser.add_argument(
            "--host-type",
            help=(
                "Produce the list for the specified host type, "
                "rather than that of the current system"
            ),
        )
        parser.add_argument(
            "project",
            help=(
                "name of the project or path to a manifest "
                "file describing the project"
            ),
        )

@cmd("build", "build a given project")
class BuildCmd(SubCmd):
    def run(self, args):
        opts = setup_build_options(args)
        if args.clean:
            for d in ["build", "installed", "extracted", "shipit"]:
                d = os.path.join(opts.scratch_dir, d)
                print("Cleaning %s..." % d)
                if os.path.exists(d):
                    shutil.rmtree(d)

        manifest = load_project(opts, args.project)

        ctx = context_from_host_tuple()
        projects = manifests_in_dependency_order(opts, manifest, ctx)

        # Accumulate the install directories so that the build steps
        # can find their dep installation
        install_dirs = []

        for m in projects:
            print("Assessing %s..." % m.name)
            fetcher = m.create_fetcher(opts, ctx)

            if args.clean:
                fetcher.clean()
            change_status = fetcher.update()
            reconfigure = change_status.build_changed()
            sources_changed = change_status.sources_changed()

            hash = fetcher.hash()
            directory = "%s-%s" % (m.name, hash)
            build_dir = os.path.join(opts.scratch_dir, "build", directory)
            inst_dir = os.path.join(opts.scratch_dir, "installed", directory)

            built_marker = os.path.join(inst_dir, ".built-by-getdeps")
            if os.path.exists(built_marker):
                with open(built_marker, "r") as f:
                    built_hash = f.read().strip()
                if built_hash != hash:
                    # Some kind of inconsistency with a prior build,
                    # let's run it again to be sure
                    os.unlink(built_marker)

            if sources_changed or reconfigure or not os.path.exists(built_marker):
                if os.path.exists(built_marker):
                    os.unlink(built_marker)
                src_dir = fetcher.get_src_dir()
                builder = m.create_builder(opts, src_dir, build_dir, inst_dir, ctx)
                builder.build(install_dirs, reconfigure=reconfigure)

                with open(built_marker, "w") as f:
                    f.write(hash)

            install_dirs.append(inst_dir)

    def setup_parser(self, parser):
        parser.add_argument(
            "project",
            help=(
                "name of the project or path to a manifest "
                "file describing the project"
            ),
        )
        parser.add_argument(
            "--clean",
            action="store_true",
            default=False,
            help=(
                "Clean up the build and installation area prior to building, "
                "causing the projects to be built from scratch"
            ),
        )

def build_argparser():
    common_args = argparse.ArgumentParser(add_help=False)
    common_args.add_argument(
        "--scratch-path", help="Where to maintain checkouts and build dirs"
    )
    common_args.add_argument(
        "--install-prefix",
        help=(
            "Where the final build products will be installed "
            "(default is [scratch-path]/installed)"
        ),
    )
    common_args.add_argument(
        "--num-jobs",
        type=int,
        help=(
            "Number of concurrent jobs to use while building. "
            "(default=number of cpu cores)"
        ),
    )

    ap = argparse.ArgumentParser(
        description="Get and build dependencies and projects", parents=[common_args]
    )
    sub = ap.add_subparsers(
        # metavar suppresses the long and ugly default list of subcommands on a
        # single line.  We still render the nicer list below where we would
        # have shown the nasty one.
        metavar="",
        title="Available commands",
        help="",
    )

    add_subcommands(sub, common_args)

    return ap

def main():
    ap = build_argparser()
    args = ap.parse_args()
    if getattr(args, "func", None) is None:
        ap.print_help()
        return 0
    try:
        return args.func(args)
    except subprocess.CalledProcessError:
        print("!! Failed", file=sys.stderr)
        return 1

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2020 Dmitry Kozlyuk <dmitry.kozliuk@gmail.com>

import os
import subprocess
import sys
import tempfile

_, tmp_root, ar, archive, output, *pmdinfogen = sys.argv
with tempfile.TemporaryDirectory(dir=tmp_root) as temp:
    proc = subprocess.run(
        # Don't use "ar p", because its output is corrupted on Windows.
        [ar, "xv", os.path.abspath(archive)], stdout=subprocess.PIPE, check=True, cwd=temp
    )
    lines = proc.stdout.decode().splitlines()
    names = [line[len("x - ") :] for line in lines]
    paths = [os.path.join(temp, name) for name in names]
    subprocess.run(pmdinfogen + paths + [output], check=True)
<commit_msg>buildtools: fix archive extraction for LLVM 8<commit_after>#!/usr/bin/env python3
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2020 Dmitry Kozlyuk <dmitry.kozliuk@gmail.com>

import os
import subprocess
import sys
import tempfile

_, tmp_root, ar, archive, output, *pmdinfogen = sys.argv
with tempfile.TemporaryDirectory(dir=tmp_root) as temp:
    run_ar = lambda command: subprocess.run(
        [ar, command, os.path.abspath(archive)],
        stdout=subprocess.PIPE, check=True, cwd=temp
    )
    # Don't use "ar p", because its output is corrupted on Windows.
    run_ar("x")
    names = run_ar("t").stdout.decode().splitlines()
    paths = [os.path.join(temp, name) for name in names]
    subprocess.run(pmdinfogen + paths + [output], check=True)
<|endoftext|>


<commit_before># ----------------------------------------------------------------------------
# Copyright (c) 2016--,  Calour development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

import unittest
from os.path import join, dirname, abspath

import numpy as np
import numpy.testing as npt

from calour.testing import Tests
import calour as ca

class TestExperiment(Tests):
    def setUp(self):
        super().setUp()
        # load the simple experiment as sparse
        self.simple = ca.read(self.simple_table, self.simple_map)
        # load the simple experiment as dense
        self.simple_dense = ca.read(self.simple_table, self.simple_map, sparse=False)
        # load the complex experiment as sparse
        self.complex = ca.read(self.complex_table, self.complex_map)

    def test_reorder(self):
        exp = self.simple
        # keep only samples 5,4
        newexp = exp.reorder([5, 4], axis=0)

        # test we didn't loose any bacteria, correct number of samples
        self.assertEqual(newexp.data.shape[0], 2)
        self.assertEqual(newexp.data.shape[1], exp.data.shape[1])

        # test both sample_metadata and data are correct
        self.assertEqual(newexp.sample_metadata['id'][0], 6)
        self.assertEqual(newexp.sample_metadata['id'][1], 5)
        sseq = 'TACGTAGGGTGCAAGCGTTAATCGGAATTACTGGGCGTAAAGCGTGCGCAGGCGGTTTTGTAAGTCTGATGTGAAATCCCCGGGCTCAACCTGGGAATTGCATTGGAGACTGCAAGGCTAGAATCTGGCAGAGGGGGGTAGAATTCCACG'
        seqpos = exp.feature_metadata.index.get_loc(sseq)
        self.assertEqual(newexp.data[0, seqpos], 6)
        self.assertEqual(newexp.data[1, seqpos], 5)

        # test inplace reordering of features
        sseq = 'TACGTAGGGTGCAAGCGTTAATCGGAATTACTGGGCGTAAAGCGTGCGCAGGCGGTTTTGTAAGTCTGATGTGAAATCCCCGGGCTCAACCTGGGAATTGCATTGGAGACTGCAAGGCTAGAATCTGGCAGAGGGGGGTAGAATTCCACG'
        seqpos = exp.feature_metadata.index.get_loc(sseq)
        newexp.reorder([seqpos, 0], axis=1, inplace=True)
        seqpos = newexp.feature_metadata.index.get_loc(sseq)
        self.assertEqual(seqpos, 0)
        self.assertEqual(newexp.data[0, 0], 6)
        self.assertEqual(newexp.data[1, 0], 5)

        # test double permuting of big dataset
        exp = self.complex
        rand_perm_samples = np.random.permutation(exp.data.shape[0])
        rand_perm_features = np.random.permutation(exp.data.shape[1])
        rev_perm_samples = np.argsort(rand_perm_samples)
        rev_perm_features = np.argsort(rand_perm_features)
        newexp = exp.reorder(rand_perm_features, axis=1)
        newexp = newexp.reorder(rand_perm_samples, axis=0, inplace=True)
        newexp = newexp.reorder(rev_perm_features, axis=1, inplace=True)
        newexp = newexp.reorder(rev_perm_samples, axis=0)
        self.assertEqual(0, np.sum(newexp.data != exp.data))
        self.assertTrue(newexp.sample_metadata.equals(exp.sample_metadata))
        self.assertTrue(newexp.feature_metadata.equals(exp.feature_metadata))

    def test_eq(self):

if __name__ == "__main__":
    unittest.main()
<commit_msg>fix unit testing<commit_after># ----------------------------------------------------------------------------
# Copyright (c) 2016--,  Calour development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

import unittest
from os.path import join, dirname, abspath

import numpy as np
import numpy.testing as npt

from calour._testing import Tests
import calour as ca

class TestExperiment(Tests):
    def setUp(self):
        super().setUp()
        # load the simple experiment as sparse
        self.simple = ca.read(self.simple_table, self.simple_map)
        # load the simple experiment as dense
        self.simple_dense = ca.read(self.simple_table, self.simple_map, sparse=False)
        # load the complex experiment as sparse
        self.complex = ca.read(self.complex_table, self.complex_map)

    def test_reorder(self):
        exp = self.simple
        # keep only samples 5,4
        newexp = exp.reorder([5, 4], axis=0)

        # test we didn't loose any bacteria, correct number of samples
        self.assertEqual(newexp.data.shape[0], 2)
        self.assertEqual(newexp.data.shape[1], exp.data.shape[1])

        # test both sample_metadata and data are correct
        self.assertEqual(newexp.sample_metadata['id'][0], 6)
        self.assertEqual(newexp.sample_metadata['id'][1], 5)
        sseq = 'TACGTAGGGTGCAAGCGTTAATCGGAATTACTGGGCGTAAAGCGTGCGCAGGCGGTTTTGTAAGTCTGATGTGAAATCCCCGGGCTCAACCTGGGAATTGCATTGGAGACTGCAAGGCTAGAATCTGGCAGAGGGGGGTAGAATTCCACG'
        seqpos = exp.feature_metadata.index.get_loc(sseq)
        self.assertEqual(newexp.data[0, seqpos], 6)
        self.assertEqual(newexp.data[1, seqpos], 5)

        # test inplace reordering of features
        sseq = 'TACGTAGGGTGCAAGCGTTAATCGGAATTACTGGGCGTAAAGCGTGCGCAGGCGGTTTTGTAAGTCTGATGTGAAATCCCCGGGCTCAACCTGGGAATTGCATTGGAGACTGCAAGGCTAGAATCTGGCAGAGGGGGGTAGAATTCCACG'
        seqpos = exp.feature_metadata.index.get_loc(sseq)
        newexp.reorder([seqpos, 0], axis=1, inplace=True)
        seqpos = newexp.feature_metadata.index.get_loc(sseq)
        self.assertEqual(seqpos, 0)
        self.assertEqual(newexp.data[0, 0], 6)
        self.assertEqual(newexp.data[1, 0], 5)

        # test double permuting of big dataset
        exp = self.complex
        rand_perm_samples = np.random.permutation(exp.data.shape[0])
        rand_perm_features = np.random.permutation(exp.data.shape[1])
        rev_perm_samples = np.argsort(rand_perm_samples)
        rev_perm_features = np.argsort(rand_perm_features)
        newexp = exp.reorder(rand_perm_features, axis=1)
        newexp = newexp.reorder(rand_perm_samples, axis=0, inplace=True)
        newexp = newexp.reorder(rev_perm_features, axis=1, inplace=True)
        newexp = newexp.reorder(rev_perm_samples, axis=0)
        self.assertEqual(0, np.sum(newexp.data != exp.data))
        self.assertTrue(newexp.sample_metadata.equals(exp.sample_metadata))
        self.assertTrue(newexp.feature_metadata.equals(exp.feature_metadata))

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>from __future__ import print_function, division
import sys
import string
import select
import socket
import Tkinter as Tk
from PIL import Image, ImageTk
import numpy as np
import zlib
import pdb
import time
import zmq

class ClientSocket:
    MAX_BUFFER = 65536
#    def __init__(self,IP="133.40.162.192", Port=3001):
    def __init__(self,IP="150.203.89.12",Port=3001):
        #NB See the prototype in macquarie-university-automation for a slightly neater start.
        ADS = (IP,Port)
        try:
            self.context = zmq.Context()
            self.client = self.context.socket(zmq.REQ)
            self.client.connect("tcp://localhost:3001")
            self.connected=True
        except:
            print('ERROR: Could not connect to server. Please check that the server is running.')
            self.connected=False

    def send_command(self, command):
        """WARNING: currently a blocking send/recv!"""
        try: self.client.send(command)
        except Exception: return 'Error sending command, connection likely lost.'
        try: return self.client.recv(self.MAX_BUFFER)
        except Exception: return 'Error receiving response'

class RHEAGui:
    current_image=0;
    IMAGE_WIDTH=320;
    IMAGE_HEIGHT=256;
    def __init__(self):
        self.client_socket = ClientSocket() #IP="192.168.2.13"
        self.win=Tk.Tk()
        self.win.title("RHEA@Subaru Injection")
        row = Tk.Frame(self.win)
        Tk.Label(row, text='Command: ').pack(side=Tk.LEFT)
        self.command = Tk.StringVar()
        command_entry = Tk.Entry(row, textvariable=self.command)
        command_entry.pack(side=Tk.RIGHT, expand=Tk.YES)
        command_entry.bind("<Return>", self.send_to_server)
        row.pack(fill=Tk.X)
        self.response_label = Tk.Label(self.win, text="[No Server Response Yet]",\
            height=6, width=40,justify=Tk.LEFT, bg='black', fg='green', anchor='w')
        self.response_label.pack(fill=Tk.X)
        imdata = np.zeros( (self.IMAGE_HEIGHT,self.IMAGE_WIDTH), dtype=np.uint8)
        image = ImageTk.PhotoImage(Image.fromarray(imdata))
        self.image_label = Tk.Label(self.win, image=image)
        self.image_label.image=image
        self.image_label.pack(fill=Tk.X)
        self.ask_for_image()

    def ask_for_image(self):
        command = "image {0:d}".format(self.current_image)
        if (self.client_socket.connected):
            response = self.client_socket.send_command(command)
            if (response.split(" ", 1)[0]=="image"):
                self.display_image(response)
            else:
                self.response_label['text']=response
        self.win.after(100, self.ask_for_image)

    def display_image(self, response):
        argv = response.split(" ",3)
        current_image=int(argv[1])
        compressed_len = int(argv[2])
        if compressed_len == 0:
            return
        if (compressed_len > len(argv[3])):
            print("ERROR: Only got {0:d} of {1:d} bytes".format(len(argv[3]), compressed_len))
        else:
            imdata = zlib.decompress(argv[3][0:compressed_len])
            if (len(imdata)!=self.IMAGE_WIDTH*self.IMAGE_HEIGHT*4):
                print("ERROR: Image size {0:d} instead of {1:d}".format(len(imdata), self.IMAGE_WIDTH*self.IMAGE_HEIGHT*4))
            else:
                imdata = np.fromstring(imdata, np.float32)
                imdata = np.array(imdata).reshape( (self.IMAGE_HEIGHT,self.IMAGE_WIDTH) )
                #Autoscale for now...
                imdata = ( (imdata - np.min(imdata))/(np.max(imdata)-np.min(imdata))*255).astype(np.uint8)
                image = ImageTk.PhotoImage(Image.fromarray(imdata))
                self.image_label['image']=image
                self.image_label.image=image

    def send_to_server(self, event):
        if (self.client_socket.connected):
            response = self.client_socket.send_command(self.command.get())
            if (response.split(" ", 1)[0]=="image"):
                self.display_image(response)
            else:
                self.response_label['text']=response
        else:
            self.response_label['text']="*** Not Connected ***"

        self.command.set("")

TheGui = RHEAGui()
TheGui.win.mainloop()

<commit_msg>Change Tcl/Tk to QT, in order to work on mac (weird Tkinter image issues)<commit_after>from __future__ import print_function, division
import sys
import string
import select
import socket
from PyQt4.QtGui import *
from PyQt4.QtCore import *
import numpy as np
import zlib
import pdb
import time
import zmq

class ClientSocket:
    MAX_BUFFER = 65536
#    def __init__(self,IP="133.40.162.192", Port=3001):
#    def __init__(self,IP="150.203.89.12",Port=3001):
    def __init__(self,IP="192.168.1.5",Port=3001):
        #NB See the prototype in macquarie-university-automation for a slightly neater start.
        ADS = (IP,Port)
        try:
            self.context = zmq.Context()
            self.client = self.context.socket(zmq.REQ)
            self.client.connect("tcp://"+IP+":3001")
            self.connected=True
        except:
            print('ERROR: Could not connect to server. Please check that the server is running.')
            self.connected=False

    def send_command(self, command):
        """WARNING: currently a blocking send/recv!"""
        try: self.client.send(command)
        except Exception: return 'Error sending command, connection likely lost.'
        try: return self.client.recv(self.MAX_BUFFER)
        except Exception: return 'Error receiving response'

class RHEAGui(QWidget):
    current_image=0;
    IMAGE_WIDTH=320;
    IMAGE_HEIGHT=256;
    def __init__(self, parent=None):
        super(RHEAGui,self).__init__(parent)
        self.client_socket = ClientSocket()
        lbl1 = QLabel('Command: ', self)
        self.lineedit = QLineEdit("")
        self.connect(self.lineedit, SIGNAL("returnPressed()"),
                        self.send_to_server)
        self.response_label = QLabel('[No Sever Response Yet]', self)
        self.response_label.setFixedWidth(320)
        self.response_label.setFixedHeight(100)
        imdata = np.zeros( (self.IMAGE_HEIGHT,self.IMAGE_WIDTH), dtype=np.uint32)
        image = QImage(imdata.tostring(),self.IMAGE_HEIGHT,self.IMAGE_WIDTH,QImage.Format_RGB32)
        self.image_label = QLabel("",self)
        self.image_label.setPixmap(QPixmap.fromImage(image))

        hbox1 = QHBoxLayout()
        hbox1.addWidget(lbl1)
        hbox1.addWidget(self.lineedit)

        layout = QVBoxLayout()
        layout.addLayout(hbox1)
        layout.addWidget(self.response_label)
        layout.addWidget(self.image_label)
        self.setLayout(layout)
        self.setWindowTitle("RHEA@Subaru Injection")
        self.stimer = QTimer()
        self.ask_for_image()

    def ask_for_image(self):
        command = "image {0:d}".format(self.current_image)
        if (self.client_socket.connected):
            response = self.client_socket.send_command(command)
            if (response.split(" ", 1)[0]=="image"):
                self.display_image(response)
            else:
                self.response_label.setText(response)
        self.stimer.singleShot(100, self.ask_for_image)

    def display_image(self, response):
        argv = response.split(" ",3)
        current_image=int(argv[1])
        compressed_len = int(argv[2])
        if compressed_len == 0:
            return
        if (compressed_len > len(argv[3])):
            print("ERROR: Only got {0:d} of {1:d} bytes".format(len(argv[3]), compressed_len))
        else:
            imdata = zlib.decompress(argv[3][0:compressed_len])
            if (len(imdata)!=self.IMAGE_WIDTH*self.IMAGE_HEIGHT*4):
                print("ERROR: Image size {0:d} instead of {1:d}".format(len(imdata), self.IMAGE_WIDTH*self.IMAGE_HEIGHT*4))
            else:
                imdata = np.fromstring(imdata, np.float32)
                imdata = np.array(imdata).reshape( (self.IMAGE_HEIGHT,self.IMAGE_WIDTH) )
                #Autoscale for now...
                imdata = ( (imdata - np.min(imdata))/np.maximum(np.max(imdata)-np.min(imdata),1)*255).astype(np.uint32)
                imdata = imdata + (imdata<<8) + (imdata<<16)
                image = QImage(imdata.tostring(),self.IMAGE_WIDTH,self.IMAGE_HEIGHT,QImage.Format_RGB32)
                self.image_label.setPixmap(QPixmap.fromImage(image))

    def send_to_server(self):
        if (self.client_socket.connected):
            response = self.client_socket.send_command(str(self.lineedit.text()))
            if (response.split(" ", 1)[0]=="image"):
                self.display_image(response)
            else:
                self.response_label.setText(response)
        else:
            self.response_label.setText("*** Not Connected ***")

        self.lineedit.setText("")

app = QApplication(sys.argv)
myapp = RHEAGui()
myapp.show()
sys.exit(app.exec_())

<|endoftext|>


<commit_before>import datetime, json, os
from cantools import config
from cantools.util import log, write
from cantools.web import fetch, send_mail
from actor import Actor
try:
	import psutil
except ImportError, e:
	pass # google crap engine (get it if you need it!)

class BotMeta(type):
	def __new__(cls, name, bases, attrs):
		bc = type.__new__(cls, name, bases, attrs)
		if name is not "Bot":
			name is not "Monitor" and log("Initializing Bot Class: %s"%(name,), important=True)
			config.pubsub.bots.update(name.lower(), bc)
		return bc

class Bot(Actor):
	num = 0
	__metaclass__ = BotMeta
	def __init__(self, server, channel, name=None):
		Bot.num += 1
		self.name = name or (self.__class__.__name__ + str(Bot.num))
		self.server = server
		self.channel = channel # often we only care about one channel
		self.channels = set()
		self._set_defaults()
		log("Bot Spawned: '%s'"%(self.name,), 2)
		channel.join(self)
		self.server.bots[self.name] = self

	def pub(self, message):
		self.server.publish({
			"message": message,
			"channel": self.channel.name
		}, self)

	def write(self, obj): # receive message from channel
		getattr(self, "on_%s"%(obj["action"],))(obj["data"])

	def _default_handler(self, action):
		def _h(*args):
			log('Bot %s handling %s: "%s"'%(self.name, action, json.dumps(args)), 3)
		return _h

	def _set_defaults(self):
		for action in ["channel", "publish", "subscribe", "unsubscribe", "pm", "error"]:
			hname = "on_%s"%(action,)
			if not hasattr(self, hname):
				setattr(self, hname, self._default_handler(action))

class Monitor(Bot):
	def __init__(self, server, channel, name="monitor"): # only one monitor
		import event # here for google crap engine
		self.current = {}
		self.alert = {}
		Bot.__init__(self, server, channel, name)
		event.timeout(config.admin.monitor.interval, self._tick)

	def _datedir(self):
		n = datetime.datetime.now()
		lp = os.path.join("logs", "monitor")
		yp = os.path.join(lp, str(n.year))
		mp = os.path.join(yp, str(n.month))
		dp = os.path.join(mp, str(n.day))
		if not os.path.isdir(lp):
			os.mkdir(lp)
		if not os.path.isdir(yp):
			os.mkdir(yp)
		if not os.path.isdir(mp):
			os.mkdir(mp)
		if not os.path.isdir(dp):
			os.mkdir(dp)
		return os.path.join(dp, str(n.hour))

	def log(self, data):
		self.pub(data)
		if config.admin.monitor.log:
			write(data, self._datedir(), True, append=True, newline=True)

	def _cpu(self):
		c = self.current["cpu"] = psutil.cpu_percent()
		if self.alert.get("cpu"):
			if c < config.admin.monitor.thresholds.cpu:
				del self.alert["cpu"]
				log("CPU calmed down")
				send_mail(config.admin.contacts, subject="High CPU", body="just ended")
		else:
			if c >= config.admin.monitor.thresholds.cpu:
				self.alert["cpu"] = True
				log("CPU just started going crazy")
				send_mail(config.admin.contacts, subject="High CPU", body="just started")

	def _tick(self):
		self._cpu()
		dioc = psutil.disk_io_counters()
		nioc = psutil.net_io_counters()
		dmon = fetch(config.admin.host, "/_report",
			config.admin.port, True, protocol=config.admin.protocol)
		self.log({
			"cpu": self.current["cpu"],
			"read": dioc.read_time,
			"write": dioc.write_time,
			"sent": nioc.bytes_sent,
			"recv": nioc.bytes_recv,
			"virtual_memory": psutil.virtual_memory().percent,
			"swap_memory": psutil.swap_memory().percent,
			"connections": len(psutil.net_connections()),
			"web_connections": dmon["web"]["connections"],
			"admin_connections": dmon["admin"]["connections"],
			"web_requests": dmon["web"]["requests"],
			"admin_requests": dmon["admin"]["requests"],
			"totals": {
				"web": {
					"connections": dmon["web"]["total_connections"],
					"requests": dmon["web"]["total_requests"],
					"rolls": dmon["web"]["rolls"]
				},
				"admin": {
					"connections": dmon["admin"]["total_connections"],
					"requests": dmon["admin"]["total_requests"],
					"rolls": dmon["admin"]["rolls"]
				}
			},
			"devices": {
				"web": dmon["web"]["devices"],
				"admin": dmon["admin"]["devices"]
			}
		})
		return True<commit_msg>admin Monitor bot: add ips (web, admin, and, if configured, proxy) to report<commit_after>import datetime, json, os
from cantools import config
from cantools.util import log, write
from cantools.web import fetch, send_mail
from actor import Actor
try:
	import psutil
except ImportError, e:
	pass # google crap engine (get it if you need it!)

class BotMeta(type):
	def __new__(cls, name, bases, attrs):
		bc = type.__new__(cls, name, bases, attrs)
		if name is not "Bot":
			name is not "Monitor" and log("Initializing Bot Class: %s"%(name,), important=True)
			config.pubsub.bots.update(name.lower(), bc)
		return bc

class Bot(Actor):
	num = 0
	__metaclass__ = BotMeta
	def __init__(self, server, channel, name=None):
		Bot.num += 1
		self.name = name or (self.__class__.__name__ + str(Bot.num))
		self.server = server
		self.channel = channel # often we only care about one channel
		self.channels = set()
		self._set_defaults()
		log("Bot Spawned: '%s'"%(self.name,), 2)
		channel.join(self)
		self.server.bots[self.name] = self

	def pub(self, message):
		self.server.publish({
			"message": message,
			"channel": self.channel.name
		}, self)

	def write(self, obj): # receive message from channel
		getattr(self, "on_%s"%(obj["action"],))(obj["data"])

	def _default_handler(self, action):
		def _h(*args):
			log('Bot %s handling %s: "%s"'%(self.name, action, json.dumps(args)), 3)
		return _h

	def _set_defaults(self):
		for action in ["channel", "publish", "subscribe", "unsubscribe", "pm", "error"]:
			hname = "on_%s"%(action,)
			if not hasattr(self, hname):
				setattr(self, hname, self._default_handler(action))

class Monitor(Bot):
	def __init__(self, server, channel, name="monitor"): # only one monitor
		import event # here for google crap engine
		self.current = {}
		self.alert = {}
		Bot.__init__(self, server, channel, name)
		event.timeout(config.admin.monitor.interval, self._tick)

	def _datedir(self):
		n = datetime.datetime.now()
		lp = os.path.join("logs", "monitor")
		yp = os.path.join(lp, str(n.year))
		mp = os.path.join(yp, str(n.month))
		dp = os.path.join(mp, str(n.day))
		if not os.path.isdir(lp):
			os.mkdir(lp)
		if not os.path.isdir(yp):
			os.mkdir(yp)
		if not os.path.isdir(mp):
			os.mkdir(mp)
		if not os.path.isdir(dp):
			os.mkdir(dp)
		return os.path.join(dp, str(n.hour))

	def log(self, data):
		self.pub(data)
		if config.admin.monitor.log:
			write(data, self._datedir(), True, append=True, newline=True)

	def _cpu(self):
		c = self.current["cpu"] = psutil.cpu_percent()
		if self.alert.get("cpu"):
			if c < config.admin.monitor.thresholds.cpu:
				del self.alert["cpu"]
				log("CPU calmed down")
				send_mail(config.admin.contacts, subject="High CPU", body="just ended")
		else:
			if c >= config.admin.monitor.thresholds.cpu:
				self.alert["cpu"] = True
				log("CPU just started going crazy")
				send_mail(config.admin.contacts, subject="High CPU", body="just started")

	def _tick(self):
		self._cpu()
		dioc = psutil.disk_io_counters()
		nioc = psutil.net_io_counters()
		dmon = fetch(config.admin.host, "/_report",
			config.admin.port, True, protocol=config.admin.protocol)
		data = {
			"cpu": self.current["cpu"],
			"read": dioc.read_time,
			"write": dioc.write_time,
			"sent": nioc.bytes_sent,
			"recv": nioc.bytes_recv,
			"virtual_memory": psutil.virtual_memory().percent,
			"swap_memory": psutil.swap_memory().percent,
			"connections": len(psutil.net_connections()),
			"web_connections": dmon["web"]["connections"],
			"admin_connections": dmon["admin"]["connections"],
			"web_requests": dmon["web"]["requests"],
			"admin_requests": dmon["admin"]["requests"],
			"totals": {
				"web": {
					"connections": dmon["web"]["total_connections"],
					"requests": dmon["web"]["total_requests"],
					"rolls": dmon["web"]["rolls"]
				},
				"admin": {
					"connections": dmon["admin"]["total_connections"],
					"requests": dmon["admin"]["total_requests"],
					"rolls": dmon["admin"]["rolls"]
				}
			},
			"devices": {
				"web": dmon["web"]["devices"],
				"admin": dmon["admin"]["devices"]
			},
			"ips": {
				"web": dmon["web"]["ips"],
				"admin": dmon["admin"]["ips"]
			}
		}
		if config.admin.monitor.proxy:
			data["ips"]["proxy"] = fetch(config.admin.host, "/_report",
				config.admin.monitor.proxy, True)["ips"]
		self.log(data)
		return True<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Carl is free software; you can redistribute it and/or modify it
# under the terms of the Revised BSD License; see LICENSE file for
# more details.

import numpy as np

from itertools import product
from sklearn.utils import check_random_state
from sklearn.utils import check_array
from scipy.interpolate import interp1d
from scipy.signal import medfilt

from .base import DistributionMixin

class Histogram(DistributionMixin):
    def __init__(self, bins=10, range=None, smoothing=None, interpolation=None,
                 random_state=None):
        super(Histogram, self).__init__(random_state=random_state)

        self.bins = bins
        self.range = range
        self.smoothing = smoothing
        self.interpolation = interpolation

    def pdf(self, X, **kwargs):
        X = check_array(X)

        if self.ndim_ == 1 and self.interpolation:
            return self.interpolation_(X[:, 0])

        all_indices = []

        for j in range(X.shape[1]):
            indices = np.searchsorted(self.edges_[j],
                                      X[:, j],
                                      side="right") - 1

            # For the last bin, the upper is inclusive
            indices[X[:, j] == self.edges_[j][-2]] -= 1
            all_indices.append(indices)

        return self.histogram_[all_indices]

    def nnlf(self, X, **kwargs):
        return -np.log(self.pdf(X, **kwargs))

    def rvs(self, n_samples, **kwargs):
        rng = check_random_state(self.random_state)

        # Draw random bins with respect to their densities
        h = (self.histogram_ / self.histogram_.sum()).ravel()
        flat_indices = np.searchsorted(np.cumsum(h),
                                       rng.rand(n_samples),
                                       side="right")

        # Build bin corners
        indices = np.unravel_index(flat_indices, self.histogram_.shape)
        indices_end = [a + 1 for a in indices]
        shape = [len(d) for d in self.edges_] + [len(self.edges_)]
        corners = np.array(list(product(*self.edges_))).reshape(shape)

        # Draw uniformly within bins
        low = corners[indices]
        high = corners[indices_end]
        u = rng.rand(*low.shape)

        return low + u * (high - low)

    def fit(self, X, y=None, sample_weight=None, **kwargs):
        # Checks
        X = check_array(X)

        if sample_weight is not None and len(sample_weight) != len(X):
            raise ValueError

        # Compute histogram and edges
        h, e = np.histogramdd(X, bins=self.bins, range=self.range,
                              weights=sample_weight, normed=True)

        # Add empty bins for out of bound samples
        for j in range(X.shape[1]):
            h = np.insert(h, 0, 0., axis=j)
            h = np.insert(h, h.shape[j], 0., axis=j)
            e[j] = np.insert(e[j], 0, -np.inf)
            e[j] = np.insert(e[j], len(e[j]), np.inf)

        if X.shape[1] == 1:
            if self.smoothing == "353qh":
                h[1:-1] = _353qh(h[1:-1])
            elif self.smoothing == "353qh_twice":
                h[1:-1] = _353qh_twice(h[1:-1])

            if self.interpolation:
                inputs = e[0][2:-1] - (e[0][2] - e[0][1]) / 2.
                inputs[0] = e[0][1]
                inputs[-1] = e[0][-2]
                outputs = h[1:-1]
                self.interpolation_ = interp1d(inputs, outputs,
                                               kind=self.interpolation,
                                               bounds_error=False,
                                               fill_value=0.)

        self.histogram_ = h
        self.edges_ = e
        self.ndim_ = X.shape[1]

        return self

    def score(self, X, y=None, **kwargs):
        raise NotImplementedError

    @property
    def ndim(self):
        return self.ndim_

def _median3(y):
    z = y.copy()
    z[0] = np.median([y[0], y[1], 3*y[1] - 2*y[2]])
    z[-1] = np.median([y[-1], y[-2], 3*y[-2] - 2*y[-3]])
    z[1:-1] = medfilt(y, 3)[1:-1]
    return z

def _median5(y):
    z = y.copy()
    z[1] = np.median(y[0:3])
    z[-2] = np.median(y[-3:])
    z[2:-2] = medfilt(y, 5)[2:-2]
    return z

def _quadratic(y):
    z = y.copy()

    for i in range(2, len(y) - 2):
        if y[i - 1] != y[i]:
            continue
        if y[i] != y[i + 1]:
            continue
        h0 = y[i - 2] - y[i]
        h1 = y[i + 2] - y[i]
        if h0 * h1 <= 0:
            continue
        j = 1
        if abs(h1) > abs(h0):
            j -= 1
        z[i] = -0.5 * y[i - 2*j] + y[i] / 0.75 + y[i + 2*j] / 6
        z[i+j] = 0.5 * (y[i + 2*j] - y[i - 2*j]) + y[i]

    return z

def _353qh(y):
    # 353 step
    z = _median3(y)
    z = _median5(z)
    z[1:-1] = _median3(z)[1:-1]

    # quadratic step
    z = _quadratic(z)

    # hanning step
    z[1:-1] = 0.25 * z[:-2] + 0.5 * z[1:-1] + 0.25 * z[2:]

    return z

def _353qh_twice(y):
    z = _353qh(y)
    r = _353qh(y - z)
    return z + r
<commit_msg>Remove (ineffective) smoothing option<commit_after># -*- coding: utf-8 -*-
#
# Carl is free software; you can redistribute it and/or modify it
# under the terms of the Revised BSD License; see LICENSE file for
# more details.

import numpy as np

from itertools import product
from sklearn.utils import check_random_state
from sklearn.utils import check_array
from scipy.interpolate import interp1d
from scipy.signal import medfilt

from .base import DistributionMixin

class Histogram(DistributionMixin):
    def __init__(self, bins=10, range=None, interpolation=None,
                 random_state=None):
        super(Histogram, self).__init__(random_state=random_state)

        self.bins = bins
        self.range = range
        self.interpolation = interpolation

    def pdf(self, X, **kwargs):
        X = check_array(X)

        if self.ndim_ == 1 and self.interpolation:
            return self.interpolation_(X[:, 0])

        all_indices = []

        for j in range(X.shape[1]):
            indices = np.searchsorted(self.edges_[j],
                                      X[:, j],
                                      side="right") - 1

            # For the last bin, the upper is inclusive
            indices[X[:, j] == self.edges_[j][-2]] -= 1
            all_indices.append(indices)

        return self.histogram_[all_indices]

    def nnlf(self, X, **kwargs):
        return -np.log(self.pdf(X, **kwargs))

    def rvs(self, n_samples, **kwargs):
        rng = check_random_state(self.random_state)

        # Draw random bins with respect to their densities
        h = (self.histogram_ / self.histogram_.sum()).ravel()
        flat_indices = np.searchsorted(np.cumsum(h),
                                       rng.rand(n_samples),
                                       side="right")

        # Build bin corners
        indices = np.unravel_index(flat_indices, self.histogram_.shape)
        indices_end = [a + 1 for a in indices]
        shape = [len(d) for d in self.edges_] + [len(self.edges_)]
        corners = np.array(list(product(*self.edges_))).reshape(shape)

        # Draw uniformly within bins
        low = corners[indices]
        high = corners[indices_end]
        u = rng.rand(*low.shape)

        return low + u * (high - low)

    def fit(self, X, y=None, sample_weight=None, **kwargs):
        # Checks
        X = check_array(X)

        if sample_weight is not None and len(sample_weight) != len(X):
            raise ValueError

        # Compute histogram and edges
        h, e = np.histogramdd(X, bins=self.bins, range=self.range,
                              weights=sample_weight, normed=True)

        # Add empty bins for out of bound samples
        for j in range(X.shape[1]):
            h = np.insert(h, 0, 0., axis=j)
            h = np.insert(h, h.shape[j], 0., axis=j)
            e[j] = np.insert(e[j], 0, -np.inf)
            e[j] = np.insert(e[j], len(e[j]), np.inf)

        if X.shape[1] == 1 and self.interpolation:
            inputs = e[0][2:-1] - (e[0][2] - e[0][1]) / 2.
            inputs[0] = e[0][1]
            inputs[-1] = e[0][-2]
            outputs = h[1:-1]
            self.interpolation_ = interp1d(inputs, outputs,
                                           kind=self.interpolation,
                                           bounds_error=False,
                                           fill_value=0.)

        self.histogram_ = h
        self.edges_ = e
        self.ndim_ = X.shape[1]

        return self

    def score(self, X, y=None, **kwargs):
        raise NotImplementedError

    @property
    def ndim(self):
        return self.ndim_
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from setuptools import setup

import sys

PY_VERSION = sys.version_info[:2]

tests_require = ['mock']

if PY_VERSION == (2, 6):
    tests_require.append('unittest2')

setup(name='ev_transpose',
      version='0.0a1',
      description='Transpose time-series gathered over csv-files.',
      author='Thomas Bach',
      tests_require=tests_require,
      test_suite='ev_transpose.tests',
      entry_points={
        'console_scripts':
            ['ev_tp_mapper=ev_transpose.mapper:mapper',
             'ev_tp_reducer=ev_transpose.reducer:reducer']
        })
<commit_msg>application doesn't fit in map/reduce framework<commit_after># -*- coding: utf-8 -*-

from setuptools import setup

import sys

PY_VERSION = sys.version_info[:2]

tests_require = ['mock']

if PY_VERSION == (2, 6):
    tests_require.append('unittest2')

setup(name='ev_transpose',
      version='0.0a1',
      description='Transpose time-series gathered over csv-files.',
      author='Thomas Bach',
      tests_require=tests_require,
      test_suite='ev_transpose.tests',
      entry_points={
        'console_scripts':
            ['ev_transpose=ev_transpose.script:main']})
<|endoftext|>


<commit_before>import ndb_models
import endpoints
from protorpc import messages
from protorpc import message_types
from protorpc import remote

from google.appengine.api import users
from google.appengine.ext import ndb

class GenericResponse(messages.Message):
  message = messages.StringField(1)

class StaffPosition(messages.Message):
  key = messages.IntegerField(1)
  position_name = messages.StringField(2)
  hourly_rate = messages.FloatField(3)

class User(messages.Message):
  name = messages.StringField(1)
  email = messages.StringField(2)

class Program(messages.Message):
  year = messages.IntegerField(1)
  name = messages.StringField(2)
  site_number_prefix = messages.StringField(3)
  status = messages.StringField(4)

class Programs(messages.Message):
  program = messages.MessageField(Program, 1, repeated=True)

def _authorize_staff():
  """Simply call this to ensure that the user has a Staff record.

  Raises:
  endpoints.UnauthorizedException if the user is not Staff.
  """
  current_user = endpoints.get_current_user()
  if current_user is None:
    raise endpoints.UnauthorizedException('Invalid token.')
  staff = ndb_models.Staff.query(
    ndb_models.Staff.email == current_user.email().lower()).get()
  if not staff:
    raise endpoints.UnauthorizedException(
      'Must be staff to use this API.')

@endpoints.api(name='roomApi',version='v1',
               auth_level=endpoints.AUTH_LEVEL.REQUIRED,
               allowed_client_ids=[endpoints.API_EXPLORER_CLIENT_ID,],
               description='Rebuilding Together Peninsula ROOM System API')
class RoomApi(remote.Service):

  # This needs an update for the new encoding for StaffPosition rates.  Per issue 238.
  # If it's used at all...
  @endpoints.method(StaffPosition,
                    GenericResponse,
                    name='staffposition.put')
  def staffposition_put(self, request):
    _authorize_staff()
    sp = ndb_models.StaffPosition(position_name=request.position_name,
                                  hourly_rate=request.hourly_rate)
    if request.key:
      sp.key = ndb.Key(ndb_models.StaffPosition, request.key)
    sp.put()
    return GenericResponse()

  @endpoints.method(Program,
                    GenericResponse,
                    name='program.put')
  def program_put(self, request):
    _authorize_staff()
    sp = ndb_models.Program(name=request.name,
                            year=request.year,
                            site_number_prefix=request.site_number_prefix,
                            status=request.status)
    sp.put()
    return GenericResponse()

  @endpoints.method(message_types.VoidMessage,
                    Programs,
                    http_method='GET',
                    name='program.list')
  def program_list(self, request):
    programs = Programs()
    for p in ndb_models.Program.query():
      programs.program.append(Program(name=p.name, year=p.year))
    return programs

application = endpoints.api_server([RoomApi])
<commit_msg>skip staff auth in dev<commit_after>import ndb_models
import endpoints
from protorpc import messages
from protorpc import message_types
from protorpc import remote

from google.appengine.api import users
from google.appengine.ext import ndb

class GenericResponse(messages.Message):
  message = messages.StringField(1)

class StaffPosition(messages.Message):
  key = messages.IntegerField(1)
  position_name = messages.StringField(2)
  hourly_rate = messages.FloatField(3)

class User(messages.Message):
  name = messages.StringField(1)
  email = messages.StringField(2)

class Program(messages.Message):
  year = messages.IntegerField(1)
  name = messages.StringField(2)
  site_number_prefix = messages.StringField(3)
  status = messages.StringField(4)

class Programs(messages.Message):
  program = messages.MessageField(Program, 1, repeated=True)

def _authorize_staff():
  """Simply call this to ensure that the user has a Staff record.

  Raises:
    endpoints.UnauthorizedException if the user is not Staff.
  """
  if os.environ['SERVER_SOFTWARE'].startswith('Development'):
    return
  current_user = endpoints.get_current_user()
  if current_user is None:
    raise endpoints.UnauthorizedException('Invalid token.')
  staff = ndb_models.Staff.query(
    ndb_models.Staff.email == current_user.email().lower()).get()
  if not staff:
    raise endpoints.UnauthorizedException(
      'Must be staff to use this API.')

@endpoints.api(name='roomApi',version='v1',
               auth_level=endpoints.AUTH_LEVEL.REQUIRED,
               allowed_client_ids=[endpoints.API_EXPLORER_CLIENT_ID,],
               description='Rebuilding Together Peninsula ROOM System API')
class RoomApi(remote.Service):

  # This needs an update for the new encoding for StaffPosition rates.  Per issue 238.
  # If it's used at all...
  @endpoints.method(StaffPosition,
                    GenericResponse,
                    name='staffposition.put')
  def staffposition_put(self, request):
    _authorize_staff()
    sp = ndb_models.StaffPosition(position_name=request.position_name,
                                  hourly_rate=request.hourly_rate)
    if request.key:
      sp.key = ndb.Key(ndb_models.StaffPosition, request.key)
    sp.put()
    return GenericResponse()

  @endpoints.method(Program,
                    GenericResponse,
                    name='program.put')
  def program_put(self, request):
    _authorize_staff()
    sp = ndb_models.Program(name=request.name,
                            year=request.year,
                            site_number_prefix=request.site_number_prefix,
                            status=request.status)
    sp.put()
    return GenericResponse()

  @endpoints.method(message_types.VoidMessage,
                    Programs,
                    http_method='GET',
                    name='program.list')
  def program_list(self, request):
    programs = Programs()
    for p in ndb_models.Program.query():
      programs.program.append(Program(name=p.name, year=p.year))
    return programs

application = endpoints.api_server([RoomApi])
<|endoftext|>