<commit_before>VERSION = (0, 0, 1, 'alpha', 0)

# Most of this code is taken form from the Django project

import os.path, re, colors

def get_svn_revision(path=None):
    """
    Returns the SVN revision in the form SVN-XXXX,
    where XXXX is the revision number.

    Returns SVN-unknown if anything goes wrong, such as an unexpected
    format of internal SVN files.

    If path is provided, it should be a directory whose SVN info you want to
    inspect. If it's not provided, this will use the root colors/ package
    directory.
    """
    rev = None
    if path is None:
        path = colors.__path__[0]
    entries_path = '%s/.svn/entries' % path

    try:
        entries = open(entries_path, 'r').read()
    except IOError:
        pass
    else:
        # Versions >= 7 of the entries file are flat text.  The first line is
        # the version number. The next set of digits after 'dir' is the revision.
        if re.match('(\d+)', entries):
            rev_match = re.search('\d+\s+dir\s+(\d+)', entries)
            if rev_match:
                rev = rev_match.groups()[0]
        # Older XML versions of the file specify revision as an attribute of
        # the first entries node.
        else:
            from xml.dom import minidom
            dom = minidom.parse(entries_path)
            rev = dom.getElementsByTagName('entry')[0].getAttribute('revision')

    if rev:
        return u'SVN-%s' % rev
    return u'SVN-unknown'

def get_version():
    version = '%s.%s' % (VERSION[0], VERSION[1])
    if VERSION[2]:
        version = '%s.%s' % (version, VERSION[2])
    if VERSION[3:] == ('alpha', 0):
        version = '%s pre-alpha' % version
    else:
        if VERSION[3] != 'final':
            version = '%s %s %s' % (version, VERSION[3], VERSION[4])
    svn_rev = get_svn_revision()
    if svn_rev != u'SVN-unknown':
        version = "%s %s" % (version, svn_rev)
    return version
<commit_msg>0.0.2 pre-alpha SVN-19<commit_after>VERSION = (0, 0, 2, 'alpha', 0)

# Most of this code is taken form from the Django project

import os.path, re, colors

def get_svn_revision(path=None):
    """
    Returns the SVN revision in the form SVN-XXXX,
    where XXXX is the revision number.

    Returns SVN-unknown if anything goes wrong, such as an unexpected
    format of internal SVN files.

    If path is provided, it should be a directory whose SVN info you want to
    inspect. If it's not provided, this will use the root colors/ package
    directory.
    """
    rev = None
    if path is None:
        path = colors.__path__[0]
    entries_path = '%s/.svn/entries' % path

    try:
        entries = open(entries_path, 'r').read()
    except IOError:
        pass
    else:
        # Versions >= 7 of the entries file are flat text.  The first line is
        # the version number. The next set of digits after 'dir' is the revision.
        if re.match('(\d+)', entries):
            rev_match = re.search('\d+\s+dir\s+(\d+)', entries)
            if rev_match:
                rev = rev_match.groups()[0]
        # Older XML versions of the file specify revision as an attribute of
        # the first entries node.
        else:
            from xml.dom import minidom
            dom = minidom.parse(entries_path)
            rev = dom.getElementsByTagName('entry')[0].getAttribute('revision')

    if rev:
        return u'SVN-%s' % rev
    return u'SVN-unknown'

def get_version():
    version = '%s.%s' % (VERSION[0], VERSION[1])
    if VERSION[2]:
        version = '%s.%s' % (version, VERSION[2])
    if VERSION[3:] == ('alpha', 0):
        version = '%s pre-alpha' % version
    else:
        if VERSION[3] != 'final':
            version = '%s %s %s' % (version, VERSION[3], VERSION[4])
    svn_rev = get_svn_revision()
    if svn_rev != u'SVN-unknown':
        version = "%s %s" % (version, svn_rev)
    return version
<|endoftext|>


<commit_before># This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Adds Beatport release and track search support to the autotagger
"""
import logging
import re
from datetime import datetime, timedelta

import requests

from beets.autotag.hooks import AlbumInfo, TrackInfo
from beets.autotag.match import Distance
from beets.plugins import BeetsPlugin

log = logging.getLogger('beets')

class BeatportAPIError(Exception):
    pass

class BeatportObject(object):
    def __init__(self, data):
        self.beatport_id = data['id']
        self.name = unicode(data['name'])
        if 'releaseDate' in data:
            self.release_date = datetime.strptime(data['releaseDate'],
                                                  '%Y-%m-%d')
        if 'artists' in data:
            self.artists = [(x['id'], unicode(x['name']))
                            for x in data['artists']]
        if 'genres' in data:
            self.genres = [unicode(x['name'])
                           for x in data['genres']]

class BeatportAPI(object):
    API_BASE = 'http://api.beatport.com/'

    @classmethod
    def get(cls, endpoint, **kwargs):
        try:
            response = requests.get(cls.API_BASE + endpoint, params=kwargs)
        except Exception as e:
            raise BeatportAPIError("Error connection to Beatport API: {}"
                                   .format(e.message))
        if not response:
            raise BeatportAPIError(
                "Error {0.status_code} for '{0.request.path_url}"
                .format(response))
        return response.json()['results']

class BeatportSearch(object):
    query = None
    release_type = None

    def __unicode__(self):
        return u'<BeatportSearch for {0} "{1}" with {2} results>'.format(
            self.release_type, self.query, len(self.results))

    def __init__(self, query, release_type='release', details=True):
        self.results = []
        self.query = query
        self.release_type = release_type
        response = BeatportAPI.get('catalog/3/search', query=query,
                                   facets=['fieldType:{0}'
                                           .format(release_type)],
                                   perPage=5)
        for item in response:
            if release_type == 'release':
                release = BeatportRelease(item)
                if details:
                    release.get_tracks()
                self.results.append(release)
            elif release_type == 'track':
                self.results.append(BeatportTrack(item))

class BeatportRelease(BeatportObject):
    API_ENDPOINT = 'catalog/3/beatport/release'

    def __unicode__(self):
        if len(self.artists) < 4:
            artist_str = ", ".join(x[1] for x in self.artists)
        else:
            artist_str = "Various Artists"
        return u"<BeatportRelease: {0} - {1} ({2})>".format(artist_str,
                                                            self.name,
                                                            self.catalog_number)

    def __init__(self, data):
        BeatportObject.__init__(self, data)
        if 'catalogNumber' in data:
            self.catalog_number = data['catalogNumber']
        if 'label' in data:
            self.label_name = data['label']['name']
        if 'category' in data:
            self.category = data['category']
        if 'slug' in data:
            self.url = "http://beatport.com/release/{0}/{1}".format(
                data['slug'], data['id'])

    @classmethod
    def from_id(cls, beatport_id):
        response = BeatportAPI.get(cls.API_ENDPOINT, id=beatport_id)
        release = BeatportRelease(response['release'])
        release.tracks = [BeatportTrack(x) for x in response['tracks']]
        return release

    def get_tracks(self):
        response = BeatportAPI.get(self.API_ENDPOINT, id=self.beatport_id)
        self.tracks = [BeatportTrack(x) for x in response['tracks']]

class BeatportTrack(BeatportObject):
    API_ENDPOINT = 'catalog/3/beatport/track'

    def __unicode__(self):
        artist_str = ", ".join(x[1] for x in self.artists)
        return u"<BeatportTrack: {0} - {1} ({2})>".format(artist_str, self.name,
                                                          self.mix_name)

    def __init__(self, data):
        BeatportObject.__init__(self, data)
        if 'title' in data:
            self.title = unicode(data['title'])
        if 'mixName' in data:
            self.mix_name = unicode(data['mixName'])
        if 'length' in data:
            self.length = timedelta(milliseconds=data['lengthMs'])
        if 'slug' in data:
            self.url = "http://beatport.com/track/{0}/{1}".format(data['slug'],
                                                                  data['id'])

    @classmethod
    def from_id(cls, beatport_id):
        response = BeatportAPI.get(cls.API_ENDPOINT, id=beatport_id)
        return BeatportTrack(response['track'])

class BeatportPlugin(BeetsPlugin):
    def __init__(self):
        super(BeatportPlugin, self).__init__()
        self.config.add({
            'source_weight': 0.5,
        })

    def album_distance(self, items, album_info, mapping):
        """Returns the beatport source weight and the maximum source weight
        for albums.
        """
        dist = Distance()
        if album_info.data_source == 'Beatport':
            dist.add('source', self.config['source_weight'].as_number())
        return dist

    def track_distance(self, item, info):
        """Returns the beatport source weight and the maximum source weight
        for individual tracks.
        """
        return Distance()  # FIXME: Need source information for tracks.

    def candidates(self, items, artist, release, va_likely):
        """Returns a list of AlbumInfo objects for beatport search results
        matching release and artist (if not various).
        """
        if va_likely:
            query = release
        else:
            query = '%s %s' % (artist, release)
        try:
            return self._get_releases(query)
        except BeatportAPIError as e:
            log.debug('Beatport API Error: %s (query: %s)' % (e, query))
            return []

    def item_candidates(self, item, artist, title):
        """Returns a list of TrackInfo objects for beatport search results
        matching title and artist.
        """
        query = '%s %s' % (artist, title)
        try:
            return self._get_tracks(query)
        except BeatportAPIError as e:
            log.debug('Beatport API Error: %s (query: %s)' % (e, query))
            return []

    def album_for_id(self, release_id):
        """Fetches a release by its Beatport ID and returns an AlbumInfo object
        or None if the release is not found.
        """
        log.debug('Searching Beatport for release %s' % str(release_id))
        match = re.search(r'(^|beatport\.com/release/.+/)(\d+)$', release_id)
        if not match:
            return None
        release = BeatportRelease.from_id(match.group(2))
        album = self._get_album_info(release)
        return album

    def track_for_id(self, track_id):
        """Fetches a track by its Beatport ID and returns a TrackInfo object
        or None if the track is not found.
        """
        log.debug('Searching Beatport for track %s' % str(track_id))
        match = re.search(r'(^|beatport\.com/track/.+/)(\d+)$', track_id)
        if not match:
            return None
        bp_track = BeatportTrack.from_id(match.group(2))
        track = self._get_track_info(bp_track)
        return track

    def _get_releases(self, query):
        """Returns a list of AlbumInfo objects for a beatport search query.
        """
        # Strip non-word characters from query. Things like "!" and "-" can
        # cause a query to return no results, even if they match the artist or
        # album title. Use `re.UNICODE` flag to avoid stripping non-english
        # word characters.
        query = re.sub(r'\W+', ' ', query, re.UNICODE)
        # Strip medium information from query, Things like "CD1" and "disk 1"
        # can also negate an otherwise positive result.
        query = re.sub(r'\b(CD|disc)\s*\d+', '', query, re.I)
        albums = [self._get_album_info(x)
                  for x in BeatportSearch(query).results]
        return albums

    def _get_album_info(self, release):
        """Returns an AlbumInfo object for a Beatport Release object.
        """
        va = len(release.artists) > 3
        artist, artist_id = self._get_artist(release.artists)
        if va:
            artist = u"Various Artists"
        tracks = [self._get_track_info(x, index=idx)
                  for idx, x in enumerate(release.tracks, 1)]

        return AlbumInfo(album=release.name, album_id=release.beatport_id,
                         artist=artist, artist_id=artist_id, tracks=tracks,
                         albumtype=release.category, va=va,
                         year=release.release_date.year,
                         month=release.release_date.month,
                         day=release.release_date.day,
                         label=release.label_name,
                         catalognum=release.catalog_number, media=u'Digital',
                         data_source=u'Beatport', data_url=release.url)

    def _get_track_info(self, track, index=None):
        """Returns a TrackInfo object for a Beatport Track object.
        """
        title = track.name
        if track.mix_name != u"Original Mix":
            title += u" ({0})".format(track.mix_name)
        artist, artist_id = self._get_artist(track.artists)
        length = track.length.total_seconds()

        return TrackInfo(title=title, track_id=track.beatport_id,
                         artist=artist, artist_id=artist_id,
                         length=length, index=index)

    def _get_artist(self, artists):
        """Returns an artist string (all artists) and an artist_id (the main
        artist) for a list of Beatport release or track artists.
        """
        artist_id = None
        bits = []
        for artist in artists:
            if not artist_id:
                artist_id = artist[0]
            name = artist[1]
            # Strip disambiguation number.
            name = re.sub(r' \(\d+\)$', '', name)
            # Move articles to the front.
            name = re.sub(r'^(.*?), (a|an|the)$', r'\2 \1', name, flags=re.I)
            bits.append(name)
        artist = ', '.join(bits).replace(' ,', ',') or None
        return artist, artist_id

    def _get_tracks(self, query):
        """Returns a list of TrackInfo objects for a Beatport query.
        """
        bp_tracks = BeatportSearch(query, release_type='track').results
        tracks = [self._get_track_info(x) for x in bp_tracks]
        return tracks
<commit_msg>Fix bug where a null value for lengthMS would crash an import. Added a fallback to parse the standard min:sec length field.<commit_after># This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Adds Beatport release and track search support to the autotagger
"""
import logging
import re
from datetime import datetime, timedelta

import requests

from beets.autotag.hooks import AlbumInfo, TrackInfo
from beets.autotag.match import Distance
from beets.plugins import BeetsPlugin

log = logging.getLogger('beets')

class BeatportAPIError(Exception):
    pass

class BeatportObject(object):
    def __init__(self, data):
        self.beatport_id = data['id']
        self.name = unicode(data['name'])
        if 'releaseDate' in data:
            self.release_date = datetime.strptime(data['releaseDate'],
                                                  '%Y-%m-%d')
        if 'artists' in data:
            self.artists = [(x['id'], unicode(x['name']))
                            for x in data['artists']]
        if 'genres' in data:
            self.genres = [unicode(x['name'])
                           for x in data['genres']]

class BeatportAPI(object):
    API_BASE = 'http://api.beatport.com/'

    @classmethod
    def get(cls, endpoint, **kwargs):
        try:
            response = requests.get(cls.API_BASE + endpoint, params=kwargs)
        except Exception as e:
            raise BeatportAPIError("Error connection to Beatport API: {}"
                                   .format(e.message))
        if not response:
            raise BeatportAPIError(
                "Error {0.status_code} for '{0.request.path_url}"
                .format(response))
        return response.json()['results']

class BeatportSearch(object):
    query = None
    release_type = None

    def __unicode__(self):
        return u'<BeatportSearch for {0} "{1}" with {2} results>'.format(
            self.release_type, self.query, len(self.results))

    def __init__(self, query, release_type='release', details=True):
        self.results = []
        self.query = query
        self.release_type = release_type
        response = BeatportAPI.get('catalog/3/search', query=query,
                                   facets=['fieldType:{0}'
                                           .format(release_type)],
                                   perPage=5)
        for item in response:
            if release_type == 'release':
                release = BeatportRelease(item)
                if details:
                    release.get_tracks()
                self.results.append(release)
            elif release_type == 'track':
                self.results.append(BeatportTrack(item))

class BeatportRelease(BeatportObject):
    API_ENDPOINT = 'catalog/3/beatport/release'

    def __unicode__(self):
        if len(self.artists) < 4:
            artist_str = ", ".join(x[1] for x in self.artists)
        else:
            artist_str = "Various Artists"
        return u"<BeatportRelease: {0} - {1} ({2})>".format(artist_str,
                                                            self.name,
                                                            self.catalog_number)

    def __init__(self, data):
        BeatportObject.__init__(self, data)
        if 'catalogNumber' in data:
            self.catalog_number = data['catalogNumber']
        if 'label' in data:
            self.label_name = data['label']['name']
        if 'category' in data:
            self.category = data['category']
        if 'slug' in data:
            self.url = "http://beatport.com/release/{0}/{1}".format(
                data['slug'], data['id'])

    @classmethod
    def from_id(cls, beatport_id):
        response = BeatportAPI.get(cls.API_ENDPOINT, id=beatport_id)
        release = BeatportRelease(response['release'])
        release.tracks = [BeatportTrack(x) for x in response['tracks']]
        return release

    def get_tracks(self):
        response = BeatportAPI.get(self.API_ENDPOINT, id=self.beatport_id)
        self.tracks = [BeatportTrack(x) for x in response['tracks']]

class BeatportTrack(BeatportObject):
    API_ENDPOINT = 'catalog/3/beatport/track'

    def __unicode__(self):
        artist_str = ", ".join(x[1] for x in self.artists)
        return u"<BeatportTrack: {0} - {1} ({2})>".format(artist_str, self.name,
                                                          self.mix_name)

    def __init__(self, data):
        BeatportObject.__init__(self, data)
        if 'title' in data:
            self.title = unicode(data['title'])
        if 'mixName' in data:
            self.mix_name = unicode(data['mixName'])
        self.length = timedelta(milliseconds=data.get('lengthMs',0) or 0)
        if self.length == 0:
            (min, sec) = data.get('length','0:0').split(':')
            self.length = timedelta(minutes=min, seconds=sec)
        if 'slug' in data:
            self.url = "http://beatport.com/track/{0}/{1}".format(data['slug'],
                                                                  data['id'])

    @classmethod
    def from_id(cls, beatport_id):
        response = BeatportAPI.get(cls.API_ENDPOINT, id=beatport_id)
        return BeatportTrack(response['track'])

class BeatportPlugin(BeetsPlugin):
    def __init__(self):
        super(BeatportPlugin, self).__init__()
        self.config.add({
            'source_weight': 0.5,
        })

    def album_distance(self, items, album_info, mapping):
        """Returns the beatport source weight and the maximum source weight
        for albums.
        """
        dist = Distance()
        if album_info.data_source == 'Beatport':
            dist.add('source', self.config['source_weight'].as_number())
        return dist

    def track_distance(self, item, info):
        """Returns the beatport source weight and the maximum source weight
        for individual tracks.
        """
        return Distance()  # FIXME: Need source information for tracks.

    def candidates(self, items, artist, release, va_likely):
        """Returns a list of AlbumInfo objects for beatport search results
        matching release and artist (if not various).
        """
        if va_likely:
            query = release
        else:
            query = '%s %s' % (artist, release)
        try:
            return self._get_releases(query)
        except BeatportAPIError as e:
            log.debug('Beatport API Error: %s (query: %s)' % (e, query))
            return []

    def item_candidates(self, item, artist, title):
        """Returns a list of TrackInfo objects for beatport search results
        matching title and artist.
        """
        query = '%s %s' % (artist, title)
        try:
            return self._get_tracks(query)
        except BeatportAPIError as e:
            log.debug('Beatport API Error: %s (query: %s)' % (e, query))
            return []

    def album_for_id(self, release_id):
        """Fetches a release by its Beatport ID and returns an AlbumInfo object
        or None if the release is not found.
        """
        log.debug('Searching Beatport for release %s' % str(release_id))
        match = re.search(r'(^|beatport\.com/release/.+/)(\d+)$', release_id)
        if not match:
            return None
        release = BeatportRelease.from_id(match.group(2))
        album = self._get_album_info(release)
        return album

    def track_for_id(self, track_id):
        """Fetches a track by its Beatport ID and returns a TrackInfo object
        or None if the track is not found.
        """
        log.debug('Searching Beatport for track %s' % str(track_id))
        match = re.search(r'(^|beatport\.com/track/.+/)(\d+)$', track_id)
        if not match:
            return None
        bp_track = BeatportTrack.from_id(match.group(2))
        track = self._get_track_info(bp_track)
        return track

    def _get_releases(self, query):
        """Returns a list of AlbumInfo objects for a beatport search query.
        """
        # Strip non-word characters from query. Things like "!" and "-" can
        # cause a query to return no results, even if they match the artist or
        # album title. Use `re.UNICODE` flag to avoid stripping non-english
        # word characters.
        query = re.sub(r'\W+', ' ', query, re.UNICODE)
        # Strip medium information from query, Things like "CD1" and "disk 1"
        # can also negate an otherwise positive result.
        query = re.sub(r'\b(CD|disc)\s*\d+', '', query, re.I)
        albums = [self._get_album_info(x)
                  for x in BeatportSearch(query).results]
        return albums

    def _get_album_info(self, release):
        """Returns an AlbumInfo object for a Beatport Release object.
        """
        va = len(release.artists) > 3
        artist, artist_id = self._get_artist(release.artists)
        if va:
            artist = u"Various Artists"
        tracks = [self._get_track_info(x, index=idx)
                  for idx, x in enumerate(release.tracks, 1)]

        return AlbumInfo(album=release.name, album_id=release.beatport_id,
                         artist=artist, artist_id=artist_id, tracks=tracks,
                         albumtype=release.category, va=va,
                         year=release.release_date.year,
                         month=release.release_date.month,
                         day=release.release_date.day,
                         label=release.label_name,
                         catalognum=release.catalog_number, media=u'Digital',
                         data_source=u'Beatport', data_url=release.url)

    def _get_track_info(self, track, index=None):
        """Returns a TrackInfo object for a Beatport Track object.
        """
        title = track.name
        if track.mix_name != u"Original Mix":
            title += u" ({0})".format(track.mix_name)
        artist, artist_id = self._get_artist(track.artists)
        length = track.length.total_seconds()

        return TrackInfo(title=title, track_id=track.beatport_id,
                         artist=artist, artist_id=artist_id,
                         length=length, index=index)

    def _get_artist(self, artists):
        """Returns an artist string (all artists) and an artist_id (the main
        artist) for a list of Beatport release or track artists.
        """
        artist_id = None
        bits = []
        for artist in artists:
            if not artist_id:
                artist_id = artist[0]
            name = artist[1]
            # Strip disambiguation number.
            name = re.sub(r' \(\d+\)$', '', name)
            # Move articles to the front.
            name = re.sub(r'^(.*?), (a|an|the)$', r'\2 \1', name, flags=re.I)
            bits.append(name)
        artist = ', '.join(bits).replace(' ,', ',') or None
        return artist, artist_id

    def _get_tracks(self, query):
        """Returns a list of TrackInfo objects for a Beatport query.
        """
        bp_tracks = BeatportSearch(query, release_type='track').results
        tracks = [self._get_track_info(x) for x in bp_tracks]
        return tracks
<|endoftext|>


<commit_before>import numpy as np
import gffutils
from collections import defaultdict
import pandas as pd
import sys

def miso_exon_to_gencode_exon(exon):
    """Convert a single miso exon to gencode gffutils database exon id

    >>> # Skipped exon (SE) or Mutually exclusive exon (MXE) ID
    >>> miso_exon_to_gencode_exon('chr2:9624561:9624679:+')
    'exon:chr2:9624561-9624679:+'
    >>> # Alt 5' splice site (pick first of alternative)
    >>> miso_exon_to_gencode_exon('chr15:42565276:42565087|42565161:-')
    'exon:chr15:42565276-42565087:-'
    >>> # Alt 3' splice site (pick first of alternative)
    >>> miso_exon_to_gencode_exon('chr2:130914199|130914248:130914158:-')
    'exon:chr2:130914199-130914158:-'
    >>> # Retained intron: start, stop separated by '-' instead of ':'
    >>> miso_exon_to_gencode_exon('chr1:906259-906386:+')
    'exon:chr1:906259-906386:+'
    """
    return 'exon:{}:{}-{}:{}'.format(*miso_exon_to_coords(exon))

def miso_id_to_exon_ids(miso_id):
    """Convert a MISO-style alternative event ID

    Split on the pipe ("|") to account for Alt 5'/3' splice site events

    # A skipped exon (SE) ID
    >>> miso_id_to_exon_ids('chr2:9624561:9624679:+@chr2:9627585:9627676:+@chr2:9628276:9628591:+')
    ['exon:chr2:9624561-9624679:+', 'exon:chr2:9627585-9627676:+', 'exon:chr2:9628276-9628591:+']
    >>> # A mutually exclusive (MXE) ID
    >>> miso_id_to_exon_ids('chr16:89288500:89288591:+@chr16:89289565:89289691:+@chr16:89291127:89291210:+@chr16:89291963:89292039:+')
    ['exon:chr16:89288500-89288591:+', 'exon:chr16:89289565-89289691:+', 'exon:chr16:89291127-89291210:+', 'exon:chr16:89291963-89292039:+']
    >>> # An Alt 5' splice site (A5SS) ID
    >>> miso_id_to_exon_ids("chr15:42565276:42565087|42565161:-@chr15:42564261:42564321:-")
    ['exon:chr15:42565276-42565087:-', 'exon:chr15:42564261-42564321:-']
    >>> # An Alt 3' splice site (A3SS) ID
    >>> miso_id_to_exon_ids('chr2:130914824:130914969:-@chr2:130914199|130914248:130914158:-')
    ['exon:chr2:130914824-130914969:-', 'exon:chr2:130914199-130914158:-']
    >>> # A retained intron (RI) ID
    >>> miso_id_to_exon_ids('chr1:906066-906138:+@chr1:906259-906386:+')
    ['exon:chr1:906066-906138:+', 'exon:chr1:906259-906386:+']
    """
    return map(miso_exon_to_gencode_exon, miso_id.split('@'))

def miso_exon_to_coords(exon):
    """

    >>> miso_exon_to_coords('chr2:130914824:130914969:-')
    ('chr2', '130914824', '130914969', '-')
    >>> # Alt 5' SS - pick the first of the alternative ends
    >>> miso_exon_to_coords('chr15:42565276:42565087|42565161:-')
    ('chr15', '42565276', '42565087', '-')
    >>> # Alt 3' SS - pick the first of the alternative starts
    >>> miso_exon_to_coords('chr2:130914199|130914248:130914158:-')
    ('chr2', '130914199', '130914158', '-')
    >>> # Retained intron
    >>> miso_exon_to_coords('chr1:906066-906138:+')
    ('chr1', '906066', '906138', '+')
    """
    strand = exon[-1]
    coords = map(lambda x: x.split('|')[0],
                 exon.split(':'))
    if '-' in coords[1]:
        start, stop = coords[1].split('-')
        coords = coords[0], start, stop, strand
    return coords[0], coords[1], coords[2], strand

def convert_miso_ids_to_everything(miso_ids, db,
                                   event_type,
                                   out_dir):
    """
    Given a list of miso IDs and a gffutils database, pull out the
    ensembl/gencode/gene name/gene type/transcript names

    @param miso_ids:
    @type miso_ids:
    @param db:
    @type db:
    @return:
    @rtype:
    """
    out_dir = out_dir.rstrip('/')
    event_type = event_type.lower()

    miso_to_ensembl = {}
    miso_to_gencode = {}
    miso_to_gene_name = {}
    miso_to_gene_type = {}
    miso_to_ensembl_transcript = {}
    miso_to_gencode_transcript = {}

    ensembl_to_miso = defaultdict(list)
    gencode_to_miso = defaultdict(list)
    gene_name_to_miso = defaultdict(list)

    for miso_id in miso_ids:
        exons = miso_id_to_exon_ids(miso_id)

        gencode = set([])
        ensembl = set([])
        gene_name = set([])
        gene_type = set([])
        gencode_transcript = set([])
        ensembl_transcript = set([])
        for e in exons:
            try:
                exon = db[e]

                gencode.update(exon.attributes['gene_id'])
                ensembl.update(
                    map(lambda x: x.split('.')[0], exon.attributes['gene_id']))
                gene_name.update(exon.attributes['gene_name'])
                gene_type.update(exon.attributes['gene_type'])
                gencode_transcript(exon.attributes['transcript_id'])
                ensembl_transcript.update(
                    map(lambda x: x.split('.')[0], exon.attributes['gene_id'])
                )
            except gffutils.FeatureNotFoundError:
                continue

        if len(gencode) > 0:

            for ens in ensembl:
                ensembl_to_miso[ens].append(miso_id)
            for g in gene_name:
                gene_name_to_miso[g].append(miso_id)
            for g in gencode:
                gencode_to_miso[g].append(miso_id)

            ensembl = ','.join(ensembl)

            gencode = ','.join(gencode)
            gene_name = ','.join(gene_name)
            gene_type = ','.join(gene_type)

            gencode_transcript = ','.join(gencode_transcript)
            ensembl_transcript = ','.join(ensembl_transcript)

            miso_to_gencode[miso_id] = gencode
            miso_to_ensembl[miso_id] = ensembl
            miso_to_gene_name[miso_id] = gene_name
            miso_to_gene_type[miso_id] = gene_type

            miso_to_gencode_transcript[miso_id] = gencode_transcript
            miso_to_ensembl_transcript[miso_id] = ensembl_transcript

        else:
            miso_to_gencode[miso_id] = np.nan
            miso_to_ensembl[miso_id] = np.nan
            miso_to_gene_name[miso_id] = np.nan
            miso_to_gene_type[miso_id] = np.nan

    miso_tos = {'gencode_gene': miso_to_gencode,
                'ensembl_gene': miso_to_ensembl,
                'gene_name': miso_to_gene_name,
                'gene_type': miso_to_gene_type,
                'gencode_transcript': miso_to_gencode_transcript,
                'ensembl_transcript': miso_to_ensembl_transcript}

    to_misos = {'ensembl_gene': ensembl_to_miso,
                'gene_name': gene_name_to_miso,
                'gencode_gene': gencode_to_miso}

    for name, d in miso_tos.iteritems():
        df = pd.DataFrame.from_dict(d, orient='index')
        df.index.name = 'event_name'
        df.columns = [name]
        tsv = '{}/miso_{}_to_{}.tsv'.format(out_dir, event_type, name)
        df.to_csv(tsv, sep='\t')
        sys.stdout.write('Wrote {}'.format(tsv))

    for name, d in to_misos.iteritems():
        tsv = '{}/{}_to_miso_{}.tsv'.format(out_dir, name, event_type)
        with open(tsv, 'w') as f:
            for k, v in d.iteritems():
                f.write('{}\t{}\n'.format(k, '\t'.join(v)))
        sys.stdout.write('Wrote {}'.format(tsv))
<commit_msg>fix no .update on a set<commit_after>import numpy as np
import gffutils
from collections import defaultdict
import pandas as pd
import sys

def miso_exon_to_gencode_exon(exon):
    """Convert a single miso exon to gencode gffutils database exon id

    >>> # Skipped exon (SE) or Mutually exclusive exon (MXE) ID
    >>> miso_exon_to_gencode_exon('chr2:9624561:9624679:+')
    'exon:chr2:9624561-9624679:+'
    >>> # Alt 5' splice site (pick first of alternative)
    >>> miso_exon_to_gencode_exon('chr15:42565276:42565087|42565161:-')
    'exon:chr15:42565276-42565087:-'
    >>> # Alt 3' splice site (pick first of alternative)
    >>> miso_exon_to_gencode_exon('chr2:130914199|130914248:130914158:-')
    'exon:chr2:130914199-130914158:-'
    >>> # Retained intron: start, stop separated by '-' instead of ':'
    >>> miso_exon_to_gencode_exon('chr1:906259-906386:+')
    'exon:chr1:906259-906386:+'
    """
    return 'exon:{}:{}-{}:{}'.format(*miso_exon_to_coords(exon))

def miso_id_to_exon_ids(miso_id):
    """Convert a MISO-style alternative event ID

    Split on the pipe ("|") to account for Alt 5'/3' splice site events

    # A skipped exon (SE) ID
    >>> miso_id_to_exon_ids('chr2:9624561:9624679:+@chr2:9627585:9627676:+@chr2:9628276:9628591:+')
    ['exon:chr2:9624561-9624679:+', 'exon:chr2:9627585-9627676:+', 'exon:chr2:9628276-9628591:+']
    >>> # A mutually exclusive (MXE) ID
    >>> miso_id_to_exon_ids('chr16:89288500:89288591:+@chr16:89289565:89289691:+@chr16:89291127:89291210:+@chr16:89291963:89292039:+')
    ['exon:chr16:89288500-89288591:+', 'exon:chr16:89289565-89289691:+', 'exon:chr16:89291127-89291210:+', 'exon:chr16:89291963-89292039:+']
    >>> # An Alt 5' splice site (A5SS) ID
    >>> miso_id_to_exon_ids("chr15:42565276:42565087|42565161:-@chr15:42564261:42564321:-")
    ['exon:chr15:42565276-42565087:-', 'exon:chr15:42564261-42564321:-']
    >>> # An Alt 3' splice site (A3SS) ID
    >>> miso_id_to_exon_ids('chr2:130914824:130914969:-@chr2:130914199|130914248:130914158:-')
    ['exon:chr2:130914824-130914969:-', 'exon:chr2:130914199-130914158:-']
    >>> # A retained intron (RI) ID
    >>> miso_id_to_exon_ids('chr1:906066-906138:+@chr1:906259-906386:+')
    ['exon:chr1:906066-906138:+', 'exon:chr1:906259-906386:+']
    """
    return map(miso_exon_to_gencode_exon, miso_id.split('@'))

def miso_exon_to_coords(exon):
    """

    >>> miso_exon_to_coords('chr2:130914824:130914969:-')
    ('chr2', '130914824', '130914969', '-')
    >>> # Alt 5' SS - pick the first of the alternative ends
    >>> miso_exon_to_coords('chr15:42565276:42565087|42565161:-')
    ('chr15', '42565276', '42565087', '-')
    >>> # Alt 3' SS - pick the first of the alternative starts
    >>> miso_exon_to_coords('chr2:130914199|130914248:130914158:-')
    ('chr2', '130914199', '130914158', '-')
    >>> # Retained intron
    >>> miso_exon_to_coords('chr1:906066-906138:+')
    ('chr1', '906066', '906138', '+')
    """
    strand = exon[-1]
    coords = map(lambda x: x.split('|')[0],
                 exon.split(':'))
    if '-' in coords[1]:
        start, stop = coords[1].split('-')
        coords = coords[0], start, stop, strand
    return coords[0], coords[1], coords[2], strand

def convert_miso_ids_to_everything(miso_ids, db,
                                   event_type,
                                   out_dir):
    """
    Given a list of miso IDs and a gffutils database, pull out the
    ensembl/gencode/gene name/gene type/transcript names

    @param miso_ids:
    @type miso_ids:
    @param db:
    @type db:
    @return:
    @rtype:
    """
    out_dir = out_dir.rstrip('/')
    event_type = event_type.lower()

    miso_to_ensembl = {}
    miso_to_gencode = {}
    miso_to_gene_name = {}
    miso_to_gene_type = {}
    miso_to_ensembl_transcript = {}
    miso_to_gencode_transcript = {}

    ensembl_to_miso = defaultdict(list)
    gencode_to_miso = defaultdict(list)
    gene_name_to_miso = defaultdict(list)

    for miso_id in miso_ids:
        exons = miso_id_to_exon_ids(miso_id)

        gencode = set([])
        ensembl = set([])
        gene_name = set([])
        gene_type = set([])
        gencode_transcript = set([])
        ensembl_transcript = set([])
        for e in exons:
            try:
                exon = db[e]

                gencode.update(exon.attributes['gene_id'])
                ensembl.update(
                    map(lambda x: x.split('.')[0], exon.attributes['gene_id']))
                gene_name.update(exon.attributes['gene_name'])
                gene_type.update(exon.attributes['gene_type'])
                gencode_transcript.update(exon.attributes['transcript_id'])
                ensembl_transcript.update(
                    map(lambda x: x.split('.')[0], exon.attributes['gene_id'])
                )
            except gffutils.FeatureNotFoundError:
                continue

        if len(gencode) > 0:

            for ens in ensembl:
                ensembl_to_miso[ens].append(miso_id)
            for g in gene_name:
                gene_name_to_miso[g].append(miso_id)
            for g in gencode:
                gencode_to_miso[g].append(miso_id)

            ensembl = ','.join(ensembl)

            gencode = ','.join(gencode)
            gene_name = ','.join(gene_name)
            gene_type = ','.join(gene_type)

            gencode_transcript = ','.join(gencode_transcript)
            ensembl_transcript = ','.join(ensembl_transcript)

            miso_to_gencode[miso_id] = gencode
            miso_to_ensembl[miso_id] = ensembl
            miso_to_gene_name[miso_id] = gene_name
            miso_to_gene_type[miso_id] = gene_type

            miso_to_gencode_transcript[miso_id] = gencode_transcript
            miso_to_ensembl_transcript[miso_id] = ensembl_transcript

        else:
            miso_to_gencode[miso_id] = np.nan
            miso_to_ensembl[miso_id] = np.nan
            miso_to_gene_name[miso_id] = np.nan
            miso_to_gene_type[miso_id] = np.nan

    miso_tos = {'gencode_gene': miso_to_gencode,
                'ensembl_gene': miso_to_ensembl,
                'gene_name': miso_to_gene_name,
                'gene_type': miso_to_gene_type,
                'gencode_transcript': miso_to_gencode_transcript,
                'ensembl_transcript': miso_to_ensembl_transcript}

    to_misos = {'ensembl_gene': ensembl_to_miso,
                'gene_name': gene_name_to_miso,
                'gencode_gene': gencode_to_miso}

    for name, d in miso_tos.iteritems():
        df = pd.DataFrame.from_dict(d, orient='index')
        df.index.name = 'event_name'
        df.columns = [name]
        tsv = '{}/miso_{}_to_{}.tsv'.format(out_dir, event_type, name)
        df.to_csv(tsv, sep='\t')
        sys.stdout.write('Wrote {}'.format(tsv))

    for name, d in to_misos.iteritems():
        tsv = '{}/{}_to_miso_{}.tsv'.format(out_dir, name, event_type)
        with open(tsv, 'w') as f:
            for k, v in d.iteritems():
                f.write('{}\t{}\n'.format(k, '\t'.join(v)))
        sys.stdout.write('Wrote {}'.format(tsv))
<|endoftext|>


<commit_before># This file is part of beets.
import time
import logging
import socket

from beets import util, config, plugins, ui, library
import pyechonest
import pyechonest.song
import pyechonest.track

log = logging.getLogger('beets')

# If a request at the EchoNest fails, we want to retry the request RETRIES
# times and wait between retries for RETRY_INTERVAL seconds.
RETRIES = 10
RETRY_INTERVAL = 10

# for converting files
import os
import tempfile
from string import Template
from subprocess import Popen
DEVNULL = open(os.devnull, 'wb')

# The attributes we can import and where to store them
# Note:  We use echonest_id (song_id) and echonest_fingerprint to speed up
# lookups.  They are not listed as attributes here.
ATTRIBUTES = {
        'energy'        : 'echonest_energy',
        'liveness'      : 'echonest_liveness',
        'speechiness'   : 'echonest_speechiness',
        'acousticness'  : 'echonest_acousticness',
        'danceability'  : 'echonest_danceability',
        'valence'       : 'echonest_valence',
        'tempo'         : 'bpm',
    }

def _splitstrip(string):
    """Split string at comma and return the stripped values as array."""
    return [ s.strip() for s in string.split(u',') ]

class EchonestMetadataPlugin(plugins.BeetsPlugin):
    _songs = {}
    _attributes = []
    _no_mapping = []

    def __init__(self):
        super(EchonestMetadataPlugin, self).__init__()
        self.config.add({
                'auto'          : True,
                'apikey'        : u'NY2KTZHQ0QDSHBAP6',
                'codegen'       : None,
                'upload'        : True,
                'convert'       : True,
            })
        for k, v in ATTRIBUTES.iteritems():
            self.config.add({k:v})

        pyechonest.config.ECHO_NEST_API_KEY = \
            config['echonest']['apikey'].get(unicode)

        if config['echonest']['codegen'].get() is not None:
            pyechonest.config.CODEGEN_BINARY_OVERRIDE = \
                config['echonest']['codegen'].get(unicode)

        self.register_listener('import_task_start', self.fetch_song_task)
        self.register_listener('import_task_apply', self.apply_metadata_task)

    def _echofun(self, func, **kwargs):
        """Wrapper for requests to the EchoNest API.  Will retry up to RETRIES
        times and wait between retries for RETRY_INTERVAL seconds.
        """
        for i in range(RETRIES):
            try:
                result = func(**kwargs)
            except pyechonest.util.EchoNestAPIError as e:
                if e.code == 3:
                    # reached access limit per minute
                    time.sleep(RETRY_INTERVAL)
                elif e.code == 5:
                    # specified identifier does not exist
                    # no use in trying again.
                    log.debug(u'echonest: {}'.format(e))
                    return None
                else:
                    log.error(u'echonest: {0}'.format(e.args[0][0]))
                    return None
            except (pyechonest.util.EchoNestIOError, socket.error) as e:
                log.warn(u'echonest: IO error: {0}'.format(e))
                time.sleep(RETRY_INTERVAL)
            else:
                break
        else:
            # If we exited the loop without breaking, then we used up all
            # our allotted retries.
            raise Exception(u'exceeded retries')
            return None
        return result

    def fingerprint(self, item):
        """Get the fingerprint for this item from the EchoNest.  If we already
        have a fingerprint, return it and don't calculate it again.
        """
        if item.get('echonest_fingerprint', None) is not None:
            try:
                code = self._echofun(pyechonest.util.codegen, filename=item.path)
                item['echonest_fingerprint'] = code[0]['code']
                item.write()
            except Exception as exc:
                log.error(u'echonest: fingerprinting failed: {0}'
                        .format(str(exc)))
                return None
        return item.get('echonest_fingerprint')

    def convert(self, item):
        """Converts an item in an unsupported media format to ogg.  Config
        pending.
        This is stolen from Jakob Schnitzers convert plugin.
        """
        fd, dest = tempfile.mkstemp(u'.ogg')
        os.close(fd)
        source = item.path
        # FIXME: use avconv?
        command = u'ffmpeg -i $source -y -acodec libvorbis -vn -aq 2 $dest'.split(u' ')
        log.info(u'echonest: encoding {0} to {1}'
                .format(util.displayable_path(source),
                util.displayable_path(dest)))
        opts = []
        for arg in command:
            arg = arg.encode('utf-8')
            opts.append(Template(arg).substitute({
                'source':   source,
                'dest':     dest
            }))

        try:
            encode = Popen(opts, close_fds=True, stderr=DEVNULL)
            encode.wait()
        except Exception as exc:
            log.error(u'echonest: encode failed: {0}'.format(str(exc)))
            util.remove(dest)
            util.prune_dirs(os.path.dirname(dest))
            return None

        if encode.returncode != 0:
            log.info(u'echonest: encoding {0} failed ({1}). Cleaning up...'
                     .format(util.displayable_path(source), encode.returncode))
            util.remove(dest)
            util.prune_dirs(os.path.dirname(dest))
            return None
        log.info(u'Finished encoding {0}'.format(util.displayable_path(source)))
        return dest

    def analyze(self, item):
        """Upload the item to the EchoNest for analysis.  May require to
        convert the item to a supported media format.
        """
        try:
            source = item.path
            if item.format.lower() not in ['wav', 'mp3', 'au', 'ogg', 'mp4', 'm4a']:
                if not config['echonest']['convert'].get(bool):
                    raise Exception(u'format {} not supported for upload'
                            .format(item.format))
                else:
                    source = self.convert(item)
                    if source is None:
                        raise Exception(u'failed to convert file'
                                .format(item.format))
            log.info(u'echonest: uploading file, be patient')
            track = self._echofun(pyechonest.track.track_from_filename,
                    filename=source)
            if track is None:
                raise Exception(u'failed to upload file')

            # Sometimes we have a track but no song.  I guess this happens for
            # new / unverified songs.  We need to 'extract' the audio_summary
            # from the track object 'manually'.  I don't know why the
            # pyechonest API handles tracks (merge audio_summary to __dict__)
            # and songs (keep audio_summary in an extra attribute)
            # differently.
            # Maybe a patch for pyechonest could help?
            ids = []
            try:
                ids = [track.song_id]
            except Exception:
                result = {}
                result['energy'] = track.energy
                result['liveness'] = track.liveness
                result['speechiness'] = track.speechiness
                result['acousticness'] = track.acousticness
                result['danceability'] = track.danceability
                result['valence'] = track.valence
                result['tempo'] = track.tempo
                result['duration'] = track.duration
                return result
            songs = self._echofun(pyechonest.song.profile,
                    ids=ids, track_ids=[track.id],
                    buckets=['audio_summary'])
            if songs is None:
                raise Exception(u'failed to retrieve info from upload')
            # No need to _pick_song, match is match
            # return self._pick_song(songs, item)
            return songs[0]
        except Exception as exc:
            log.error(u'echonest: analysis failed: {0}'.format(str(exc)))
            return None

    def identify(self, item):
        """Try to identify the song at the EchoNest.
        """
        try:
            code = self.fingerprint(item)
            if code is None:
                raise Exception(u'can not identify without a fingerprint')
            songs = self._echofun(pyechonest.song.identify, code=code)
            if not songs:
                raise Exception(u'no songs found')
            return max(songs, key=lambda s: s.score)
        except Exception as exc:
            log.error(u'echonest: identification failed: {0}'.format(str(exc)))
            return None

    def _pick_song(self, songs, item):
        """Helper method to pick the best matching song from a list of songs
        returned by the EchoNest.  Compares artist, title and duration.  If
        the artist and title match and the duration difference is <= 1.0
        seconds, it's considered a match.
        """
        pick = None
        if songs:
            min_dist = item.length
            for song in songs:
                if song.artist_name.lower() == item.artist.lower() \
                        and song.title.lower() == item.title.lower():
                    dist = abs(item.length - song.audio_summary['duration'])
                    if dist < min_dist:
                        min_dist = dist
                        pick = song
            if min_dist > 1.0:
                return None
        return pick

    def search(self, item):
        """Search the item at the EchoNest by artist and title.
        """
        try:
            songs = self._echofun(pyechonest.song.search, title=item.title,
                    results=100, artist=item.artist,
                    buckets=['id:musicbrainz', 'tracks'])
            pick = self._pick_song(songs, item)
            if pick is None:
                raise Exception(u'no (matching) songs found')
            return pick
        except Exception as exc:
            log.error(u'echonest: search failed: {0}'.format(str(exc)))
            return None

    def profile(self, item):
        """Do a lookup on the EchoNest by MusicBrainz ID.
        """
        try:
            if item.get('echonest_id', None) is None:
                if not item.mb_trackid:
                    raise Exception(u'musicbrainz ID not available')
                mbid = 'musicbrainz:track:{0}'.format(item.mb_trackid)
                track = self._echofun(pyechonest.track.track_from_id, identifier=mbid)
                if not track:
                    raise Exception(u'could not get track from ID')
                ids = track.song_id
            else:
                ids = item.get('echonest_id')
            songs = self._echofun(pyechonest.song.profile, ids=ids,
                    buckets=['id:musicbrainz', 'audio_summary'])
            if not songs:
                raise Exception(u'could not get songs from track ID')
            # It seems like _pick_song fails even if we have a good match.  So
            # just return the first song.
            # return self._pick_song(songs, item)
            return songs[0]
        except Exception as exc:
            log.debug(u'echonest: profile failed: {0}'.format(str(exc)))
            return None

    def fetch_song(self, item):
        """Try all methods, to get a matching song object from the EchoNest.
        """
        methods = [self.profile, self.search]
        if config['echonest']['codegen'].get() is not None:
            methods.append(self.identify)
        if config['echonest']['upload'].get(bool):
            methods.append(self.analyze)
        for method in methods:
            try:
                song = method(item)
                if not song is None:
                    if isinstance(song, pyechonest.song.Song):
                        log.debug(u'echonest: got song through {0}: {1} - {2} [{3}]'
                                  .format(method.im_func.func_name,
                                  song.artist_name, song.title,
                                  song.audio_summary['duration']))
                    else: # it's our dict filled from a track object
                        log.debug(u'echonest: got song through {0}: {1} - {2} [{3}]'
                                  .format(method.im_func.func_name,
                                  item.artist, item.title,
                                  song['duration']))
                    return song
            except Exception as exc:
                log.debug(u'echonest: profile failed: {0}'.format(str(exc)))
        return None

    def apply_metadata(self, item):
        """Copy the metadata from the EchoNest to the item.
        """
        if item.path in self._songs:
            # song can be a dict
            if isinstance(self._songs[item.path], pyechonest.song.Song):
                item.echonest_id = self._songs[item.path].id
                values = self._songs[item.path].audio_summary
            else:
                values = self._songs[item.path]
            for k, v in values.iteritems():
                if ATTRIBUTES.has_key(k) and ATTRIBUTES[k] is not None:
                    log.debug(u'echonest: metadata: {0} = {1}'
                            .format(ATTRIBUTES[k], v))
                    item[ATTRIBUTES[k]] = v
            if config['import']['write'].get(bool):
                log.info(u'echonest: writing metadata: {0}'
                         .format(util.displayable_path(item.path)))
                item.write()
                if item._lib:
                    item.store()
        else:
            log.warn(u'echonest: no metadata available')

    def requires_update(self, item):
        """Check if this item requires an update from the EchoNest aka data is
        missing.
        """
        for k, v in ATTRIBUTES.iteritems():
            if v is None:
                continue
            if item.get(v, None) is None:
                return True
        log.info(u'echonest: no update required')
        return False

    def fetch_song_task(self, task, session):
        items = task.items if task.is_album else [task.item]
        for item in items:
            song = self.fetch_song(item)
            if not song is None:
                self._songs[item.path] = song

    def apply_metadata_task(self, task, session):
        for item in task.imported_items():
            self.apply_metadata(item)

    def commands(self):
        cmd = ui.Subcommand('echonest',
            help='Fetch metadata from the EchoNest')
        cmd.parser.add_option('-f', '--force', dest='force',
            action='store_true', default=False,
            help='(re-)download information from the EchoNest')

        def func(lib, opts, args):
            self.config.set_args(opts)
            for item in lib.items(ui.decargs(args)):
                log.info(u'echonest: {0} - {1} [{2}]'.format(item.artist,
                        item.title, item.length))
                if self.config['force'] or self.requires_update(item):
                    song = self.fetch_song(item)
                    if not song is None:
                        self._songs[item.path] = song
                    self.apply_metadata(item)

        cmd.func = func
        return [cmd]

# eof
<commit_msg>workaround: onetime after analyze, the wrong song was returned.  double check that.  fall back to track data.<commit_after># This file is part of beets.
import time
import logging
import socket

from beets import util, config, plugins, ui, library
import pyechonest
import pyechonest.song
import pyechonest.track

log = logging.getLogger('beets')

# If a request at the EchoNest fails, we want to retry the request RETRIES
# times and wait between retries for RETRY_INTERVAL seconds.
RETRIES = 10
RETRY_INTERVAL = 10

# for converting files
import os
import tempfile
from string import Template
from subprocess import Popen
DEVNULL = open(os.devnull, 'wb')

# The attributes we can import and where to store them
# Note:  We use echonest_id (song_id) and echonest_fingerprint to speed up
# lookups.  They are not listed as attributes here.
ATTRIBUTES = {
        'energy'        : 'echonest_energy',
        'liveness'      : 'echonest_liveness',
        'speechiness'   : 'echonest_speechiness',
        'acousticness'  : 'echonest_acousticness',
        'danceability'  : 'echonest_danceability',
        'valence'       : 'echonest_valence',
        'tempo'         : 'bpm',
    }

def _splitstrip(string):
    """Split string at comma and return the stripped values as array."""
    return [ s.strip() for s in string.split(u',') ]

class EchonestMetadataPlugin(plugins.BeetsPlugin):
    _songs = {}
    _attributes = []
    _no_mapping = []

    def __init__(self):
        super(EchonestMetadataPlugin, self).__init__()
        self.config.add({
                'auto'          : True,
                'apikey'        : u'NY2KTZHQ0QDSHBAP6',
                'codegen'       : None,
                'upload'        : True,
                'convert'       : True,
            })
        for k, v in ATTRIBUTES.iteritems():
            self.config.add({k:v})

        pyechonest.config.ECHO_NEST_API_KEY = \
            config['echonest']['apikey'].get(unicode)

        if config['echonest']['codegen'].get() is not None:
            pyechonest.config.CODEGEN_BINARY_OVERRIDE = \
                config['echonest']['codegen'].get(unicode)

        self.register_listener('import_task_start', self.fetch_song_task)
        self.register_listener('import_task_apply', self.apply_metadata_task)

    def _echofun(self, func, **kwargs):
        """Wrapper for requests to the EchoNest API.  Will retry up to RETRIES
        times and wait between retries for RETRY_INTERVAL seconds.
        """
        for i in range(RETRIES):
            try:
                result = func(**kwargs)
            except pyechonest.util.EchoNestAPIError as e:
                if e.code == 3:
                    # reached access limit per minute
                    time.sleep(RETRY_INTERVAL)
                elif e.code == 5:
                    # specified identifier does not exist
                    # no use in trying again.
                    log.debug(u'echonest: {}'.format(e))
                    return None
                else:
                    log.error(u'echonest: {0}'.format(e.args[0][0]))
                    return None
            except (pyechonest.util.EchoNestIOError, socket.error) as e:
                log.warn(u'echonest: IO error: {0}'.format(e))
                time.sleep(RETRY_INTERVAL)
            else:
                break
        else:
            # If we exited the loop without breaking, then we used up all
            # our allotted retries.
            raise Exception(u'exceeded retries')
            return None
        return result

    def fingerprint(self, item):
        """Get the fingerprint for this item from the EchoNest.  If we already
        have a fingerprint, return it and don't calculate it again.
        """
        if item.get('echonest_fingerprint', None) is not None:
            try:
                code = self._echofun(pyechonest.util.codegen, filename=item.path)
                item['echonest_fingerprint'] = code[0]['code']
                item.write()
            except Exception as exc:
                log.error(u'echonest: fingerprinting failed: {0}'
                        .format(str(exc)))
                return None
        return item.get('echonest_fingerprint')

    def convert(self, item):
        """Converts an item in an unsupported media format to ogg.  Config
        pending.
        This is stolen from Jakob Schnitzers convert plugin.
        """
        fd, dest = tempfile.mkstemp(u'.ogg')
        os.close(fd)
        source = item.path
        # FIXME: use avconv?
        command = u'ffmpeg -i $source -y -acodec libvorbis -vn -aq 2 $dest'.split(u' ')
        log.info(u'echonest: encoding {0} to {1}'
                .format(util.displayable_path(source),
                util.displayable_path(dest)))
        opts = []
        for arg in command:
            arg = arg.encode('utf-8')
            opts.append(Template(arg).substitute({
                'source':   source,
                'dest':     dest
            }))

        try:
            encode = Popen(opts, close_fds=True, stderr=DEVNULL)
            encode.wait()
        except Exception as exc:
            log.error(u'echonest: encode failed: {0}'.format(str(exc)))
            util.remove(dest)
            util.prune_dirs(os.path.dirname(dest))
            return None

        if encode.returncode != 0:
            log.info(u'echonest: encoding {0} failed ({1}). Cleaning up...'
                     .format(util.displayable_path(source), encode.returncode))
            util.remove(dest)
            util.prune_dirs(os.path.dirname(dest))
            return None
        log.info(u'Finished encoding {0}'.format(util.displayable_path(source)))
        return dest

    def analyze(self, item):
        """Upload the item to the EchoNest for analysis.  May require to
        convert the item to a supported media format.
        """
        try:
            source = item.path
            if item.format.lower() not in ['wav', 'mp3', 'au', 'ogg', 'mp4', 'm4a']:
                if not config['echonest']['convert'].get(bool):
                    raise Exception(u'format {} not supported for upload'
                            .format(item.format))
                else:
                    source = self.convert(item)
                    if source is None:
                        raise Exception(u'failed to convert file'
                                .format(item.format))
            log.info(u'echonest: uploading file, be patient')
            track = self._echofun(pyechonest.track.track_from_filename,
                    filename=source)
            if track is None:
                raise Exception(u'failed to upload file')

            # Sometimes we have a track but no song.  I guess this happens for
            # new / unverified songs.  We need to 'extract' the audio_summary
            # from the track object 'manually'.  I don't know why the
            # pyechonest API handles tracks (merge audio_summary to __dict__)
            # and songs (keep audio_summary in an extra attribute)
            # differently.
            # Maybe a patch for pyechonest could help?
            from_track = {}
            from_track['energy'] = track.energy
            from_track['liveness'] = track.liveness
            from_track['speechiness'] = track.speechiness
            from_track['acousticness'] = track.acousticness
            from_track['danceability'] = track.danceability
            from_track['valence'] = track.valence
            from_track['tempo'] = track.tempo
            from_track['duration'] = track.duration
            ids = []
            try:
                ids = [track.song_id]
            except Exception:
                return from_track
            songs = self._echofun(pyechonest.song.profile,
                    ids=ids, track_ids=[track.id], limit=100,
                    buckets=['audio_summary'])
            if songs is None:
                raise Exception(u'failed to retrieve info from upload')
            pick = self._pick_song(songs, item)
            if pick is None:
                return from_track
            return pick
        except Exception as exc:
            log.error(u'echonest: analysis failed: {0}'.format(str(exc)))
            return None

    def identify(self, item):
        """Try to identify the song at the EchoNest.
        """
        try:
            code = self.fingerprint(item)
            if code is None:
                raise Exception(u'can not identify without a fingerprint')
            songs = self._echofun(pyechonest.song.identify, code=code)
            if not songs:
                raise Exception(u'no songs found')
            return max(songs, key=lambda s: s.score)
        except Exception as exc:
            log.error(u'echonest: identification failed: {0}'.format(str(exc)))
            return None

    def _pick_song(self, songs, item):
        """Helper method to pick the best matching song from a list of songs
        returned by the EchoNest.  Compares artist, title and duration.  If
        the artist and title match and the duration difference is <= 1.0
        seconds, it's considered a match.
        """
        pick = None
        if songs:
            min_dist = item.length
            for song in songs:
                if song.artist_name.lower() == item.artist.lower() \
                        and song.title.lower() == item.title.lower():
                    dist = abs(item.length - song.audio_summary['duration'])
                    if dist < min_dist:
                        min_dist = dist
                        pick = song
            if min_dist > 1.0:
                return None
        return pick

    def search(self, item):
        """Search the item at the EchoNest by artist and title.
        """
        try:
            songs = self._echofun(pyechonest.song.search, title=item.title,
                    results=100, artist=item.artist,
                    buckets=['id:musicbrainz', 'tracks'])
            pick = self._pick_song(songs, item)
            if pick is None:
                raise Exception(u'no (matching) songs found')
            return pick
        except Exception as exc:
            log.error(u'echonest: search failed: {0}'.format(str(exc)))
            return None

    def profile(self, item):
        """Do a lookup on the EchoNest by MusicBrainz ID.
        """
        try:
            if item.get('echonest_id', None) is None:
                if not item.mb_trackid:
                    raise Exception(u'musicbrainz ID not available')
                mbid = 'musicbrainz:track:{0}'.format(item.mb_trackid)
                track = self._echofun(pyechonest.track.track_from_id, identifier=mbid)
                if not track:
                    raise Exception(u'could not get track from ID')
                ids = track.song_id
            else:
                ids = item.get('echonest_id')
            songs = self._echofun(pyechonest.song.profile, ids=ids,
                    buckets=['id:musicbrainz', 'audio_summary'])
            if not songs:
                raise Exception(u'could not get songs from track ID')
            # It seems like _pick_song fails even if we have a good match.  So
            # just return the first song.
            # return self._pick_song(songs, item)
            return songs[0]
        except Exception as exc:
            log.debug(u'echonest: profile failed: {0}'.format(str(exc)))
            return None

    def fetch_song(self, item):
        """Try all methods, to get a matching song object from the EchoNest.
        """
        methods = [self.profile, self.search]
        if config['echonest']['codegen'].get() is not None:
            methods.append(self.identify)
        if config['echonest']['upload'].get(bool):
            methods.append(self.analyze)
        for method in methods:
            try:
                song = method(item)
                if not song is None:
                    if isinstance(song, pyechonest.song.Song):
                        log.debug(u'echonest: got song through {0}: {1} - {2} [{3}]'
                                  .format(method.im_func.func_name,
                                  song.artist_name, song.title,
                                  song.audio_summary['duration']))
                    else: # it's our dict filled from a track object
                        log.debug(u'echonest: got song through {0}: {1} - {2} [{3}]'
                                  .format(method.im_func.func_name,
                                  item.artist, item.title,
                                  song['duration']))
                    return song
            except Exception as exc:
                log.debug(u'echonest: profile failed: {0}'.format(str(exc)))
        return None

    def apply_metadata(self, item):
        """Copy the metadata from the EchoNest to the item.
        """
        if item.path in self._songs:
            # song can be a dict
            if isinstance(self._songs[item.path], pyechonest.song.Song):
                item.echonest_id = self._songs[item.path].id
                values = self._songs[item.path].audio_summary
            else:
                values = self._songs[item.path]
            for k, v in values.iteritems():
                if ATTRIBUTES.has_key(k) and ATTRIBUTES[k] is not None:
                    log.debug(u'echonest: metadata: {0} = {1}'
                            .format(ATTRIBUTES[k], v))
                    item[ATTRIBUTES[k]] = v
            if config['import']['write'].get(bool):
                log.info(u'echonest: writing metadata: {0}'
                         .format(util.displayable_path(item.path)))
                item.write()
                if item._lib:
                    item.store()
        else:
            log.warn(u'echonest: no metadata available')

    def requires_update(self, item):
        """Check if this item requires an update from the EchoNest aka data is
        missing.
        """
        for k, v in ATTRIBUTES.iteritems():
            if v is None:
                continue
            if item.get(v, None) is None:
                return True
        log.info(u'echonest: no update required')
        return False

    def fetch_song_task(self, task, session):
        items = task.items if task.is_album else [task.item]
        for item in items:
            song = self.fetch_song(item)
            if not song is None:
                self._songs[item.path] = song

    def apply_metadata_task(self, task, session):
        for item in task.imported_items():
            self.apply_metadata(item)

    def commands(self):
        cmd = ui.Subcommand('echonest',
            help='Fetch metadata from the EchoNest')
        cmd.parser.add_option('-f', '--force', dest='force',
            action='store_true', default=False,
            help='(re-)download information from the EchoNest')

        def func(lib, opts, args):
            self.config.set_args(opts)
            for item in lib.items(ui.decargs(args)):
                log.info(u'echonest: {0} - {1} [{2}]'.format(item.artist,
                        item.title, item.length))
                if self.config['force'] or self.requires_update(item):
                    song = self.fetch_song(item)
                    if not song is None:
                        self._songs[item.path] = song
                    self.apply_metadata(item)

        cmd.func = func
        return [cmd]

# eof
<|endoftext|>


<commit_before>from django.db import transaction
from rest_framework import serializers
from rest_framework.reverse import reverse

from conversion_job.models import Extent, ConversionJob, GISFormat, GISOption
from converters.converter import Options
from manager.job_manager import ConversionJobManager
from rest_api.serializer_helpers import ModelSideValidationMixin
from shared import JobStatus

class StatusHyperlinkSerializer(serializers.HyperlinkedRelatedField):
    read_only = True
    view_name = 'conversion_job_result-detail'

    def get_url(self, obj, view_name, request, format):   # pragma: nocover
        url = reverse(viewname=self.view_name, kwargs={'rq_job_id': obj.pk}, request=request)
        return url

class ExtentSerializer(ModelSideValidationMixin, serializers.ModelSerializer):
    polyfile = serializers.FileField(max_length=None, allow_empty_file=True, allow_null=True)

    class Meta:
        model = Extent
        fields = ('west', 'south', 'east', 'north', 'polyfile')

class GISFormatListSerializer(serializers.ListSerializer):
    def to_representation(self, data):
        return data.values_list('format', flat=True)

    def to_internal_value(self, data):
        return super().to_internal_value([{'format': value} for value in data])

class GISFormatSerializer(serializers.ModelSerializer):
    class Meta:
        model = GISFormat
        fields = ('format',)
        list_serializer_class = GISFormatListSerializer

class GISOptionSerializer(serializers.ModelSerializer):
    class Meta:
        model = GISOption
        fields = ('coordinate_reference_system', 'detail_level',)

class ConversionJobSerializer(serializers.ModelSerializer):
    extent = ExtentSerializer(many=False)
    gis_formats = GISFormatSerializer(required=False, many=True)
    gis_options = GISOptionSerializer()
    status = StatusHyperlinkSerializer(source='rq_job_id', read_only=True)

    def create(self, validated_data):
        gis_formats = validated_data.pop('gis_formats')
        formats = [d['format'] for d in gis_formats]
        with transaction.atomic():
            validated_data['gis_options_id'] = GISOption.objects.create(**validated_data.pop('gis_options')).id
            validated_data['extent_id'] = Extent.objects.create(**validated_data.pop('extent')).id
            conversion_job = super().create(validated_data)
            for gis_format_dict in gis_formats:
                GISFormat.objects.create(conversion_job=conversion_job, **gis_format_dict)
            rq_job = self._enqueue_rq_job(
                geometry=conversion_job.extent.get_geometry(),
                format_options=Options(output_formats=formats),
                callback_url=conversion_job.callback_url,
                output_directory=conversion_job.output_directory,
            )
            conversion_job.rq_job_id = rq_job.id
            conversion_job.status = JobStatus.QUEUED.technical_representation
            conversion_job.save()
        return conversion_job

    def _enqueue_rq_job(self, geometry, format_options, callback_url, output_directory):
        cm = ConversionJobManager(geometry=geometry, format_options=format_options)
        return cm.start_conversion(callback_url, output_directory)

    class Meta:
        model = ConversionJob
        fields = ('rq_job_id', 'callback_url', 'status', 'gis_formats', 'gis_options', 'extent')
        depth = 1
        read_only_fields = ('rq_job_id', 'status',)

# Status-only serializers

class DownloadURLField(serializers.HyperlinkedRelatedField):
    view_name = 'gisformat-download-result'

    def get_url(self, obj, view_name, request, format):
        gis_format = GISFormat.objects.get(pk=obj.pk)
        if not gis_format.get_result_file_path():
            return None

        url = reverse(viewname=self.view_name, kwargs={'pk': obj.pk}, request=request)
        return url

class GISFormatStatusSerializer(serializers.ModelSerializer):
    progress = serializers.CharField(source='get_progress_display')
    result_url = DownloadURLField(source='pk', read_only=True)

    class Meta:
        model = GISFormat
        fields = ('format', 'progress', 'result_url', )

class ConversionJobStatusSerializer(serializers.ModelSerializer):
    gis_formats = GISFormatStatusSerializer(many=True)
    status = serializers.CharField(source='get_status_display')

    class Meta:
        model = ConversionJob
        fields = ('rq_job_id', 'status', 'progress', 'gis_formats')
        read_only_fields = fields
<commit_msg>create format task models after job model<commit_after>from django.db import transaction
from rest_framework import serializers
from rest_framework.reverse import reverse

from conversion_job.models import Extent, ConversionJob, GISFormat, GISOption
from converters.converter import Options
from manager.job_manager import ConversionJobManager
from rest_api.serializer_helpers import ModelSideValidationMixin
from shared import JobStatus

class StatusHyperlinkSerializer(serializers.HyperlinkedRelatedField):
    read_only = True
    view_name = 'conversion_job_result-detail'

    def get_url(self, obj, view_name, request, format):   # pragma: nocover
        url = reverse(viewname=self.view_name, kwargs={'rq_job_id': obj.pk}, request=request)
        return url

class ExtentSerializer(ModelSideValidationMixin, serializers.ModelSerializer):
    polyfile = serializers.FileField(max_length=None, allow_empty_file=True, allow_null=True)

    class Meta:
        model = Extent
        fields = ('west', 'south', 'east', 'north', 'polyfile')

class GISFormatListSerializer(serializers.ListSerializer):
    def to_representation(self, data):
        return data.values_list('format', flat=True)

    def to_internal_value(self, data):
        return super().to_internal_value([{'format': value} for value in data])

class GISFormatSerializer(serializers.ModelSerializer):
    class Meta:
        model = GISFormat
        fields = ('format',)
        list_serializer_class = GISFormatListSerializer

class GISOptionSerializer(serializers.ModelSerializer):
    class Meta:
        model = GISOption
        fields = ('coordinate_reference_system', 'detail_level',)

class ConversionJobSerializer(serializers.ModelSerializer):
    extent = ExtentSerializer(many=False)
    gis_formats = GISFormatSerializer(required=False, many=True)
    gis_options = GISOptionSerializer()
    status = StatusHyperlinkSerializer(source='rq_job_id', read_only=True)

    def create(self, validated_data):
        gis_formats = validated_data.pop('gis_formats')
        formats = [d['format'] for d in gis_formats]
        with transaction.atomic():
            validated_data['gis_options_id'] = GISOption.objects.create(**validated_data.pop('gis_options')).id
            validated_data['extent_id'] = Extent.objects.create(**validated_data.pop('extent')).id
            conversion_job = super().create(validated_data)
            rq_job = self._enqueue_rq_job(
                geometry=conversion_job.extent.get_geometry(),
                format_options=Options(output_formats=formats),
                callback_url=conversion_job.callback_url,
                output_directory=conversion_job.output_directory,
            )
            conversion_job.rq_job_id = rq_job.id
            conversion_job.status = JobStatus.QUEUED.technical_representation
            conversion_job.save()
            for gis_format_dict in gis_formats:
                GISFormat.objects.create(conversion_job=conversion_job, **gis_format_dict)
        return conversion_job

    def _enqueue_rq_job(self, geometry, format_options, callback_url, output_directory):
        cm = ConversionJobManager(geometry=geometry, format_options=format_options)
        return cm.start_conversion(callback_url, output_directory)

    class Meta:
        model = ConversionJob
        fields = ('rq_job_id', 'callback_url', 'status', 'gis_formats', 'gis_options', 'extent')
        depth = 1
        read_only_fields = ('rq_job_id', 'status',)

# Status-only serializers

class DownloadURLField(serializers.HyperlinkedRelatedField):
    view_name = 'gisformat-download-result'

    def get_url(self, obj, view_name, request, format):
        gis_format = GISFormat.objects.get(pk=obj.pk)
        if not gis_format.get_result_file_path():
            return None

        url = reverse(viewname=self.view_name, kwargs={'pk': obj.pk}, request=request)
        return url

class GISFormatStatusSerializer(serializers.ModelSerializer):
    progress = serializers.CharField(source='get_progress_display')
    result_url = DownloadURLField(source='pk', read_only=True)

    class Meta:
        model = GISFormat
        fields = ('format', 'progress', 'result_url', )

class ConversionJobStatusSerializer(serializers.ModelSerializer):
    gis_formats = GISFormatStatusSerializer(many=True)
    status = serializers.CharField(source='get_status_display')

    class Meta:
        model = ConversionJob
        fields = ('rq_job_id', 'status', 'progress', 'gis_formats')
        read_only_fields = fields
<|endoftext|>