<commit_before>from __future__ import print_function
import math
import numpy as np

class Survey:
    def __init__(self):
        pass

    def __get_species_in_subject__(self,aggregations):
        """
        use Ali's and Margaret's code to determine how many species are a given subject
        and return those X top species
        :return:
        """
        num_species = int(np.median(aggregations["num species"]))
        assert(num_species >= 1)
        # sort the species by the number of votes
        species_by_vote = []

        for species_id in aggregations:
            if species_id not in ["num users","num species"]:
                species_by_vote.append((species_id,aggregations[species_id]["num votes"]))
        sorted_species = sorted(species_by_vote,key = lambda x:x[1],reverse=True)

        return sorted_species[:num_species]

    def __shannon_entropy__(self,probabilities):
        return -sum([p*math.log(p) for p in probabilities])

    def __calc__pielou__(self,aggregations):
        """
        return a row for survey tasks with subject id and pielou index
        :param aggregations:
        :return:
        """
        # for development - a few bad aggregations have crept into the db (for small projects) so skip them
        if max(aggregations["num species"]) == 0:
            return ""

        num_users = aggregations["num users"]
        probabilities = []
        for details in aggregations.values():
            # list => "num species", int => "num users"
            if type(details) in [list,int]:
                continue

            probabilities.append(details["num votes"]/float(num_users))

        num_species = len(probabilities)

        if num_species == 1:
            pielou_index = 0
        else:
            shannon = self.__shannon_entropy__(probabilities)
            pielou_index = shannon/math.log(num_species)

        return pielou_index

    def __species_annotation__(self,aggregation_so_far,annotation):
        """
        for a given user's annotation - go through each species that the user marked and process that annotation
        :param aggregation_so_far: aggregations so far for a single subject
        :param annotation: the next annotation for that subject
        :return:
        """
        for species_ann in annotation:
            species = species_ann["choice"]

            # is the first classification we've encountered for this species?
            if species not in aggregation_so_far:
                aggregation_so_far[species] = {"num votes": 1,"followup":{}}

            else:
                aggregation_so_far[species]["num votes"] += 1

            # go through each of the follow up questions
            for question,answer in species_ann["answers"].items():
                # is this the first time we've seen this particular followup question
                # should probably always happen the first time we see the species (since most, if not
                # all of the time, all of the follow up questions should be required)
                if question not in aggregation_so_far[species]["followup"]:
                    aggregation_so_far[species]["followup"][question] = {}

                # if list - multiple answers are allowed
                if isinstance(answer,list):
                    for ans in answer:
                        # first time we've seen this particular answer?
                        if ans not in aggregation_so_far[species]["followup"]:
                            aggregation_so_far[species]["followup"][question][ans] = 1
                        else:
                            aggregation_so_far[species]["followup"][question][ans] += 1
                else:
                    # first time we've seen this particular answer
                    if answer not in aggregation_so_far[species]["followup"]:
                        aggregation_so_far[species]["followup"][question][answer] = 1
                    else:
                        aggregation_so_far[species]["followup"][question][answer] += 1

        return aggregation_so_far

    def __aggregate__(self,raw_classifications,aggregated_results={}):

        # todo - figure out why this is happening
        if raw_classifications == {}:
            return dict()

        assert len(raw_classifications.keys()) == 1

        task_id = raw_classifications.keys()[0]

        for subject_id in raw_classifications[task_id]:
            # just to save on some typing
            annotation_dictionary = raw_classifications[task_id][subject_id]
            # some annotations appear to be empty - those should just be skipped
            # as a result, don't calculate "num users" just yet
            subject_results = {"num species":[]}

            num_users = 0

            for annotations_per_user in annotation_dictionary.values():
                # how many species did this user report?
                num_species = len([a for a in annotations_per_user if a != []])
                # if the user just reported nothing - skip
                if num_species == 0:
                    continue

                num_users += 1

                subject_results["num species"].append(num_species)
                # assert False

                for ann in annotations_per_user:
                    # not sure why we have sometimes getting empty annotations but it happens - if so, skip
                    if ann == []:
                        continue

                    subject_results = self.__species_annotation__(subject_results,ann)

            # save the results and record how many people have seen this particular subject
            # on the off chance that all of the annotations were empty, skip it
            if subject_results["num species"] != []:
                subject_results["num users"] = num_users
                subject_results["num species in image"] = self.__get_species_in_subject__(subject_results)
                subject_results["pielou index"] = self.__calc__pielou__(subject_results)

                aggregated_results[subject_id] = subject_results

        return aggregated_results
<commit_msg>debugging<commit_after>from __future__ import print_function
import math
import numpy as np

class Survey:
    def __init__(self):
        pass

    def __get_species_in_subject__(self,aggregations):
        """
        use Ali's and Margaret's code to determine how many species are a given subject
        and return those X top species
        :return:
        """
        num_species = int(np.median(aggregations["num species"]))
        assert(num_species >= 1)
        # sort the species by the number of votes
        species_by_vote = []

        for species_id in aggregations:
            if species_id not in ["num users","num species"]:
                species_by_vote.append((species_id,aggregations[species_id]["num votes"]))
        sorted_species = sorted(species_by_vote,key = lambda x:x[1],reverse=True)

        return sorted_species[:num_species]

    def __shannon_entropy__(self,probabilities):
        return -sum([p*math.log(p) for p in probabilities])

    def __calc__pielou__(self,aggregations):
        """
        return a row for survey tasks with subject id and pielou index
        :param aggregations:
        :return:
        """
        # for development - a few bad aggregations have crept into the db (for small projects) so skip them
        if max(aggregations["num species"]) == 0:
            return ""

        num_users = aggregations["num users"]
        probabilities = []
        for details in aggregations.values():
            # list => "num species", int => "num users"
            if type(details) in [list,int]:
                continue

            probabilities.append(details["num votes"]/float(num_users))

        num_species = len(probabilities)

        if num_species == 1:
            pielou_index = 0
        else:
            shannon = self.__shannon_entropy__(probabilities)
            pielou_index = shannon/math.log(num_species)

        return pielou_index

    def __species_annotation__(self,aggregation_so_far,annotation):
        """
        for a given user's annotation - go through each species that the user marked and process that annotation
        :param aggregation_so_far: aggregations so far for a single subject
        :param annotation: the next annotation for that subject
        :return:
        """
        for species_ann in annotation:
            species = species_ann["choice"]

            # is the first classification we've encountered for this species?
            if species not in aggregation_so_far:
                aggregation_so_far[species] = {"num votes": 1,"followup":{}}

            else:
                aggregation_so_far[species]["num votes"] += 1

            # go through each of the follow up questions
            for question,answer in species_ann["answers"].items():
                # is this the first time we've seen this particular followup question
                # should probably always happen the first time we see the species (since most, if not
                # all of the time, all of the follow up questions should be required)
                if question not in aggregation_so_far[species]["followup"]:
                    aggregation_so_far[species]["followup"][question] = {}

                # if list - multiple answers are allowed
                if isinstance(answer,list):
                    for ans in answer:
                        # first time we've seen this particular answer?
                        if ans not in aggregation_so_far[species]["followup"]:
                            aggregation_so_far[species]["followup"][question][ans] = 1
                        else:
                            aggregation_so_far[species]["followup"][question][ans] += 1
                else:
                    # first time we've seen this particular answer
                    if answer not in aggregation_so_far[species]["followup"]:
                        aggregation_so_far[species]["followup"][question][answer] = 1
                    else:
                        aggregation_so_far[species]["followup"][question][answer] += 1

        return aggregation_so_far

    def __aggregate__(self,raw_classifications,aggregated_results={}):

        # todo - figure out why this is happening
        if raw_classifications == {}:
            return dict()

        assert len(raw_classifications.keys()) == 1

        task_id = raw_classifications.keys()[0]

        for subject_id in raw_classifications[task_id]:
            # just to save on some typing
            annotation_dictionary = raw_classifications[task_id][subject_id]
            # some annotations appear to be empty - those should just be skipped
            # as a result, don't calculate "num users" just yet
            subject_results = {"num species":[]}

            num_users = 0

            for annotations_per_user in annotation_dictionary.values():
                # how many species did this user report?
                num_species = len([a for a in annotations_per_user if a != []])
                # if the user just reported nothing - skip
                if num_species == 0:
                    continue

                num_users += 1

                subject_results["num species"].append(num_species)
                # assert False

                # go through each individual species annotation from user
                for ann in annotations_per_user:
                    # not sure why we have sometimes getting empty annotations but it happens - if so, skip
                    if ann == []:
                        continue

                    subject_results = self.__species_annotation__(subject_results,ann)

            # save the results and record how many people have seen this particular subject
            # on the off chance that all of the annotations were empty, skip it
            if subject_results["num species"] != []:
                subject_results["num users"] = num_users
                subject_results["num species in image"] = self.__get_species_in_subject__(subject_results)
                subject_results["pielou index"] = self.__calc__pielou__(subject_results)

                aggregated_results[subject_id] = subject_results
            if subject_id == 873807:
                print(aggregated_results[subject_id])
                assert False

        return aggregated_results
<|endoftext|>


<commit_before>from django.urls import path
from django.views.generic import TemplateView

from . import views

app_name = "core"
urlpatterns = [
    # path(r'^test-404/$', 'core.views.error404'),
    path('events/', views.events, name='events'),
    path('events/map/', views.events_map, name='events_map'),
    path('events/calendar.ics', views.events_ical, name='icalendar'),
    path('resources/', views.resources, name='resources'),
    path('newsletter/', views.newsletter, name='newsletter'),
    path('faq/', views.faq, name='faq'),
    path('foundation/', views.foundation, name='foundation'),
    path('foundation/governing-document/', views.governing_document, name='foundation-governing-document'),
    path('contribute/', views.contribute, name='contribute'),
    path('2015/', views.year_2015, name='year_2015'),
    path('2016-2017/', views.year_2016_2017, name='year_2016_2017'),
    path('terms-conditions/', views.terms_conditions, name='terms-conditions'),
    path('privacy-cookies/', views.privacy_cookies, name='privacy-cookies'),
    # path(r'^workshop-box/$', views.workshop_box, name='workshop-box'),
    path('coc/', views.coc, name='coc'),
    # path(r'^crowdfunding-donors/$', views.crowdfunding_donors, name='crowdfunding-donors'),
    path('server-error/', views.server_error, name='server_error'),
    path('<slug:page_url>/', views.event, name='event'),
    path('robots.txt', TemplateView.as_view(
        template_name="core/txt/robots.txt", content_type='text/plain'), name="robots"),
    path('', views.index, name='index'),
    path('google3ef9938c7b93b707.html', TemplateView.as_view(
        template_name="google3ef9938c7b93b707.html", content_type='text/plain'), name="google_site_verification"),
]
<commit_msg>Remove text/plain in google file URL<commit_after>from django.urls import path
from django.views.generic import TemplateView

from . import views

app_name = "core"
urlpatterns = [
    # path(r'^test-404/$', 'core.views.error404'),
    path('events/', views.events, name='events'),
    path('events/map/', views.events_map, name='events_map'),
    path('events/calendar.ics', views.events_ical, name='icalendar'),
    path('resources/', views.resources, name='resources'),
    path('newsletter/', views.newsletter, name='newsletter'),
    path('faq/', views.faq, name='faq'),
    path('foundation/', views.foundation, name='foundation'),
    path('foundation/governing-document/', views.governing_document, name='foundation-governing-document'),
    path('contribute/', views.contribute, name='contribute'),
    path('2015/', views.year_2015, name='year_2015'),
    path('2016-2017/', views.year_2016_2017, name='year_2016_2017'),
    path('terms-conditions/', views.terms_conditions, name='terms-conditions'),
    path('privacy-cookies/', views.privacy_cookies, name='privacy-cookies'),
    # path(r'^workshop-box/$', views.workshop_box, name='workshop-box'),
    path('coc/', views.coc, name='coc'),
    # path(r'^crowdfunding-donors/$', views.crowdfunding_donors, name='crowdfunding-donors'),
    path('server-error/', views.server_error, name='server_error'),
    path('<slug:page_url>/', views.event, name='event'),
    path('robots.txt', TemplateView.as_view(
        template_name="core/txt/robots.txt", content_type='text/plain'), name="robots"),
    path('', views.index, name='index'),
    path('google3ef9938c7b93b707.html', TemplateView.as_view(
        template_name="google3ef9938c7b93b707.html"), name="google_site_verification"),
]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Open metadata information in a text editor to let the user edit it.
"""
from __future__ import division, absolute_import, print_function

from beets import plugins
from beets import util
from beets import ui
from beets.dbcore import types
from beets.importer import action
from beets.ui.commands import _do_query, PromptChoice
from copy import deepcopy
import subprocess
import yaml
from tempfile import NamedTemporaryFile
import os
import six

# These "safe" types can avoid the format/parse cycle that most fields go
# through: they are safe to edit with native YAML types.
SAFE_TYPES = (types.Float, types.Integer, types.Boolean)

class ParseError(Exception):
    """The modified file is unreadable. The user should be offered a chance to
    fix the error.
    """

def edit(filename, log):
    """Open `filename` in a text editor.
    """
    cmd = util.shlex_split(util.editor_command())
    cmd.append(filename)
    log.debug(u'invoking editor command: {!r}', cmd)
    try:
        subprocess.call(cmd)
    except OSError as exc:
        raise ui.UserError(u'could not run editor command {!r}: {}'.format(
            cmd[0], exc
        ))

def dump(arg):
    """Dump a sequence of dictionaries as YAML for editing.
    """
    return yaml.safe_dump_all(
        arg,
        allow_unicode=True,
        default_flow_style=False,
    )

def load(s):
    """Read a sequence of YAML documents back to a list of dictionaries
    with string keys.

    Can raise a `ParseError`.
    """
    try:
        out = []
        for d in yaml.load_all(s):
            if not isinstance(d, dict):
                raise ParseError(
                    u'each entry must be a dictionary; found {}'.format(
                        type(d).__name__
                    )
                )

            # Convert all keys to strings. They started out as strings,
            # but the user may have inadvertently messed this up.
            out.append({six.text_type(k): v for k, v in d.items()})

    except yaml.YAMLError as e:
        raise ParseError(u'invalid YAML: {}'.format(e))
    return out

def _safe_value(obj, key, value):
    """Check whether the `value` is safe to represent in YAML and trust as
    returned from parsed YAML.

    This ensures that values do not change their type when the user edits their
    YAML representation.
    """
    typ = obj._type(key)
    return isinstance(typ, SAFE_TYPES) and isinstance(value, typ.model_type)

def flatten(obj, fields):
    """Represent `obj`, a `dbcore.Model` object, as a dictionary for
    serialization. Only include the given `fields` if provided;
    otherwise, include everything.

    The resulting dictionary's keys are strings and the values are
    safely YAML-serializable types.
    """
    # Format each value.
    d = {}
    for key in obj.keys():
        value = obj[key]
        if _safe_value(obj, key, value):
            # A safe value that is faithfully representable in YAML.
            d[key] = value
        else:
            # A value that should be edited as a string.
            d[key] = obj.formatted()[key]

    # Possibly filter field names.
    if fields:
        return {k: v for k, v in d.items() if k in fields}
    else:
        return d

def apply_(obj, data):
    """Set the fields of a `dbcore.Model` object according to a
    dictionary.

    This is the opposite of `flatten`. The `data` dictionary should have
    strings as values.
    """
    for key, value in data.items():
        if _safe_value(obj, key, value):
            # A safe value *stayed* represented as a safe type. Assign it
            # directly.
            obj[key] = value
        else:
            # Either the field was stringified originally or the user changed
            # it from a safe type to an unsafe one. Parse it as a string.
            obj.set_parse(key, six.text_type(value))

class EditPlugin(plugins.BeetsPlugin):

    def __init__(self):
        super(EditPlugin, self).__init__()

        self.config.add({
            # The default fields to edit.
            'albumfields': 'album albumartist',
            'itemfields': 'track title artist album',

            # Silently ignore any changes to these fields.
            'ignore_fields': 'id path',
        })

        self.register_listener('before_choose_candidate',
                               self.before_choose_candidate_listener)

    def commands(self):
        edit_command = ui.Subcommand(
            'edit',
            help=u'interactively edit metadata'
        )
        edit_command.parser.add_option(
            u'-f', u'--field',
            metavar='FIELD',
            action='append',
            help=u'edit this field also',
        )
        edit_command.parser.add_option(
            u'--all',
            action='store_true', dest='all',
            help=u'edit all fields',
        )
        edit_command.parser.add_album_option()
        edit_command.func = self._edit_command
        return [edit_command]

    def _edit_command(self, lib, opts, args):
        """The CLI command function for the `beet edit` command.
        """
        # Get the objects to edit.
        query = ui.decargs(args)
        items, albums = _do_query(lib, query, opts.album, False)
        objs = albums if opts.album else items
        if not objs:
            ui.print_(u'Nothing to edit.')
            return

        # Get the fields to edit.
        if opts.all:
            fields = None
        else:
            fields = self._get_fields(opts.album, opts.field)
        self.edit(opts.album, objs, fields)

    def _get_fields(self, album, extra):
        """Get the set of fields to edit.
        """
        # Start with the configured base fields.
        if album:
            fields = self.config['albumfields'].as_str_seq()
        else:
            fields = self.config['itemfields'].as_str_seq()

        # Add the requested extra fields.
        if extra:
            fields += extra

        # Ensure we always have the `id` field for identification.
        fields.append('id')

        return set(fields)

    def edit(self, album, objs, fields):
        """The core editor function.

        - `album`: A flag indicating whether we're editing Items or Albums.
        - `objs`: The `Item`s or `Album`s to edit.
        - `fields`: The set of field names to edit (or None to edit
          everything).
        """
        # Present the YAML to the user and let her change it.
        success = self.edit_objects(objs, fields)

        # Save the new data.
        if success:
            self.save_changes(objs)

    def edit_objects(self, objs, fields):
        """Dump a set of Model objects to a file as text, ask the user
        to edit it, and apply any changes to the objects.

        Return a boolean indicating whether the edit succeeded.
        """
        # Get the content to edit as raw data structures.
        old_data = [flatten(o, fields) for o in objs]

        # Set up a temporary file with the initial data for editing.
        new = NamedTemporaryFile(mode='w', suffix='.yaml', delete=False)
        old_str = dump(old_data)
        new.write(old_str)
        new.close()

        # Loop until we have parseable data and the user confirms.
        try:
            while True:
                # Ask the user to edit the data.
                edit(new.name, self._log)

                # Read the data back after editing and check whether anything
                # changed.
                with open(new.name) as f:
                    new_str = f.read()
                if new_str == old_str:
                    ui.print_(u"No changes; aborting.")
                    return False

                # Parse the updated data.
                try:
                    new_data = load(new_str)
                except ParseError as e:
                    ui.print_(u"Could not read data: {}".format(e))
                    if ui.input_yn(u"Edit again to fix? (Y/n)", True):
                        continue
                    else:
                        return False

                # Show the changes.
                # If the objects are not on the DB yet, we need a copy of their
                # original state for show_model_changes.
                objs_old = [deepcopy(obj) if not obj._db else None
                            for obj in objs]
                self.apply_data(objs, old_data, new_data)
                changed = False
                for obj, obj_old in zip(objs, objs_old):
                    changed |= ui.show_model_changes(obj, obj_old)
                if not changed:
                    ui.print_(u'No changes to apply.')
                    return False

                # Confirm the changes.
                choice = ui.input_options(
                    (u'continue Editing', u'apply', u'cancel')
                )
                if choice == u'a':  # Apply.
                    return True
                elif choice == u'c':  # Cancel.
                    return False
                elif choice == u'e':  # Keep editing.
                    # Reset the temporary changes to the objects.
                    for obj in objs:
                        obj.read()
                    continue

        # Remove the temporary file before returning.
        finally:
            os.remove(new.name)

    def apply_data(self, objs, old_data, new_data):
        """Take potentially-updated data and apply it to a set of Model
        objects.

        The objects are not written back to the database, so the changes
        are temporary.
        """
        if len(old_data) != len(new_data):
            self._log.warn(u'number of objects changed from {} to {}',
                           len(old_data), len(new_data))

        obj_by_id = {o.id: o for o in objs}
        ignore_fields = self.config['ignore_fields'].as_str_seq()
        for old_dict, new_dict in zip(old_data, new_data):
            # Prohibit any changes to forbidden fields to avoid
            # clobbering `id` and such by mistake.
            forbidden = False
            for key in ignore_fields:
                if old_dict.get(key) != new_dict.get(key):
                    self._log.warn(u'ignoring object whose {} changed', key)
                    forbidden = True
                    break
            if forbidden:
                continue

            id_ = int(old_dict['id'])
            apply_(obj_by_id[id_], new_dict)

    def save_changes(self, objs):
        """Save a list of updated Model objects to the database.
        """
        # Save to the database and possibly write tags.
        for ob in objs:
            if ob._dirty:
                self._log.debug(u'saving changes to {}', ob)
                ob.try_sync(ui.should_write(), ui.should_move())

    # Methods for interactive importer execution.

    def before_choose_candidate_listener(self, session, task):
        """Append an "Edit" choice and an "edit Candidates" choice (if
        there are candidates) to the interactive importer prompt.
        """
        choices = [PromptChoice('d', 'eDit', self.importer_edit)]
        if task.candidates:
            choices.append(PromptChoice('c', 'edit Candidates',
                                        self.importer_edit_candidate))

        return choices

    def importer_edit(self, session, task):
        """Callback for invoking the functionality during an interactive
        import session on the *original* item tags.
        """
        # Assign temporary ids to the Items.
        for i, obj in enumerate(task.items):
            obj.id = i + 1

        # Present the YAML to the user and let her change it.
        fields = self._get_fields(album=False, extra=[])
        success = self.edit_objects(task.items, fields)

        # Remove temporary ids.
        for obj in task.items:
            obj.id = None

        # Save the new data.
        if success:
            # Return action.RETAG, which makes the importer write the tags
            # to the files if needed without re-applying metadata.
            return action.RETAG
        else:
            # Edit cancelled / no edits made. Revert changes.
            for obj in task.items:
                obj.read()

    def importer_edit_candidate(self, session, task):
        """Callback for invoking the functionality during an interactive
        import session on a *candidate*. The candidate's metadata is
        applied to the original items.
        """
        # Prompt the user for a candidate.
        sel = ui.input_options([], numrange=(1, len(task.candidates)))
        # Force applying the candidate on the items.
        task.match = task.candidates[sel - 1]
        task.apply_metadata()

        return self.importer_edit(session, task)
<commit_msg>Read and write edit plugin yaml as utf-8<commit_after># -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Open metadata information in a text editor to let the user edit it.
"""
from __future__ import division, absolute_import, print_function

from beets import plugins
from beets import util
from beets import ui
from beets.dbcore import types
from beets.importer import action
from beets.ui.commands import _do_query, PromptChoice
from copy import deepcopy
import codecs
import subprocess
import yaml
from tempfile import NamedTemporaryFile
import os
import six

# These "safe" types can avoid the format/parse cycle that most fields go
# through: they are safe to edit with native YAML types.
SAFE_TYPES = (types.Float, types.Integer, types.Boolean)

class ParseError(Exception):
    """The modified file is unreadable. The user should be offered a chance to
    fix the error.
    """

def edit(filename, log):
    """Open `filename` in a text editor.
    """
    cmd = util.shlex_split(util.editor_command())
    cmd.append(filename)
    log.debug(u'invoking editor command: {!r}', cmd)
    try:
        subprocess.call(cmd)
    except OSError as exc:
        raise ui.UserError(u'could not run editor command {!r}: {}'.format(
            cmd[0], exc
        ))

def dump(arg):
    """Dump a sequence of dictionaries as YAML for editing.
    """
    return yaml.safe_dump_all(
        arg,
        allow_unicode=True,
        default_flow_style=False,
    )

def load(s):
    """Read a sequence of YAML documents back to a list of dictionaries
    with string keys.

    Can raise a `ParseError`.
    """
    try:
        out = []
        for d in yaml.load_all(s):
            if not isinstance(d, dict):
                raise ParseError(
                    u'each entry must be a dictionary; found {}'.format(
                        type(d).__name__
                    )
                )

            # Convert all keys to strings. They started out as strings,
            # but the user may have inadvertently messed this up.
            out.append({six.text_type(k): v for k, v in d.items()})

    except yaml.YAMLError as e:
        raise ParseError(u'invalid YAML: {}'.format(e))
    return out

def _safe_value(obj, key, value):
    """Check whether the `value` is safe to represent in YAML and trust as
    returned from parsed YAML.

    This ensures that values do not change their type when the user edits their
    YAML representation.
    """
    typ = obj._type(key)
    return isinstance(typ, SAFE_TYPES) and isinstance(value, typ.model_type)

def flatten(obj, fields):
    """Represent `obj`, a `dbcore.Model` object, as a dictionary for
    serialization. Only include the given `fields` if provided;
    otherwise, include everything.

    The resulting dictionary's keys are strings and the values are
    safely YAML-serializable types.
    """
    # Format each value.
    d = {}
    for key in obj.keys():
        value = obj[key]
        if _safe_value(obj, key, value):
            # A safe value that is faithfully representable in YAML.
            d[key] = value
        else:
            # A value that should be edited as a string.
            d[key] = obj.formatted()[key]

    # Possibly filter field names.
    if fields:
        return {k: v for k, v in d.items() if k in fields}
    else:
        return d

def apply_(obj, data):
    """Set the fields of a `dbcore.Model` object according to a
    dictionary.

    This is the opposite of `flatten`. The `data` dictionary should have
    strings as values.
    """
    for key, value in data.items():
        if _safe_value(obj, key, value):
            # A safe value *stayed* represented as a safe type. Assign it
            # directly.
            obj[key] = value
        else:
            # Either the field was stringified originally or the user changed
            # it from a safe type to an unsafe one. Parse it as a string.
            obj.set_parse(key, six.text_type(value))

class EditPlugin(plugins.BeetsPlugin):

    def __init__(self):
        super(EditPlugin, self).__init__()

        self.config.add({
            # The default fields to edit.
            'albumfields': 'album albumartist',
            'itemfields': 'track title artist album',

            # Silently ignore any changes to these fields.
            'ignore_fields': 'id path',
        })

        self.register_listener('before_choose_candidate',
                               self.before_choose_candidate_listener)

    def commands(self):
        edit_command = ui.Subcommand(
            'edit',
            help=u'interactively edit metadata'
        )
        edit_command.parser.add_option(
            u'-f', u'--field',
            metavar='FIELD',
            action='append',
            help=u'edit this field also',
        )
        edit_command.parser.add_option(
            u'--all',
            action='store_true', dest='all',
            help=u'edit all fields',
        )
        edit_command.parser.add_album_option()
        edit_command.func = self._edit_command
        return [edit_command]

    def _edit_command(self, lib, opts, args):
        """The CLI command function for the `beet edit` command.
        """
        # Get the objects to edit.
        query = ui.decargs(args)
        items, albums = _do_query(lib, query, opts.album, False)
        objs = albums if opts.album else items
        if not objs:
            ui.print_(u'Nothing to edit.')
            return

        # Get the fields to edit.
        if opts.all:
            fields = None
        else:
            fields = self._get_fields(opts.album, opts.field)
        self.edit(opts.album, objs, fields)

    def _get_fields(self, album, extra):
        """Get the set of fields to edit.
        """
        # Start with the configured base fields.
        if album:
            fields = self.config['albumfields'].as_str_seq()
        else:
            fields = self.config['itemfields'].as_str_seq()

        # Add the requested extra fields.
        if extra:
            fields += extra

        # Ensure we always have the `id` field for identification.
        fields.append('id')

        return set(fields)

    def edit(self, album, objs, fields):
        """The core editor function.

        - `album`: A flag indicating whether we're editing Items or Albums.
        - `objs`: The `Item`s or `Album`s to edit.
        - `fields`: The set of field names to edit (or None to edit
          everything).
        """
        # Present the YAML to the user and let her change it.
        success = self.edit_objects(objs, fields)

        # Save the new data.
        if success:
            self.save_changes(objs)

    def edit_objects(self, objs, fields):
        """Dump a set of Model objects to a file as text, ask the user
        to edit it, and apply any changes to the objects.

        Return a boolean indicating whether the edit succeeded.
        """
        # Get the content to edit as raw data structures.
        old_data = [flatten(o, fields) for o in objs]

        # Set up a temporary file with the initial data for editing.
        if six.PY2:
            new = NamedTemporaryFile(mode='w', suffix='.yaml', delete=False)
        else:
            new = NamedTemporaryFile(mode='w', suffix='.yaml', delete=False,
                                     encoding='utf-8')
        old_str = dump(old_data)
        new.write(old_str)
        new.close()

        # Loop until we have parseable data and the user confirms.
        try:
            while True:
                # Ask the user to edit the data.
                edit(new.name, self._log)

                # Read the data back after editing and check whether anything
                # changed.
                with codecs.open(new.name, encoding='utf-8') as f:
                    new_str = f.read()
                if new_str == old_str:
                    ui.print_(u"No changes; aborting.")
                    return False

                # Parse the updated data.
                try:
                    new_data = load(new_str)
                except ParseError as e:
                    ui.print_(u"Could not read data: {}".format(e))
                    if ui.input_yn(u"Edit again to fix? (Y/n)", True):
                        continue
                    else:
                        return False

                # Show the changes.
                # If the objects are not on the DB yet, we need a copy of their
                # original state for show_model_changes.
                objs_old = [deepcopy(obj) if not obj._db else None
                            for obj in objs]
                self.apply_data(objs, old_data, new_data)
                changed = False
                for obj, obj_old in zip(objs, objs_old):
                    changed |= ui.show_model_changes(obj, obj_old)
                if not changed:
                    ui.print_(u'No changes to apply.')
                    return False

                # Confirm the changes.
                choice = ui.input_options(
                    (u'continue Editing', u'apply', u'cancel')
                )
                if choice == u'a':  # Apply.
                    return True
                elif choice == u'c':  # Cancel.
                    return False
                elif choice == u'e':  # Keep editing.
                    # Reset the temporary changes to the objects.
                    for obj in objs:
                        obj.read()
                    continue

        # Remove the temporary file before returning.
        finally:
            os.remove(new.name)

    def apply_data(self, objs, old_data, new_data):
        """Take potentially-updated data and apply it to a set of Model
        objects.

        The objects are not written back to the database, so the changes
        are temporary.
        """
        if len(old_data) != len(new_data):
            self._log.warn(u'number of objects changed from {} to {}',
                           len(old_data), len(new_data))

        obj_by_id = {o.id: o for o in objs}
        ignore_fields = self.config['ignore_fields'].as_str_seq()
        for old_dict, new_dict in zip(old_data, new_data):
            # Prohibit any changes to forbidden fields to avoid
            # clobbering `id` and such by mistake.
            forbidden = False
            for key in ignore_fields:
                if old_dict.get(key) != new_dict.get(key):
                    self._log.warn(u'ignoring object whose {} changed', key)
                    forbidden = True
                    break
            if forbidden:
                continue

            id_ = int(old_dict['id'])
            apply_(obj_by_id[id_], new_dict)

    def save_changes(self, objs):
        """Save a list of updated Model objects to the database.
        """
        # Save to the database and possibly write tags.
        for ob in objs:
            if ob._dirty:
                self._log.debug(u'saving changes to {}', ob)
                ob.try_sync(ui.should_write(), ui.should_move())

    # Methods for interactive importer execution.

    def before_choose_candidate_listener(self, session, task):
        """Append an "Edit" choice and an "edit Candidates" choice (if
        there are candidates) to the interactive importer prompt.
        """
        choices = [PromptChoice('d', 'eDit', self.importer_edit)]
        if task.candidates:
            choices.append(PromptChoice('c', 'edit Candidates',
                                        self.importer_edit_candidate))

        return choices

    def importer_edit(self, session, task):
        """Callback for invoking the functionality during an interactive
        import session on the *original* item tags.
        """
        # Assign temporary ids to the Items.
        for i, obj in enumerate(task.items):
            obj.id = i + 1

        # Present the YAML to the user and let her change it.
        fields = self._get_fields(album=False, extra=[])
        success = self.edit_objects(task.items, fields)

        # Remove temporary ids.
        for obj in task.items:
            obj.id = None

        # Save the new data.
        if success:
            # Return action.RETAG, which makes the importer write the tags
            # to the files if needed without re-applying metadata.
            return action.RETAG
        else:
            # Edit cancelled / no edits made. Revert changes.
            for obj in task.items:
                obj.read()

    def importer_edit_candidate(self, session, task):
        """Callback for invoking the functionality during an interactive
        import session on a *candidate*. The candidate's metadata is
        applied to the original items.
        """
        # Prompt the user for a candidate.
        sel = ui.input_options([], numrange=(1, len(task.candidates)))
        # Force applying the candidate on the items.
        task.match = task.candidates[sel - 1]
        task.apply_metadata()

        return self.importer_edit(session, task)
<|endoftext|>


<commit_before>"""
beam_aligner
Author: Russell Klopfer
"""

class Alignment(object):
    def __init__(self, final_node, source_seq, target_seq):
        """
        Construct a new alignment from the given parameters.

        :param final_node:
        :param source_seq:
        :param target_seq:
        :return:
        :rtype: Alignment
        """
        self.nodes = final_node.flatten()
        self.nodes.reverse()

        self.source_seq = source_seq
        self.target_seq = target_seq
        self.cost = final_node.cost

    def size(self):
        return len(self.nodes)

    def get_source(self, align_x):
        return self.source_seq[self.nodes[align_x].sourcePos]

    def get_target(self, align_x):
        return self.target_seq[self.nodes[align_x].targetPos]

    def get_type(self, align_x):
        return self.nodes[align_x].align_type

    def __str__(self):
        return ("size={} len(source)={}, len(target)={}, cost={}"
                .format(self.size(), len(self.source_seq), len(self.target_seq), self.cost)
                )

    def pretty_print(self):
        pretty = self.__str__() + "\n"
        for i in xrange(self.size()):
            a_type = self.get_type(i)
            source = '' if a_type == AlignmentType.INS else self.get_source(i)
            target = '' if a_type == AlignmentType.DEL else self.get_target(i)
            pretty += "{:<30}{:^10}{:>30}\n".format(source, a_type, target)
        return pretty

class AlignmentType:
    def __init__(self):
        raise AssertionError("Cannot instantiate this 'enum' type")

    START = "START"
    MATCH = "MATCH"
    SUB = "SUB"
    INS = "INS"
    DEL = "DEL"

class AlignmentNode(object):
    def __init__(self, align_type, previous, source_pos, target_pos, cost):
        """

        :rtype: AlignmentNode
        """
        self.align_type = align_type
        self.previous = previous
        self.cost = cost
        self.sourcePos = source_pos
        self.targetPos = target_pos

    def flatten(self):
        """
        Flatten the linked nodes into a list.
        :return:
        :rtype: list
        """
        flat = []
        current = self
        while current != Aligner.START_NODE:
            flat.append(current)
            current = current.previous
        return flat

    def is_source_end(self, source):
        return self.sourcePos >= len(source) - 1

    def is_target_end(self, target):
        return self.targetPos >= len(target) - 1

    def __eq__(self, other):
        return (
            self.previous == other.previous and
            self.cost == other.cost and
            self.sourcePos == other.sourcePos and
            self.targetPos == other.targetPos
        )

    def __lt__(self, other):
        return self.cost < other.cost

    def __le__(self, other):
        return self.cost <= other.cost

    def __str__(self):
        return "{type: %s, source_pos: %d, target_pos: %d, cost: %d}" % (
            self.align_type, self.sourcePos, self.targetPos, self.cost)

class Aligner(object):
    # Constants
    START_NODE = AlignmentNode(AlignmentType.START, None, -1, -1, 0)
    MAX_BEAM_SIZE = 50000

    def __init__(self, beam_size, sub_cost=1, ins_cost=1, del_cost=1):
        """
        Construct a new aligner with the given parameters.

        :param sub_cost: cost of substituting one token for another
        :param ins_cost: cost of inserting a token (something in target that isn't in source)
        :param del_cost: cost of deleting a token (something in source that isn't in target)
        :return: a new Aligner object
        :rtype: Aligner
        """
        assert beam_size > 0, "beam_size must be > 0"
        self.beam_size = beam_size
        self.sub_cost = sub_cost
        self.ins_cost = ins_cost
        self.del_cost = del_cost

    def __str__(self):
        return ("beam_size: {}, sub_cost: {}, ins_cost: {}, del_cost: {}".
                format(self.beam_size, self.sub_cost, self.ins_cost, self.del_cost))

    @staticmethod
    def __print_heap(heap, n=None):
        """
        Prints the top-n elements of the heap.

        :param heap:
        :param n:
        :return: None
        """
        n = len(heap) if n is None else min(n, len(heap))
        print "HEAP ({}) [[[".format(len(heap))
        for i in xrange(n):
            print heap[i]
        print "]]]"

    @staticmethod
    def __add_new_node(node_list, node):
        node_list.append(node)

    @staticmethod
    def __prune(node_list, top_n):
        return sorted(node_list, key=lambda node: node.cost)[:min(top_n, len(node_list))]

    def align(self, source, target):
        """

        :param source:
        :param target:
        :return:
        :rtype: Alignment
        """
        heap = [Aligner.START_NODE]

        while heap[0].sourcePos < len(source) - 1 or heap[0].targetPos < len(target) - 1:
            # Aligner.__print_heap(heap, 10)
            node_list = []
            for node in heap:
                self.__populate_nodes(node_list, node, source, target)
            heap = Aligner.__prune(node_list, self.beam_size)

        return Alignment(heap[0], source, target)

    def __populate_nodes(self, node_list, previous_node, source, target):
        source_x = previous_node.sourcePos
        target_x = previous_node.targetPos
        source_finished = source_x >= len(source) - 1
        target_finished = target_x >= len(target) - 1

        def insertion():
            return AlignmentNode(AlignmentType.INS, previous_node, source_x, target_x + 1,
                                 previous_node.cost + self.ins_cost)

        def deletion():
            return AlignmentNode(AlignmentType.DEL, previous_node, source_x + 1, target_x,
                                 previous_node.cost + self.del_cost)

        def match():
            return AlignmentNode(AlignmentType.MATCH, previous_node, source_x + 1, target_x + 1,
                                 previous_node.cost)

        def substitution():
            return AlignmentNode(AlignmentType.SUB, previous_node, source_x + 1, target_x + 1,
                                 previous_node.cost + self.sub_cost)

        # we're at the end of the alignment already
        if source_finished and target_finished:
            return

        # we're at the end of the source sequence, this must be an insertion
        if source_finished:
            Aligner.__add_new_node(node_list, insertion())
            return

        # we're at the end of the target sequence, this must be a deletion
        if target_finished:
            Aligner.__add_new_node(node_list, deletion())
            return

        # match
        if source[source_x + 1] == target[target_x + 1]:
            Aligner.__add_new_node(node_list, match())
        # sub
        else:
            Aligner.__add_new_node(node_list, substitution())

        # always allow for insertions
        Aligner.__add_new_node(node_list, insertion())
        # and deletions
        Aligner.__add_new_node(node_list, deletion())

def __find_reasonable_beam(source, target):
    """
    Get a reasonable beam size for the given source and target. This method may or may not be cheap.

    :param source:
    :param target:
    :return:
    :rtype: int
    """
    source_len = len(source)
    target_len = len(target)

    # print "source_len={} target_len={}".format(source_len, target_len)
    max_dist = 0
    for source_x in xrange(source_len):
        try:
            target_x = target.index(source[source_x])
        except ValueError:
            continue
        # print "source_x={} target_x={}".format(source_x, target_x)
        max_dist = max(abs(target_x - source_x), max_dist)

    # this is too small
    return 100 + max_dist
    # if max dist is huge you're probably fucked anyway
    # return 3 ** max_dist

def __sigmoid(v):
    return 1. / (1. + 2. ** -v)

def __find_reasonable_params(source, target):
    source_len = len(source)
    target_len = len(target)

    # beam_size = __find_reasonable_beam(source, target)
    beam_size = 200

    # if they're equal length, give some power to SUB
    if source_len == target_len:
        return beam_size, .9, 1, 1

    del_cost = 1 + __sigmoid(target_len - source_len)
    ins_cost = 3 - del_cost
    # sub_cost = 1.5
    sub_cost = min(del_cost, ins_cost) * 1.1

    return beam_size, sub_cost, ins_cost, del_cost

def construct_reasonable_aligner(source, target):
    return Aligner(*__find_reasonable_params(source, target))

def align(source, target):
    return construct_reasonable_aligner(source, target).align(source, target)
<commit_msg>better logging<commit_after>"""
beam_aligner
Author: Russell Klopfer
"""

class Alignment(object):
    def __init__(self, final_node, source_seq, target_seq):
        """
        Construct a new alignment from the given parameters.

        :param final_node:
        :param source_seq:
        :param target_seq:
        :return:
        :rtype: Alignment
        """
        self.nodes = final_node.flatten()
        self.nodes.reverse()

        self.source_seq = source_seq
        self.target_seq = target_seq
        self.cost = final_node.cost

    def size(self):
        return len(self.nodes)

    def get_source(self, align_x):
        return self.source_seq[self.nodes[align_x].sourcePos]

    def get_target(self, align_x):
        return self.target_seq[self.nodes[align_x].targetPos]

    def get_type(self, align_x):
        return self.nodes[align_x].align_type

    def __str__(self):
        return ("size={} len(source)={}, len(target)={}, cost={}"
                .format(self.size(), len(self.source_seq), len(self.target_seq), self.cost)
                )

    def pretty_print(self):
        pretty = self.__str__() + "\n"
        for i in xrange(self.size()):
            a_type = self.get_type(i)
            source = '' if a_type == AlignmentType.INS else self.get_source(i)
            target = '' if a_type == AlignmentType.DEL else self.get_target(i)
            pretty += "{:<30}{:^10}{:>30}\n".format(source, a_type, target)
        return pretty

class AlignmentType:
    def __init__(self):
        raise AssertionError("Cannot instantiate this 'enum' type")

    START = "START"
    MATCH = "MATCH"
    SUB = "SUB"
    INS = "INS"
    DEL = "DEL"

class AlignmentNode(object):
    def __init__(self, align_type, previous, source_pos, target_pos, cost):
        """

        :rtype: AlignmentNode
        """
        self.align_type = align_type
        self.previous = previous
        self.cost = cost
        self.sourcePos = source_pos
        self.targetPos = target_pos

    def flatten(self):
        """
        Flatten the linked nodes into a list.
        :return:
        :rtype: list
        """
        flat = []
        current = self
        while current != Aligner.START_NODE:
            flat.append(current)
            current = current.previous
        return flat

    def is_source_end(self, source):
        return self.sourcePos >= len(source) - 1

    def is_target_end(self, target):
        return self.targetPos >= len(target) - 1

    def __eq__(self, other):
        return (
            self.previous == other.previous and
            self.cost == other.cost and
            self.sourcePos == other.sourcePos and
            self.targetPos == other.targetPos
        )

    def __lt__(self, other):
        return self.cost < other.cost

    def __le__(self, other):
        return self.cost <= other.cost

    def __str__(self):
        return "{type: %s, source_pos: %d, target_pos: %d, cost: %d}" % (
            self.align_type, self.sourcePos, self.targetPos, self.cost)

class Aligner(object):
    # Constants
    START_NODE = AlignmentNode(AlignmentType.START, None, -1, -1, 0)
    MAX_BEAM_SIZE = 50000

    def __init__(self, beam_size, sub_cost=1, ins_cost=1, del_cost=1):
        """
        Construct a new aligner with the given parameters.

        :param sub_cost: cost of substituting one token for another
        :param ins_cost: cost of inserting a token (something in target that isn't in source)
        :param del_cost: cost of deleting a token (something in source that isn't in target)
        :return: a new Aligner object
        :rtype: Aligner
        """
        assert beam_size > 0, "beam_size must be > 0"
        self.beam_size = beam_size
        self.sub_cost = sub_cost
        self.ins_cost = ins_cost
        self.del_cost = del_cost

    def __str__(self):
        return ("beam_size: {}, sub_cost: {}, ins_cost: {}, del_cost: {}".
                format(self.beam_size, self.sub_cost, self.ins_cost, self.del_cost))

    @staticmethod
    def __print_heap(heap, source, target, n=None):
        """
        Prints the top-n elements of the heap.

        :param heap:
        :param n:
        :return: None
        """
        n = len(heap) if n is None else min(n, len(heap))
        print "HEAP ({}) [[[".format(len(heap))
        for i in xrange(n):
            print "{}.) {}\n".format(i, Alignment(heap[i], source, target).pretty_print())
        print "]]]"

    @staticmethod
    def __add_new_node(node_list, node):
        node_list.append(node)

    @staticmethod
    def __prune(node_list, top_n):
        return sorted(node_list, key=lambda node: node.cost)[:min(top_n, len(node_list))]

    def align(self, source, target):
        """

        :param source:
        :param target:
        :return:
        :rtype: Alignment
        """
        heap = [Aligner.START_NODE]

        while heap[0].sourcePos < len(source) - 1 or heap[0].targetPos < len(target) - 1:
            Aligner.__print_heap(heap, source, target, 5)
            node_list = []
            for node in heap:
                self.__populate_nodes(node_list, node, source, target)
            heap = Aligner.__prune(node_list, self.beam_size)

        return Alignment(heap[0], source, target)

    def __populate_nodes(self, node_list, previous_node, source, target):
        source_x = previous_node.sourcePos
        target_x = previous_node.targetPos
        source_finished = source_x >= len(source) - 1
        target_finished = target_x >= len(target) - 1

        def insertion():
            return AlignmentNode(AlignmentType.INS, previous_node, source_x, target_x + 1,
                                 previous_node.cost + self.ins_cost)

        def deletion():
            return AlignmentNode(AlignmentType.DEL, previous_node, source_x + 1, target_x,
                                 previous_node.cost + self.del_cost)

        def match():
            return AlignmentNode(AlignmentType.MATCH, previous_node, source_x + 1, target_x + 1,
                                 previous_node.cost)

        def substitution():
            return AlignmentNode(AlignmentType.SUB, previous_node, source_x + 1, target_x + 1,
                                 previous_node.cost + self.sub_cost)

        # we're at the end of the alignment already
        if source_finished and target_finished:
            return

        # we're at the end of the source sequence, this must be an insertion
        if source_finished:
            Aligner.__add_new_node(node_list, insertion())
            return

        # we're at the end of the target sequence, this must be a deletion
        if target_finished:
            Aligner.__add_new_node(node_list, deletion())
            return

        # match
        if source[source_x + 1] == target[target_x + 1]:
            Aligner.__add_new_node(node_list, match())
        # sub
        else:
            Aligner.__add_new_node(node_list, substitution())

        # always allow for insertions
        Aligner.__add_new_node(node_list, insertion())
        # and deletions
        Aligner.__add_new_node(node_list, deletion())

def __find_reasonable_beam(source, target):
    """
    Get a reasonable beam size for the given source and target. This method may or may not be cheap.

    :param source:
    :param target:
    :return:
    :rtype: int
    """
    source_len = len(source)
    target_len = len(target)

    # print "source_len={} target_len={}".format(source_len, target_len)
    max_dist = 0
    for source_x in xrange(source_len):
        try:
            target_x = target.index(source[source_x])
        except ValueError:
            continue
        # print "source_x={} target_x={}".format(source_x, target_x)
        max_dist = max(abs(target_x - source_x), max_dist)

    # this is too small
    return 100 + max_dist
    # if max dist is huge you're probably fucked anyway
    # return 3 ** max_dist

def __sigmoid(v):
    return 1. / (1. + 2. ** -v)

def __find_reasonable_params(source, target):
    source_len = len(source)
    target_len = len(target)

    # beam_size = __find_reasonable_beam(source, target)
    beam_size = 200

    # if they're equal length, give some power to SUB
    if source_len == target_len:
        return beam_size, .9, 1, 1

    del_cost = 1 + __sigmoid(target_len - source_len)
    ins_cost = 3 - del_cost
    # sub_cost = 1.5
    sub_cost = min(del_cost, ins_cost) * 1.1

    return beam_size, sub_cost, ins_cost, del_cost

def construct_reasonable_aligner(source, target):
    return Aligner(*__find_reasonable_params(source, target))

def align(source, target):
    return construct_reasonable_aligner(source, target).align(source, target)
<|endoftext|>


<commit_before># encoding: utf8
from __future__ import unicode_literals, print_function

from os import path

from ..language import Language
from ..attrs import LANG
from ..tokens import Doc

from .language_data import *

class Japanese(Language):
    lang = 'ja'

    def make_doc(self, text):
        try:
            from janome.tokenizer import Tokenizer
        except ImportError:
            raise ImportError("The Japanese tokenizer requires the Janome library: "
                              "https://github.com/mocobeta/janome")
        words = [x.surface for x in Tokenizer().tokenize(text)]
        return Doc(self.vocab, words=words, spaces=[False]*len(words))
<commit_msg>adding export japanese<commit_after># encoding: utf8
from __future__ import unicode_literals, print_function

from os import path

from ..language import Language
from ..attrs import LANG
from ..tokens import Doc

from .language_data import *

class Japanese(Language):
    lang = 'ja'

    def make_doc(self, text):
        try:
            from janome.tokenizer import Tokenizer
        except ImportError:
            raise ImportError("The Japanese tokenizer requires the Janome library: "
                              "https://github.com/mocobeta/janome")
        words = [x.surface for x in Tokenizer().tokenize(text)]
        return Doc(self.vocab, words=words, spaces=[False]*len(words))

EXPORT = Japanese<|endoftext|>


<commit_before>#!/usr/bin/python
import sys
import os
import tempfile
from optparse import OptionParser
from sfa.util.config import Config
import sfa.util.xmlrpcprotocol as xmlrpcprotocol
import sfa.util.misc as misc
from sfa.trust.certificate import Keypair, Certificate
from sfa.trust.credential import Credential
from sfa.trust.gid import GID
from sfa.trust.hierarchy import Hierarchy

def get_server(url=None, port=None, keyfile=None, certfile=None,verbose=False):
    """
    returns an xmlrpc connection to the service a the specified
    address
    """
    if url:
        url_parts = url.split(":")
        if len(url_parts) >1:
            pass
        else:
            url = "http://%(url)s:%(port)s" % locals()
    else:
        # connect to registry by default
        config = Config()
        addr, port = config.SFA_REGISTRY_HOST, config.SFA_REGISTRY_PORT
        url = "http://%(addr)s:%(port)s" % locals()

    if verbose:
        print "Contacting registry at: %(url)s" % locals()

    server = xmlrpcprotocol.get_server(url, keyfile, certfile)
    return server

def create_default_dirs():
    config = Config()
    hierarchy = Hierarchy()
    config_dir = config.config_path
    trusted_certs_dir = config.get_trustedroots_dir()
    authorities_dir = hierarchy.basedir
    all_dirs = [config_dir, trusted_certs_dir, authorities_dir]
    for dir in all_dirs:
        if not os.path.exists(dir):
            os.makedirs(dir)

def get_node_key(registry=None, verbose=False):
    # this call requires no authentication,
    # so we can generate a random keypair here
    subject="component"
    keyfile = tempfile.mktemp()
    certfile = tempfile.mktemp()
    key = Keypair(create=True)
    key.save_to_file(keyfile)
    cert = Certificate(subject=subject)
    cert.set_issuer(key=key, subject=subject)
    cert.set_pubkey(key)
    cert.sign()
    cert.save_to_file(certfile)

    registry = get_server(url = registry, keyfile=keyfile, certfile=certfile)
    registry.get_key()

def create_server_keypair(keyfile=None, certfile=None, hrn="component", verbose=False):
    """
    create the server key/cert pair in the right place
    """
    key = Keypair(filename=keyfile)
    key.save_to_file(keyfile)
    cert = Certificate(subject=hrn)
    cert.set_issuer(key=key, subject=hrn)
    cert.set_pubkey(key)
    cert.sign()
    cert.save_to_file(certfile, save_parents=True)

def get_credential(registry=None, verbose=False):
    config = Config()
    hierarchy = Hierarchy()
    key_dir= hierarchy.basedir
    data_dir = config.data_path
    config_dir = config.config_path
    credfile = data_dir + os.sep + 'node.cred'
    # check for existing credential
    if os.path.exists(credfile):
        if verbose:
            print "Loading Credential from %(credfile)s " % locals()
        cred = Credential(filename=credfile).save_to_string(save_parents=True)
    else:
        if verbose:
            print "Getting credential from registry"
        # make sure node private key exists
        node_pkey_file = config_dir + os.sep + "node.key"
        node_gid_file = config_dir + os.sep + "node.gid"
        if not os.path.exists(node_pkey_file) or \
           not os.path.exists(node_gid_file):
            get_node_key(registry=registry, verbose=verbose)

        gid = GID(filename=node_gid_file)
        hrn = gid.get_hrn()
        # create server key and certificate
        keyfile =data_dir + os.sep + "server.key"
        certfile = data_dir + os.sep + "server.cert"
        key = Keypair(filename=node_pkey_file)
        key.save_to_file(keyfile)
        create_server_keypair(keyfile, certfile, hrn, verbose)

        # get credential from registry
        registry = get_server(url=registry, keyfile=keyfile, certfile=certfile)
        cert_str = cert.save_to_string(save_parents=True)
        cred = registry.get_self_credential(cert_str, 'node', hrn)
        Credential(string=cred).save_to_file(credfile)

    return cred

def get_trusted_certs(registry=None, verbose=False):
    """
    refresh our list of trusted certs.
    """
    # define useful variables
    config = Config()
    data_dir = config.data_path
    trusted_certs_dir = config.get_trustedroots_dir()
    keyfile = data_dir + os.sep + "server.key"
    certfile = data_dir + os.sep + "server.cert"
    node_gid_file = config_dir + os.sep + "node.gid"
    node_gid = GID(filename=node_gid_file)
    hrn = node_gid.get_hrn()
    # get credential
    cred = get_credential(registry=registry, verbose=verbose)
    # make sure server key cert pair exists
    create_server_keypair(keyfile=keyfile, certfile=certfile, hrn=hrn, verbose=verbose)
    registry = get_server(url=registry, keyfile=keyfile, certfile=certfile)
    # get the trusted certs and save them in the right place
    if verbose:
        print "Getting trusted certs from registry"
    trusted_certs = registry.get_trusted_certs(cred)
    trusted_gid_names = []
    for gid_str in trusted_certs:
        gid = GID(string=gid_str)
        gid.decode()
        trusted_gid_names.append(gid.get_hrn())
        gid_filename = trusted_certs_dir + os.sep + gid.get_hrn() + ".gid"
        if verbose:
            print "Writing GID for %s as %s" % (gid.get_hrn(), gid_filename)
        gid.save_to_file(gid_filename, save_parents=True)

    # remove old certs
    all_gids_names = os.listdir(trusted_certs_dir)
    for gid_name in all_gids_names:
        if gid_name not in trusted_gid_names:
            os.unlink(trusted_certs_dir + os.sep + gid_name)

def get_gids(registry=None, verbose=False):
    """
    Get the gid for all instantiated slices on this node and store it
    in /etc/sfa/slice.gid in the slice's filesystem
    """
    # define useful variables
    config = Config()
    data_dir = config.data_path
    trusted_certs_dir = config.get_trustedroots_dir()
    keyfile = data_dir + os.sep + "server.key"
    certfile = data_dir + os.sep + "server.cert"
    node_gid_file = config_dir + os.sep + "node.gid"
    node_gid = GID(filename=node_gid_file)
    hrn = node_gid.get_hrn()
    interface_hrn = config.SFA_INTERFACE_HRN
    # get credential
    cred = get_credential(registry=registry, verbose=verbose)
    # make sure server key cert pair exists
    create_server_keypair(keyfile=keyfile, certfile=certfile, hrn=hrn, verbose=verbose)
    registry = get_server(url=registry, keyfile=keyfile, certfile=certfile)

    if verbose:
        print "Getting current slices on this node"
    # get a list of slices on this node
    from sfa.plc.api import ComponentAPI
    api = ComponentAPI()
    slicenames = api.nodemanger.GetXIDs().keys()
    slicename_to_hrn = lambda name: ".".join([interface_hrn, name.replace('_', '.')])
    hrns = map(slicename_to_hrn, slicenames)

    if verbose:
        print "Getting gids for slices on this node from registry"
    # get the gids
    # and save them in the right palce
    records = registry.get_gids(cred, hrns)
    for record in records:
        # if this isnt a slice record skip it
        if not record['type'] == 'slice':
            continue
        slicename = misc.hrn_to_pl_slicename(record['hrn'])
        # if this slice isnt really instatiated skip it
        if not os.path.exists("/vservers/%(slicename)s" % locals()):
            continue

        # save the slice gid in /etc/sfa/ in the vservers filesystem
        vserver_path = "/vserver/%(slicename)s" % locals()
        gid = record['gid']
        slice_gid_filename = os.sep.join([vserver_path, config_dir, "slice.gid"])
        if verbose:
            print "Saving GID for %(slicename)s as %(slice_gid_filename)s" % locals()
        GID(string=gid).save_to_file(slice_gid_filename, save_parents=True)
        # save the node gid in /etc/sfa
        node_gid_filename = os.sep.join([vserver_path, config_dir, "node.gid"])
        if verbose:
            print "Saving node GID for %(slicename)s as %(slice_gid_filename)s" % locals()
        node_gid.save_to_file(node_gid_filename, save_parents=True)

def dispatch(options, args):

    create_default_dirs()
    if options.key:
        if options.verbose:
            print "Getting the component's pkey"
        get_node_key(registry=options.registry, verbose=options.verbose)
    if options.certs:
        if options.verbose:
            print "Getting the component's trusted certs"
        get_certs(verbose=options.verbose)
    if options.gids:
        if options.verbose:
            print "Geting the component's GIDs"
        get_gids(verbose=options.verbose)

def main():
    args = sys.argv
    prog_name = args[0]
    parser = OptionParser(usage="%(prog_name)s [options]" % locals())
    parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
                      default=False, help="Be verbose")
    parser.add_option("-r", "--registry", dest="registry", default=None,
                      help="Url of registry to contact")
    parser.add_option("-k", "--key", dest="key", action="store_true",
                     default=False,
                     help="Get the node's pkey from the registry")
    parser.add_option("-c", "--certs", dest="certs", action="store_true",
                      default=False,
                      help="Get the trusted certs from the registry")
    parser.add_option("-g", "--gids", dest="gids", action="store_true",
                      default=False,
                      help="Get gids for all the slices on the component")

    (options, args) = parser.parse_args()

    dispatch(options, args)

if __name__ == '__main__':
    main()
<commit_msg>get_certs() should be get_trusted_certs()<commit_after>#!/usr/bin/python
import sys
import os
import tempfile
from optparse import OptionParser
from sfa.util.config import Config
import sfa.util.xmlrpcprotocol as xmlrpcprotocol
import sfa.util.misc as misc
from sfa.trust.certificate import Keypair, Certificate
from sfa.trust.credential import Credential
from sfa.trust.gid import GID
from sfa.trust.hierarchy import Hierarchy

def get_server(url=None, port=None, keyfile=None, certfile=None,verbose=False):
    """
    returns an xmlrpc connection to the service a the specified
    address
    """
    if url:
        url_parts = url.split(":")
        if len(url_parts) >1:
            pass
        else:
            url = "http://%(url)s:%(port)s" % locals()
    else:
        # connect to registry by default
        config = Config()
        addr, port = config.SFA_REGISTRY_HOST, config.SFA_REGISTRY_PORT
        url = "http://%(addr)s:%(port)s" % locals()

    if verbose:
        print "Contacting registry at: %(url)s" % locals()

    server = xmlrpcprotocol.get_server(url, keyfile, certfile)
    return server

def create_default_dirs():
    config = Config()
    hierarchy = Hierarchy()
    config_dir = config.config_path
    trusted_certs_dir = config.get_trustedroots_dir()
    authorities_dir = hierarchy.basedir
    all_dirs = [config_dir, trusted_certs_dir, authorities_dir]
    for dir in all_dirs:
        if not os.path.exists(dir):
            os.makedirs(dir)

def get_node_key(registry=None, verbose=False):
    # this call requires no authentication,
    # so we can generate a random keypair here
    subject="component"
    keyfile = tempfile.mktemp()
    certfile = tempfile.mktemp()
    key = Keypair(create=True)
    key.save_to_file(keyfile)
    cert = Certificate(subject=subject)
    cert.set_issuer(key=key, subject=subject)
    cert.set_pubkey(key)
    cert.sign()
    cert.save_to_file(certfile)

    registry = get_server(url = registry, keyfile=keyfile, certfile=certfile)
    registry.get_key()

def create_server_keypair(keyfile=None, certfile=None, hrn="component", verbose=False):
    """
    create the server key/cert pair in the right place
    """
    key = Keypair(filename=keyfile)
    key.save_to_file(keyfile)
    cert = Certificate(subject=hrn)
    cert.set_issuer(key=key, subject=hrn)
    cert.set_pubkey(key)
    cert.sign()
    cert.save_to_file(certfile, save_parents=True)

def get_credential(registry=None, verbose=False):
    config = Config()
    hierarchy = Hierarchy()
    key_dir= hierarchy.basedir
    data_dir = config.data_path
    config_dir = config.config_path
    credfile = data_dir + os.sep + 'node.cred'
    # check for existing credential
    if os.path.exists(credfile):
        if verbose:
            print "Loading Credential from %(credfile)s " % locals()
        cred = Credential(filename=credfile).save_to_string(save_parents=True)
    else:
        if verbose:
            print "Getting credential from registry"
        # make sure node private key exists
        node_pkey_file = config_dir + os.sep + "node.key"
        node_gid_file = config_dir + os.sep + "node.gid"
        if not os.path.exists(node_pkey_file) or \
           not os.path.exists(node_gid_file):
            get_node_key(registry=registry, verbose=verbose)

        gid = GID(filename=node_gid_file)
        hrn = gid.get_hrn()
        # create server key and certificate
        keyfile =data_dir + os.sep + "server.key"
        certfile = data_dir + os.sep + "server.cert"
        key = Keypair(filename=node_pkey_file)
        key.save_to_file(keyfile)
        create_server_keypair(keyfile, certfile, hrn, verbose)

        # get credential from registry
        registry = get_server(url=registry, keyfile=keyfile, certfile=certfile)
        cert_str = cert.save_to_string(save_parents=True)
        cred = registry.get_self_credential(cert_str, 'node', hrn)
        Credential(string=cred).save_to_file(credfile)

    return cred

def get_trusted_certs(registry=None, verbose=False):
    """
    refresh our list of trusted certs.
    """
    # define useful variables
    config = Config()
    data_dir = config.data_path
    trusted_certs_dir = config.get_trustedroots_dir()
    keyfile = data_dir + os.sep + "server.key"
    certfile = data_dir + os.sep + "server.cert"
    node_gid_file = config_dir + os.sep + "node.gid"
    node_gid = GID(filename=node_gid_file)
    hrn = node_gid.get_hrn()
    # get credential
    cred = get_credential(registry=registry, verbose=verbose)
    # make sure server key cert pair exists
    create_server_keypair(keyfile=keyfile, certfile=certfile, hrn=hrn, verbose=verbose)
    registry = get_server(url=registry, keyfile=keyfile, certfile=certfile)
    # get the trusted certs and save them in the right place
    if verbose:
        print "Getting trusted certs from registry"
    trusted_certs = registry.get_trusted_certs(cred)
    trusted_gid_names = []
    for gid_str in trusted_certs:
        gid = GID(string=gid_str)
        gid.decode()
        trusted_gid_names.append(gid.get_hrn())
        gid_filename = trusted_certs_dir + os.sep + gid.get_hrn() + ".gid"
        if verbose:
            print "Writing GID for %s as %s" % (gid.get_hrn(), gid_filename)
        gid.save_to_file(gid_filename, save_parents=True)

    # remove old certs
    all_gids_names = os.listdir(trusted_certs_dir)
    for gid_name in all_gids_names:
        if gid_name not in trusted_gid_names:
            os.unlink(trusted_certs_dir + os.sep + gid_name)

def get_gids(registry=None, verbose=False):
    """
    Get the gid for all instantiated slices on this node and store it
    in /etc/sfa/slice.gid in the slice's filesystem
    """
    # define useful variables
    config = Config()
    data_dir = config.data_path
    trusted_certs_dir = config.get_trustedroots_dir()
    keyfile = data_dir + os.sep + "server.key"
    certfile = data_dir + os.sep + "server.cert"
    node_gid_file = config_dir + os.sep + "node.gid"
    node_gid = GID(filename=node_gid_file)
    hrn = node_gid.get_hrn()
    interface_hrn = config.SFA_INTERFACE_HRN
    # get credential
    cred = get_credential(registry=registry, verbose=verbose)
    # make sure server key cert pair exists
    create_server_keypair(keyfile=keyfile, certfile=certfile, hrn=hrn, verbose=verbose)
    registry = get_server(url=registry, keyfile=keyfile, certfile=certfile)

    if verbose:
        print "Getting current slices on this node"
    # get a list of slices on this node
    from sfa.plc.api import ComponentAPI
    api = ComponentAPI()
    slicenames = api.nodemanger.GetXIDs().keys()
    slicename_to_hrn = lambda name: ".".join([interface_hrn, name.replace('_', '.')])
    hrns = map(slicename_to_hrn, slicenames)

    if verbose:
        print "Getting gids for slices on this node from registry"
    # get the gids
    # and save them in the right palce
    records = registry.get_gids(cred, hrns)
    for record in records:
        # if this isnt a slice record skip it
        if not record['type'] == 'slice':
            continue
        slicename = misc.hrn_to_pl_slicename(record['hrn'])
        # if this slice isnt really instatiated skip it
        if not os.path.exists("/vservers/%(slicename)s" % locals()):
            continue

        # save the slice gid in /etc/sfa/ in the vservers filesystem
        vserver_path = "/vserver/%(slicename)s" % locals()
        gid = record['gid']
        slice_gid_filename = os.sep.join([vserver_path, config_dir, "slice.gid"])
        if verbose:
            print "Saving GID for %(slicename)s as %(slice_gid_filename)s" % locals()
        GID(string=gid).save_to_file(slice_gid_filename, save_parents=True)
        # save the node gid in /etc/sfa
        node_gid_filename = os.sep.join([vserver_path, config_dir, "node.gid"])
        if verbose:
            print "Saving node GID for %(slicename)s as %(slice_gid_filename)s" % locals()
        node_gid.save_to_file(node_gid_filename, save_parents=True)

def dispatch(options, args):

    create_default_dirs()
    if options.key:
        if options.verbose:
            print "Getting the component's pkey"
        get_node_key(registry=options.registry, verbose=options.verbose)
    if options.certs:
        if options.verbose:
            print "Getting the component's trusted certs"
        get_trusted_certs(verbose=options.verbose)
    if options.gids:
        if options.verbose:
            print "Geting the component's GIDs"
        get_gids(verbose=options.verbose)

def main():
    args = sys.argv
    prog_name = args[0]
    parser = OptionParser(usage="%(prog_name)s [options]" % locals())
    parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
                      default=False, help="Be verbose")
    parser.add_option("-r", "--registry", dest="registry", default=None,
                      help="Url of registry to contact")
    parser.add_option("-k", "--key", dest="key", action="store_true",
                     default=False,
                     help="Get the node's pkey from the registry")
    parser.add_option("-c", "--certs", dest="certs", action="store_true",
                      default=False,
                      help="Get the trusted certs from the registry")
    parser.add_option("-g", "--gids", dest="gids", action="store_true",
                      default=False,
                      help="Get gids for all the slices on the component")

    (options, args) = parser.parse_args()

    dispatch(options, args)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from django.conf.urls import patterns, include, url
# Uncomment the next two lines to enable the admin:
from django.contrib import admin
from core import views
# import django_bitcoin
admin.autodiscover()

# Routers provide an easy way of automatically determining the URL conf.
# router = routers.DefaultRouter()
# router.register(r'users', UserViewSet)
# router.register(r'wallets', WalletViewSet)
# router.register(r'address', BitcoinAddressViewSet)
urlpatterns = patterns(
    '',
    # Examples:
    url(r'^admin/', include(admin.site.urls)),
    url(r'^', include(admin.site.urls)),
    url(r'^bitcoin/', include('django_bitcoin.urls')),
    url(r'^api/', include('bitcoin_api.urls')),
    url(r'^docs/', include('rest_framework_swagger.urls')),
    url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')),
    url(r'^api-token-auth/', 'rest_framework_jwt.views.obtain_jwt_token'),
    # overwirtte some views of userena
    url(r'^member/', include('member.urls')),
    url(r'^member/', include('userena.urls')),
    url(r'^home/', views.home),

)
<commit_msg>no message<commit_after>from django.conf.urls import patterns, include, url
# Uncomment the next two lines to enable the admin:
from django.contrib import admin
from core import views
# import django_bitcoin
admin.autodiscover()

# Routers provide an easy way of automatically determining the URL conf.
# router = routers.DefaultRouter()
# router.register(r'users', UserViewSet)
# router.register(r'wallets', WalletViewSet)
# router.register(r'address', BitcoinAddressViewSet)
urlpatterns = patterns(
    '',
    # Examples:
    url(r'^admin/', include(admin.site.urls)),
#     url(r'^', include(admin.site.urls)),
    url(r'^bitcoin/', include('django_bitcoin.urls')),
    url(r'^api/', include('bitcoin_api.urls')),
    url(r'^docs/', include('rest_framework_swagger.urls')),
    url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')),
    url(r'^api-token-auth/', 'rest_framework_jwt.views.obtain_jwt_token'),
    # overwirtte some views of userena
    url(r'^member/', include('member.urls')),
    url(r'^member/', include('userena.urls')),
    url(r'^home/', views.home),

)
<|endoftext|>


<commit_before># This file is part of beets.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Adds support for ipfs. Requires go-ipfs and a running ipfs daemon
"""
from beets import ui
from beets.plugins import BeetsPlugin

import subprocess
import shutil
import os

class IPFSPlugin(BeetsPlugin):

    def __init__(self):
        super(IPFSPlugin, self).__init__()

    def commands(self):
        cmd = ui.Subcommand('ipfs',
                            help='interact with ipfs')
        cmd.parser.add_option('-a', '--add', dest='add',
                                    action='store_true',
                                    help='Add to ipfs')
        cmd.parser.add_option('-g', '--get', dest='get',
                                    action='store_true',
                                    help='Get from ipfs')
        cmd.parser.add_option('-p', '--publish', dest='publish',
                                    action='store_true',
                                    help='Publish local library to ipfs')
        cmd.parser.add_option('-i', '--import', dest='_import',
                                    action='store_true',
                                    help='Import remote library from ipfs')

        def func(lib, opts, args):
            if opts.add:
                for album in lib.albums(ui.decargs(args)):
                    self.ipfs_add(album)
                    album.store()

            if opts.get:
                self.ipfs_get(lib, ui.decargs(args))

            if opts.publish:
                self.ipfs_publish(lib)

            if opts._import:
                self.ipfs_import(lib, ui.decargs(args))

        cmd.func = func
        return [cmd]

    def ipfs_add(self, lib):
        try:
            album_dir = lib.item_dir()
        except AttributeError:
            return
        self._log.info('Adding {0} to ipfs', album_dir)

        _proc = subprocess.Popen(["ipfs", "add", "-q", "-p", "-r", album_dir],
                                 stdout=subprocess.PIPE)
        count = 0
        while True:
            line = _proc.stdout.readline().strip()

            if line != '':
                if count < len(lib.items()):
                    item = lib.items()[count]
                    self._log.info("item: {0}", line)
                    item.ipfs = line
                    item.store()
                    count += 1
                else:
                    self._log.info("album: {0}", line)
                    lib.ipfs = line
            else:
                break
        return True

    def ipfs_get(self, lib, _hash):
        try:
            subprocess.check_output(["ipfs", "get", _hash[0]],
                                    stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as err:
            self._log.error('Failed to get {0} from ipfs.\n{1}',
                            _hash[0], err.output)
            return False

        self._log.info('Getting {0} from ipfs', _hash[0])
        imp = ui.commands.TerminalImportSession(lib, loghandler=None,
                                                query=None, paths=_hash)
        imp.run()
        shutil.rmtree(_hash[0])

    def ipfs_publish(self, lib):
        _proc = subprocess.Popen(["ipfs", "add", "-q", "-p", lib.path],
                                 stdout=subprocess.PIPE)
        self._log.info("hash of library: {0}", _proc.stdout.readline())

    def ipfs_import(self, lib, _hash):
        # TODO: should be able to tag libraries, for example by nicks
        subprocess.Popen(["ipfs", "get", _hash[0]])
        path = os.path.dirname(lib.path) + "/" + _hash[0] + ".db"
        shutil.move(_hash[0], path)
<commit_msg>Add right files when adding folders with[...]<commit_after># This file is part of beets.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Adds support for ipfs. Requires go-ipfs and a running ipfs daemon
"""
from beets import ui
from beets.plugins import BeetsPlugin

import subprocess
import shutil
import os

class IPFSPlugin(BeetsPlugin):

    def __init__(self):
        super(IPFSPlugin, self).__init__()

    def commands(self):
        cmd = ui.Subcommand('ipfs',
                            help='interact with ipfs')
        cmd.parser.add_option('-a', '--add', dest='add',
                                    action='store_true',
                                    help='Add to ipfs')
        cmd.parser.add_option('-g', '--get', dest='get',
                                    action='store_true',
                                    help='Get from ipfs')
        cmd.parser.add_option('-p', '--publish', dest='publish',
                                    action='store_true',
                                    help='Publish local library to ipfs')
        cmd.parser.add_option('-i', '--import', dest='_import',
                                    action='store_true',
                                    help='Import remote library from ipfs')

        def func(lib, opts, args):
            if opts.add:
                for album in lib.albums(ui.decargs(args)):
                    self.ipfs_add(album)
                    album.store()

            if opts.get:
                self.ipfs_get(lib, ui.decargs(args))

            if opts.publish:
                self.ipfs_publish(lib)

            if opts._import:
                self.ipfs_import(lib, ui.decargs(args))

        cmd.func = func
        return [cmd]

    def ipfs_add(self, lib):
        try:
            album_dir = lib.item_dir()
        except AttributeError:
            return
        self._log.info('Adding {0} to ipfs', album_dir)

        _proc = subprocess.Popen(["ipfs", "add", "-q", "-r", album_dir],
                                 stdout=subprocess.PIPE)

        all_lines = _proc.stdout.readlines()
        length = len(all_lines)

        for linenr, line in enumerate(all_lines):
            if linenr == length-1:
                # last printed line is the album hash
                self._log.info("album: {0}", line)
                lib.ipfs = line
            else:
                try:
                    item = lib.items()[linenr]
                    self._log.info("item: {0}", line)
                    item.ipfs = line
                    item.store()
                except IndexError:
                    # if there's non music files in the to-add folder they'll
                    # get ignored here
                    pass

        return True

    def ipfs_get(self, lib, _hash):
        try:
            subprocess.check_output(["ipfs", "get", _hash[0]],
                                    stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as err:
            self._log.error('Failed to get {0} from ipfs.\n{1}',
                            _hash[0], err.output)
            return False

        self._log.info('Getting {0} from ipfs', _hash[0])
        imp = ui.commands.TerminalImportSession(lib, loghandler=None,
                                                query=None, paths=_hash)
        imp.run()
        shutil.rmtree(_hash[0])

    def ipfs_publish(self, lib):
        _proc = subprocess.Popen(["ipfs", "add", "-q", "-p", lib.path],
                                 stdout=subprocess.PIPE)
        self._log.info("hash of library: {0}", _proc.stdout.readline())

    def ipfs_import(self, lib, _hash):
        # TODO: should be able to tag libraries, for example by nicks
        subprocess.Popen(["ipfs", "get", _hash[0]])
        path = os.path.dirname(lib.path) + "/" + _hash[0] + ".db"
        shutil.move(_hash[0], path)
<|endoftext|>