<commit_before>import bisect
import numpy as np

from ms2ldaviz.celery_tasks import app
from basicviz.models import Experiment,Mass2MotifInstance,MotifMatch

def get_experiment_features(experiment):
    motifs = experiment.mass2motif_set.all()
    feature_instances = Mass2MotifInstance.objects.filter(mass2motif__in = motifs)
    features = set([fi.feature for fi in feature_instances])
    return features

@app.task
def match_motifs_set(experiment_id,base_experiment_id,min_score_to_save = 0.5):

    # Get the expeiment objetcs
    experiment = Experiment.objects.get(id = experiment_id)
    base_experiment = Experiment.objects.get(id = base_experiment_id)

    # get the fs objects
    fs = experiment.featureset
    base_fs = base_experiment.featureset

    print fs,base_fs

    # get the features used in each experiment
    features = get_experiment_features(experiment)
    base_features = get_experiment_features(base_experiment)

    base_feature_name_dict = {}
    for feature in base_features:
        base_feature_name_dict[feature.name] = feature

    feature_map = {}
    if fs == base_fs:
        # the task is easy because both sets of motifs are in the same set
        for feature in features:
            if feature in base_features:
                feature_map[feature] = [feature]
    elif fs.name == 'binned_01' and base_fs.name == 'binned_005':
        # allow the comparison of things with wider fs with the original motifs
        for feature in features:
            ftype = feature.name.split('_')[0]
            if ftype == 'fragment' or ftype == 'loss':
                # work out the names of the two 005 features
                lowval = feature.min_mz
                highval = feature.max_mz
                fname1 = ftype + "_{:.4f}".format(lowval + 0.005/2.0)
                fname2 = ftype + "_{:.4f}".format(highval - 0.005/2.0)
                temp = []
                if fname1 in base_feature_name_dict:
                    temp.append(base_feature_name_dict[fname1])
                if fname2 in base_feature_name_dict:
                    temp.append(base_feature_name_dict[fname2])
                if len(temp) > 0:
                    feature_map[feature] = temp
            else if ftype == 'mzdiff':
                temp = []
                fname1 = ftype + "_{:.4f}".format(feature.min_mz)
                fname2 = ftype + "_{:.4f}".format(feature.max_mz)
                if feature.name in base_feature_name_dict:
                    temp.append(base_feature_name_dict[feature.name])
                if fname1 in base_feature_name_dict:
                    temp.append(base_feature_name_dict[fname1])
                if fname2 in base_feature_name_dict:
                    temp.append(base_feature_name_dict[fname2])
                if len(temp) > 0:
                    feature_map[feature] = temp
    elif fs.name == 'binned_005' and basefs.name == 'binned_01':
        # Each will only match to one, but the matches will appear more than once
        for feature in features:
            ftype = feature.name.split('_')[0]
            if ftype == 'fragent' or ftype == 'loss':
                lowval = feature.min_mz
                highval = feature.max_mz
                # Each feature will map perfectly into the first or second half of
                # one of the base features
                # we just need to work out if it is a first half or a second half one
                # Easiest way is to check if upper lor lower val is a name in the other set
                uppername = ftype+ "_{:.4f}".format(highval)
                if uppername in base_feature_name_dict:
                    feature_map[feature] = base_feature_name_dict[uppername]
                else:
                    lowername = ftype+ "_{:.4f}".format(lowval)
                    if lowername in base_feature_name_dict:
                        feature_map[lowername] = base_feature_name_dict[lowername]
            else if ftype == 'mzdiff':
                if feature.name in base_feature_name_dict:
                    feature_map[feature] = [base_feature_name_dict[feature.name]]
                else:
                    # Complicated case: 0.005 diffs either align with the centre
                    # of the 0.01 diff or at either end
                    # if the latter, we try linking to the 0.01 feature above and below
                    temp = []
                    fmz = float(feature.name.split('_')[1])
                    drop_name = 'mzdiff_{:.4f}'.format(fmz - 0.005)
                    up_name = 'mzdiff_{:.4f}'.format(fmz + 0.005)
                    if drop_name in base_feature_name_dict:
                        temp.append(base_feature_name_dict[drop_name])
                    if up_name in base_feature_name_dict:
                        temp.append(base_feature_name_dict[up_name])
                    if len(temp) > 0:
                        feature_map[feature] = temp
    else:
        # Ensure backward compatibility
        return match_motifs(experiment_id,base_experiment_id,min_score_to_save = min_score_to_save)

    mzd_features = filter(lambda x: x.name.startswith('mzdiff'),feature_map.keys())
    for f in mzd_features:
        print f,feature_map[f]

    print "Found matches of {} out of {} features".format(len(feature_map),len(features))

    # get the motifs
    motifs = experiment.mass2motif_set.all()
    motif_dict = {}
    motif_norms = {}

    base_motifs = base_experiment.mass2motif_set.all()
    base_motif_dict = {}
    base_motif_norms = {}

    # get the instances and compute the normalisation terms
    mfs = Mass2MotifInstance.objects.filter(mass2motif__in = motifs)
    base_mfs = Mass2MotifInstance.objects.filter(mass2motif__in = base_motifs)

    # Store them in dicts for quick access
    for mf in mfs:
        motif = mf.mass2motif
        if not motif in motif_dict:
            motif_dict[motif] = {}
            motif_norms[motif] = 0.0
        motif_dict[motif][mf.feature] = mf.probability
        motif_norms[motif] += mf.probability ** 2

    for mf in base_mfs:
        motif = mf.mass2motif
        if not motif in base_motif_dict:
            base_motif_dict[motif] = {}
            base_motif_norms[motif] = 0.0
        base_motif_dict[motif][mf.feature] = mf.probability
        base_motif_norms[motif] += mf.probability ** 2

    for motif in motif_norms:
        motif_norms[motif] = np.sqrt(motif_norms[motif])
    for motif in base_motif_norms:
        base_motif_norms[motif] = np.sqrt(base_motif_norms[motif])

    # compute the cosine scores
    matches = []
    for motif in motif_dict.keys():
        best_score = 0.0
        best_base = None
        best_list = None
        for base_motif in base_motif_dict.keys():
            match_list = []
            score = 0.0
            # Note: with matching now, one feature could hit two base features. In this case
            # we can't use it twice or the normalisation will mess up. So, we use the assignment
            # that gives the max score
            for feature,probability in motif_dict[motif].items():
                temp_list = None
                if feature in feature_map:
                    map_features = feature_map[feature]
                    for map_feature in map_features:
                        temp_score = 0.0
                        if map_feature in base_motif_dict[base_motif]:
                            map_probability = base_motif_dict[base_motif][map_feature]
                            t = probability * map_probability
                            if t >= temp_score:
                                temp_score = probability * map_probability
                                temp_list = (feature,probability,map_feature,map_probability)
                    match_list.append(temp_list)
                    score += temp_score

            score /= motif_norms[motif]
            score /= base_motif_norms[base_motif]

            if score > best_score: # only keep the best one from this query motif
                best_score = score
                best_base = base_motif
                best_list = match_list
        if best_score >= min_score_to_save:
            matches.append((motif,best_base,best_score,best_list))
            print motif,best_score,best_base

    for match in matches:
        frommotif = match[0]
        tomotif = match[1]
        score = match[2]
        mm,status = MotifMatch.objects.get_or_create(frommotif = frommotif,tomotif = tomotif)
        mm.score = score
        mm.save()

@app.task
def match_motifs(experiment_id,base_experiment_id,min_score_to_save = 0.5):

    experiment = Experiment.objects.get(id = experiment_id)
    base_experiment = Experiment.objects.get(id = base_experiment_id)

    # Get the features associated with this experiment for matching
    # features = experiment.feature_set.all().order_by('min_mz')
    # base_features = base_experiment.feature_set.all().order_by('min_mz')
    features = get_experiment_features(experiment)
    base_features = get_experiment_features(base_experiment)
    features = sorted(features,key = lambda x: x.min_mz)
    base_features = sorted(base_features,key = lambda x: x.min_mz)

    print len(features)
    print len(base_features)

    feature_name_dict = {}
    base_feature_name_dict = {}
    for feature in features:
        feature_name_dict[feature.name] = feature
    for feature in base_features:
        base_feature_name_dict[feature.name] = feature

    # Split into features and losses in case we need to match by mz
    base_fragments = [f for f in base_features if f.name.startswith('fragment')]
    base_losses = [f for f in base_features if f.name.startswith('loss')]

    # The following code is just for testing should be removed...

    # base_mz = [float(f.name.split('_')[1]) for f in base_fragments]
    # for i,f in enumerate(base_fragments):
    #     if not f.min_mz:
    #         f.min_mz = base_mz[i] - 5e-6 * base_mz[i]
    #     if not f.max_mz:
    #         f.max_mz = base_mz[i] + 5e-6 * base_mz[i]

    # base_mz = [float(f.name.split('_')[1]) for f in base_losses]
    # for i,f in enumerate(base_losses):
    #     if not f.min_mz:
    #         f.min_mz = base_mz[i] - 5e-6 * base_mz[i]
    #     if not f.max_mz:
    #         f.max_mz = base_mz[i] + 5e-6 * base_mz[i]

    # base_fragments = sorted(base_fragments,key = lambda x: x.min_mz)
    # base_losses = sorted(base_losses,key = lambda x: x.min_mz)

    # END OF TESTING BLOCK

    base_fragment_min_mz = [f.min_mz for f in base_fragments]
    base_fragment_max_mz = [f.max_mz for f in base_fragments]

    base_loss_min_mz = [f.min_mz for f in base_losses]
    base_loss_max_mz = [f.max_mz for f in base_losses]

    feature_map = {}

    # Here we map the features across the experiments.
    # The code first tries to match based on the name
    # and then reverts to finding the lightest feature
    # in the base set that includes the feature from the
    # original set.

    for feature_name in feature_name_dict:
        feature = feature_name_dict[feature_name]
        if feature_name in base_feature_name_dict:
            feature_map[feature] = base_feature_name_dict[feature_name]
        else:
            feature_type = feature_name.split('_')[0]
            feature_mz = float(feature_name.split('_')[1])
            if feature_type == 'fragment':
                if feature_mz >= base_fragment_min_mz[0] and feature_mz <= base_fragment_max_mz[-1]:
                    pos = bisect.bisect_right(base_fragment_min_mz,feature_mz) - 1
                    if feature_mz <= base_fragment_max_mz[pos]:
                        feature_map[feature] = base_fragments[pos]
            elif feature_type == 'loss':
                if feature_mz >= base_loss_min_mz[0] and feature_mz <= base_loss_max_mz[-1]:
                    pos = bisect.bisect_right(base_loss_min_mz,feature_mz) - 1
                    if feature_mz <= base_loss_max_mz[pos]:
                        feature_map[feature] = base_losses[pos]

    # Extract the motifs from each experiment
    motifs = experiment.mass2motif_set.all()
    motif_dict = {}
    motif_norms = {}

    base_motifs = base_experiment.mass2motif_set.all()
    base_motif_dict = {}
    base_motif_norms = {}

    mfs = Mass2MotifInstance.objects.filter(mass2motif__in = motifs)
    base_mfs = Mass2MotifInstance.objects.filter(mass2motif__in = base_motifs)

    # Store them in dicts for quick access
    for mf in mfs:
        motif = mf.mass2motif
        if not motif in motif_dict:
            motif_dict[motif] = {}
            motif_norms[motif] = 0.0
        motif_dict[motif][mf.feature] = mf.probability
        motif_norms[motif] += mf.probability ** 2

    for mf in base_mfs:
        motif = mf.mass2motif
        if not motif in base_motif_dict:
            base_motif_dict[motif] = {}
            base_motif_norms[motif] = 0.0
        base_motif_dict[motif][mf.feature] = mf.probability
        base_motif_norms[motif] += mf.probability ** 2

    for motif in motif_norms:
        motif_norms[motif] = np.sqrt(motif_norms[motif])
    for motif in base_motif_norms:
        base_motif_norms[motif] = np.sqrt(base_motif_norms[motif])

    # Compute the cosine scores between all pairs
    matches = []
    for motif in motif_dict.keys():
        best_score = 0.0
        best_base = None
        best_list = None
        for base_motif in base_motif_dict.keys():
            match_list = []
            score = 0.0
            for feature,probability in motif_dict[motif].items():
                if feature in feature_map:
                    map_feature = feature_map[feature]
                    if map_feature in base_motif_dict[base_motif]:
                        map_probability = base_motif_dict[base_motif][map_feature]
                        score += probability * map_probability
                        match_list.append((feature,probability,map_feature,map_probability))

            score /= motif_norms[motif]
            score /= base_motif_norms[base_motif]

            if score > best_score:
                best_score = score
                best_base = base_motif
                best_list = match_list
        if best_score >= min_score_to_save:
            matches.append((motif,best_base,best_score,best_list))
            print motif,best_score,best_base

    for match in matches:
        frommotif = match[0]
        tomotif = match[1]
        score = match[2]
        mm,status = MotifMatch.objects.get_or_create(frommotif = frommotif,tomotif = tomotif)
        mm.score = score
        mm.save()
<commit_msg>fixed mzdiff feature matching<commit_after>import bisect
import numpy as np

from ms2ldaviz.celery_tasks import app
from basicviz.models import Experiment,Mass2MotifInstance,MotifMatch

def get_experiment_features(experiment):
    motifs = experiment.mass2motif_set.all()
    feature_instances = Mass2MotifInstance.objects.filter(mass2motif__in = motifs)
    features = set([fi.feature for fi in feature_instances])
    return features

@app.task
def match_motifs_set(experiment_id,base_experiment_id,min_score_to_save = 0.5):

    # Get the expeiment objetcs
    experiment = Experiment.objects.get(id = experiment_id)
    base_experiment = Experiment.objects.get(id = base_experiment_id)

    # get the fs objects
    fs = experiment.featureset
    base_fs = base_experiment.featureset

    print fs,base_fs

    # get the features used in each experiment
    features = get_experiment_features(experiment)
    base_features = get_experiment_features(base_experiment)

    base_feature_name_dict = {}
    for feature in base_features:
        base_feature_name_dict[feature.name] = feature

    feature_map = {}
    if fs == base_fs:
        # the task is easy because both sets of motifs are in the same set
        for feature in features:
            if feature in base_features:
                feature_map[feature] = [feature]
    elif fs.name == 'binned_01' and base_fs.name == 'binned_005':
        # allow the comparison of things with wider fs with the original motifs
        for feature in features:
            ftype = feature.name.split('_')[0]
            if ftype == 'fragment' or ftype == 'loss':
                # work out the names of the two 005 features
                lowval = feature.min_mz
                highval = feature.max_mz
                fname1 = ftype + "_{:.4f}".format(lowval + 0.005/2.0)
                fname2 = ftype + "_{:.4f}".format(highval - 0.005/2.0)
                temp = []
                if fname1 in base_feature_name_dict:
                    temp.append(base_feature_name_dict[fname1])
                if fname2 in base_feature_name_dict:
                    temp.append(base_feature_name_dict[fname2])
                if len(temp) > 0:
                    feature_map[feature] = temp
            elif ftype == 'mzdiff':
                temp = []
                fname1 = ftype + "_{:.4f}".format(feature.min_mz)
                fname2 = ftype + "_{:.4f}".format(feature.max_mz)
                if feature.name in base_feature_name_dict:
                    temp.append(base_feature_name_dict[feature.name])
                if fname1 in base_feature_name_dict:
                    temp.append(base_feature_name_dict[fname1])
                if fname2 in base_feature_name_dict:
                    temp.append(base_feature_name_dict[fname2])
                if len(temp) > 0:
                    feature_map[feature] = temp
    elif fs.name == 'binned_005' and basefs.name == 'binned_01':
        # Each will only match to one, but the matches will appear more than once
        for feature in features:
            ftype = feature.name.split('_')[0]
            if ftype == 'fragent' or ftype == 'loss':
                lowval = feature.min_mz
                highval = feature.max_mz
                # Each feature will map perfectly into the first or second half of
                # one of the base features
                # we just need to work out if it is a first half or a second half one
                # Easiest way is to check if upper lor lower val is a name in the other set
                uppername = ftype+ "_{:.4f}".format(highval)
                if uppername in base_feature_name_dict:
                    feature_map[feature] = base_feature_name_dict[uppername]
                else:
                    lowername = ftype+ "_{:.4f}".format(lowval)
                    if lowername in base_feature_name_dict:
                        feature_map[lowername] = base_feature_name_dict[lowername]
            elif ftype == 'mzdiff':
                if feature.name in base_feature_name_dict:
                    feature_map[feature] = [base_feature_name_dict[feature.name]]
                else:
                    # Complicated case: 0.005 diffs either align with the centre
                    # of the 0.01 diff or at either end
                    # if the latter, we try linking to the 0.01 feature above and below
                    temp = []
                    fmz = float(feature.name.split('_')[1])
                    drop_name = 'mzdiff_{:.4f}'.format(fmz - 0.005)
                    up_name = 'mzdiff_{:.4f}'.format(fmz + 0.005)
                    if drop_name in base_feature_name_dict:
                        temp.append(base_feature_name_dict[drop_name])
                    if up_name in base_feature_name_dict:
                        temp.append(base_feature_name_dict[up_name])
                    if len(temp) > 0:
                        feature_map[feature] = temp
    else:
        # Ensure backward compatibility
        return match_motifs(experiment_id,base_experiment_id,min_score_to_save = min_score_to_save)

    mzd_features = filter(lambda x: x.name.startswith('mzdiff'),feature_map.keys())
    for f in mzd_features:
        print f,feature_map[f]

    print "Found matches of {} out of {} features".format(len(feature_map),len(features))

    # get the motifs
    motifs = experiment.mass2motif_set.all()
    motif_dict = {}
    motif_norms = {}

    base_motifs = base_experiment.mass2motif_set.all()
    base_motif_dict = {}
    base_motif_norms = {}

    # get the instances and compute the normalisation terms
    mfs = Mass2MotifInstance.objects.filter(mass2motif__in = motifs)
    base_mfs = Mass2MotifInstance.objects.filter(mass2motif__in = base_motifs)

    # Store them in dicts for quick access
    for mf in mfs:
        motif = mf.mass2motif
        if not motif in motif_dict:
            motif_dict[motif] = {}
            motif_norms[motif] = 0.0
        motif_dict[motif][mf.feature] = mf.probability
        motif_norms[motif] += mf.probability ** 2

    for mf in base_mfs:
        motif = mf.mass2motif
        if not motif in base_motif_dict:
            base_motif_dict[motif] = {}
            base_motif_norms[motif] = 0.0
        base_motif_dict[motif][mf.feature] = mf.probability
        base_motif_norms[motif] += mf.probability ** 2

    for motif in motif_norms:
        motif_norms[motif] = np.sqrt(motif_norms[motif])
    for motif in base_motif_norms:
        base_motif_norms[motif] = np.sqrt(base_motif_norms[motif])

    # compute the cosine scores
    matches = []
    for motif in motif_dict.keys():
        best_score = 0.0
        best_base = None
        best_list = None
        for base_motif in base_motif_dict.keys():
            match_list = []
            score = 0.0
            # Note: with matching now, one feature could hit two base features. In this case
            # we can't use it twice or the normalisation will mess up. So, we use the assignment
            # that gives the max score
            for feature,probability in motif_dict[motif].items():
                temp_list = None
                if feature in feature_map:
                    map_features = feature_map[feature]
                    for map_feature in map_features:
                        temp_score = 0.0
                        if map_feature in base_motif_dict[base_motif]:
                            map_probability = base_motif_dict[base_motif][map_feature]
                            t = probability * map_probability
                            if t >= temp_score:
                                temp_score = probability * map_probability
                                temp_list = (feature,probability,map_feature,map_probability)
                    match_list.append(temp_list)
                    score += temp_score

            score /= motif_norms[motif]
            score /= base_motif_norms[base_motif]

            if score > best_score: # only keep the best one from this query motif
                best_score = score
                best_base = base_motif
                best_list = match_list
        if best_score >= min_score_to_save:
            matches.append((motif,best_base,best_score,best_list))
            print motif,best_score,best_base

    for match in matches:
        frommotif = match[0]
        tomotif = match[1]
        score = match[2]
        mm,status = MotifMatch.objects.get_or_create(frommotif = frommotif,tomotif = tomotif)
        mm.score = score
        mm.save()

@app.task
def match_motifs(experiment_id,base_experiment_id,min_score_to_save = 0.5):

    experiment = Experiment.objects.get(id = experiment_id)
    base_experiment = Experiment.objects.get(id = base_experiment_id)

    # Get the features associated with this experiment for matching
    # features = experiment.feature_set.all().order_by('min_mz')
    # base_features = base_experiment.feature_set.all().order_by('min_mz')
    features = get_experiment_features(experiment)
    base_features = get_experiment_features(base_experiment)
    features = sorted(features,key = lambda x: x.min_mz)
    base_features = sorted(base_features,key = lambda x: x.min_mz)

    print len(features)
    print len(base_features)

    feature_name_dict = {}
    base_feature_name_dict = {}
    for feature in features:
        feature_name_dict[feature.name] = feature
    for feature in base_features:
        base_feature_name_dict[feature.name] = feature

    # Split into features and losses in case we need to match by mz
    base_fragments = [f for f in base_features if f.name.startswith('fragment')]
    base_losses = [f for f in base_features if f.name.startswith('loss')]

    # The following code is just for testing should be removed...

    # base_mz = [float(f.name.split('_')[1]) for f in base_fragments]
    # for i,f in enumerate(base_fragments):
    #     if not f.min_mz:
    #         f.min_mz = base_mz[i] - 5e-6 * base_mz[i]
    #     if not f.max_mz:
    #         f.max_mz = base_mz[i] + 5e-6 * base_mz[i]

    # base_mz = [float(f.name.split('_')[1]) for f in base_losses]
    # for i,f in enumerate(base_losses):
    #     if not f.min_mz:
    #         f.min_mz = base_mz[i] - 5e-6 * base_mz[i]
    #     if not f.max_mz:
    #         f.max_mz = base_mz[i] + 5e-6 * base_mz[i]

    # base_fragments = sorted(base_fragments,key = lambda x: x.min_mz)
    # base_losses = sorted(base_losses,key = lambda x: x.min_mz)

    # END OF TESTING BLOCK

    base_fragment_min_mz = [f.min_mz for f in base_fragments]
    base_fragment_max_mz = [f.max_mz for f in base_fragments]

    base_loss_min_mz = [f.min_mz for f in base_losses]
    base_loss_max_mz = [f.max_mz for f in base_losses]

    feature_map = {}

    # Here we map the features across the experiments.
    # The code first tries to match based on the name
    # and then reverts to finding the lightest feature
    # in the base set that includes the feature from the
    # original set.

    for feature_name in feature_name_dict:
        feature = feature_name_dict[feature_name]
        if feature_name in base_feature_name_dict:
            feature_map[feature] = base_feature_name_dict[feature_name]
        else:
            feature_type = feature_name.split('_')[0]
            feature_mz = float(feature_name.split('_')[1])
            if feature_type == 'fragment':
                if feature_mz >= base_fragment_min_mz[0] and feature_mz <= base_fragment_max_mz[-1]:
                    pos = bisect.bisect_right(base_fragment_min_mz,feature_mz) - 1
                    if feature_mz <= base_fragment_max_mz[pos]:
                        feature_map[feature] = base_fragments[pos]
            elif feature_type == 'loss':
                if feature_mz >= base_loss_min_mz[0] and feature_mz <= base_loss_max_mz[-1]:
                    pos = bisect.bisect_right(base_loss_min_mz,feature_mz) - 1
                    if feature_mz <= base_loss_max_mz[pos]:
                        feature_map[feature] = base_losses[pos]

    # Extract the motifs from each experiment
    motifs = experiment.mass2motif_set.all()
    motif_dict = {}
    motif_norms = {}

    base_motifs = base_experiment.mass2motif_set.all()
    base_motif_dict = {}
    base_motif_norms = {}

    mfs = Mass2MotifInstance.objects.filter(mass2motif__in = motifs)
    base_mfs = Mass2MotifInstance.objects.filter(mass2motif__in = base_motifs)

    # Store them in dicts for quick access
    for mf in mfs:
        motif = mf.mass2motif
        if not motif in motif_dict:
            motif_dict[motif] = {}
            motif_norms[motif] = 0.0
        motif_dict[motif][mf.feature] = mf.probability
        motif_norms[motif] += mf.probability ** 2

    for mf in base_mfs:
        motif = mf.mass2motif
        if not motif in base_motif_dict:
            base_motif_dict[motif] = {}
            base_motif_norms[motif] = 0.0
        base_motif_dict[motif][mf.feature] = mf.probability
        base_motif_norms[motif] += mf.probability ** 2

    for motif in motif_norms:
        motif_norms[motif] = np.sqrt(motif_norms[motif])
    for motif in base_motif_norms:
        base_motif_norms[motif] = np.sqrt(base_motif_norms[motif])

    # Compute the cosine scores between all pairs
    matches = []
    for motif in motif_dict.keys():
        best_score = 0.0
        best_base = None
        best_list = None
        for base_motif in base_motif_dict.keys():
            match_list = []
            score = 0.0
            for feature,probability in motif_dict[motif].items():
                if feature in feature_map:
                    map_feature = feature_map[feature]
                    if map_feature in base_motif_dict[base_motif]:
                        map_probability = base_motif_dict[base_motif][map_feature]
                        score += probability * map_probability
                        match_list.append((feature,probability,map_feature,map_probability))

            score /= motif_norms[motif]
            score /= base_motif_norms[base_motif]

            if score > best_score:
                best_score = score
                best_base = base_motif
                best_list = match_list
        if best_score >= min_score_to_save:
            matches.append((motif,best_base,best_score,best_list))
            print motif,best_score,best_base

    for match in matches:
        frommotif = match[0]
        tomotif = match[1]
        score = match[2]
        mm,status = MotifMatch.objects.get_or_create(frommotif = frommotif,tomotif = tomotif)
        mm.score = score
        mm.save()
<|endoftext|>


<commit_before>"""
Load a shared library.
"""
import os
import sys
import ctypes
import ctypes.util
import logging
import subprocess

from . import utils, DEFAULT_EXTENSION

logger = logging.getLogger(__name__)

class LoadLibrary(object):

    def __init__(self, path, libtype='cdll'):
        """Load a shared library.

        For example, a C/C++, FORTRAN, CLR, Java, Delphi, LabVIEW, ... library.

        Based on the value of `libtype` or the file extension this class will load the
        shared library as a:

            * :class:`~ctypes.CDLL` if `libtype` is ``'cdll'``,
            * :class:`~ctypes.WinDLL` if `libtype` is ``'windll'``,
            * :class:`~ctypes.OleDLL` if `libtype` is ``'oledll'``,
            * `System.Reflection.Assembly <Assembly_>`_ if `libtype` is ``'net'``, or a
            * :class:`~.py4j.java_gateway.JavaGateway` if `libtype` is ``'java'``.

        .. _Assembly: https://msdn.microsoft.com/en-us/library/system.reflection.assembly(v=vs.110).aspx

        Parameters
        ----------
        path : :class:`str`
            The path to the shared library.

            The search order for finding the shared library is:

                1. assume that a full or a relative (to the current working directory)
                   path is specified,
                2. use :func:`ctypes.util.find_library` to find the shared library file,
                3. search :data:`sys.path`, then
                4. search :data:`os.environ['PATH'] <os.environ>` to find the shared library.

        libtype : :class:`str`, optional
            The library type. The following values are currently supported:

            * ``'cdll'`` -- for a library that uses the __cdecl calling convention
            * ``'windll'`` or ``'oledll'`` -- for a __stdcall calling convention
            * ``'net'`` -- for Microsoft's .NET Framework (Common Language Runtime)
            * ``'java'`` -- for a Java archive, ``.jar``, or Java byte code, ``.class``, file

            Default is ``'cdll'``.

            .. note::
               Since the ``.jar`` or ``.class`` extension uniquely defines a Java library,
               the `libtype` will automatically be set to ``'java'`` if `path` ends with
               ``.jar`` or ``.class``.

        Raises
        ------
        IOError
            If the shared library cannot be loaded.
        TypeError
            If `libtype` is not a supported library type.
        """

        # a reference to the shared library
        self._lib = None

        # a reference to the .NET Runtime Assembly
        self._assembly = None

        # a reference to the Py4J JavaGateway
        self._gateway = None

        # fixes Issue #8, if `path` is a <class 'pathlib.Path'> object
        path = str(path)

        # create a new reference to `path` just in case the
        # DEFAULT_EXTENSION is appended below so that the
        # ctypes.util.find_library function call will use the
        # unmodified value of `path`
        _path = path

        # assume a default extension if no extension was provided
        ext = os.path.splitext(_path)[1]
        if not ext:
            _path += DEFAULT_EXTENSION

        # the .jar or .class extension uniquely defines a Java library
        if ext in ('.jar', '.class'):
            libtype = 'java'

        self._path = os.path.abspath(_path)
        if not os.path.isfile(self._path):
            # for find_library use the original 'path' value since it may be a library name
            # without any prefix like 'lib', suffix like '.so', '.dylib' or version number
            self._path = ctypes.util.find_library(path)
            if self._path is None:  # then search sys.path and os.environ['PATH']
                success = False
                search_dirs = sys.path + os.environ['PATH'].split(os.pathsep)
                for directory in search_dirs:
                    p = os.path.join(directory, _path)
                    if os.path.isfile(p):
                        self._path = p
                        success = True
                        break
                if not success:
                    raise IOError('Cannot find the shared library "{}"'.format(path))

        libtype = libtype.lower()
        if libtype == 'cdll':
            self._lib = ctypes.CDLL(self._path)
        elif libtype == 'windll':
            self._lib = ctypes.WinDLL(self._path)
        elif libtype == 'oledll':
            self._lib = ctypes.OleDLL(self._path)
        elif libtype == 'java':
            if not utils.is_py4j_installed():
                raise IOError('Cannot load a Java file because Py4J is not installed.\n'
                              'To install Py4J run: pip install py4j')

            from py4j.version import __version__
            from py4j.java_gateway import JavaGateway, GatewayParameters

            # the address and port to use to host the py4j.GatewayServer
            address = '127.0.0.1'
            port = utils.get_available_port()

            # find the py4j JAR (needed to import py4j.GatewayServer on the Java side)
            root = os.path.dirname(sys.executable)
            filename = 'py4j'+__version__+'.jar'
            py4j_jar = os.path.join(root, 'share', 'py4j', filename)
            if not os.path.isfile(py4j_jar):
                root = os.path.dirname(root)  # then check one folder up (for unix or venv)
                py4j_jar = os.path.join(root, 'share', 'py4j', filename)
                if not os.path.isfile(py4j_jar):
                    py4j_jar = os.environ.get('PY4J_JAR', '')  # then check the environment variable
                    if not os.path.isfile(py4j_jar):
                        raise IOError('Cannot find {0}\nCreate a PY4J_JAR environment '
                                      'variable to be equal to the full path to {0}'.format(filename))

            # build the java command
            wrapper = os.path.join(os.path.dirname(__file__), 'py4j-wrapper.jar')
            cmd = ['java', '-cp', py4j_jar + os.pathsep + wrapper, 'Py4JWrapper', str(port)]

            # from the URLClassLoader documentation:
            #   Any URL that ends with a '/' is assumed to refer to a directory. Otherwise, the URL
            #   is assumed to refer to a JAR file which will be downloaded and opened as needed.
            if ext == '.jar':
                cmd.append(self._path)
            else:  # it is a .class file
                cmd.append(os.path.dirname(self._path) + '/')

            try:
                # start the py4j.GatewayServer, cannot use subprocess.call() because it blocks
                subprocess.Popen(cmd, stderr=sys.stderr, stdout=sys.stdout)
                err = None
            except IOError:
                err = 'You must have a Java Runtime Environment installed and available on PATH'

            if err:
                raise IOError(err)

            utils.wait_for_server(address, port, 5.0)

            self._gateway = JavaGateway(
                gateway_parameters=GatewayParameters(address=address, port=port)
            )

            self._lib = self._gateway.jvm

        elif libtype == 'net':
            if not utils.is_pythonnet_installed():
                raise IOError('Cannot load a .NET Assembly because pythonnet is not installed.\n'
                              'To install pythonnet run: pip install pythonnet')

            import clr
            try:
                # By default, pythonnet can only load libraries that are for .NET 4.0+.
                #
                # In order to allow pythonnet to load a library from .NET <4.0 the
                # useLegacyV2RuntimeActivationPolicy property needs to be enabled
                # in a <python-executable>.config file. If the following statement
                # raises a FileLoadException then attempt to create the configuration
                # file that has the property enabled and then notify the user why
                # loading the library failed and ask them to re-run their Python
                # script to load the .NET library.
                self._assembly = clr.System.Reflection.Assembly.LoadFile(self._path)

            except clr.System.IO.FileLoadException as err:
                # Example error message that can occur if the library is for .NET <4.0,
                # and the useLegacyV2RuntimeActivationPolicy is not enabled:
                #
                # " Mixed mode assembly is built against version 'v2.0.50727' of the
                #  runtime and cannot be loaded in the 4.0 runtime without additional
                #  configuration information. "
                if str(err).startswith('Mixed mode assembly'):
                    status, msg = utils.check_dot_net_config(sys.executable)
                    if not status == 0:
                        raise IOError(msg)
                    else:
                        update_msg = 'Checking .NET config returned "{}" '.format(msg)
                        update_msg += 'and still cannot load library.\n'
                        update_msg += str(err)
                        raise IOError(update_msg)
                raise IOError('The above "System.IO.FileLoadException" is not handled.\n')

            # the shared library must be available in sys.path
            head, tail = os.path.split(self._path)
            sys.path.append(head)

            # don't include the library extension
            clr.AddReference(os.path.splitext(tail)[0])

            # import namespaces, create instances of classes or reference a System.Type[] object
            dotnet = dict()
            for t in self._assembly.GetTypes():
                if t.Namespace is not None:
                    mod = __import__(t.Namespace)
                    if mod.__name__ not in dotnet:
                        dotnet[mod.__name__] = mod
                else:
                    try:
                        dotnet[t.Name] = self._assembly.CreateInstance(t.FullName)
                    except:
                        if t.Name not in dotnet:
                            dotnet[t.Name] = t
            self._lib = DotNet(dotnet, self._path)

        else:
            raise TypeError('Cannot load libtype={}'.format(libtype))
        logger.debug('Loaded ' + self._path)

    def __repr__(self):
        return '<{} id={:#x} libtype={} path={}>'.format(
            self.__class__.__name__, id(self), self._lib.__class__.__name__, self._path)

    def __del__(self):
        if self._gateway is not None:
            self._gateway.shutdown()
            logger.debug('shutdown py4j.GatewayServer')

    @property
    def assembly(self):
        """
        Returns a reference to the `.NET Runtime Assembly <NET_>`_ object, *only if
        the shared library is a .NET Framework*, otherwise returns :data:`None`.

        .. tip::
           The `JetBrains dotPeek`_ program can be used to reliably decompile any
           .NET Assembly in to the equivalent source code.

        .. _NET: https://msdn.microsoft.com/en-us/library/system.reflection.assembly(v=vs.110).aspx
        .. _JetBrains dotPeek: https://www.jetbrains.com/decompiler/
        """
        return self._assembly

    @property
    def gateway(self):
        """
        Returns the :class:`~py4j.java_gateway.JavaGateway` object, *only if
        the shared library is a Java archive*, otherwise returns :data:`None`.
        """
        return self._gateway

    @property
    def lib(self):
        """Returns the reference to the loaded library object.

        For example, if `libtype` is

            * ``'cdll'`` then a :class:`~ctypes.CDLL` object
            * ``'windll'`` then a :class:`~ctypes.WinDLL` object
            * ``'oledll'`` then a :class:`~ctypes.OleDLL` object
            * ``'net'`` then a :class:`~.load_library.DotNet` object
            * ``'java'`` then a :class:`~py4j.java_gateway.JVMView` object
        """
        return self._lib

    @property
    def path(self):
        """:class:`str`: The path to the shared library file."""
        return self._path

class DotNet(object):

    def __init__(self, dot_net_dict, path):
        """Contains the namespace_ modules, classes and `System.Type`_ objects of a .NET Assembly.

        Do not instantiate directly.

        .. _namespace: https://msdn.microsoft.com/en-us/library/z2kcy19k.aspx
        .. _System.Type: https://msdn.microsoft.com/en-us/library/system.type(v=vs.110).aspx
        """
        self.__dict__.update(dot_net_dict)
        self._path = path

    def __repr__(self):
        return '<{} id={:#x} path={}>'.format(
            self.__class__.__name__, id(self), self._path)
<commit_msg>change note:: to tip::<commit_after>"""
Load a shared library.
"""
import os
import sys
import ctypes
import ctypes.util
import logging
import subprocess

from . import utils, DEFAULT_EXTENSION

logger = logging.getLogger(__name__)

class LoadLibrary(object):

    def __init__(self, path, libtype='cdll'):
        """Load a shared library.

        For example, a C/C++, FORTRAN, CLR, Java, Delphi, LabVIEW, ... library.

        Based on the value of `libtype` or the file extension this class will load the
        shared library as a:

            * :class:`~ctypes.CDLL` if `libtype` is ``'cdll'``,
            * :class:`~ctypes.WinDLL` if `libtype` is ``'windll'``,
            * :class:`~ctypes.OleDLL` if `libtype` is ``'oledll'``,
            * `System.Reflection.Assembly <Assembly_>`_ if `libtype` is ``'net'``, or a
            * :class:`~.py4j.java_gateway.JavaGateway` if `libtype` is ``'java'``.

        .. _Assembly: https://msdn.microsoft.com/en-us/library/system.reflection.assembly(v=vs.110).aspx

        Parameters
        ----------
        path : :class:`str`
            The path to the shared library.

            The search order for finding the shared library is:

                1. assume that a full or a relative (to the current working directory)
                   path is specified,
                2. use :func:`ctypes.util.find_library` to find the shared library file,
                3. search :data:`sys.path`, then
                4. search :data:`os.environ['PATH'] <os.environ>` to find the shared library.

        libtype : :class:`str`, optional
            The library type. The following values are currently supported:

            * ``'cdll'`` -- for a library that uses the __cdecl calling convention
            * ``'windll'`` or ``'oledll'`` -- for a __stdcall calling convention
            * ``'net'`` -- for Microsoft's .NET Framework (Common Language Runtime)
            * ``'java'`` -- for a Java archive, ``.jar``, or Java byte code, ``.class``, file

            Default is ``'cdll'``.

            .. tip::
               Since the ``.jar`` or ``.class`` extension uniquely defines a Java library,
               the `libtype` will automatically be set to ``'java'`` if `path` ends with
               ``.jar`` or ``.class``.

        Raises
        ------
        IOError
            If the shared library cannot be loaded.
        TypeError
            If `libtype` is not a supported library type.
        """

        # a reference to the shared library
        self._lib = None

        # a reference to the .NET Runtime Assembly
        self._assembly = None

        # a reference to the Py4J JavaGateway
        self._gateway = None

        # fixes Issue #8, if `path` is a <class 'pathlib.Path'> object
        path = str(path)

        # create a new reference to `path` just in case the
        # DEFAULT_EXTENSION is appended below so that the
        # ctypes.util.find_library function call will use the
        # unmodified value of `path`
        _path = path

        # assume a default extension if no extension was provided
        ext = os.path.splitext(_path)[1]
        if not ext:
            _path += DEFAULT_EXTENSION

        # the .jar or .class extension uniquely defines a Java library
        if ext in ('.jar', '.class'):
            libtype = 'java'

        self._path = os.path.abspath(_path)
        if not os.path.isfile(self._path):
            # for find_library use the original 'path' value since it may be a library name
            # without any prefix like 'lib', suffix like '.so', '.dylib' or version number
            self._path = ctypes.util.find_library(path)
            if self._path is None:  # then search sys.path and os.environ['PATH']
                success = False
                search_dirs = sys.path + os.environ['PATH'].split(os.pathsep)
                for directory in search_dirs:
                    p = os.path.join(directory, _path)
                    if os.path.isfile(p):
                        self._path = p
                        success = True
                        break
                if not success:
                    raise IOError('Cannot find the shared library "{}"'.format(path))

        libtype = libtype.lower()
        if libtype == 'cdll':
            self._lib = ctypes.CDLL(self._path)
        elif libtype == 'windll':
            self._lib = ctypes.WinDLL(self._path)
        elif libtype == 'oledll':
            self._lib = ctypes.OleDLL(self._path)
        elif libtype == 'java':
            if not utils.is_py4j_installed():
                raise IOError('Cannot load a Java file because Py4J is not installed.\n'
                              'To install Py4J run: pip install py4j')

            from py4j.version import __version__
            from py4j.java_gateway import JavaGateway, GatewayParameters

            # the address and port to use to host the py4j.GatewayServer
            address = '127.0.0.1'
            port = utils.get_available_port()

            # find the py4j JAR (needed to import py4j.GatewayServer on the Java side)
            root = os.path.dirname(sys.executable)
            filename = 'py4j'+__version__+'.jar'
            py4j_jar = os.path.join(root, 'share', 'py4j', filename)
            if not os.path.isfile(py4j_jar):
                root = os.path.dirname(root)  # then check one folder up (for unix or venv)
                py4j_jar = os.path.join(root, 'share', 'py4j', filename)
                if not os.path.isfile(py4j_jar):
                    py4j_jar = os.environ.get('PY4J_JAR', '')  # then check the environment variable
                    if not os.path.isfile(py4j_jar):
                        raise IOError('Cannot find {0}\nCreate a PY4J_JAR environment '
                                      'variable to be equal to the full path to {0}'.format(filename))

            # build the java command
            wrapper = os.path.join(os.path.dirname(__file__), 'py4j-wrapper.jar')
            cmd = ['java', '-cp', py4j_jar + os.pathsep + wrapper, 'Py4JWrapper', str(port)]

            # from the URLClassLoader documentation:
            #   Any URL that ends with a '/' is assumed to refer to a directory. Otherwise, the URL
            #   is assumed to refer to a JAR file which will be downloaded and opened as needed.
            if ext == '.jar':
                cmd.append(self._path)
            else:  # it is a .class file
                cmd.append(os.path.dirname(self._path) + '/')

            try:
                # start the py4j.GatewayServer, cannot use subprocess.call() because it blocks
                subprocess.Popen(cmd, stderr=sys.stderr, stdout=sys.stdout)
                err = None
            except IOError:
                err = 'You must have a Java Runtime Environment installed and available on PATH'

            if err:
                raise IOError(err)

            utils.wait_for_server(address, port, 5.0)

            self._gateway = JavaGateway(
                gateway_parameters=GatewayParameters(address=address, port=port)
            )

            self._lib = self._gateway.jvm

        elif libtype == 'net':
            if not utils.is_pythonnet_installed():
                raise IOError('Cannot load a .NET Assembly because pythonnet is not installed.\n'
                              'To install pythonnet run: pip install pythonnet')

            import clr
            try:
                # By default, pythonnet can only load libraries that are for .NET 4.0+.
                #
                # In order to allow pythonnet to load a library from .NET <4.0 the
                # useLegacyV2RuntimeActivationPolicy property needs to be enabled
                # in a <python-executable>.config file. If the following statement
                # raises a FileLoadException then attempt to create the configuration
                # file that has the property enabled and then notify the user why
                # loading the library failed and ask them to re-run their Python
                # script to load the .NET library.
                self._assembly = clr.System.Reflection.Assembly.LoadFile(self._path)

            except clr.System.IO.FileLoadException as err:
                # Example error message that can occur if the library is for .NET <4.0,
                # and the useLegacyV2RuntimeActivationPolicy is not enabled:
                #
                # " Mixed mode assembly is built against version 'v2.0.50727' of the
                #  runtime and cannot be loaded in the 4.0 runtime without additional
                #  configuration information. "
                if str(err).startswith('Mixed mode assembly'):
                    status, msg = utils.check_dot_net_config(sys.executable)
                    if not status == 0:
                        raise IOError(msg)
                    else:
                        update_msg = 'Checking .NET config returned "{}" '.format(msg)
                        update_msg += 'and still cannot load library.\n'
                        update_msg += str(err)
                        raise IOError(update_msg)
                raise IOError('The above "System.IO.FileLoadException" is not handled.\n')

            # the shared library must be available in sys.path
            head, tail = os.path.split(self._path)
            sys.path.append(head)

            # don't include the library extension
            clr.AddReference(os.path.splitext(tail)[0])

            # import namespaces, create instances of classes or reference a System.Type[] object
            dotnet = dict()
            for t in self._assembly.GetTypes():
                if t.Namespace is not None:
                    mod = __import__(t.Namespace)
                    if mod.__name__ not in dotnet:
                        dotnet[mod.__name__] = mod
                else:
                    try:
                        dotnet[t.Name] = self._assembly.CreateInstance(t.FullName)
                    except:
                        if t.Name not in dotnet:
                            dotnet[t.Name] = t
            self._lib = DotNet(dotnet, self._path)

        else:
            raise TypeError('Cannot load libtype={}'.format(libtype))
        logger.debug('Loaded ' + self._path)

    def __repr__(self):
        return '<{} id={:#x} libtype={} path={}>'.format(
            self.__class__.__name__, id(self), self._lib.__class__.__name__, self._path)

    def __del__(self):
        if self._gateway is not None:
            self._gateway.shutdown()
            logger.debug('shutdown py4j.GatewayServer')

    @property
    def assembly(self):
        """
        Returns a reference to the `.NET Runtime Assembly <NET_>`_ object, *only if
        the shared library is a .NET Framework*, otherwise returns :data:`None`.

        .. tip::
           The `JetBrains dotPeek`_ program can be used to reliably decompile any
           .NET Assembly in to the equivalent source code.

        .. _NET: https://msdn.microsoft.com/en-us/library/system.reflection.assembly(v=vs.110).aspx
        .. _JetBrains dotPeek: https://www.jetbrains.com/decompiler/
        """
        return self._assembly

    @property
    def gateway(self):
        """
        Returns the :class:`~py4j.java_gateway.JavaGateway` object, *only if
        the shared library is a Java archive*, otherwise returns :data:`None`.
        """
        return self._gateway

    @property
    def lib(self):
        """Returns the reference to the loaded library object.

        For example, if `libtype` is

            * ``'cdll'`` then a :class:`~ctypes.CDLL` object
            * ``'windll'`` then a :class:`~ctypes.WinDLL` object
            * ``'oledll'`` then a :class:`~ctypes.OleDLL` object
            * ``'net'`` then a :class:`~.load_library.DotNet` object
            * ``'java'`` then a :class:`~py4j.java_gateway.JVMView` object
        """
        return self._lib

    @property
    def path(self):
        """:class:`str`: The path to the shared library file."""
        return self._path

class DotNet(object):

    def __init__(self, dot_net_dict, path):
        """Contains the namespace_ modules, classes and `System.Type`_ objects of a .NET Assembly.

        Do not instantiate directly.

        .. _namespace: https://msdn.microsoft.com/en-us/library/z2kcy19k.aspx
        .. _System.Type: https://msdn.microsoft.com/en-us/library/system.type(v=vs.110).aspx
        """
        self.__dict__.update(dot_net_dict)
        self._path = path

    def __repr__(self):
        return '<{} id={:#x} path={}>'.format(
            self.__class__.__name__, id(self), self._path)
<|endoftext|>


<commit_before>"""
Support for models' internal Translation class.
"""

## TO DO: this is messy and needs to be cleaned up

from django.contrib.admin import StackedInline, ModelAdmin
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from django.db.models import signals
from django.db.models.base import ModelBase
from django.dispatch.dispatcher import connect
from multilingual.languages import *
from multilingual.exceptions import TranslationDoesNotExist
from multilingual.fields import TranslationForeignKey
from multilingual.manipulators import add_multilingual_manipulators
from multilingual import manager

from new import instancemethod

class MultilingualStackedInline(StackedInline):
    template = "admin/edit_inline_translations_newforms.html"

def translation_save_translated_fields(instance, **kwargs):
    """
    Save all the translations of instance in post_save signal handler.
    """
    if not hasattr(instance, '_translation_cache'):
        return
    for l_id, translation in instance._translation_cache.iteritems():
        # set the translation ID just in case the translation was
        # created while instance was not stored in the DB yet

        # note: we're using _get_pk_val here even though it is
        # private, since that's the most reliable way to get the value
        # on older Django (pk property did not exist yet)
        translation.master_id = instance._get_pk_val()
        translation.save()

def translation_overwrite_previous(instance, **kwargs):
    """
    Delete previously existing translation with the same master and
    language_id values.  To be called by translation model's pre_save
    signal.

    This most probably means I am abusing something here trying to use
    Django inline editor.  Oh well, it will have to go anyway when we
    move to newforms.
    """
    qs = instance.__class__.objects
    try:
        qs = qs.filter(master=instance.master).filter(language_id=instance.language_id)
        qs.delete()
    except ObjectDoesNotExist:
        # We are probably loading a fixture that defines translation entities
        # before their master entity.
        pass

def fill_translation_cache(instance):
    """
    Fill the translation cache using information received in the
    instance objects as extra fields.

    You can not do this in post_init because the extra fields are
    assigned by QuerySet.iterator after model initialization.
    """

    if hasattr(instance, '_translation_cache'):
        # do not refill the cache
        return

    instance._translation_cache = {}
    for language_id in get_language_id_list():
        # see if translation for language_id was in the query
        field_alias = get_translated_field_alias('id', language_id)
        if getattr(instance, field_alias, None) is not None:
            field_names = [f.attname for f in instance._meta.translation_model._meta.fields]

            # if so, create a translation object and put it in the cache
            field_data = {}
            for fname in field_names:
                field_data[fname] = getattr(instance,
                                            get_translated_field_alias(fname, language_id))

            translation = instance._meta.translation_model(**field_data)
            instance._translation_cache[language_id] = translation

    # In some situations an (existing in the DB) object is loaded
    # without using the normal QuerySet.  In such case fallback to
    # loading the translations using a separate query.

    # Unfortunately, this is indistinguishable from the situation when
    # an object does not have any translations.  Oh well, we'll have
    # to live with this for the time being.
    if len(instance._translation_cache.keys()) == 0:
        for translation in instance.translations.all():
            instance._translation_cache[translation.language_id] = translation

class TranslatedFieldProxy(property):
    def __init__(self, field_name, alias, field, language_id=None):
        self.field_name = field_name
        self.field = field
        self.admin_order_field = alias
        self.language_id = language_id

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self

        return getattr(obj, 'get_' + self.field_name)(self.language_id)

    def __set__(self, obj, value):
        language_id = self.language_id

        return getattr(obj, 'set_' + self.field_name)(value, self.language_id)

    short_description = property(lambda self: self.field.short_description)

def getter_generator(field_name, short_description):
    """
    Generate get_'field name' method for field field_name.
    """
    def get_translation_field(self, language_id_or_code=None):
        try:
            return getattr(self.get_translation(language_id_or_code), field_name)
        except TranslationDoesNotExist:
            return None
    get_translation_field.short_description = short_description
    return get_translation_field

def setter_generator(field_name):
    """
    Generate set_'field name' method for field field_name.
    """
    def set_translation_field(self, value, language_id_or_code=None):
        setattr(self.get_translation(language_id_or_code, True),
                field_name, value)
    set_translation_field.short_description = "set " + field_name
    return set_translation_field

def get_translation(self, language_id_or_code,
                    create_if_necessary=False):
    """
    Get a translation instance for the given language_id_or_code.

    If it does not exist, either create one or raise the
    TranslationDoesNotExist exception, depending on the
    create_if_necessary argument.
    """

    # fill the cache if necessary
    self.fill_translation_cache()

    language_id = get_language_id_from_id_or_code(language_id_or_code, False)
    if language_id is None:
        language_id = getattr(self, '_default_language', None)
    if language_id is None:
        language_id = get_default_language()

    if language_id not in self._translation_cache:
        if not create_if_necessary:
            raise TranslationDoesNotExist(language_id)
        new_translation = self._meta.translation_model(master=self,
                                                       language_id=language_id)
        self._translation_cache[language_id] = new_translation
    return self._translation_cache.get(language_id, None)

class Translation:
    """
    A superclass for translatablemodel.Translation inner classes.
    """

    def contribute_to_class(cls, main_cls, name):
        """
        Handle the inner 'Translation' class.
        """

        # delay the creation of the *Translation until the master model is
        # fully created
        connect(cls.finish_multilingual_class, signal=signals.class_prepared,
                sender=main_cls, weak=False)

        # connect the post_save signal on master class to a handler
        # that saves translations
        connect(translation_save_translated_fields, signal=signals.post_save,
                sender=main_cls)

    contribute_to_class = classmethod(contribute_to_class)

    def create_translation_attrs(cls, main_cls):
        """
        Creates get_'field name'(language_id) and set_'field
        name'(language_id) methods for all the translation fields.
        Adds the 'field name' properties too.

        Returns the translated_fields hash used in field lookups, see
        multilingual.query.  It maps field names to (field,
        language_id) tuples.
        """
        translated_fields = {}

        for fname, field in cls.__dict__.items():
            if isinstance(field, models.fields.Field):
                translated_fields[fname] = (field, None)

                # add get_'fname' and set_'fname' methods to main_cls
                getter = getter_generator(fname, getattr(field, 'verbose_name', fname))
                setattr(main_cls, 'get_' + fname, getter)

                setter = setter_generator(fname)
                setattr(main_cls, 'set_' + fname, setter)

                # add the 'fname' proxy property that allows reads
                # from and writing to the appropriate translation
                setattr(main_cls, fname,
                        TranslatedFieldProxy(fname, fname, field))

                # create the 'fname'_'language_code' proxy properties
                for language_id in get_language_id_list():
                    language_code = get_language_code(language_id)
                    fname_lng = fname + '_' + language_code.replace('-', '_')
                    translated_fields[fname_lng] = (field, language_id)
                    setattr(main_cls, fname_lng,
                            TranslatedFieldProxy(fname, fname_lng, field,
                                                 language_id))
        return translated_fields
    create_translation_attrs = classmethod(create_translation_attrs)

    def get_unique_fields(cls):
        """
        Return a list of fields with "unique" attribute, which needs to
        be augmented by the language.
        """
        unique_fields = []

        for fname, field in cls.__dict__.items():
            if isinstance(field, models.fields.Field):
                if getattr(field,'unique',False):
                    try:
                        field.unique = False
                    except AttributeError:
                        # newer Django defines unique as a property
                        # that uses _unique to store data.  We're
                        # jumping over the fence by setting _unique,
                        # so this sucks, but this happens early enough
                        # to be safe.
                        field._unique = False
                    unique_fields.append(fname)
        return unique_fields
    get_unique_fields = classmethod(get_unique_fields)

    def finish_multilingual_class(cls, *args, **kwargs):
        """
        Create a model with translations of a multilingual class.
        """

        main_cls = kwargs['sender']
        translation_model_name = main_cls.__name__ + "Translation"

        # create the model with all the translatable fields
        unique = [('language_id', 'master')]
        for f in cls.get_unique_fields():
            unique.append(('language_id',f))

        class TransMeta:
            pass

        try:
            meta = cls.Meta
        except AttributeError:
            meta = TransMeta

        meta.ordering = ('language_id',)
        meta.unique_together = tuple(unique)
        meta.app_label = main_cls._meta.app_label
        if not hasattr(meta, 'db_table'):
            meta.db_table = main_cls._meta.db_table + '_translation'

        trans_attrs = cls.__dict__.copy()
        trans_attrs['Meta'] = meta
        trans_attrs['language_id'] = models.IntegerField(blank=False, null=False, core=True,
                                                         choices=get_language_choices(),
                                                         db_index=True)

        edit_inline = True

        trans_attrs['master'] = TranslationForeignKey(main_cls, blank=False, null=False,
                                                      edit_inline=edit_inline,
                                                      related_name='translations',
                                                      num_in_admin=get_language_count(),
                                                      min_num_in_admin=get_language_count(),
                                                      num_extra_on_change=0)
        trans_attrs['__str__'] = lambda self: ("%s object, language_code=%s"
                                               % (translation_model_name,
                                                  get_language_code(self.language_id)))

        trans_model = ModelBase(translation_model_name, (models.Model,), trans_attrs)
        trans_model._meta.translated_fields = cls.create_translation_attrs(main_cls)

        _old_init_name_map = main_cls._meta.__class__.init_name_map
        def init_name_map(self):
            cache = _old_init_name_map(self)
            for name, field_and_lang_id in trans_model._meta.translated_fields.items():
                #import sys; sys.stderr.write('TM %r\n' % trans_model)
                cache[name] = (field_and_lang_id[0], trans_model, True, False)
            return cache
        main_cls._meta.init_name_map = instancemethod(init_name_map,
                                                      main_cls._meta,
                                                      main_cls._meta.__class__)

        main_cls._meta.translation_model = trans_model
        main_cls.get_translation = get_translation
        main_cls.fill_translation_cache = fill_translation_cache

        # Note: don't fill the translation cache in post_init, as all
        # the extra values selected by QAddTranslationData will be
        # assigned AFTER init()
#        connect(fill_translation_cache, signal=signals.post_init,
#                sender=main_cls)

        # connect the pre_save signal on translation class to a
        # function removing previous translation entries.
        connect(translation_overwrite_previous, signal=signals.pre_save,
                sender=trans_model, weak=False)

    finish_multilingual_class = classmethod(finish_multilingual_class)

def install_translation_library():
    # modify ModelBase.__new__ so that it understands how to handle the
    # 'Translation' inner class

    if getattr(ModelBase, '_multilingual_installed', False):
        # don't install it twice
        return

    _old_new = ModelBase.__new__

    def multilingual_modelbase_new(cls, name, bases, attrs):
        if 'Translation' in attrs:
            if not issubclass(attrs['Translation'], Translation):
                raise ValueError, ("%s.Translation must be a subclass "
                                   + " of multilingual.Translation.") % (name,)

            # Make sure that if the class specifies objects then it is
            # a subclass of our Manager.
            #
            # Don't check other managers since someone might want to
            # have a non-multilingual manager, but assigning a
            # non-multilingual manager to objects would be a common
            # mistake.
            if ('objects' in attrs) and (not isinstance(attrs['objects'], manager.Manager)):
                raise ValueError, ("Model %s specifies translations, " +
                                   "so its 'objects' manager must be " +
                                   "a subclass of multilingual.Manager.") % (name,)

            # Change the default manager to multilingual.Manager.
            if not 'objects' in attrs:
                attrs['objects'] = manager.Manager()

            # Install a hack to let add_multilingual_manipulators know
            # this is a translatable model
            attrs['is_translation_model'] = lambda self: True

        return _old_new(cls, name, bases, attrs)
    ModelBase.__new__ = staticmethod(multilingual_modelbase_new)
    ModelBase._multilingual_installed = True

    # Override ModelAdmin.__new__ to create automatic inline
    # editor for multilingual models.
    class MultiType(type):
        pass
    _old_admin_new = ModelAdmin.__new__
    def multilingual_modeladmin_new(cls, model, admin_site):
        if isinstance(model.objects, manager.Manager):
            X = MultiType('X',(MultilingualStackedInline,),
                     {'model':model._meta.translation_model,
                      'fk_name':'master',
                      'extra':get_language_count(),
                      'max_num':get_language_count()})
            if cls.inlines:
                for inline in cls.inlines:
                    if X.__class__ == inline.__class__:
                        cls.inlines.remove(inline)
                        break
                cls.inlines.append(X)
            else:
                cls.inlines = [X]
        return _old_admin_new(cls, model, admin_site)
    ModelAdmin.__new__ = staticmethod(multilingual_modeladmin_new)

# install the library
install_translation_library()
<commit_msg>Factor out the dynamic generation of the translation ModelAdmin class.<commit_after>"""
Support for models' internal Translation class.
"""

## TO DO: this is messy and needs to be cleaned up

from django.contrib.admin import StackedInline, ModelAdmin
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from django.db.models import signals
from django.db.models.base import ModelBase
from django.dispatch.dispatcher import connect
from multilingual.languages import *
from multilingual.exceptions import TranslationDoesNotExist
from multilingual.fields import TranslationForeignKey
from multilingual.manipulators import add_multilingual_manipulators
from multilingual import manager

from new import instancemethod

class MultilingualStackedInline(StackedInline):
    template = "admin/edit_inline_translations_newforms.html"

def translation_save_translated_fields(instance, **kwargs):
    """
    Save all the translations of instance in post_save signal handler.
    """
    if not hasattr(instance, '_translation_cache'):
        return
    for l_id, translation in instance._translation_cache.iteritems():
        # set the translation ID just in case the translation was
        # created while instance was not stored in the DB yet

        # note: we're using _get_pk_val here even though it is
        # private, since that's the most reliable way to get the value
        # on older Django (pk property did not exist yet)
        translation.master_id = instance._get_pk_val()
        translation.save()

def translation_overwrite_previous(instance, **kwargs):
    """
    Delete previously existing translation with the same master and
    language_id values.  To be called by translation model's pre_save
    signal.

    This most probably means I am abusing something here trying to use
    Django inline editor.  Oh well, it will have to go anyway when we
    move to newforms.
    """
    qs = instance.__class__.objects
    try:
        qs = qs.filter(master=instance.master).filter(language_id=instance.language_id)
        qs.delete()
    except ObjectDoesNotExist:
        # We are probably loading a fixture that defines translation entities
        # before their master entity.
        pass

def fill_translation_cache(instance):
    """
    Fill the translation cache using information received in the
    instance objects as extra fields.

    You can not do this in post_init because the extra fields are
    assigned by QuerySet.iterator after model initialization.
    """

    if hasattr(instance, '_translation_cache'):
        # do not refill the cache
        return

    instance._translation_cache = {}
    for language_id in get_language_id_list():
        # see if translation for language_id was in the query
        field_alias = get_translated_field_alias('id', language_id)
        if getattr(instance, field_alias, None) is not None:
            field_names = [f.attname for f in instance._meta.translation_model._meta.fields]

            # if so, create a translation object and put it in the cache
            field_data = {}
            for fname in field_names:
                field_data[fname] = getattr(instance,
                                            get_translated_field_alias(fname, language_id))

            translation = instance._meta.translation_model(**field_data)
            instance._translation_cache[language_id] = translation

    # In some situations an (existing in the DB) object is loaded
    # without using the normal QuerySet.  In such case fallback to
    # loading the translations using a separate query.

    # Unfortunately, this is indistinguishable from the situation when
    # an object does not have any translations.  Oh well, we'll have
    # to live with this for the time being.
    if len(instance._translation_cache.keys()) == 0:
        for translation in instance.translations.all():
            instance._translation_cache[translation.language_id] = translation

class TranslatedFieldProxy(property):
    def __init__(self, field_name, alias, field, language_id=None):
        self.field_name = field_name
        self.field = field
        self.admin_order_field = alias
        self.language_id = language_id

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self

        return getattr(obj, 'get_' + self.field_name)(self.language_id)

    def __set__(self, obj, value):
        language_id = self.language_id

        return getattr(obj, 'set_' + self.field_name)(value, self.language_id)

    short_description = property(lambda self: self.field.short_description)

def getter_generator(field_name, short_description):
    """
    Generate get_'field name' method for field field_name.
    """
    def get_translation_field(self, language_id_or_code=None):
        try:
            return getattr(self.get_translation(language_id_or_code), field_name)
        except TranslationDoesNotExist:
            return None
    get_translation_field.short_description = short_description
    return get_translation_field

def setter_generator(field_name):
    """
    Generate set_'field name' method for field field_name.
    """
    def set_translation_field(self, value, language_id_or_code=None):
        setattr(self.get_translation(language_id_or_code, True),
                field_name, value)
    set_translation_field.short_description = "set " + field_name
    return set_translation_field

def get_translation(self, language_id_or_code,
                    create_if_necessary=False):
    """
    Get a translation instance for the given language_id_or_code.

    If it does not exist, either create one or raise the
    TranslationDoesNotExist exception, depending on the
    create_if_necessary argument.
    """

    # fill the cache if necessary
    self.fill_translation_cache()

    language_id = get_language_id_from_id_or_code(language_id_or_code, False)
    if language_id is None:
        language_id = getattr(self, '_default_language', None)
    if language_id is None:
        language_id = get_default_language()

    if language_id not in self._translation_cache:
        if not create_if_necessary:
            raise TranslationDoesNotExist(language_id)
        new_translation = self._meta.translation_model(master=self,
                                                       language_id=language_id)
        self._translation_cache[language_id] = new_translation
    return self._translation_cache.get(language_id, None)

class Translation:
    """
    A superclass for translatablemodel.Translation inner classes.
    """

    def contribute_to_class(cls, main_cls, name):
        """
        Handle the inner 'Translation' class.
        """

        # delay the creation of the *Translation until the master model is
        # fully created
        connect(cls.finish_multilingual_class, signal=signals.class_prepared,
                sender=main_cls, weak=False)

        # connect the post_save signal on master class to a handler
        # that saves translations
        connect(translation_save_translated_fields, signal=signals.post_save,
                sender=main_cls)

    contribute_to_class = classmethod(contribute_to_class)

    def create_translation_attrs(cls, main_cls):
        """
        Creates get_'field name'(language_id) and set_'field
        name'(language_id) methods for all the translation fields.
        Adds the 'field name' properties too.

        Returns the translated_fields hash used in field lookups, see
        multilingual.query.  It maps field names to (field,
        language_id) tuples.
        """
        translated_fields = {}

        for fname, field in cls.__dict__.items():
            if isinstance(field, models.fields.Field):
                translated_fields[fname] = (field, None)

                # add get_'fname' and set_'fname' methods to main_cls
                getter = getter_generator(fname, getattr(field, 'verbose_name', fname))
                setattr(main_cls, 'get_' + fname, getter)

                setter = setter_generator(fname)
                setattr(main_cls, 'set_' + fname, setter)

                # add the 'fname' proxy property that allows reads
                # from and writing to the appropriate translation
                setattr(main_cls, fname,
                        TranslatedFieldProxy(fname, fname, field))

                # create the 'fname'_'language_code' proxy properties
                for language_id in get_language_id_list():
                    language_code = get_language_code(language_id)
                    fname_lng = fname + '_' + language_code.replace('-', '_')
                    translated_fields[fname_lng] = (field, language_id)
                    setattr(main_cls, fname_lng,
                            TranslatedFieldProxy(fname, fname_lng, field,
                                                 language_id))
        return translated_fields
    create_translation_attrs = classmethod(create_translation_attrs)

    def get_unique_fields(cls):
        """
        Return a list of fields with "unique" attribute, which needs to
        be augmented by the language.
        """
        unique_fields = []

        for fname, field in cls.__dict__.items():
            if isinstance(field, models.fields.Field):
                if getattr(field,'unique',False):
                    try:
                        field.unique = False
                    except AttributeError:
                        # newer Django defines unique as a property
                        # that uses _unique to store data.  We're
                        # jumping over the fence by setting _unique,
                        # so this sucks, but this happens early enough
                        # to be safe.
                        field._unique = False
                    unique_fields.append(fname)
        return unique_fields
    get_unique_fields = classmethod(get_unique_fields)

    def finish_multilingual_class(cls, *args, **kwargs):
        """
        Create a model with translations of a multilingual class.
        """

        main_cls = kwargs['sender']
        translation_model_name = main_cls.__name__ + "Translation"

        # create the model with all the translatable fields
        unique = [('language_id', 'master')]
        for f in cls.get_unique_fields():
            unique.append(('language_id',f))

        class TransMeta:
            pass

        try:
            meta = cls.Meta
        except AttributeError:
            meta = TransMeta

        meta.ordering = ('language_id',)
        meta.unique_together = tuple(unique)
        meta.app_label = main_cls._meta.app_label
        if not hasattr(meta, 'db_table'):
            meta.db_table = main_cls._meta.db_table + '_translation'

        trans_attrs = cls.__dict__.copy()
        trans_attrs['Meta'] = meta
        trans_attrs['language_id'] = models.IntegerField(blank=False, null=False, core=True,
                                                         choices=get_language_choices(),
                                                         db_index=True)

        edit_inline = True

        trans_attrs['master'] = TranslationForeignKey(main_cls, blank=False, null=False,
                                                      edit_inline=edit_inline,
                                                      related_name='translations',
                                                      num_in_admin=get_language_count(),
                                                      min_num_in_admin=get_language_count(),
                                                      num_extra_on_change=0)
        trans_attrs['__str__'] = lambda self: ("%s object, language_code=%s"
                                               % (translation_model_name,
                                                  get_language_code(self.language_id)))

        trans_model = ModelBase(translation_model_name, (models.Model,), trans_attrs)
        trans_model._meta.translated_fields = cls.create_translation_attrs(main_cls)

        _old_init_name_map = main_cls._meta.__class__.init_name_map
        def init_name_map(self):
            cache = _old_init_name_map(self)
            for name, field_and_lang_id in trans_model._meta.translated_fields.items():
                #import sys; sys.stderr.write('TM %r\n' % trans_model)
                cache[name] = (field_and_lang_id[0], trans_model, True, False)
            return cache
        main_cls._meta.init_name_map = instancemethod(init_name_map,
                                                      main_cls._meta,
                                                      main_cls._meta.__class__)

        main_cls._meta.translation_model = trans_model
        main_cls.get_translation = get_translation
        main_cls.fill_translation_cache = fill_translation_cache

        # Note: don't fill the translation cache in post_init, as all
        # the extra values selected by QAddTranslationData will be
        # assigned AFTER init()
#        connect(fill_translation_cache, signal=signals.post_init,
#                sender=main_cls)

        # connect the pre_save signal on translation class to a
        # function removing previous translation entries.
        connect(translation_overwrite_previous, signal=signals.pre_save,
                sender=trans_model, weak=False)

    finish_multilingual_class = classmethod(finish_multilingual_class)

def install_translation_library():
    # modify ModelBase.__new__ so that it understands how to handle the
    # 'Translation' inner class

    if getattr(ModelBase, '_multilingual_installed', False):
        # don't install it twice
        return

    _old_new = ModelBase.__new__

    def multilingual_modelbase_new(cls, name, bases, attrs):
        if 'Translation' in attrs:
            if not issubclass(attrs['Translation'], Translation):
                raise ValueError, ("%s.Translation must be a subclass "
                                   + " of multilingual.Translation.") % (name,)

            # Make sure that if the class specifies objects then it is
            # a subclass of our Manager.
            #
            # Don't check other managers since someone might want to
            # have a non-multilingual manager, but assigning a
            # non-multilingual manager to objects would be a common
            # mistake.
            if ('objects' in attrs) and (not isinstance(attrs['objects'], manager.Manager)):
                raise ValueError, ("Model %s specifies translations, " +
                                   "so its 'objects' manager must be " +
                                   "a subclass of multilingual.Manager.") % (name,)

            # Change the default manager to multilingual.Manager.
            if not 'objects' in attrs:
                attrs['objects'] = manager.Manager()

            # Install a hack to let add_multilingual_manipulators know
            # this is a translatable model
            attrs['is_translation_model'] = lambda self: True

        return _old_new(cls, name, bases, attrs)
    ModelBase.__new__ = staticmethod(multilingual_modelbase_new)
    ModelBase._multilingual_installed = True

    # Override ModelAdmin.__new__ to create automatic inline
    # editor for multilingual models.
    class MultiType(type):
        pass
    _old_admin_new = ModelAdmin.__new__
    def multilingual_modeladmin_new(cls, model, admin_site):
        if isinstance(model.objects, manager.Manager):
            X = cls.get_translation_modeladmin(model)
            if cls.inlines:
                for inline in cls.inlines:
                    if X.__class__ == inline.__class__:
                        cls.inlines.remove(inline)
                        break
                cls.inlines.append(X)
            else:
                cls.inlines = [X]
        return _old_admin_new(cls, model, admin_site)
    def get_translation_modeladmin(cls, model):
            X = MultiType('X',(MultilingualStackedInline,),
                     {'model':model._meta.translation_model,
                      'fk_name':'master',
                      'extra':get_language_count(),
                      'max_num':get_language_count()})
            return X
    ModelAdmin.__new__ = staticmethod(multilingual_modeladmin_new)
    ModelAdmin.get_translation_modeladmin = classmethod(get_translation_modeladmin)

# install the library
install_translation_library()
<|endoftext|>


<commit_before># -*-: coding: utf-8 -*-

import requests
import json
import re

from ..audio import Track

LANGS = {
    u"jp": (u"jp", u"Japanese"),
    u"en": (u"en", u"English"),
}

def _get_json_by_link(link):
    url = u'''http://vgmdb.info/%s''' % link
    return _get_json(url)

def _get_json(url):
    resp = requests.get(url, timeout=5)
    if resp.status_code != 200:
        raise Exception("Failed to search from VGMdb! Error Code = %d" % resp.status_code)

    d = resp.json()

    return d

# Functions with Search API
def search_all(keyword):
    d = _get_json_by_link(u'''search/"%s"?format=json''' % keyword)
    _validate_search_result(d)

    return d[u"results"]

def search_albums(keyword):
    d = _get_json_by_link(u'''search/albums/"%s"?format=json''' % keyword)
    _validate_search_result(d)

    return d[u"results"]

def search_artists(keyword):
    d = _get_json_by_link(u'''search/artists/"%s"?format=json''' % keyword)
    _validate_search_result(d)

    return d[u"results"]

def search_orgs(keyword):
    d = _get_json_by_link(u'''search/orgs/"%s"?format=json''' % keyword)
    _validate_search_result(d)

    return d[u"results"]

def search_products(keyword):
    d = _get_json_by_link(u'''search/products/"%s"?format=json''' % keyword)
    _validate_search_result(d)

    return d[u"results"]

def _validate_search_result(search_result):
    for key in [u"results", u"sections"]:
        if key not in search_result:
            raise ValueError("Invalid search result: field '%s' required!" % key)

    for key in search_result[u"sections"]:
        if key not in search_result[u"results"]:
            raise ValueError("Invalid search result: in field 'results', field '%s' required!"
                             % key)

# Functions with Infomation API
def get_album(id):
    d = _get_json_by_link(u'''album/%s''' % id)
    return d

def get_artist(id):
    d = _get_json_by_link(u'''artist/%s''' % id)
    return d

def get_org(id):
    d = _get_json_by_link(u'''org/%s''' % id)
    return d

def get_product(id):
    d = _get_json_by_link(u'''product/%s''' % id)
    return d

def get_event(id):
    d = _get_json_by_link(u'''event/%s''' % id)
    return d

# Utility
def disc(num):
    return u"Disc %s" % num

# Functions that producing data from JSON (dict)
def album_get_cover_picture(album_info, size="full"):
    if u"covers" not in album_info:
        return None

    for cover_dict in album_info[u"covers"]:
        if cover_dict[u"name"].lower() in [u"front", u"cover", u"folder"]:
            return cover_dict[size]

def album_tracks(album_info, discname, lang=u"en"):
    for disc in album_info[u"discs"]:
        if disc[u"name"] == discname:
            tracks = []
            for track in disc[u"tracks"]:
                tracks.append(track[u"names"].get(
                    LANGS[lang][1],
                    track[u"names"][u"English"]
                ))
            return tracks

def update(album_info, track, lang=u"en"):
    if not isinstance(track, Track):
        raise TypeError("Instance is not a Meta object")

    update_album_title(album_info, track, lang=lang)
    # update_artist(album_info, track, lang=lang)
    update_album_artist(album_info, track, lang=lang)
    update_catalog(album_info, track, lang=lang)
    update_category(album_info, track, lang=lang)
    update_cover_picture(album_info, track, lang=lang)
    update_title(album_info, track, lang=lang)

    return

def update_album_title(album_info, track, lang=u"en"):
    track.ALBUM = album_info[u"names"].get(
        LANGS[lang][0],
        album_info[u"names"][u"en"]
    )

def update_album_artist(album_info, track, lang=u"en"):
    composers = u""
    for composer in album_info[u"composers"]:
        composers += u", %s" % composer[u"names"].get(
            LANGS[lang][0],
            composer[u"names"][u"en"]
        )
    track.ALBUMARTIST = composers[2:]

def update_catalog(album_info, track, lang=u"en"):
    if u"catalog" in album_info:
        track[u"CATALOG"] = album_info[u"catalog"]

def update_category(album_info, track, lang=u"en"):
    if u"category" in album_info:
        track[u"CATEGORY"] = album_info[u"category"]

def update_cover_picture(album_info, track, lang=u"en"):
    track[u"_picture"] = album_get_cover_picture(album_info)

def update_title(album_info, track, lang=u"en"):
    if track.DISCNUMBER is None:
        discname = u"Disc 1"
    else:
        discname = disc(track.DISCNUMBER)

    track_names = album_tracks(album_info, discname, lang=lang)
    track.TITLE = track_names[int(track.TRACKNUMBER) - 1]

def album_detail(album_info, lang=u"English", lang_short=u"en"):
    detail_string = u""

    if lang_short in album_info[u"names"]:
        detail_string += u"TITLE : %s\n" % album_info[u"names"][lang_short]
    else:
        detail_string += u"TITLE : %s\n" % album_info[u"name"]

    detail_string += u"\nCOMPOSER :\n"

    for composer in album_info[u"composers"]:
        if lang_short in composer[u"names"]:
            detail_string += u"%s\n" % composer[u"names"][lang_short]
        else:
            detail_string += u"%s\n" % composer[u"names"][u"en"]

    for disc in album_info[u"discs"]:
        detail_string += u"\nIn : %s\n" % disc[u"name"]
        for track_id, track in enumerate(disc[u"tracks"]):
            detail_string += u"  %s : %s\n" % (
                str(track_id + 1).zfill(2),
                track[u"names"][lang] if lang in track[u"names"] else track[u"names"]["English"])

    return detail_string

# Functions for details
def print_album_detail(album_info, lang=u"English", lang_short=u"en"):
    print(album_detail(album_info, lang, lang_short))

<commit_msg>Fixed an mistake, for wrong importing.<commit_after># -*-: coding: utf-8 -*-

import requests
import json
import re

from ..audio import AudioTrack

LANGS = {
    u"jp": (u"jp", u"Japanese"),
    u"en": (u"en", u"English"),
}

def _get_json_by_link(link):
    url = u'''http://vgmdb.info/%s''' % link
    return _get_json(url)

def _get_json(url):
    resp = requests.get(url, timeout=5)
    if resp.status_code != 200:
        raise Exception("Failed to search from VGMdb! Error Code = %d" % resp.status_code)

    d = resp.json()

    return d

# Functions with Search API
def search_all(keyword):
    d = _get_json_by_link(u'''search/"%s"?format=json''' % keyword)
    _validate_search_result(d)

    return d[u"results"]

def search_albums(keyword):
    d = _get_json_by_link(u'''search/albums/"%s"?format=json''' % keyword)
    _validate_search_result(d)

    return d[u"results"]

def search_artists(keyword):
    d = _get_json_by_link(u'''search/artists/"%s"?format=json''' % keyword)
    _validate_search_result(d)

    return d[u"results"]

def search_orgs(keyword):
    d = _get_json_by_link(u'''search/orgs/"%s"?format=json''' % keyword)
    _validate_search_result(d)

    return d[u"results"]

def search_products(keyword):
    d = _get_json_by_link(u'''search/products/"%s"?format=json''' % keyword)
    _validate_search_result(d)

    return d[u"results"]

def _validate_search_result(search_result):
    for key in [u"results", u"sections"]:
        if key not in search_result:
            raise ValueError("Invalid search result: field '%s' required!" % key)

    for key in search_result[u"sections"]:
        if key not in search_result[u"results"]:
            raise ValueError("Invalid search result: in field 'results', field '%s' required!"
                             % key)

# Functions with Infomation API
def get_album(id):
    d = _get_json_by_link(u'''album/%s''' % id)
    return d

def get_artist(id):
    d = _get_json_by_link(u'''artist/%s''' % id)
    return d

def get_org(id):
    d = _get_json_by_link(u'''org/%s''' % id)
    return d

def get_product(id):
    d = _get_json_by_link(u'''product/%s''' % id)
    return d

def get_event(id):
    d = _get_json_by_link(u'''event/%s''' % id)
    return d

# Utility
def disc(num):
    return u"Disc %s" % num

# Functions that producing data from JSON (dict)
def album_get_cover_picture(album_info, size="full"):
    if u"covers" not in album_info:
        return None

    for cover_dict in album_info[u"covers"]:
        if cover_dict[u"name"].lower() in [u"front", u"cover", u"folder"]:
            return cover_dict[size]

def album_tracks(album_info, discname, lang=u"en"):
    for disc in album_info[u"discs"]:
        if disc[u"name"] == discname:
            tracks = []
            for track in disc[u"tracks"]:
                tracks.append(track[u"names"].get(
                    LANGS[lang][1],
                    track[u"names"][u"English"]
                ))
            return tracks

def update(album_info, track, lang=u"en"):
    if not isinstance(track, AudioTrack):
        raise TypeError("Instance is not a Meta object")

    update_album_title(album_info, track, lang=lang)
    # update_artist(album_info, track, lang=lang)
    update_album_artist(album_info, track, lang=lang)
    update_catalog(album_info, track, lang=lang)
    update_category(album_info, track, lang=lang)
    update_cover_picture(album_info, track, lang=lang)
    update_title(album_info, track, lang=lang)

    return

def update_album_title(album_info, track, lang=u"en"):
    track.ALBUM = album_info[u"names"].get(
        LANGS[lang][0],
        album_info[u"names"][u"en"]
    )

def update_album_artist(album_info, track, lang=u"en"):
    composers = u""
    for composer in album_info[u"composers"]:
        composers += u", %s" % composer[u"names"].get(
            LANGS[lang][0],
            composer[u"names"][u"en"]
        )
    track.ALBUMARTIST = composers[2:]

def update_catalog(album_info, track, lang=u"en"):
    if u"catalog" in album_info:
        track[u"CATALOG"] = album_info[u"catalog"]

def update_category(album_info, track, lang=u"en"):
    if u"category" in album_info:
        track[u"CATEGORY"] = album_info[u"category"]

def update_cover_picture(album_info, track, lang=u"en"):
    track[u"_picture"] = album_get_cover_picture(album_info)

def update_title(album_info, track, lang=u"en"):
    if track.DISCNUMBER is None:
        discname = u"Disc 1"
    else:
        discname = disc(track.DISCNUMBER)

    track_names = album_tracks(album_info, discname, lang=lang)
    track.TITLE = track_names[int(track.TRACKNUMBER) - 1]

def album_detail(album_info, lang=u"English", lang_short=u"en"):
    detail_string = u""

    if lang_short in album_info[u"names"]:
        detail_string += u"TITLE : %s\n" % album_info[u"names"][lang_short]
    else:
        detail_string += u"TITLE : %s\n" % album_info[u"name"]

    detail_string += u"\nCOMPOSER :\n"

    for composer in album_info[u"composers"]:
        if lang_short in composer[u"names"]:
            detail_string += u"%s\n" % composer[u"names"][lang_short]
        else:
            detail_string += u"%s\n" % composer[u"names"][u"en"]

    for disc in album_info[u"discs"]:
        detail_string += u"\nIn : %s\n" % disc[u"name"]
        for track_id, track in enumerate(disc[u"tracks"]):
            detail_string += u"  %s : %s\n" % (
                str(track_id + 1).zfill(2),
                track[u"names"][lang] if lang in track[u"names"] else track[u"names"]["English"])

    return detail_string

# Functions for details
def print_album_detail(album_info, lang=u"English", lang_short=u"en"):
    print(album_detail(album_info, lang, lang_short))

<|endoftext|>