<commit_before>import gensim
import os
import argparse
import pandas as pd
import networkx as nx
import matplotlib.pyplot as plt

from networkx.drawing.nx_agraph import graphviz_layout
from chinese_whispers import chinese_whispers, aggregate_clusters
from gensim.models.poincare import PoincareModel
from nltk.corpus import wordnet as wn

def process_input(taxonomy):
    """ Read the taxonomy and generate a networkx graph """

    # Read the taxonomy as a dataframe
    print('Reading input...')
    df = pd.read_csv(
        taxonomy,
        sep='\t',
        header=None,
        names=['hyponym', 'hypernym'],
        usecols=[1, 2],
    )

    # Construct the networkx graph
    print('Constructing networkx graph...')
    graph = nx.DiGraph()
    for hypernym, hyponym in zip(list(df['hypernym']), list(df['hyponym'])):
        # Simplify the compound words by replacing the whitespaces with underscores
        if ' ' in hypernym:
            hypernym = '_'.join(hypernym.split())
        if ' ' in hyponym:
            hyponym = '_'.join(hyponym.split())
        graph.add_edge(hypernym, hyponym)

    return graph

def display_taxonomy(graph):
    """ Display the taxonomy in a hierarchical layout """

    pos = graphviz_layout(graph, prog='dot', args="-Grankdir=LR")
    plt.figure(3, figsize=(48, 144))
    nx.draw(graph, pos, with_labels=True, arrows=True)
    plt.show()

def load_vectors(embedding):
    """ Load word vectors. """

    embedding_dir = '/home/5aly/taxi/distributed_semantics/embeddings/'

    if embedding == "wiki2M":
        model = gensim.models.KeyedVectors.load_word2vec_format(embedding_dir + 'crawl-300d-2M.vec', binary=False)
    elif embedding == "wiki1M_subword":
        model = gensim.models.KeyedVectors.load_word2vec_format(
            embedding_dir + 'wiki-news-300d-1M-subword.vec', binary=False
        )
    elif embedding == "own_w2v":
        model = gensim.models.KeyedVectors.load(embedding_dir + 'own_embeddings_w2v')
    elif embedding == "poincare":
        model = PoincareModel.load(embedding_dir + 'embeddings_poincare_wordnet')

    return model

def create_children_clusters(w2v_model, graph, embedding, depth=100):
    """ This function returns a dictionary where corresponding to each key(node) is a graph of its children """
    clustered_graph = {}
    for node in graph.nodes():
        clustered_graph[node] = nx.Graph()
        successors = [s.lower() for s in graph.successors(node)]

        for successor in successors:
            word_in_vocab = False
            if embedding == "poincare":
                word_senses = wn.synsets(successor)  # Get all the senses of the given node
                for sense in word_senses:
                    try:
                        for word, score in w2v_model.kv.most_similar(sense.name(), topn=depth):
                            word = word.split('.')[0]  # convert the word from poincare format to normal string
                            word_in_vocab = True
                            if word.lower() in successors:
                                clustered_graph[node].add_edge(successor, word.lower())
                    except KeyError:
                        continue
            else:
                try:
                    for word, score in w2v_model.most_similar(successor, topn=depth):
                        word_in_vocab = True
                        if word.lower() in successors:
                            clustered_graph[node].add_edge(successor, word.lower())
                except KeyError:
                    pass

            if not word_in_vocab:  # If the word in not in vocabulary, check using the substring based method
                successor_terms = successor.split('_')
                root_terms = [successor_terms[0], successor_terms[-1]]
                if node in root_terms:
                    clustered_graph[node].add_node(successor)

    return clustered_graph

def remove_clusters(model, nx_graph, embedding, depth=100):
    """ Removes the less related and small clusters from the graph """

    print('Removing small clusters..')
    g_clustered = create_children_clusters(model, nx_graph, embedding, depth)
    removed_clusters = []

    nodes, clusters, size_ratio = [], [], []
    for node, graph in g_clustered.items():
        gc = chinese_whispers(graph, weighting='top', iterations=60)
        try:  # Get the length of the largest cluster
            max_cluster_size = len(max(aggregate_clusters(gc).values(), key=len))
        except ValueError:
            continue

        # Calculate the size ratio of all the clusters which are smaller than the largest
        for _, cluster in aggregate_clusters(gc).items():
            if len(cluster) < max_cluster_size:
                nodes.append(node)
                clusters.append(cluster)
                size_ratio.append(len(cluster) / max_cluster_size)

    # Sort the small clusters according to their size_ratio
    sorted_node_clusters = [(node, cluster) for _, cluster, node in sorted(zip(size_ratio, clusters, nodes))]
    if len(sorted_node_clusters) > 10:
        sorted_node_clusters = sorted_node_clusters[:10]

    for node, cluster in sorted_node_clusters:  # detach only the smallest 10 clusters in the entire taxonomy
        removed_clusters.append(cluster)
        for item in cluster:
            nx_graph.remove_edge(node, item)

    return nx_graph, removed_clusters

def calculate_similarity(w2v_model, parent, family, cluster, embedding):

    # Similarity between the parent and a cluster
    parent_similarity = 0
    for item in cluster:
        if embedding == "poincare":
            max_similarity = 0
            item_senses = wn.synsets(item)
            parent_senses = wn.synsets(parent)
            for parent_sense in parent_senses:
                for item_sense in item_senses:
                    try:
                        similarity = w2v_model.kv.similarity(parent_sense.name(), item_sense.name())
                        if similarity > max_similarity:
                            max_similarity = similarity
                    except KeyError as e:
                        if parent_sense.name() in e:
                            break
                        else:
                            continue
            parent_similarity += max_similarity
        else:
            try:
                parent_similarity += w2v_model.similarity(parent, item)
            except KeyError:  # skip the terms not in vocabulary
                continue
    parent_similarity /= len(cluster)

    # Similarity between a family and a cluster
    family_similarity = 0
    for f_item in family:
        for c_item in cluster:
            if embedding == "poincare":
                max_similarity = 0
                f_senses = wn.synsets(f_item)
                c_senses = wn.synsets(c_item)
                for f_sense in f_senses:
                    for c_sense in c_senses:
                        try:
                            similarity = w2v_model.kv.similarity(f_sense.name(), c_sense.name())
                            if similarity > max_similarity:
                                max_similarity = similarity
                        except KeyError as e:
                            if f_sense.name() in e:
                                break
                            else:
                                continue
                family_similarity += max_similarity
            else:
                try:
                    family_similarity += w2v_model.similarity(f_item, c_item)
                except KeyError:  # skip the terms not in vocabulary
                    continue
    family_similarity /= (len(family) * len(cluster))

    # Final score is the average of both the similarities
    return (parent_similarity + family_similarity) / 2

def tune_result(g_improved):
    """ Filter the results i.e. remove all the isolated nodes and nodes with blank labels """

    print('Tuning the result...')

    if '' in g_improved.nodes():
        g_improved.remove_node('')

    hypernyms = {x[0] for x in g_improved.edges()}
    isolated_nodes = list(nx.isolates(g_improved))
    for isolated_node in isolated_nodes:
        terms = isolated_node.split('_')
        if terms[-1] in hypernyms:
            g_improved.add_edge(terms[-1], isolated_node)
        elif terms[0] in hypernyms:
            g_improved.add_edge(terms[0], isolated_node)
        else:
            g_improved.remove_node(isolated_node)

    return g_improved

def apply_distributional_semantics(nx_graph, mode, embeddings, depth, iterations):
    # Load the pre-trained vectors
    print('Loading', embeddings, 'embeddings...')
    w2v_model = load_vectors(embeddings)
    print('Loaded.')

    print('\n\nApplying distributional semantics...')
    g_improved = nx_graph.copy()
    for i in range(1, iterations + 1):
        print('\nIteration %d/%d:' % (i, iterations))

        # Remove small clusters
        g_improved, removed_clusters = remove_clusters(w2v_model, g_improved, embeddings, depth)
        print('Removed %d clusters.' % (len(removed_clusters)))

        # Reattach the removed clusters
        if mode == 'reattach':
            print('Reattaching removed clusters...')
            g_detached = create_children_clusters(w2v_model, g_improved, embeddings, depth)
            for cluster in removed_clusters:
                max_score = 0
                max_score_node = ''
                for node, graph in g_detached.items():
                    gc = chinese_whispers(graph, weighting='top', iterations=60)
                    for label, family in aggregate_clusters(gc).items():
                        score = calculate_similarity(w2v_model, node, family, cluster, embeddings)
                        if score > max_score:
                            max_score = score
                            max_score_node = node
                for item in cluster:
                    g_improved.add_edge(max_score_node, item)
        print('Done.')

        # Tune the result
        g_improved = tune_result(g_improved)
        print('Tuned.')

    return g_improved

def save_result(result, path, mode):
    print('\n\nSaving the result...')
    df_improved = pd.DataFrame(list(result.edges()), columns=['hypernym', 'hyponym'])
    df_improved = df_improved[df_improved.columns.tolist()[::-1]]

    # Replace the underscores with blanks
    df_improved['hyponym'] = df_improved['hyponym'].apply(lambda x: x.replace('_', ' '))
    df_improved['hypernym'] = df_improved['hypernym'].apply(lambda x: x.replace('_', ' '))

    result_path = os.path.splitext(path)
    output_path = 'taxi_output/distributional_semantics/' + str(result_path[0].split('/')[-1]) + '-semantic'
    if mode == 'only_removal':
        output_path += '-removal'
    output_path += result_path[1]

    df_improved.to_csv(output_path, sep='\t', header=False)
    print('Output saved at:', output_path)

def main(taxonomy, mode, embeddings, depth, iterations):

    # Read the input
    graph = process_input(taxonomy)

    # Distributional Semantics
    g_improved = apply_distributional_semantics(graph, mode, embeddings, depth, iterations)

    # Save the results
    save_result(g_improved, taxonomy, mode)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Distributional Semantics for Taxonomy')
    parser.add_argument('-t', '--taxonomy', required=True, help='Input file containing the taxonomy')
    parser.add_argument(
        '-e', '--embeddings', required=True, type=str,
        choices=['poincare', 'wiki2M', 'wiki1M_subword', 'own_w2v'],
        help='Classifier architecture of the system.'
    )
    parser.add_argument('-m', '--mode', default='reattach', choices=['only_removal', 'reattach'])
    parser.add_argument(
        '-d', '--depth', type=int, default=100,
        help='Number of results to return while checking for most similar nodes of a term.'
    )
    parser.add_argument('-i', '--iterations', type=int, default=1, help='Number of iterations.')
    args = parser.parse_args()

    print('Input File:', args.taxonomy)
    print('Embeddings:', args.embeddings)
    print('Mode:', args.mode)

    main(args.taxonomy, args.mode, args.embeddings, args.depth, args.iterations)
<commit_msg>Fix issue of same cluster getting removed again<commit_after>import gensim
import os
import argparse
import pandas as pd
import networkx as nx
import matplotlib.pyplot as plt

from networkx.drawing.nx_agraph import graphviz_layout
from chinese_whispers import chinese_whispers, aggregate_clusters
from gensim.models.poincare import PoincareModel
from nltk.corpus import wordnet as wn

def process_input(taxonomy):
    """ Read the taxonomy and generate a networkx graph """

    # Read the taxonomy as a dataframe
    print('Reading input...')
    df = pd.read_csv(
        taxonomy,
        sep='\t',
        header=None,
        names=['hyponym', 'hypernym'],
        usecols=[1, 2],
    )

    # Construct the networkx graph
    print('Constructing networkx graph...')
    graph = nx.DiGraph()
    for hypernym, hyponym in zip(list(df['hypernym']), list(df['hyponym'])):
        # Simplify the compound words by replacing the whitespaces with underscores
        if ' ' in hypernym:
            hypernym = '_'.join(hypernym.split())
        if ' ' in hyponym:
            hyponym = '_'.join(hyponym.split())
        graph.add_edge(hypernym, hyponym)

    return graph

def display_taxonomy(graph):
    """ Display the taxonomy in a hierarchical layout """

    pos = graphviz_layout(graph, prog='dot', args="-Grankdir=LR")
    plt.figure(3, figsize=(48, 144))
    nx.draw(graph, pos, with_labels=True, arrows=True)
    plt.show()

def load_vectors(embedding):
    """ Load word vectors. """

    embedding_dir = '/home/5aly/taxi/distributed_semantics/embeddings/'

    if embedding == "wiki2M":
        model = gensim.models.KeyedVectors.load_word2vec_format(embedding_dir + 'crawl-300d-2M.vec', binary=False)
    elif embedding == "wiki1M_subword":
        model = gensim.models.KeyedVectors.load_word2vec_format(
            embedding_dir + 'wiki-news-300d-1M-subword.vec', binary=False
        )
    elif embedding == "own_w2v":
        model = gensim.models.KeyedVectors.load(embedding_dir + 'own_embeddings_w2v')
    elif embedding == "poincare":
        model = PoincareModel.load(embedding_dir + 'embeddings_poincare_wordnet')

    return model

def create_children_clusters(w2v_model, graph, embedding, depth=100):
    """ This function returns a dictionary where corresponding to each key(node) is a graph of its children """
    clustered_graph = {}
    for node in graph.nodes():
        clustered_graph[node] = nx.Graph()
        successors = [s.lower() for s in graph.successors(node)]

        for successor in successors:
            word_in_vocab = False
            if embedding == "poincare":
                word_senses = wn.synsets(successor)  # Get all the senses of the given node
                for sense in word_senses:
                    try:
                        for word, score in w2v_model.kv.most_similar(sense.name(), topn=depth):
                            word = word.split('.')[0]  # convert the word from poincare format to normal string
                            word_in_vocab = True
                            if word.lower() in successors:
                                clustered_graph[node].add_edge(successor, word.lower())
                    except KeyError:
                        continue
            else:
                try:
                    for word, score in w2v_model.most_similar(successor, topn=depth):
                        word_in_vocab = True
                        if word.lower() in successors:
                            clustered_graph[node].add_edge(successor, word.lower())
                except KeyError:
                    pass

            if not word_in_vocab:  # If the word in not in vocabulary, check using the substring based method
                successor_terms = successor.split('_')
                root_terms = [successor_terms[0], successor_terms[-1]]
                if node in root_terms:
                    clustered_graph[node].add_node(successor)

    return clustered_graph

def remove_clusters(model, nx_graph, embedding, clusters_touched, depth=100):
    """ Removes the less related and small clusters from the graph """

    print('Removing small clusters..')
    g_clustered = create_children_clusters(model, nx_graph, embedding, depth)
    removed_clusters = []

    nodes, clusters, size_ratio = [], [], []
    for node, graph in g_clustered.items():
        gc = chinese_whispers(graph, weighting='top', iterations=60)
        try:  # Get the length of the largest cluster
            max_cluster_size = len(max(aggregate_clusters(gc).values(), key=len))
        except ValueError:
            continue

        # Calculate the size ratio of all the clusters which are smaller than the largest
        for _, cluster in aggregate_clusters(gc).items():
            if len(cluster) < max_cluster_size and cluster not in clusters_touched:
                nodes.append(node)
                clusters.append(cluster)
                size_ratio.append(len(cluster) / max_cluster_size)

    # Sort the small clusters according to their size_ratio
    sorted_node_clusters = [(node, cluster) for _, cluster, node in sorted(zip(size_ratio, clusters, nodes))]
    if len(sorted_node_clusters) > 10:
        sorted_node_clusters = sorted_node_clusters[:10]

    for node, cluster in sorted_node_clusters:  # detach only the smallest 10 clusters in the entire taxonomy
        removed_clusters.append(cluster)
        for item in cluster:
            nx_graph.remove_edge(node, item)

    return nx_graph, removed_clusters

def calculate_similarity(w2v_model, parent, family, cluster, embedding):

    # Similarity between the parent and a cluster
    parent_similarity = 0
    for item in cluster:
        if embedding == "poincare":
            max_similarity = 0
            item_senses = wn.synsets(item)
            parent_senses = wn.synsets(parent)
            for parent_sense in parent_senses:
                for item_sense in item_senses:
                    try:
                        similarity = w2v_model.kv.similarity(parent_sense.name(), item_sense.name())
                        if similarity > max_similarity:
                            max_similarity = similarity
                    except KeyError as e:
                        if parent_sense.name() in e:
                            break
                        else:
                            continue
            parent_similarity += max_similarity
        else:
            try:
                parent_similarity += w2v_model.similarity(parent, item)
            except KeyError:  # skip the terms not in vocabulary
                continue
    parent_similarity /= len(cluster)

    # Similarity between a family and a cluster
    family_similarity = 0
    for f_item in family:
        for c_item in cluster:
            if embedding == "poincare":
                max_similarity = 0
                f_senses = wn.synsets(f_item)
                c_senses = wn.synsets(c_item)
                for f_sense in f_senses:
                    for c_sense in c_senses:
                        try:
                            similarity = w2v_model.kv.similarity(f_sense.name(), c_sense.name())
                            if similarity > max_similarity:
                                max_similarity = similarity
                        except KeyError as e:
                            if f_sense.name() in e:
                                break
                            else:
                                continue
                family_similarity += max_similarity
            else:
                try:
                    family_similarity += w2v_model.similarity(f_item, c_item)
                except KeyError:  # skip the terms not in vocabulary
                    continue
    family_similarity /= (len(family) * len(cluster))

    # Final score is the average of both the similarities
    return (parent_similarity + family_similarity) / 2

def tune_result(g_improved):
    """ Filter the results i.e. remove all the isolated nodes and nodes with blank labels """

    print('Tuning the result...')

    if '' in g_improved.nodes():
        g_improved.remove_node('')

    hypernyms = {x[0] for x in g_improved.edges()}
    isolated_nodes = list(nx.isolates(g_improved))
    for isolated_node in isolated_nodes:
        terms = isolated_node.split('_')
        if terms[-1] in hypernyms:
            g_improved.add_edge(terms[-1], isolated_node)
        elif terms[0] in hypernyms:
            g_improved.add_edge(terms[0], isolated_node)
        else:
            g_improved.remove_node(isolated_node)

    return g_improved

def apply_distributional_semantics(nx_graph, mode, embeddings, depth, iterations):
    # Load the pre-trained vectors
    print('Loading', embeddings, 'embeddings...')
    w2v_model = load_vectors(embeddings)
    print('Loaded.')

    print('\n\nApplying distributional semantics...')
    g_improved = nx_graph.copy()
    clusters_touched = []
    for i in range(1, iterations + 1):
        print('\nIteration %d/%d:' % (i, iterations))

        # Remove small clusters
        g_improved, removed_clusters = remove_clusters(w2v_model, g_improved, embeddings, clusters_touched, depth)
        print('Removed %d clusters.' % (len(removed_clusters)))
        print('Clusters Removed:', removed_clusters)
        clusters_touched.append(removed_clusters)  # To ensure that the same cluster does not get removed again

        # Reattach the removed clusters
        if mode == 'reattach':
            print('Reattaching removed clusters...')
            g_detached = create_children_clusters(w2v_model, g_improved, embeddings, depth)
            for cluster in removed_clusters:
                max_score = 0
                max_score_node = ''
                for node, graph in g_detached.items():
                    gc = chinese_whispers(graph, weighting='top', iterations=60)
                    for label, family in aggregate_clusters(gc).items():
                        score = calculate_similarity(w2v_model, node, family, cluster, embeddings)
                        if score > max_score:
                            max_score = score
                            max_score_node = node
                for item in cluster:
                    g_improved.add_edge(max_score_node, item)
        print('Done.')

        # Tune the result
        g_improved = tune_result(g_improved)
        print('Tuned.')

    return g_improved

def save_result(result, path, mode):
    print('\n\nSaving the result...')
    df_improved = pd.DataFrame(list(result.edges()), columns=['hypernym', 'hyponym'])
    df_improved = df_improved[df_improved.columns.tolist()[::-1]]

    # Replace the underscores with blanks
    df_improved['hyponym'] = df_improved['hyponym'].apply(lambda x: x.replace('_', ' '))
    df_improved['hypernym'] = df_improved['hypernym'].apply(lambda x: x.replace('_', ' '))

    result_path = os.path.splitext(path)
    output_path = 'taxi_output/distributional_semantics/' + str(result_path[0].split('/')[-1]) + '-semantic'
    if mode == 'only_removal':
        output_path += '-removal'
    output_path += result_path[1]

    df_improved.to_csv(output_path, sep='\t', header=False)
    print('Output saved at:', output_path)

def main(taxonomy, mode, embeddings, depth, iterations):

    # Read the input
    graph = process_input(taxonomy)

    # Distributional Semantics
    g_improved = apply_distributional_semantics(graph, mode, embeddings, depth, iterations)

    # Save the results
    save_result(g_improved, taxonomy, mode)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Distributional Semantics for Taxonomy')
    parser.add_argument('-t', '--taxonomy', required=True, help='Input file containing the taxonomy')
    parser.add_argument(
        '-e', '--embeddings', required=True, type=str,
        choices=['poincare', 'wiki2M', 'wiki1M_subword', 'own_w2v'],
        help='Classifier architecture of the system.'
    )
    parser.add_argument('-m', '--mode', default='reattach', choices=['only_removal', 'reattach'])
    parser.add_argument(
        '-d', '--depth', type=int, default=100,
        help='Number of results to return while checking for most similar nodes of a term.'
    )
    parser.add_argument('-i', '--iterations', type=int, default=1, help='Number of iterations.')
    args = parser.parse_args()

    print('Input File:', args.taxonomy)
    print('Embeddings:', args.embeddings)
    print('Mode:', args.mode)

    main(args.taxonomy, args.mode, args.embeddings, args.depth, args.iterations)
<|endoftext|>


<commit_before>"""
Music downloader module
"""

from __future__ import unicode_literals
import eyed3
import zerorpc
import subprocess
import os
import yaml

config = yaml.load(open("../config.yaml").read())

class YtDownloader(object):
    """
    Download music from youtube
    """

    def __init__(self, save_dir):
        self.save_dir = save_dir

    def save(self, url, metadata):
        filename = metadata["artist"] + "-" + metadata["title"]
        # absolute filename
        filename = os.path.join(self.save_dir, filename)
        command = ["youtube-dl", "-x", "--audio-format", "mp3",
                   "--audio-quality", "0", "--no-playlist"]
        command += ["--output", filename + ".%(ext)s"]
        command += [url]

        retcode = subprocess.call(command)

        if retcode == 1:
            return "err"
        else:
            # Write Metadata to file
            print("Writing metadata...")
            song = eyed3.load(filename + ".mp3")
            song.tag.artist = unicode(metadata["artist"])
            song.tag.title = unicode(metadata["title"])
            song.tag.save()
            return "ok"

s = zerorpc.Server(YtDownloader(config["download_dir"]))
s.bind("tcp://0.0.0.0:" + str(config["rpc_port"]))
s.run()
<commit_msg>Add switch to use ffmpeg as default for downloader<commit_after>"""
Music downloader module
"""

from __future__ import unicode_literals
import eyed3
import zerorpc
import subprocess
import os
import yaml

config = yaml.load(open("../config.yaml").read())

class YtDownloader(object):
    """
    Download music from youtube
    """

    def __init__(self, save_dir):
        self.save_dir = save_dir

    def save(self, url, metadata):
        filename = metadata["artist"] + "-" + metadata["title"]
        # absolute filename
        filename = os.path.join(self.save_dir, filename)
        command = ["youtube-dl", "-x", "--audio-format", "mp3",
                   "--audio-quality", "0", "--no-playlist", "--prefer-ffmpeg"]
        command += ["--output", filename + ".%(ext)s"]
        command += [url]

        retcode = subprocess.call(command)

        if retcode == 1:
            return "err"
        else:
            # Write Metadata to file
            print("Writing metadata...")
            song = eyed3.load(filename + ".mp3")
            song.tag.artist = unicode(metadata["artist"])
            song.tag.title = unicode(metadata["title"])
            song.tag.save()
            return "ok"

s = zerorpc.Server(YtDownloader(config["download_dir"]))
s.bind("tcp://0.0.0.0:" + str(config["rpc_port"]))
s.run()
<|endoftext|>


<commit_before>import numpy as np
import sys, os
import nrrd

def sphere(shape, radius, position):
    # assume shape and position are both a 3-tuple of int or float
    # the units are pixels / voxels (px for short)
    # radius is a int or float in px
    semisizes = (radius,) * 3

    #ignore divide by zero
    np.seterr(divide='ignore', invalid='ignore')

    # genereate the grid for the support points
    # centered at the position indicated by position
    grid = [slice(-x0, dim - x0) for x0, dim in zip(position, shape)]
    position = np.ogrid[grid]
    # calculate the distance of all points from `position` center
    # scaled by the radius
    arr = np.zeros(shape, dtype=float)
    for x_i, semisize in zip(position, semisizes):
        arr += (np.abs(x_i / semisize) ** 2)
    # the inner part of the sphere will have distance below 1
    return arr <= 1.0

scale=1

if (len(sys.argv) < 2):
    print('Error: missing arguments!')
    print('e.g. python swc2nrrd.py template.nrrd neuron.swc Image.nrrd [width] [scale]')
else:
    Itemp = str(sys.argv[1])
    Iswc = str(sys.argv[2])
    Iout = str(sys.argv[3])
    bounded = True

    w = 0
    if (len(sys.argv) > 4):
      w=np.int32(sys.argv[4])

    if (len(sys.argv) > 5):
      scale=np.int32(sys.argv[5])
      bounded = False

    print('Loading %s...'% (Itemp))
    tempData1, tempHeader1 = nrrd.read(Itemp)

    print('Loading %s...'% (Iswc))
    with open(Iswc) as fI:
        swcIn = fI.readlines()

    lineDict = {}
    for thisLine in swcIn:
        if thisLine[0]!='#':
            splitLine = thisLine.split(" ")
            lineDict[int(splitLine[0])] = {'position':np.array([splitLine[2],splitLine[3],splitLine[4]],dtype=np.float),
                                      'radius':splitLine[5],
                                      'parent':int(splitLine[6])}
    extent=[1000,1000,1000]

    if bounded:
        for i in range(3):
            extent[i] = np.max([(np.max([x['position'][i] for x in lineDict.values()])/scale).astype(np.int),np.shape(tempData1)[i]])
    else:
        for i in range(3):
            extent[i] = (np.max([x['position'][i] for x in lineDict.values()])/scale).astype(np.int)

    print(extent)

    outputImg = np.zeros(extent,dtype=np.uint8)

    r=0

    for thisDict in lineDict.values():
        p = np.round(np.divide(np.divide(thisDict['position'],[tempHeader1['space directions'][0][0],tempHeader1['space directions'][1][1],tempHeader1['space directions'][2][2]]),scale)).astype(np.int)

        if w==0 and float(thisDict['radius'])>0:
            r=float(thisDict['radius'])
        else:
            r=w
        point = np.multiply(sphere(extent, r, p),np.uint8(255)).astype(np.uint8)
        outputImg = np.maximum(outputImg, point).astype(np.uint8)

    nrrd.write(Iout, np.uint8(outputImg), header=tempHeader1)
    print('saved to ' + Iout)
<commit_msg>speeding up for basic points<commit_after>import numpy as np
import sys, os
import nrrd

def sphere(shape, radius, position):
    # assume shape and position are both a 3-tuple of int or float
    # the units are pixels / voxels (px for short)
    # radius is a int or float in px
    semisizes = (radius,) * 3

    #ignore divide by zero
    np.seterr(divide='ignore', invalid='ignore')

    # genereate the grid for the support points
    # centered at the position indicated by position
    grid = [slice(-x0, dim - x0) for x0, dim in zip(position, shape)]
    position = np.ogrid[grid]
    # calculate the distance of all points from `position` center
    # scaled by the radius
    arr = np.zeros(shape, dtype=float)
    for x_i, semisize in zip(position, semisizes):
        arr += (np.abs(x_i / semisize) ** 2)
    # the inner part of the sphere will have distance below 1
    return arr <= 1.0

scale=1

if (len(sys.argv) < 2):
    print('Error: missing arguments!')
    print('e.g. python swc2nrrd.py template.nrrd neuron.swc Image.nrrd [width] [scale]')
else:
    Itemp = str(sys.argv[1])
    Iswc = str(sys.argv[2])
    Iout = str(sys.argv[3])
    bounded = True

    w = 0
    if (len(sys.argv) > 4):
      w=np.int32(sys.argv[4])

    if (len(sys.argv) > 5):
      scale=np.int32(sys.argv[5])
      bounded = False

    print('Loading %s...'% (Itemp))
    tempData1, tempHeader1 = nrrd.read(Itemp)

    print('Loading %s...'% (Iswc))
    with open(Iswc) as fI:
        swcIn = fI.readlines()

    lineDict = {}
    for thisLine in swcIn:
        if thisLine[0]!='#':
            splitLine = thisLine.split(" ")
            lineDict[int(splitLine[0])] = {'position':np.array([splitLine[2],splitLine[3],splitLine[4]],dtype=np.float),
                                      'radius':splitLine[5],
                                      'parent':int(splitLine[6])}
    extent=[1000,1000,1000]

    if bounded:
        for i in range(3):
            extent[i] = np.max([(np.max([x['position'][i] for x in lineDict.values()])/scale).astype(np.int),np.shape(tempData1)[i]])
    else:
        for i in range(3):
            extent[i] = (np.max([x['position'][i] for x in lineDict.values()])/scale).astype(np.int)

    print(extent)

    outputImg = np.zeros(extent,dtype=np.uint8)

    r=0

    for thisDict in lineDict.values():
        p = np.round(np.divide(np.divide(thisDict['position'],[tempHeader1['space directions'][0][0],tempHeader1['space directions'][1][1],tempHeader1['space directions'][2][2]]),scale)).astype(np.int)

        if w==0 and float(thisDict['radius'])>0:
            r=float(thisDict['radius'])
        else:
            r=w
        if r==0:
            outputImg[p]=np.uint8(255)
        else:
            point = np.multiply(sphere(extent, r, p),np.uint8(255)).astype(np.uint8)
            outputImg = np.maximum(outputImg, point).astype(np.uint8)

    nrrd.write(Iout, np.uint8(outputImg), header=tempHeader1)
    print('saved to ' + Iout)
<|endoftext|>


<commit_before>import os
import logging
import envoy
from .ants_error import ANTSError
from .similarity_metrics import *
from .template import create_template
from .warp_transform import WarpTransform

def register(path, output, metric=None, reference=None):
    ANTS(output, metric=metric, reference=reference).register(path)

class ANTS(object):
    """ANTS wraps the ANTS filters.
    """

    def __init__(self, output=None, metric=None, reference=None):
        """Initializes this ANTS instance.

        :param output: the output directory (default is the input directory)
        :param metric: the similarity metric (default is cross-correlation)
        :param reference: the fixed reference image (default is generated)
        """
        self.output = self.reference = None
        if not metric:
            metric = CC()
        self.metric = metric
        if output:
            self.output = output
        if reference:
            self.reference = os.path.abspath(reference)

    def register(self, path):
        """Registers the images in the given directory using a symmetric diffeomorphic deformation.

        :param path: the directory containing the images to register
        """
        files = os.listdir(path)
        if self.output:
            output = self.output
            if not os.path.exists(output):
                os.makedirs(output)
            # Link to the source images, if necessary.
            if not os.path.samefile(path, output):
                self._link_files(path, output, files)
        else:
            output = path
        logging.info("Registering the images in %s..." % output)
        cwd = os.getcwd()
        os.chdir(output)
        try:
            if not self.reference:
                self.reference = create_template(self.metric, files)
            for moving in files:
                WarpTransform(moving, self.reference, self.metric).apply()
        finally:
            os.chdir(cwd)
        logging.info("Registered the images in %s." % output)

    def _link_files(self, source, destination, files):
        """
        Links the given files in the source directory to the destination.
        """
        logging.info("Linking the %(source)s source images to the output %(destination)s..." % {'source': source, 'destination': destination})
        for f in files:
            dest = os.path.join(destination, f)
            if not os.path.exists(dest):
                src = os.path.join(source, f)
                os.symlink(src, dest)

    <commit_msg>Make link source path absolute.<commit_after>import os
import logging
import envoy
from .ants_error import ANTSError
from .similarity_metrics import *
from .template import create_template
from .warp_transform import WarpTransform

def register(path, output, metric=None, reference=None):
    ANTS(output, metric=metric, reference=reference).register(path)

class ANTS(object):
    """ANTS wraps the ANTS filters.
    """

    def __init__(self, output=None, metric=None, reference=None):
        """Initializes this ANTS instance.

        :param output: the output directory (default is the input directory)
        :param metric: the similarity metric (default is cross-correlation)
        :param reference: the fixed reference image (default is generated)
        """
        self.output = self.reference = None
        if not metric:
            metric = CC()
        self.metric = metric
        if output:
            self.output = output
        if reference:
            self.reference = os.path.abspath(reference)

    def register(self, path):
        """Registers the images in the given directory using a symmetric diffeomorphic deformation.

        :param path: the directory containing the images to register
        """
        files = os.listdir(path)
        if self.output:
            output = self.output
            if not os.path.exists(output):
                os.makedirs(output)
            # Link to the source images, if necessary.
            if not os.path.samefile(path, output):
                self._link_files(path, output, files)
        else:
            output = path
        logging.info("Registering the images in %s..." % output)
        cwd = os.getcwd()
        os.chdir(output)
        try:
            if not self.reference:
                self.reference = create_template(self.metric, files)
            for moving in files:
                WarpTransform(moving, self.reference, self.metric).apply()
        finally:
            os.chdir(cwd)
        logging.info("Registered the images in %s." % output)

    def _link_files(self, source, destination, files):
        """
        Links the given files in the source directory to the destination.
        """
        logging.info("Linking the %(source)s source images to the output %(destination)s..." % {'source': source, 'destination': destination})
        source = os.path.abspath(source)
        for f in files:
            dest = os.path.join(destination, f)
            if not os.path.exists(dest):
                src = os.path.join(source, f)
                os.symlink(src, dest)

    <|endoftext|>


<commit_before>import os

from .PYPY import PYTHON_FS
from .python import PythonExecutor
from judgeenv import env

try:
    from cptbox.syscalls import sys_mkdir, sys_unlink
except ImportError:
    pass

class Executor(PythonExecutor):
    command = env['runtime'].get('pypy3')
    test_program = "print(__import__('sys').stdin.read(), end='')"
    name = 'PYPY3'
    fs = PYTHON_FS + [command] + ([env['runtime']['pypy3dir']] if 'pypy3dir' in env['runtime'] else [])

    def get_security(self):
        sec = super(Executor, self).get_security()

        def unsafe_pypy3dir(debugger):
            # Relies on the fact this user can't access here.
            return debugger.readstr(debugger.uarg0).startswith(env['runtime']['pypy3dir'])

        if not os.access(env['runtime']['pypy3dir'], os.W_OK):
            sec[sys_mkdir] = unsafe_pypy3dir
            sec[sys_unlink] = unsafe_pypy3dir
        return sec

initialize = Executor.initialize
<commit_msg>Why even bother trying to import *nix stuff on Windows?<commit_after>import os

from .PYPY import PYTHON_FS
from .python import PythonExecutor
from judgeenv import env

class Executor(PythonExecutor):
    command = env['runtime'].get('pypy3')
    test_program = "print(__import__('sys').stdin.read(), end='')"
    name = 'PYPY3'
    fs = PYTHON_FS + [command] + ([env['runtime']['pypy3dir']] if 'pypy3dir' in env['runtime'] else [])

    def get_security(self):
        from cptbox.syscalls import sys_mkdir, sys_unlink

        sec = super(Executor, self).get_security()

        def unsafe_pypy3dir(debugger):
            # Relies on the fact this user can't access here.
            return debugger.readstr(debugger.uarg0).startswith(env['runtime']['pypy3dir'])

        if not os.access(env['runtime']['pypy3dir'], os.W_OK):
            sec[sys_mkdir] = unsafe_pypy3dir
            sec[sys_unlink] = unsafe_pypy3dir
        return sec

initialize = Executor.initialize
<|endoftext|>


<commit_before>import functools
import json
import os
import subprocess
import sys
from time import time

import click
import yaml

from .. import config, exceptions, settings
from ..utils import check_call, check_output, is_windows

DOT_ALDRYN_FILE_NOT_FOUND = (
    "Divio Cloud configuration file '{}' or '{}' could not be found!\n"
    "Please make sure you're in a Divio Cloud project folder and the "
    "file exists.\n\n"
    "You can create a new configuration file for an existing project "
    "with the `divio app configure` command.".format(
        settings.ALDRYN_DOT_FILE, settings.DIVIO_DOT_FILE
    )
)

def get_project_settings(path=None, silent=False):
    project_home = get_application_home(path, silent=silent)
    try:
        old_file_found = False
        if os.path.exists(
            os.path.join(project_home, settings.ALDRYN_DOT_FILE)
        ):
            path = os.path.join(project_home, settings.ALDRYN_DOT_FILE)
            old_file_found = True

        if os.path.exists(os.path.join(project_home, settings.DIVIO_DOT_FILE)):
            path = os.path.join(project_home, settings.DIVIO_DOT_FILE)
            if old_file_found:
                click.secho(
                    "Warning: Old ({}) and new ({}) divio configuration files found at the same time. The new one will be used.".format(
                        settings.ALDRYN_DOT_FILE, settings.DIVIO_DOT_FILE
                    ),
                    fg="yellow",
                )

        with open(path) as fh:
            return json.load(fh)
    except (TypeError, OSError):
        raise click.ClickException(DOT_ALDRYN_FILE_NOT_FOUND)
    except json.decoder.JSONDecodeError:
        click.secho("Error: Unexpected value in {}".format(path), fg="red")
        sys.exit(1)

def get_application_home(path=None, silent=False):
    """
    find project root by traversing up the tree looking for
    the configuration file
    """
    previous_path = None
    current_path = path or os.getcwd()
    global_config_path = config.get_global_config_path()

    # loop until we're at the root of the volume
    while current_path != previous_path:
        # check if configuration file exists in current directory
        dotfile = os.path.join(current_path, settings.ALDRYN_DOT_FILE)
        if os.path.exists(dotfile) and dotfile != global_config_path:
            return current_path

        # check if configuration file exists in current directory
        dotfile = os.path.join(current_path, settings.DIVIO_DOT_FILE)
        if os.path.exists(dotfile) and dotfile != global_config_path:
            return current_path

        # traversing up the tree
        previous_path = current_path
        current_path = os.path.abspath(os.path.join(current_path, os.pardir))
    if silent:
        return
    raise click.ClickException(DOT_ALDRYN_FILE_NOT_FOUND)

UNIX_DOCKER_COMPOSE_FILENAME = "docker-compose.yml"
WINDOWS_DOCKER_COMPOSE_FILENAME = "docker-compose-windows.yml"

def get_docker_compose_cmd(path):
    if is_windows():
        docker_compose_filename = WINDOWS_DOCKER_COMPOSE_FILENAME
        ensure_windows_docker_compose_file_exists(path)
    else:
        docker_compose_filename = UNIX_DOCKER_COMPOSE_FILENAME

    docker_compose_filename = os.path.join(path, docker_compose_filename)

    if not os.path.isfile(docker_compose_filename):
        raise RuntimeError(
            "Warning: Could not find a 'docker-compose.yml' file."
        )

    conf = config.Config()
    cmd = conf.config.get("docker-compose", ["docker-compose"])
    docker_compose_base = cmd + ["-f", docker_compose_filename, " --log-level", "DEBUG"]

    def docker_compose(*commands):
        return docker_compose_base + [cmd for cmd in commands]

    return docker_compose

def ensure_windows_docker_compose_file_exists(path):
    """
    Unfortunately, docker-compose is not yet officially released
     for Windows There's still some rough edges, and volume
     configuration is one. There's also some open issues in
     boot2docker for windows which makes things difficult.

    We have to change the volume specifications to a very specific
     format:

     - absolute paths: relative one's are not yet supported
     - currently only works if the project is running on the C:\\ drive
     - unix style paths: need to replace '\' with '/'
     - paths have to start with /c/ instead of C:\\ otherwise
        docker-compose gets confused because they use : as separation

    Example:
      unix format:  .:/app:rw
      cwd:          C:\\Users\\aldryn\\acme-portfolio
      windows:     /c/Users/aldryn/acme-portfolio:/app:rw

    Hope that's all. And of course, I'm sorry.
    """

    windows_path = os.path.join(path, WINDOWS_DOCKER_COMPOSE_FILENAME)
    if os.path.isfile(windows_path):
        return

    unix_path = os.path.join(path, UNIX_DOCKER_COMPOSE_FILENAME)
    if not os.path.isfile(unix_path):
        # TODO: use correct exit from click
        click.secho(
            "docker-compose.yml not found at {}".format(unix_path), fg="red"
        )
        sys.exit(1)

    with open(unix_path, "r") as fh:
        conf = yaml.load(fh, Loader=yaml.SafeLoader)

    for component, sections in conf.items():
        if "volumes" not in sections:
            continue
        volumes = []
        for volume in sections["volumes"]:
            parts = volume.split(":")
            if len(parts) == 2:
                old_host, container = parts
                mode = None
            else:
                old_host, container, mode = parts

            # assuming relative path's for old_host
            new_host = os.path.abspath(os.path.join(path, old_host))
            # replace C:\ with /c/, because, docker on windows
            new_host = new_host.replace("C:\\", "/c/")
            # change to unix paths
            new_host = new_host.replace("\\", "/")
            new_volume = [new_host, container]
            if mode:
                new_volume.append(mode)
            volumes.append(":".join(new_volume))

        conf[component]["volumes"] = volumes

    with open(windows_path, "w+") as fh:
        yaml.safe_dump(conf, fh)

def get_db_container_id(path, raise_on_missing=True, prefix="DEFAULT"):
    """
    Returns the container id for a running database with a given prefix.
    """
    try:
        docker_compose = get_docker_compose_cmd(path)
    except RuntimeError:
        raise exceptions.DivioException(
            "docker-compose.yml not found. Unable to find database container"
        )
    should_check_oldstyle = False
    output = None

    try:
        output = check_output(
            docker_compose("ps", "-q", "database_{}".format(prefix).lower()),
            catch=False,
            stderr=open(os.devnull, "w"),
        ).rstrip(os.linesep)
        if not output:
            # This behavior was briefly used in docker-compose v2.
            # The output can be empty if the container does not exist.
            # For information: https://github.com/docker/compose-cli/issues/1893
            should_check_oldstyle = True
    except subprocess.CalledProcessError:
        # A not existing service will result in an error.
        should_check_oldstyle = True

    if should_check_oldstyle:
        try:
            output = check_output(docker_compose("ps", "-q", "db")).rstrip(
                os.linesep
            )
        except subprocess.CalledProcessError:
            # A not existing service will result in an error.
            pass

    if not output and raise_on_missing:
        raise exceptions.DivioException("Unable to find database container")
    return output

def start_database_server(docker_compose, prefix):
    start_db = time()
    click.secho(" ---> Starting local database server")
    click.secho("      ", nl=False)
    docker_compose_config = DockerComposeConfig(docker_compose)
    if (
        "database_{}".format(prefix).lower()
        in docker_compose_config.get_services()
    ):
        check_call(
            docker_compose("up", "-d", "database_{}".format(prefix).lower())
        )
    else:
        check_call(docker_compose("up", "-d", "db"))
    click.secho("      [{}s]".format(int(time() - start_db)))

class DockerComposeConfig(object):
    def __init__(self, docker_compose):
        super(DockerComposeConfig, self).__init__()
        self.config = yaml.load(
            check_output(docker_compose("config")), Loader=yaml.SafeLoader
        )

    def get_services(self):
        return self.config.get("services", {})

    def has_service(self, service):
        return service in self.get_services().keys()

    def has_volume_mount(self, service, remote_path):
        """
        Services may look like the following depending on the OS:

        - /home/user/some/path:/app:rw
        - C:\\whatever\\windows\\path:/app:rw (windows)
        """
        try:
            service_config = self.get_services()[service]
        except KeyError:
            return False

        for mount in service_config.get("volumes", []):
            # docker compose < 2
            if not isinstance(mount, dict):
                bits = mount.strip().split(":")
                if len(bits) > 2 and bits[-2] == remote_path:
                    return True
            else:
                if mount["target"] == remote_path:
                    return True

def allow_remote_id_override(func):
    """Adds an identifier option to the command, and gets the proper id"""

    @functools.wraps(func)
    def read_remote_id(remote_id, *args, **kwargs):
        ERROR_MSG = (
            "This command requires a Divio Cloud Project id. Please "
            "provide one with the --remote-id option or call the "
            "command from a project directory."
        )

        if not remote_id:
            try:
                remote_id = get_project_settings(silent=True)["id"]
            except KeyError:
                raise click.ClickException(ERROR_MSG)
            else:
                if not remote_id:
                    raise click.ClickException(ERROR_MSG)
        return func(remote_id, *args, **kwargs)

    return click.option(
        "--remote-id",
        "remote_id",
        default=None,
        type=int,
        help="Remote Project ID to use for project commands. "
        "Defaults to the project in the current directory using the "
        "configuration file.",
    )(read_remote_id)

def get_service_type(identifier, path=None):
    """
    Retrieves the service type based on the `SERVICE_MANAGER` environment
    variable of a services from the docker-compose file.
    """
    project_home = get_application_home(path)
    try:
        docker_compose = get_docker_compose_cmd(project_home)
    except RuntimeError:
        # Docker-compose does not exist
        click.secho(
            "Warning: docker-compose.yml does not exist. Can not get the service type without!",
            fg="red",
        )
        sys.exit(1)
    docker_compose_config = DockerComposeConfig(docker_compose)
    services = docker_compose_config.get_services()
    if (
        identifier in services
        and "environment" in services[identifier]
        and "SERVICE_MANAGER" in services[identifier]["environment"]
    ):
        return services[identifier]["environment"]["SERVICE_MANAGER"]

    raise RuntimeError("Can not get service type")

def get_db_type(prefix, path=None):
    """
    Utility function to wrap `get_service_type` to search for databases so we
    can properly fall back to PostgreSQL in case of old structures.
    """
    try:
        db_type = get_service_type(
            "database_{}".format(prefix.lower()), path=path
        )
    except RuntimeError:
        # legacy section. we try to look for the db, if it does not exist, fail
        try:
            docker_compose = get_docker_compose_cmd(path)
        except RuntimeError:
            # Docker-compose does not exist
            click.secho(
                "Warning: docker-compose.yml does not exist. Can not get the service type without!",
                fg="red",
            )
            sys.exit(1)
        docker_compose_config = DockerComposeConfig(docker_compose)
        if not docker_compose_config.has_service("db"):
            click.secho(
                "The local database container must be called "
                "`database_default`, must define the `SERVICE_MANAGER` "
                "environment variable and must mount the project directory "
                "from the host to the /app directory of the container."
                "\n\nSee https://docs.divio.com/en/latest/reference/docker-docker-compose/#database-default",
                fg="red",
            )
            sys.exit(1)
        else:
            # Fall back to database for legacy docker-compose files
            db_type = "fsm-postgres"
    return db_type
<commit_msg>dashes<commit_after>import functools
import json
import os
import subprocess
import sys
from time import time

import click
import yaml

from .. import config, exceptions, settings
from ..utils import check_call, check_output, is_windows

DOT_ALDRYN_FILE_NOT_FOUND = (
    "Divio Cloud configuration file '{}' or '{}' could not be found!\n"
    "Please make sure you're in a Divio Cloud project folder and the "
    "file exists.\n\n"
    "You can create a new configuration file for an existing project "
    "with the `divio app configure` command.".format(
        settings.ALDRYN_DOT_FILE, settings.DIVIO_DOT_FILE
    )
)

def get_project_settings(path=None, silent=False):
    project_home = get_application_home(path, silent=silent)
    try:
        old_file_found = False
        if os.path.exists(
            os.path.join(project_home, settings.ALDRYN_DOT_FILE)
        ):
            path = os.path.join(project_home, settings.ALDRYN_DOT_FILE)
            old_file_found = True

        if os.path.exists(os.path.join(project_home, settings.DIVIO_DOT_FILE)):
            path = os.path.join(project_home, settings.DIVIO_DOT_FILE)
            if old_file_found:
                click.secho(
                    "Warning: Old ({}) and new ({}) divio configuration files found at the same time. The new one will be used.".format(
                        settings.ALDRYN_DOT_FILE, settings.DIVIO_DOT_FILE
                    ),
                    fg="yellow",
                )

        with open(path) as fh:
            return json.load(fh)
    except (TypeError, OSError):
        raise click.ClickException(DOT_ALDRYN_FILE_NOT_FOUND)
    except json.decoder.JSONDecodeError:
        click.secho("Error: Unexpected value in {}".format(path), fg="red")
        sys.exit(1)

def get_application_home(path=None, silent=False):
    """
    find project root by traversing up the tree looking for
    the configuration file
    """
    previous_path = None
    current_path = path or os.getcwd()
    global_config_path = config.get_global_config_path()

    # loop until we're at the root of the volume
    while current_path != previous_path:
        # check if configuration file exists in current directory
        dotfile = os.path.join(current_path, settings.ALDRYN_DOT_FILE)
        if os.path.exists(dotfile) and dotfile != global_config_path:
            return current_path

        # check if configuration file exists in current directory
        dotfile = os.path.join(current_path, settings.DIVIO_DOT_FILE)
        if os.path.exists(dotfile) and dotfile != global_config_path:
            return current_path

        # traversing up the tree
        previous_path = current_path
        current_path = os.path.abspath(os.path.join(current_path, os.pardir))
    if silent:
        return
    raise click.ClickException(DOT_ALDRYN_FILE_NOT_FOUND)

UNIX_DOCKER_COMPOSE_FILENAME = "docker-compose.yml"
WINDOWS_DOCKER_COMPOSE_FILENAME = "docker-compose-windows.yml"

def get_docker_compose_cmd(path):
    if is_windows():
        docker_compose_filename = WINDOWS_DOCKER_COMPOSE_FILENAME
        ensure_windows_docker_compose_file_exists(path)
    else:
        docker_compose_filename = UNIX_DOCKER_COMPOSE_FILENAME

    docker_compose_filename = os.path.join(path, docker_compose_filename)

    if not os.path.isfile(docker_compose_filename):
        raise RuntimeError(
            "Warning: Could not find a 'docker-compose.yml' file."
        )

    conf = config.Config()
    cmd = conf.config.get("docker-compose", ["docker-compose"])
    docker_compose_base = cmd + ["-f", docker_compose_filename, "--log-level", "DEBUG"]

    def docker_compose(*commands):
        return docker_compose_base + [cmd for cmd in commands]

    return docker_compose

def ensure_windows_docker_compose_file_exists(path):
    """
    Unfortunately, docker-compose is not yet officially released
     for Windows There's still some rough edges, and volume
     configuration is one. There's also some open issues in
     boot2docker for windows which makes things difficult.

    We have to change the volume specifications to a very specific
     format:

     - absolute paths: relative one's are not yet supported
     - currently only works if the project is running on the C:\\ drive
     - unix style paths: need to replace '\' with '/'
     - paths have to start with /c/ instead of C:\\ otherwise
        docker-compose gets confused because they use : as separation

    Example:
      unix format:  .:/app:rw
      cwd:          C:\\Users\\aldryn\\acme-portfolio
      windows:     /c/Users/aldryn/acme-portfolio:/app:rw

    Hope that's all. And of course, I'm sorry.
    """

    windows_path = os.path.join(path, WINDOWS_DOCKER_COMPOSE_FILENAME)
    if os.path.isfile(windows_path):
        return

    unix_path = os.path.join(path, UNIX_DOCKER_COMPOSE_FILENAME)
    if not os.path.isfile(unix_path):
        # TODO: use correct exit from click
        click.secho(
            "docker-compose.yml not found at {}".format(unix_path), fg="red"
        )
        sys.exit(1)

    with open(unix_path, "r") as fh:
        conf = yaml.load(fh, Loader=yaml.SafeLoader)

    for component, sections in conf.items():
        if "volumes" not in sections:
            continue
        volumes = []
        for volume in sections["volumes"]:
            parts = volume.split(":")
            if len(parts) == 2:
                old_host, container = parts
                mode = None
            else:
                old_host, container, mode = parts

            # assuming relative path's for old_host
            new_host = os.path.abspath(os.path.join(path, old_host))
            # replace C:\ with /c/, because, docker on windows
            new_host = new_host.replace("C:\\", "/c/")
            # change to unix paths
            new_host = new_host.replace("\\", "/")
            new_volume = [new_host, container]
            if mode:
                new_volume.append(mode)
            volumes.append(":".join(new_volume))

        conf[component]["volumes"] = volumes

    with open(windows_path, "w+") as fh:
        yaml.safe_dump(conf, fh)

def get_db_container_id(path, raise_on_missing=True, prefix="DEFAULT"):
    """
    Returns the container id for a running database with a given prefix.
    """
    try:
        docker_compose = get_docker_compose_cmd(path)
    except RuntimeError:
        raise exceptions.DivioException(
            "docker-compose.yml not found. Unable to find database container"
        )
    should_check_oldstyle = False
    output = None

    try:
        output = check_output(
            docker_compose("ps", "-q", "database_{}".format(prefix).lower()),
            catch=False,
            stderr=open(os.devnull, "w"),
        ).rstrip(os.linesep)
        if not output:
            # This behavior was briefly used in docker-compose v2.
            # The output can be empty if the container does not exist.
            # For information: https://github.com/docker/compose-cli/issues/1893
            should_check_oldstyle = True
    except subprocess.CalledProcessError:
        # A not existing service will result in an error.
        should_check_oldstyle = True

    if should_check_oldstyle:
        try:
            output = check_output(docker_compose("ps", "-q", "db")).rstrip(
                os.linesep
            )
        except subprocess.CalledProcessError:
            # A not existing service will result in an error.
            pass

    if not output and raise_on_missing:
        raise exceptions.DivioException("Unable to find database container")
    return output

def start_database_server(docker_compose, prefix):
    start_db = time()
    click.secho(" ---> Starting local database server")
    click.secho("      ", nl=False)
    docker_compose_config = DockerComposeConfig(docker_compose)
    if (
        "database_{}".format(prefix).lower()
        in docker_compose_config.get_services()
    ):
        check_call(
            docker_compose("up", "-d", "database_{}".format(prefix).lower())
        )
    else:
        check_call(docker_compose("up", "-d", "db"))
    click.secho("      [{}s]".format(int(time() - start_db)))

class DockerComposeConfig(object):
    def __init__(self, docker_compose):
        super(DockerComposeConfig, self).__init__()
        self.config = yaml.load(
            check_output(docker_compose("config")), Loader=yaml.SafeLoader
        )

    def get_services(self):
        return self.config.get("services", {})

    def has_service(self, service):
        return service in self.get_services().keys()

    def has_volume_mount(self, service, remote_path):
        """
        Services may look like the following depending on the OS:

        - /home/user/some/path:/app:rw
        - C:\\whatever\\windows\\path:/app:rw (windows)
        """
        try:
            service_config = self.get_services()[service]
        except KeyError:
            return False

        for mount in service_config.get("volumes", []):
            # docker compose < 2
            if not isinstance(mount, dict):
                bits = mount.strip().split(":")
                if len(bits) > 2 and bits[-2] == remote_path:
                    return True
            else:
                if mount["target"] == remote_path:
                    return True

def allow_remote_id_override(func):
    """Adds an identifier option to the command, and gets the proper id"""

    @functools.wraps(func)
    def read_remote_id(remote_id, *args, **kwargs):
        ERROR_MSG = (
            "This command requires a Divio Cloud Project id. Please "
            "provide one with the --remote-id option or call the "
            "command from a project directory."
        )

        if not remote_id:
            try:
                remote_id = get_project_settings(silent=True)["id"]
            except KeyError:
                raise click.ClickException(ERROR_MSG)
            else:
                if not remote_id:
                    raise click.ClickException(ERROR_MSG)
        return func(remote_id, *args, **kwargs)

    return click.option(
        "--remote-id",
        "remote_id",
        default=None,
        type=int,
        help="Remote Project ID to use for project commands. "
        "Defaults to the project in the current directory using the "
        "configuration file.",
    )(read_remote_id)

def get_service_type(identifier, path=None):
    """
    Retrieves the service type based on the `SERVICE_MANAGER` environment
    variable of a services from the docker-compose file.
    """
    project_home = get_application_home(path)
    try:
        docker_compose = get_docker_compose_cmd(project_home)
    except RuntimeError:
        # Docker-compose does not exist
        click.secho(
            "Warning: docker-compose.yml does not exist. Can not get the service type without!",
            fg="red",
        )
        sys.exit(1)
    docker_compose_config = DockerComposeConfig(docker_compose)
    services = docker_compose_config.get_services()
    if (
        identifier in services
        and "environment" in services[identifier]
        and "SERVICE_MANAGER" in services[identifier]["environment"]
    ):
        return services[identifier]["environment"]["SERVICE_MANAGER"]

    raise RuntimeError("Can not get service type")

def get_db_type(prefix, path=None):
    """
    Utility function to wrap `get_service_type` to search for databases so we
    can properly fall back to PostgreSQL in case of old structures.
    """
    try:
        db_type = get_service_type(
            "database_{}".format(prefix.lower()), path=path
        )
    except RuntimeError:
        # legacy section. we try to look for the db, if it does not exist, fail
        try:
            docker_compose = get_docker_compose_cmd(path)
        except RuntimeError:
            # Docker-compose does not exist
            click.secho(
                "Warning: docker-compose.yml does not exist. Can not get the service type without!",
                fg="red",
            )
            sys.exit(1)
        docker_compose_config = DockerComposeConfig(docker_compose)
        if not docker_compose_config.has_service("db"):
            click.secho(
                "The local database container must be called "
                "`database_default`, must define the `SERVICE_MANAGER` "
                "environment variable and must mount the project directory "
                "from the host to the /app directory of the container."
                "\n\nSee https://docs.divio.com/en/latest/reference/docker-docker-compose/#database-default",
                fg="red",
            )
            sys.exit(1)
        else:
            # Fall back to database for legacy docker-compose files
            db_type = "fsm-postgres"
    return db_type
<|endoftext|>


<commit_before>import sys
import math
import numpy as np
from PIL import Image, ImageOps, ImageMath
import imageSegmentation
import cv2

from scipy import ndimage
from skimage.measure import structural_similarity as ssim

def mse(imageA, imageB):
	err = np.sum((imageA.astype("float") - imageB.astype("float")) ** 2)
	err /= float(imageA.shape[0] * imageA.shape[1])

	# return the MSE, the lower the error, the more "similar"
	# the two images are
	return err

# quick hack to get even image dimensions
def get_even_image(im_dimensions):
	x, y, _ = im_dimensions

	if x % 2 == 0 and y % 2 == 0:
		return (x, y)
	else:
		return (x, y - 1)

def divide_into_two(img):
	(x, y) = get_even_image(img.shape)
	top_half = img[0:x, 0:y/2]
	bottom_half = ndimage.rotate(img[0:x, y/2:y], 180)
	top_half = cv2.cvtColor(top_half, cv2.COLOR_BGR2GRAY)
	bottom_half = cv2.cvtColor(bottom_half, cv2.COLOR_BGR2GRAY)
	return (top_half, bottom_half)

# higher the SSIM metric, the more symmetric the image is
def get_symmetry(img):
	(top_half, bottom_half) = divide_into_two(img)
	return ssim(top_half, bottom_half)

# def rotateImage(image, angle):
#   image_center = tuple(np.array(image.shape)/2)
#   rot_mat = cv2.getRotationMatrix2D((image_center[0], image_center[1]),angle,1.0)
#   result = cv2.warpAffine(image, rot_mat, image.shape,flags=cv2.INTER_LINEAR)
#   return result

# class Symmetry:

# 	def __init__(self):
# 		print "hello"

# 	def preprocess(outputfile):
# 		img = cv2.imread(outputfile)
# 		gray = cv2.imread(outputfile,0)
# 		ret,thresh = cv2.threshold(gray,127,255,1)
# 		contours,h = cv2.findContours(thresh,1,2)
# 		cv2.drawContours(img, contours, -1, (0,255,0), 3)
# 		cv2.imshow("window title", img)

if __name__ == '__main__':
	input_filename = sys.argv[1]
		#outputfile = sys.argv[2]

		# bounded_image = plot_bounded_box(input_filename)
  # 	bounded_image.show()
	# im = Image.open(input_filename).convert('LA')  # convert to grayscale
	# [x, y] = im.size
	# #box_top = (0, 0, x, y / 2)
	# top_half = im.crop((0, 0, x, y / 2))
	# bottom_half = ImageOps.flip(im.crop((0,y/2+1,x,y))) # reflect image vertically
	# print "here"
	# print mse(top_half, bottom_half)
	# im.show()
	# top_half.show()
	# bottom_half.show()

	img = cv2.imread(input_filename)
	(x, y) = get_even_image(img.shape)
	top_half = img[0:x, 0:y/2]
	bottom_half = ndimage.rotate(img[0:x, y/2:y], 180)
	top_half = cv2.cvtColor(top_half, cv2.COLOR_BGR2GRAY)
	bottom_half = cv2.cvtColor(bottom_half, cv2.COLOR_BGR2GRAY)
	print ssim(top_half, bottom_half)
	cv2.imshow("top half", top_half)
	cv2.imshow("bottom half", bottom_half)
	cv2.waitKey(0)
		#bottom_half
		#imageSegmentation.main(inputfile,outputfile,3)

		#img = Image.open(outputfile)
		#preprocess(outputfile)

<commit_msg>add second symmetry metric<commit_after>import cv2
import numpy as np
import sys

from scipy import ndimage
from skimage.measure import structural_similarity as ssim

def mse(imageA, imageB):
	# the 'Mean Squared Error' between the two images is the
	# sum of the squared difference between the two images;
	# NOTE: the two images must have the same dimension
	err = np.sum((imageA.astype("float") - imageB.astype("float")) ** 2)
	err /= float(imageA.shape[0] * imageA.shape[1])

	# return the MSE, the lower the error, the more "similar"
	# the two images are
	return err

# quick hack to get even image dimensions
def get_even_image(im_dimensions):
	x, y, _ = im_dimensions

	if x % 2 == 0 and y % 2 == 0:
		return (x, y)
	else:
		return (x, y - 1)

def divide_into_two(img):
	(x, y) = get_even_image(img.shape)
	top_half = img[0:x, 0:y/2]
	bottom_half = ndimage.rotate(img[0:x, y/2:y], 180)
	top_half = cv2.cvtColor(top_half, cv2.COLOR_BGR2GRAY)
	bottom_half = cv2.cvtColor(bottom_half, cv2.COLOR_BGR2GRAY)
	return (top_half, bottom_half)

# higher the SSIM metric, the more symmetric the image is
def get_symmetry_ssim(img):
	(top_half, bottom_half) = divide_into_two(img)
	return ssim(top_half, bottom_half)

# higher the mse metric, the less symmetric the image is
def get_symmetry_mse(img):
	(top_half, bottom_half) = divide_into_two(img)
	return mse(top_half, bottom_half)

# def rotateImage(image, angle):
#   image_center = tuple(np.array(image.shape)/2)
#   rot_mat = cv2.getRotationMatrix2D((image_center[0], image_center[1]),angle,1.0)
#   result = cv2.warpAffine(image, rot_mat, image.shape,flags=cv2.INTER_LINEAR)
#   return result

# class Symmetry:

# 	def __init__(self):
# 		print "hello"

# 	def preprocess(outputfile):
# 		img = cv2.imread(outputfile)
# 		gray = cv2.imread(outputfile,0)
# 		ret,thresh = cv2.threshold(gray,127,255,1)
# 		contours,h = cv2.findContours(thresh,1,2)
# 		cv2.drawContours(img, contours, -1, (0,255,0), 3)
# 		cv2.imshow("window title", img)

if __name__ == '__main__':
	input_filename = sys.argv[1]
		#outputfile = sys.argv[2]

		# bounded_image = plot_bounded_box(input_filename)
  # 	bounded_image.show()
	# im = Image.open(input_filename).convert('LA')  # convert to grayscale
	# [x, y] = im.size
	# #box_top = (0, 0, x, y / 2)
	# top_half = im.crop((0, 0, x, y / 2))
	# bottom_half = ImageOps.flip(im.crop((0,y/2+1,x,y))) # reflect image vertically
	# print "here"
	# print mse(top_half, bottom_half)
	# im.show()
	# top_half.show()
	# bottom_half.show()

	img = cv2.imread(input_filename)
	(x, y) = get_even_image(img.shape)
	(top_half, bottom_half) = divide_into_two(img)
	# top_half = img[0:x, 0:y/2]
	# bottom_half = ndimage.rotate(img[0:x, y/2:y], 180)
	# top_half = cv2.cvtColor(top_half, cv2.COLOR_BGR2GRAY)
	# bottom_half = cv2.cvtColor(bottom_half, cv2.COLOR_BGR2GRAY)
	print get_symmetry_ssim(img)
	print get_symmetry_mse(img)
	cv2.imshow("top half", top_half)
	cv2.imshow("bottom half", bottom_half)
	cv2.waitKey(0)
		#bottom_half
		#imageSegmentation.main(inputfile,outputfile,3)

		#img = Image.open(outputfile)
		#preprocess(outputfile)

<|endoftext|>


<commit_before>"""
Inlinefunc

This is a simple inline text language for use to custom-format text
in Evennia. It is applied BEFORE ANSI/MUX parsing is applied.

To activate Inlinefunc, settings.INLINEFUNC_ENABLED must be set.

The format is straightforward:

{funcname([arg1,arg2,...]) text {/funcname

Example:
    "This is {pad(50,c,-) a center-padded text{/pad of width 50."
    ->
    "This is -------------- a center-padded text--------------- of width 50."

This can be inserted in any text, operated on by the parse_inlinefunc
function.  funcname() (no space is allowed between the name and the
argument tuple) is picked from a selection of valid functions from
settings.INLINEFUNC_MODULES.

Commands can be nested, and will applied inside-out. For correct
parsing their end-tags must match the starting tags in reverse order.

Example:
    "The time is {pad(30){time(){/time{/padright now."
    ->
    "The time is         Oct 25, 11:09         right now."

An inline function should have the following call signature:

    def funcname(text, *args)

where the text is always the part between {funcname(args) and
{/funcname and the *args are taken from the appropriate part of the
call. It is important that the inline function properly clean the
incoming args, checking their type and replacing them with sane
defaults if needed. If impossible to resolve, the unmodified text
should be returned. The inlinefunc should never cause a traceback.

"""

import re
from django.conf import settings
from src.utils import utils

# inline functions

def pad(text, *args, **kwargs):
    "Pad to width. pad(text, width=78, align='c', fillchar=' ')"
    width = 78
    align = 'c'
    fillchar = ' '
    for iarg, arg in enumerate(args):
        if iarg == 0:
            width = int(arg) if arg.isdigit() else width
        elif iarg == 1:
            align = arg if arg in ('c', 'l', 'r') else align
        elif iarg == 2:
            fillchar = arg[0]
        else:
            break
    return utils.pad(text, width=width, align=align, fillchar=fillchar)

def crop(text, *args, **kwargs):
    "Crop to width. crop(text, width=78, suffix='[...]')"
    width = 78
    suffix = "[...]"
    for iarg, arg in enumerate(args):
        if iarg == 0:
            width = int(arg) if arg.isdigit() else width
        elif iarg == 1:
            suffix = arg
        else:
            break
    return utils.crop(text, width=width, suffix=suffix)

def wrap(text, *args, **kwargs):
    "Wrap/Fill text to width. fill(text, width=78, indent=0)"
    width = 78
    indent = 0
    for iarg, arg in enumerate(args):
        if iarg == 0:
            width = int(arg) if arg.isdigit() else width
        elif iarg == 1:
            indent = int(arg) if arg.isdigit() else indent
    return utils.wrap(text, width=width, indent=indent)

def time(text, *args, **kwargs):
    "Inserts current time"
    import time
    strformat = "%h %d, %H:%M"
    if args and args[0]:
        strformat = str(args[0])
    return time.strftime(strformat)

def you(text, *args, **kwargs):
    "Inserts your name"
    name = "You"
    sess = kwargs.get("session")
    if sess and sess.puppet:
        name = sess.puppet.key
    return name

# load functions from module (including this one, if using default settings)
_INLINE_FUNCS = {}
for module in utils.make_iter(settings.INLINEFUNC_MODULES):
    _INLINE_FUNCS.update(utils.all_from_module(module))
_INLINE_FUNCS.pop("inline_func_parse", None)

# dynamically build regexes for found functions
_RE_FUNCFULL = r"\{%s\((.*?)\)(.*?){/%s"
_RE_FUNCSTART = r"\{((?:%s))"
_RE_FUNCEND = r"\{/((?:%s))"
_RE_FUNCSPLIT = r"(\{/*(?:%s)(?:\(.*?\))*)"
_RE_FUNCCLEAN = r"\{%s\(.*?\)|\{/%s"

_INLINE_FUNCS = dict((key, (func, re.compile(_RE_FUNCFULL % (key, key), re.DOTALL &  re.MULTILINE)))
                          for key, func in _INLINE_FUNCS.items() if callable(func))
_FUNCSPLIT_REGEX = re.compile(_RE_FUNCSPLIT % r"|".join([key for key in _INLINE_FUNCS]), re.DOTALL & re.MULTILINE)
_FUNCSTART_REGEX = re.compile(_RE_FUNCSTART % r"|".join([key for key in _INLINE_FUNCS]), re.DOTALL & re.MULTILINE)
_FUNCEND_REGEX = re.compile(_RE_FUNCEND % r"|".join([key for key in _INLINE_FUNCS]), re.DOTALL & re.MULTILINE)
_FUNCCLEAN_REGEX = re.compile("|".join([_RE_FUNCCLEAN % (key, key) for key in _INLINE_FUNCS]), re.DOTALL & re.MULTILINE)

# inline parser functions

def _execute_inline_function(funcname, text, session):
    """
    Get the enclosed text between {funcname(...) and {/funcname
    and execute the inline function to replace the whole block
    with the result.
    Note that this lookup is "dumb" - we just grab the first end
    tag we find. So to work correctly this function must be called
    "inside out" on a nested function tree, so each call only works
    on a "flat" tag.
    """
    def subfunc(match):
        "replace the entire block with the result of the function call"
        args = [part.strip() for part in match.group(1).split(",")]
        intext = match.group(2)
        kwargs = {"session":session}
        return _INLINE_FUNCS[funcname][0](intext, *args, **kwargs)
    return _INLINE_FUNCS[funcname][1].sub(subfunc, text)

def parse_inlinefunc(text, strip=False, session=None):
    """
    Parse inline function-replacement.

    strip - remove all supported inlinefuncs from text
    session - session calling for the parsing
    """

    if strip:
        # strip all functions
        return _FUNCCLEAN_REGEX.sub("", text)

    stack = []
    for part in _FUNCSPLIT_REGEX.split(text):
        endtag = _FUNCEND_REGEX.match(part)
        if endtag:
            # an end tag
            endname = endtag.group(1)
            while stack:
                new_part = stack.pop()
                part = new_part + part  # add backwards -> fowards
                starttag = _FUNCSTART_REGEX.match(new_part)
                if starttag:
                    startname = starttag.group(1)
                    if startname == endname:
                        part = _execute_inline_function(startname, part, session)
                        break
        stack.append(part)
    return "".join(stack)

def _test():
    # this should all be handled
    s = "This is a text with a{pad(78,c,-)text {pad(5)of{/pad {pad(30)nice{/pad size{/pad inside {pad(4,l)it{/pad."
    s2 = "This is a text with a----------------text   of               nice              size---------------- inside it  ."
    t = parse_inlinefunc(s)
    assert(t == s2)
    return t
<commit_msg>Made it so that inlinefunc also accepts single function calls (no end tag) - in this case the ingoing text string will be the empty string. This is useful for functions like {time() that makes no use of the text input - you can now use just {time() instead of having to use {time(){/time.<commit_after>"""
Inlinefunc

This is a simple inline text language for use to custom-format text
in Evennia. It is applied BEFORE ANSI/MUX parsing is applied.

To activate Inlinefunc, settings.INLINEFUNC_ENABLED must be set.

The format is straightforward:

{funcname([arg1,arg2,...]) text {/funcname

Example:
    "This is {pad(50,c,-) a center-padded text{/pad of width 50."
    ->
    "This is -------------- a center-padded text--------------- of width 50."

This can be inserted in any text, operated on by the parse_inlinefunc
function.  funcname() (no space is allowed between the name and the
argument tuple) is picked from a selection of valid functions from
settings.INLINEFUNC_MODULES.

Commands can be nested, and will applied inside-out. For correct
parsing their end-tags must match the starting tags in reverse order.

Example:
    "The time is {pad(30){time(){/time{/padright now."
    ->
    "The time is         Oct 25, 11:09         right now."

An inline function should have the following call signature:

    def funcname(text, *args)

where the text is always the part between {funcname(args) and
{/funcname and the *args are taken from the appropriate part of the
call. It is important that the inline function properly clean the
incoming args, checking their type and replacing them with sane
defaults if needed. If impossible to resolve, the unmodified text
should be returned. The inlinefunc should never cause a traceback.

"""

import re
from django.conf import settings
from src.utils import utils

# inline functions

def pad(text, *args, **kwargs):
    "Pad to width. pad(text, width=78, align='c', fillchar=' ')"
    width = 78
    align = 'c'
    fillchar = ' '
    for iarg, arg in enumerate(args):
        if iarg == 0:
            width = int(arg) if arg.isdigit() else width
        elif iarg == 1:
            align = arg if arg in ('c', 'l', 'r') else align
        elif iarg == 2:
            fillchar = arg[0]
        else:
            break
    return utils.pad(text, width=width, align=align, fillchar=fillchar)

def crop(text, *args, **kwargs):
    "Crop to width. crop(text, width=78, suffix='[...]')"
    width = 78
    suffix = "[...]"
    for iarg, arg in enumerate(args):
        if iarg == 0:
            width = int(arg) if arg.isdigit() else width
        elif iarg == 1:
            suffix = arg
        else:
            break
    return utils.crop(text, width=width, suffix=suffix)

def wrap(text, *args, **kwargs):
    "Wrap/Fill text to width. fill(text, width=78, indent=0)"
    width = 78
    indent = 0
    for iarg, arg in enumerate(args):
        if iarg == 0:
            width = int(arg) if arg.isdigit() else width
        elif iarg == 1:
            indent = int(arg) if arg.isdigit() else indent
    return utils.wrap(text, width=width, indent=indent)

def time(text, *args, **kwargs):
    "Inserts current time"
    import time
    strformat = "%h %d, %H:%M"
    if args and args[0]:
        strformat = str(args[0])
    return time.strftime(strformat)

def you(text, *args, **kwargs):
    "Inserts your name"
    name = "You"
    sess = kwargs.get("session")
    if sess and sess.puppet:
        name = sess.puppet.key
    return name

# load functions from module (including this one, if using default settings)
_INLINE_FUNCS = {}
for module in utils.make_iter(settings.INLINEFUNC_MODULES):
    _INLINE_FUNCS.update(utils.all_from_module(module))
_INLINE_FUNCS.pop("inline_func_parse", None)

# dynamically build regexes for found functions
_RE_FUNCFULL = r"\{%s\((.*?)\)(.*?){/%s"
_RE_FUNCFULL_SINGLE = r"\{%s\((.*?)\)"
_RE_FUNCSTART = r"\{((?:%s))"
_RE_FUNCEND = r"\{/((?:%s))"
_RE_FUNCSPLIT = r"(\{/*(?:%s)(?:\(.*?\))*)"
_RE_FUNCCLEAN = r"\{%s\(.*?\)|\{/%s"

_INLINE_FUNCS = dict((key, (func, re.compile(_RE_FUNCFULL % (key, key), re.DOTALL &  re.MULTILINE),
                                  re.compile(_RE_FUNCFULL_SINGLE % key, re.DOTALL & re.MULTILINE)))
                          for key, func in _INLINE_FUNCS.items() if callable(func))
_FUNCSPLIT_REGEX = re.compile(_RE_FUNCSPLIT % r"|".join([key for key in _INLINE_FUNCS]), re.DOTALL & re.MULTILINE)
_FUNCSTART_REGEX = re.compile(_RE_FUNCSTART % r"|".join([key for key in _INLINE_FUNCS]), re.DOTALL & re.MULTILINE)
_FUNCEND_REGEX = re.compile(_RE_FUNCEND % r"|".join([key for key in _INLINE_FUNCS]), re.DOTALL & re.MULTILINE)
_FUNCCLEAN_REGEX = re.compile("|".join([_RE_FUNCCLEAN % (key, key) for key in _INLINE_FUNCS]), re.DOTALL & re.MULTILINE)

# inline parser functions

def _execute_inline_function(funcname, text, session):
    """
    Get the enclosed text between {funcname(...) and {/funcname
    and execute the inline function to replace the whole block
    with the result.
    Note that this lookup is "dumb" - we just grab the first end
    tag we find. So to work correctly this function must be called
    "inside out" on a nested function tree, so each call only works
    on a "flat" tag.
    """
    def subfunc(match):
        "replace the entire block with the result of the function call"
        args = [part.strip() for part in match.group(1).split(",")]
        intext = match.group(2)
        kwargs = {"session":session}
        return _INLINE_FUNCS[funcname][0](intext, *args, **kwargs)
    return _INLINE_FUNCS[funcname][1].sub(subfunc, text)

def _execute_inline_single_function(funcname, text, session):
    """
    Get the arguments of a single function call (no matching end tag)
    and execute it with an empty text input.
    """
    def subfunc(match):
        "replace the single call with the result of the function call"
        args = [part.strip() for part in match.group(1).split(",")]
        kwargs = {"session":session}
        return _INLINE_FUNCS[funcname][0]("", *args, **kwargs)
    return _INLINE_FUNCS[funcname][2].sub(subfunc, text)

def parse_inlinefunc(text, strip=False, session=None):
    """
    Parse inline function-replacement.

    strip - remove all supported inlinefuncs from text
    session - session calling for the parsing
    """

    if strip:
        # strip all functions
        return _FUNCCLEAN_REGEX.sub("", text)

    stack = []
    for part in _FUNCSPLIT_REGEX.split(text):
        endtag = _FUNCEND_REGEX.match(part)
        if endtag:
            # an end tag
            endname = endtag.group(1)
            while stack:
                new_part = stack.pop()
                part = new_part + part  # add backwards -> fowards
                starttag = _FUNCSTART_REGEX.match(new_part)
                if starttag:
                    startname = starttag.group(1)
                    if startname == endname:
                        part = _execute_inline_function(startname, part, session)
                        break
        stack.append(part)
    # handle single functions without matching end tags; these are treated
    # as being called with an empty string as text argument.
    outstack = []
    for part in _FUNCSPLIT_REGEX.split("".join(stack)):
        starttag = _FUNCSTART_REGEX.match(part)
        if starttag:
            startname = starttag.group(1)
            part = _execute_inline_single_function(startname, part, session)
        outstack.append(part)

    return "".join(outstack)

def _test():
    # this should all be handled
    s = "This is a text with a{pad(78,c,-)text {pad(5)of{/pad {pad(30)nice{/pad size{/pad inside {pad(4,l)it{/pad."
    s2 = "This is a text with a----------------text   of               nice              size---------------- inside it  ."
    t = parse_inlinefunc(s)
    assert(t == s2)
    return t
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##
##
## This file is part of Indico.
## Copyright (C) 2002 - 2014 European Organization for Nuclear Research (CERN).
##
## Indico is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 3 of the
## License, or (at your option) any later version.
##
## Indico is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Indico;if not, see <http://www.gnu.org/licenses/>.

"""
String manipulation functions
"""

import re
import unicodedata
from lxml import html, etree

import markdown
import bleach

BLEACH_ALLOWED_TAGS = bleach.ALLOWED_TAGS + ['sup', 'sub', 'small']

def unicodeOrNone(string):
    return None if string is None else string.decode('utf-8')

def remove_accents(text, reencode=True):
    if not isinstance(text, unicode):
        text = text.decode('utf-8')
    result = u''.join((c for c in unicodedata.normalize('NFD', text) if unicodedata.category(c) != 'Mn'))
    if reencode:
        return result.encode('utf-8')
    else:
        return result

def fix_broken_string(text):
    try:
        text = text.decode('utf-8')
    except UnicodeDecodeError:
        try:
            text = text.decode('latin1')
        except UnicodeDecodeError:
            text = unicode(text, 'utf-8', errors='replace')
    return text.encode('utf-8')

def remove_non_alpha(text):
    return ''.join(c for c in text if c.isalnum())

def html_line_breaks(text):
    return '
' + text.replace('\n\n', '

').replace('\n', '
') + '
'

def truncate(text, max_size, ellipsis='...', encoding='utf-8'):
    """
    Truncate text, taking unicode chars into account
    """
    encode = False

    if isinstance(text, str):
        encode = True
        text = text.decode(encoding)

    if len(text) > max_size:
        text = text[:max_size] + ellipsis

    if encode:
        text = text.encode(encoding)

    return text

def permissive_format(text, params):
    """
    Format text using params from dictionary. Function is resistant to missing parentheses
    """
    for k, v in params.iteritems():
        text = text.replace("{" + k + "}", str(v))
    return text

def remove_extra_spaces(text):
    """
    Removes multiple spaces within text and removes whitespace around the text
    'Text     with    spaces ' becomes 'Text with spaces'
    """
    pattern = re.compile(r"  +")
    return pattern.sub(' ', text).strip()

def remove_tags(text):
    """
    Removes html-like tags from given text. Tag names aren't checked,
    <no-valid-tag></no-valid-tag> pair will be removed.
    """
    pattern = re.compile(r"<(\w|\/)[^<\s\"']*?>")
    return remove_extra_spaces(pattern.sub(' ', text))

def render_markdown(text):
    """ Mako markdown to html filter """
    return markdown.markdown(bleach.clean(text, tags=BLEACH_ALLOWED_TAGS)).encode('utf-8')

def sanitize_for_platypus(text):
    """Sanitize HTML to be used in platypus"""
    tags = ['b', 'br', 'em', 'font', 'i', 'img', 'strike', 'strong', 'sub', 'sup', 'u', 'span', 'div', 'p']
    attrs = {
        'font': ['size', 'face', 'color'],
        'img': ['src', 'width', 'height', 'valign']
    }
    res = bleach.clean(text, tags=tags, attributes=attrs, strip=True)
    # Convert to XHTML
    doc = html.fromstring(res)
    return etree.tostring(doc)
<commit_msg>[IMP] unicode filter for templates<commit_after># -*- coding: utf-8 -*-
##
##
## This file is part of Indico.
## Copyright (C) 2002 - 2014 European Organization for Nuclear Research (CERN).
##
## Indico is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 3 of the
## License, or (at your option) any later version.
##
## Indico is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Indico;if not, see <http://www.gnu.org/licenses/>.

"""
String manipulation functions
"""

import re
import unicodedata
from lxml import html, etree

import markdown
import bleach

BLEACH_ALLOWED_TAGS = bleach.ALLOWED_TAGS + ['sup', 'sub', 'small']

def encode_if_unicode(s):
    return s.encode('utf-8') if isinstance(s, unicode) else s

def unicodeOrNone(s):
    return None if s is None else s.decode('utf-8')

def remove_accents(text, reencode=True):
    if not isinstance(text, unicode):
        text = text.decode('utf-8')
    result = u''.join((c for c in unicodedata.normalize('NFD', text) if unicodedata.category(c) != 'Mn'))
    if reencode:
        return result.encode('utf-8')
    else:
        return result

def fix_broken_string(text):
    try:
        text = text.decode('utf-8')
    except UnicodeDecodeError:
        try:
            text = text.decode('latin1')
        except UnicodeDecodeError:
            text = unicode(text, 'utf-8', errors='replace')
    return text.encode('utf-8')

def remove_non_alpha(text):
    return ''.join(c for c in text if c.isalnum())

def html_line_breaks(text):
    return '
' + text.replace('\n\n', '

').replace('\n', '
') + '
'

def truncate(text, max_size, ellipsis='...', encoding='utf-8'):
    """
    Truncate text, taking unicode chars into account
    """
    encode = False

    if isinstance(text, str):
        encode = True
        text = text.decode(encoding)

    if len(text) > max_size:
        text = text[:max_size] + ellipsis

    if encode:
        text = text.encode(encoding)

    return text

def permissive_format(text, params):
    """
    Format text using params from dictionary. Function is resistant to missing parentheses
    """
    for k, v in params.iteritems():
        text = text.replace("{" + k + "}", str(v))
    return text

def remove_extra_spaces(text):
    """
    Removes multiple spaces within text and removes whitespace around the text
    'Text     with    spaces ' becomes 'Text with spaces'
    """
    pattern = re.compile(r"  +")
    return pattern.sub(' ', text).strip()

def remove_tags(text):
    """
    Removes html-like tags from given text. Tag names aren't checked,
    <no-valid-tag></no-valid-tag> pair will be removed.
    """
    pattern = re.compile(r"<(\w|\/)[^<\s\"']*?>")
    return remove_extra_spaces(pattern.sub(' ', text))

def render_markdown(text):
    """ Mako markdown to html filter """
    return markdown.markdown(bleach.clean(text, tags=BLEACH_ALLOWED_TAGS)).encode('utf-8')

def sanitize_for_platypus(text):
    """Sanitize HTML to be used in platypus"""
    tags = ['b', 'br', 'em', 'font', 'i', 'img', 'strike', 'strong', 'sub', 'sup', 'u', 'span', 'div', 'p']
    attrs = {
        'font': ['size', 'face', 'color'],
        'img': ['src', 'width', 'height', 'valign']
    }
    res = bleach.clean(text, tags=tags, attributes=attrs, strip=True)
    # Convert to XHTML
    doc = html.fromstring(res)
    return etree.tostring(doc)
<|endoftext|>