<commit_before>import gzip
from collections import defaultdict
import progressbar
import re
import StringIO

from xbrowse import Family, Individual
from xbrowse.core import constants

def family_from_indiv_id_list(indiv_id_list, project_id, family_id):
    indivs = [Individual({'project_id': project_id, 'family_id': family_id, 'indiv_id': indiv_id}) for indiv_id in indiv_id_list]
    family = Family({'project_id': project_id, 'family_id': family_id})
    for indiv in indivs:
        family.add_individual(indiv)
    return family

def combine_annot_groups(annot_dict, **kwargs):

    combiner = kwargs.get('combiner', sum)
    initial_val = kwargs.get('initial_val', 0)

    group_values = {group['slug']: initial_val for group in constants.ANNOTATION_GROUPS}

    for k, v in annot_dict.items():
        group = constants.ANNOTATION_GROUP_REVERSE_MAP[k]
        group_values[group] = combiner([group_values[group], v])

    return group_values

def get_gene_from_arg(arg, reference):
    """
    Return ensembl ID from arg, which can be ensembl id or gene id
    """
    if reference.is_valid_gene_id(arg):
        return arg
    elif reference.gene_id_from_symbol(arg):
        return reference.gene_id_from_symbol(arg)
    else:
        return None

def get_progressbar(maxval, title=None):
    widgets = [
        progressbar.Percentage(), ' ',
        progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
        progressbar.ETA()
    ]
    if title:
        widgets.insert(0, '%s: ' % title)
    p = progressbar.ProgressBar(widgets=widgets, maxval=maxval)
    p.start()
    return p

def family_variant_from_full_variant(variant, family):
    """
    Extract a family variant from full variant
    It's just a copy, with the following changes:
    -- All genotypes from non-family individuals are removed
    -- variant['alt_allele_count'] is updated to alt allele count in family indivs

    Note that I refactored this from copy.deepcopy for performance
    """

    new_genotypes = {}
    for indiv_id in family['individuals']:
        new_genotypes[indiv_id] = variant['genotypes'][indiv_id]

    new_variant = variant.copy()
    new_variant['genotypes'] = new_genotypes
    new_variant['alt_allele_count'] = get_alt_allele_count(new_variant)

    return new_variant

def clip_individuals_from_variant(variant, indiv_id_list):
    """
    Extract a family variant from full variant
    It's just a copy, with the following changes:
    -- All genotypes from non-family individuals are removed
    -- variant['alt_allele_count'] is updated to alt allele count in indiv_id_list

    Note that I refactored this from copy.deepcopy for performance
    """

    new_genotypes = {}
    for indiv_id in indiv_id_list:
        new_genotypes[indiv_id] = variant['genotypes'][indiv_id]

    new_variant = variant.copy()
    new_variant['genotypes'] = new_genotypes
    new_variant['alt_allele_count'] = get_alt_allele_count(new_variant)

    return new_variant

def get_alt_allele_count(variant):
    """
    Get alt allele count for a variant
    """
    aac = 0
    for indiv_id, genotype in variant.get_genotypes():
        if genotype.num_alt is not None:
            aac += genotype.num_alt
    return aac

def alt_allele_count_map(variant_list):
    """
    Gets a map of indiv_id -> alt allele count from the variants in variant_list
    TODO: can we use collections.counter?
    """
    aac_map = defaultdict(int)
    for variant in variant_list:
        for indiv_id, genotype in variant.get_genotypes():
            if genotype.num_alt:
                aac_map[indiv_id] += genotype.num_alt
    return aac_map

# TODO: move to reference
def get_gene_id_from_str(s, reference):
    """
    Given an arbitrary string s, see if you can use reference
    to get a gene ID (ensembl ID)
    """
    if reference.is_valid_gene_id(s):
        return s
    elif reference.get_gene_id_from_symbol(s.lower()):
        return reference.get_gene_id_from_symbol(s.lower())
    else:
        return None

def get_aaf(variant):
    """
    Get alt allele frequency from a variant
    TODO: consider sex chromosomes
    TODO: tests
    """
    num_ref, num_alt = 0,0
    for indiv_id, genotype in variant.get_genotypes():
        if genotype.num_alt == 0:
            num_ref += 2
        elif genotype.num_alt > 0:
            num_alt += genotype.num_alt
    return float(num_alt) / float(num_ref + num_alt)

def is_variant_relevant_for_individuals(variant, indiv_id_list):
    """
    """
    has_missing = False
    has_refref = False
    for indiv_id in indiv_id_list:
    # this catches missing or an alt allele
        num_alt = variant.get_genotype(indiv_id).num_alt
        if num_alt > 0:
            return True
        elif num_alt is None:
            has_missing = True
        elif num_alt == 0:
            has_refref = True
    if has_missing and has_refref:
        return True
    return False

class CompressedFile(file):
    def __init__(self, *args, **kwargs):
        file.__init__(self, *args, **kwargs)
        self.tell_progress = self.tell

def compressed_file(file_path):
    """
    Return handle to a file, whether compressed or not
    gzip.open() if file_path ends in .gz
    Otherwise basic file handle
    """
    if file_path.endswith('.gz'):
        f = gzip.open(file_path)
        f.tell_progress = f.fileobj.tell
        return f
    else:
        return CompressedFile(file_path)

def slugify(s, separator='_', replace_dot=False):
    """Simplified, custom implementation of the functionality in the awesome-slugify python module.
    A custom approach was needed because awesome-slugify only supports one char as the separator, for example '-' or '_'
    but here we keep both '-' and '_', while replacing all other special chars with '_'.

    Args:
        s: string to slugify (eg. remove special chars)
        separator: the char to use in place of special characters
    Return:
        string with all characters except [a-Z\-_.] replaced with '_'
    """
    try:
        regexp = '[^a-zA-Z0-9\-_.]+' if not replace_dot else '[^a-zA-Z0-9\-_]+'
        words = re.split(regexp, s)
    except Exception as e:
        print("ERROR: string '%s' caused: %s" % (e, s))
        raise

    return separator.join(filter(None, words))

# make encoded values as human-readable as possible
ES_FIELD_NAME_ESCAPE_CHAR = '$'
ES_FIELD_NAME_BAD_LEADING_CHARS = set(['_', '-', '+', ES_FIELD_NAME_ESCAPE_CHAR])
ES_FIELD_NAME_SPECIAL_CHAR_MAP = {
    '.': '_$dot$_',
    ',': '_$comma$_',
    '#': '_$hash$_',
    '*': '_$star$_',
    '(': '_$lp$_',
    ')': '_$rp$_',
    '[': '_$lsb$_',
    ']': '_$rsb$_',
    '{': '_$lcb$_',
    '}': '_$rcb$_',
}

def _encode_name(s):
    """Applies a reversable encoding to the special chars in the given name or id string, and returns the result.
    Among other things, the encoded string is a valid elasticsearch or mongodb field name.

    See:
    https://discuss.elastic.co/t/special-characters-in-field-names/10658/2
    https://discuss.elastic.co/t/illegal-characters-in-elasticsearch-field-names/17196/2
    """
    s = s.replace(ES_FIELD_NAME_ESCAPE_CHAR, 2 * ES_FIELD_NAME_ESCAPE_CHAR)
    for original_value, encoded in ES_FIELD_NAME_SPECIAL_CHAR_MAP.items():
        s = s.replace(original_value, encoded)
    if s[0] in ES_FIELD_NAME_BAD_LEADING_CHARS:
        s = ES_FIELD_NAME_ESCAPE_CHAR + s
    return s

def _decode_name(s):
    """Decodes a name or id string that was encoded by #_encode_name(..) and returns the original string"""

    if s.startswith(ES_FIELD_NAME_ESCAPE_CHAR):
        s = s[1:]
    for original_value, encoded in ES_FIELD_NAME_SPECIAL_CHAR_MAP.items():
        s = s.replace(encoded, original_value)
    s = s.replace(2*ES_FIELD_NAME_ESCAPE_CHAR, ES_FIELD_NAME_ESCAPE_CHAR)
    return s

<commit_msg>fix search by gene symbol<commit_after>import gzip
from collections import defaultdict
import progressbar
import re
import StringIO

from xbrowse import Family, Individual
from xbrowse.core import constants

def family_from_indiv_id_list(indiv_id_list, project_id, family_id):
    indivs = [Individual({'project_id': project_id, 'family_id': family_id, 'indiv_id': indiv_id}) for indiv_id in indiv_id_list]
    family = Family({'project_id': project_id, 'family_id': family_id})
    for indiv in indivs:
        family.add_individual(indiv)
    return family

def combine_annot_groups(annot_dict, **kwargs):

    combiner = kwargs.get('combiner', sum)
    initial_val = kwargs.get('initial_val', 0)

    group_values = {group['slug']: initial_val for group in constants.ANNOTATION_GROUPS}

    for k, v in annot_dict.items():
        group = constants.ANNOTATION_GROUP_REVERSE_MAP[k]
        group_values[group] = combiner([group_values[group], v])

    return group_values

def get_gene_from_arg(arg, reference):
    """
    Return ensembl ID from arg, which can be ensembl id or gene id
    """
    if reference.is_valid_gene_id(arg):
        return arg
    elif reference.gene_id_from_symbol(arg):
        return reference.gene_id_from_symbol(arg)
    else:
        return None

def get_progressbar(maxval, title=None):
    widgets = [
        progressbar.Percentage(), ' ',
        progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
        progressbar.ETA()
    ]
    if title:
        widgets.insert(0, '%s: ' % title)
    p = progressbar.ProgressBar(widgets=widgets, maxval=maxval)
    p.start()
    return p

def family_variant_from_full_variant(variant, family):
    """
    Extract a family variant from full variant
    It's just a copy, with the following changes:
    -- All genotypes from non-family individuals are removed
    -- variant['alt_allele_count'] is updated to alt allele count in family indivs

    Note that I refactored this from copy.deepcopy for performance
    """

    new_genotypes = {}
    for indiv_id in family['individuals']:
        new_genotypes[indiv_id] = variant['genotypes'][indiv_id]

    new_variant = variant.copy()
    new_variant['genotypes'] = new_genotypes
    new_variant['alt_allele_count'] = get_alt_allele_count(new_variant)

    return new_variant

def clip_individuals_from_variant(variant, indiv_id_list):
    """
    Extract a family variant from full variant
    It's just a copy, with the following changes:
    -- All genotypes from non-family individuals are removed
    -- variant['alt_allele_count'] is updated to alt allele count in indiv_id_list

    Note that I refactored this from copy.deepcopy for performance
    """

    new_genotypes = {}
    for indiv_id in indiv_id_list:
        new_genotypes[indiv_id] = variant['genotypes'][indiv_id]

    new_variant = variant.copy()
    new_variant['genotypes'] = new_genotypes
    new_variant['alt_allele_count'] = get_alt_allele_count(new_variant)

    return new_variant

def get_alt_allele_count(variant):
    """
    Get alt allele count for a variant
    """
    aac = 0
    for indiv_id, genotype in variant.get_genotypes():
        if genotype.num_alt is not None:
            aac += genotype.num_alt
    return aac

def alt_allele_count_map(variant_list):
    """
    Gets a map of indiv_id -> alt allele count from the variants in variant_list
    TODO: can we use collections.counter?
    """
    aac_map = defaultdict(int)
    for variant in variant_list:
        for indiv_id, genotype in variant.get_genotypes():
            if genotype.num_alt:
                aac_map[indiv_id] += genotype.num_alt
    return aac_map

# TODO: move to reference
def get_gene_id_from_str(s, reference):
    """
    Given an arbitrary string s, see if you can use reference
    to get a gene ID (ensembl ID)
    """
    if reference.is_valid_gene_id(s):
        return s
    elif reference.get_gene_id_from_symbol(s.upper()):
        return reference.get_gene_id_from_symbol(s.upper())
    else:
        return None

def get_aaf(variant):
    """
    Get alt allele frequency from a variant
    TODO: consider sex chromosomes
    TODO: tests
    """
    num_ref, num_alt = 0,0
    for indiv_id, genotype in variant.get_genotypes():
        if genotype.num_alt == 0:
            num_ref += 2
        elif genotype.num_alt > 0:
            num_alt += genotype.num_alt
    return float(num_alt) / float(num_ref + num_alt)

def is_variant_relevant_for_individuals(variant, indiv_id_list):
    """
    """
    has_missing = False
    has_refref = False
    for indiv_id in indiv_id_list:
    # this catches missing or an alt allele
        num_alt = variant.get_genotype(indiv_id).num_alt
        if num_alt > 0:
            return True
        elif num_alt is None:
            has_missing = True
        elif num_alt == 0:
            has_refref = True
    if has_missing and has_refref:
        return True
    return False

class CompressedFile(file):
    def __init__(self, *args, **kwargs):
        file.__init__(self, *args, **kwargs)
        self.tell_progress = self.tell

def compressed_file(file_path):
    """
    Return handle to a file, whether compressed or not
    gzip.open() if file_path ends in .gz
    Otherwise basic file handle
    """
    if file_path.endswith('.gz'):
        f = gzip.open(file_path)
        f.tell_progress = f.fileobj.tell
        return f
    else:
        return CompressedFile(file_path)

def slugify(s, separator='_', replace_dot=False):
    """Simplified, custom implementation of the functionality in the awesome-slugify python module.
    A custom approach was needed because awesome-slugify only supports one char as the separator, for example '-' or '_'
    but here we keep both '-' and '_', while replacing all other special chars with '_'.

    Args:
        s: string to slugify (eg. remove special chars)
        separator: the char to use in place of special characters
    Return:
        string with all characters except [a-Z\-_.] replaced with '_'
    """
    try:
        regexp = '[^a-zA-Z0-9\-_.]+' if not replace_dot else '[^a-zA-Z0-9\-_]+'
        words = re.split(regexp, s)
    except Exception as e:
        print("ERROR: string '%s' caused: %s" % (e, s))
        raise

    return separator.join(filter(None, words))

# make encoded values as human-readable as possible
ES_FIELD_NAME_ESCAPE_CHAR = '$'
ES_FIELD_NAME_BAD_LEADING_CHARS = set(['_', '-', '+', ES_FIELD_NAME_ESCAPE_CHAR])
ES_FIELD_NAME_SPECIAL_CHAR_MAP = {
    '.': '_$dot$_',
    ',': '_$comma$_',
    '#': '_$hash$_',
    '*': '_$star$_',
    '(': '_$lp$_',
    ')': '_$rp$_',
    '[': '_$lsb$_',
    ']': '_$rsb$_',
    '{': '_$lcb$_',
    '}': '_$rcb$_',
}

def _encode_name(s):
    """Applies a reversable encoding to the special chars in the given name or id string, and returns the result.
    Among other things, the encoded string is a valid elasticsearch or mongodb field name.

    See:
    https://discuss.elastic.co/t/special-characters-in-field-names/10658/2
    https://discuss.elastic.co/t/illegal-characters-in-elasticsearch-field-names/17196/2
    """
    s = s.replace(ES_FIELD_NAME_ESCAPE_CHAR, 2 * ES_FIELD_NAME_ESCAPE_CHAR)
    for original_value, encoded in ES_FIELD_NAME_SPECIAL_CHAR_MAP.items():
        s = s.replace(original_value, encoded)
    if s[0] in ES_FIELD_NAME_BAD_LEADING_CHARS:
        s = ES_FIELD_NAME_ESCAPE_CHAR + s
    return s

def _decode_name(s):
    """Decodes a name or id string that was encoded by #_encode_name(..) and returns the original string"""

    if s.startswith(ES_FIELD_NAME_ESCAPE_CHAR):
        s = s[1:]
    for original_value, encoded in ES_FIELD_NAME_SPECIAL_CHAR_MAP.items():
        s = s.replace(encoded, original_value)
    s = s.replace(2*ES_FIELD_NAME_ESCAPE_CHAR, ES_FIELD_NAME_ESCAPE_CHAR)
    return s

<|endoftext|>


<commit_before>""" Example smearing script

This script:
  * Reads in mc spectra from hdf5
  * Smears spectra, default is to use weighted Gaussian method, but can
    also use specify random Gaussian method via command line
  * Smeared spectrum is saved to the same directory with ``_#value#ly`` or
    ``_#value#rs`` added to the file name

Examples:
  To smear hdf5 file ``example.hdf5`` using the random Gaussian method::

    $ python dump_smeared.py --smear_method "random" /path/to/example.hdf5

  This will create the smeared hdf5 file ``/path/to/example_200ly.hdf5``.

.. note:: Valid smear methods include:

  * "weight", default
  * "random"
"""

import echidna.output.store as store
import echidna.core.smear as smear
import os

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("-m", "--smear_method", nargs='?', const="weight",
                        type=str, default="weight",
                        help="specify the smearing method to use")
    parser.add_argument("-r", "--energy_resolution", default=None, type=float,
                        help="specify energy resolution "
                        "e.g. 0.05 for 5 percent")
    parser.add_argument("-l", "--light_yield", default=200., type=float,
                        help="specify light yield"
                        "e.g. 200 for 200 NHit/MeV")
    parser.add_argument("-g", "--gaus", dest="gaus", action="store_true",
                        help="Apply gaussian PDF")
    parser.add_argument("-d", "--dest", default=None, type=str,
                        help="specify destination directory")
    parser.add_argument("path", type=str,
                        help="specify path to hdf5 file")
    parser.set_defaults(gaus=False)
    args = parser.parse_args()

    if args.dest:
        if os.path.isdir(args.dest):
            directory = args.dest
            if directory[-1] != "/":
                directory += "/"
        else:
            raise ValueError("%s does not exist" % args.dest)
    else:
        directory = os.path.dirname(args.path)+"/"  # strip filename
    # strip directory and extension
    filename = os.path.splitext(os.path.basename(args.path))[0]

    if args.energy_resolution:
        if args.gaus:
            energy_smear = smear.EnergySmearRes(poisson=False)
        else:
            energy_smear = smear.EnergySmearRes(poisson=True)
        energy_smear.set_resolution(args.energy_resolution)
    else:  # use light yield
        if args.gaus:
            energy_smear = smear.EnergySmearLY(poisson=False)
        else:
            energy_smear = smear.EnergySmearLY(poisson=True)
        energy_smear.set_resolution(args.light_yield)
    spectrum = store.load(args.path)

    if args.smear_method == "weight":  # Use default smear method
        for par in spectrum.get_config().get_pars():
            if "energy" in par:
                energy_par = par
                spectrum = energy_smear.weighted_smear(spectrum,
                                                       par=energy_par)
    elif args.smear_method == "random":
        for par in spectrum.get_config().get_pars():
            if "energy" in par:
                energy_par = par
                spectrum = energy_smear.random_smear(spectrum,
                                                     par=energy_par)
    else:  # Not a valid smear method
        parser.error(args.smear_method + " is not a valid smear method")

    if args.energy_resolution:
        str_rs = str(args.energy_resolution)
        filename = directory + filename + "_" + str_rs + "rs.hdf5"
    else:
        str_ly = str(args.light_yield).rstrip('.0')
        filename = directory + filename + "_" + str_ly + "ly.hdf5"
    store.dump(filename, spectrum)
<commit_msg>Addresses #146<commit_after>""" Example smearing script

This script:
  * Reads in mc spectra from hdf5
  * Smears spectra, default is to use weighted Gaussian method, but can
    also use specify random Gaussian method via command line
  * Smeared spectrum is saved to the same directory with ``_#value#ly`` or
    ``_#value#rs`` added to the file name

Examples:
  To smear hdf5 file ``example.hdf5`` using the random Gaussian method::

    $ python dump_smeared.py --smear_method "random" /path/to/example.hdf5

  This will create the smeared hdf5 file ``/path/to/example_200ly.hdf5``.

.. note:: Valid smear methods include:

  * "weight", default
  * "random"
"""

import echidna.output.store as store
import echidna.core.smear as smear
import os

def main(args):
    """Smears energy and dumps spectra.

    Args:
      args (Namespace): Container for arguments. See
        >>> python dump_smeared_energy.py -h
    """
    if args.dest:
        if os.path.isdir(args.dest):
            directory = args.dest
            if directory[-1] != "/":
                directory += "/"
        else:
            raise ValueError("%s does not exist" % args.dest)
    else:
        directory = os.path.dirname(args.path)+"/"  # strip filename
    # strip directory and extension
    filename = os.path.splitext(os.path.basename(args.path))[0]

    if args.energy_resolution:
        if args.gaus:
            energy_smear = smear.EnergySmearRes(poisson=False)
        else:
            energy_smear = smear.EnergySmearRes(poisson=True)
        energy_smear.set_resolution(args.energy_resolution)
    else:  # use light yield
        if args.gaus:
            energy_smear = smear.EnergySmearLY(poisson=False)
        else:
            energy_smear = smear.EnergySmearLY(poisson=True)
        energy_smear.set_resolution(args.light_yield)
    spectrum = store.load(args.path)

    if args.smear_method == "weight":  # Use default smear method
        for par in spectrum.get_config().get_pars():
            if "energy" in par:
                energy_par = par
                spectrum = energy_smear.weighted_smear(spectrum,
                                                       par=energy_par)
    elif args.smear_method == "random":
        for par in spectrum.get_config().get_pars():
            if "energy" in par:
                energy_par = par
                spectrum = energy_smear.random_smear(spectrum,
                                                     par=energy_par)
    else:  # Not a valid smear method
        parser.error(args.smear_method + " is not a valid smear method")

    if args.energy_resolution:
        str_rs = str(args.energy_resolution)
        filename = directory + filename + "_" + str_rs + "rs.hdf5"
    else:
        str_ly = str(args.light_yield).rstrip('.0')
        filename = directory + filename + "_" + str_ly + "ly.hdf5"
    store.dump(filename, spectrum)

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("-m", "--smear_method", nargs='?', const="weight",
                        type=str, default="weight",
                        help="specify the smearing method to use")
    parser.add_argument("-r", "--energy_resolution", default=None, type=float,
                        help="specify energy resolution "
                        "e.g. 0.05 for 5 percent")
    parser.add_argument("-l", "--light_yield", default=200., type=float,
                        help="specify light yield"
                        "e.g. 200 for 200 NHit/MeV")
    parser.add_argument("-g", "--gaus", dest="gaus", action="store_true",
                        help="Apply gaussian PDF")
    parser.add_argument("-d", "--dest", default=None, type=str,
                        help="specify destination directory")
    parser.add_argument("path", type=str,
                        help="specify path to hdf5 file")
    parser.set_defaults(gaus=False)
    args = parser.parse_args()
    _logger = utilities.start_logging()

    try:
        main(args)
    except Exception:
        _logger.exception("echidna terminated because of the following error.")
<|endoftext|>


<commit_before>"""
Description : This file implements the Spell algorithm for log parsing
Author      : LogPAI team
License     : MIT
"""

import re
import os
import sys
import numpy as np
import pandas as pd
import hashlib
from datetime import datetime

class LCSObject:
    """ Class object to store a log group with the same template
    """
    def __init__(self, logTemplate='', logIDL=[]):
        self.logTemplate = logTemplate
        self.logIDL = logIDL

class Node:
    """ A node in prefix tree data structure
    """
    def __init__(self, token='', templateNo=0):
        self.logClust = None
        self.token = token
        self.templateNo = templateNo
        self.childD = dict()

class LogParser:
    """ LogParser class

    Attributes
    ----------
        path : the path of the input file
        logName : the file name of the input file
        savePath : the path of the output file
        tau : how much percentage of tokens matched to merge a log message
    """
    def __init__(self, indir='./', outdir='./result/', log_format=None, tau=0.5, rex=[]):
        self.path = indir
        self.logName = None
        self.savePath = outdir
        self.tau = tau
        self.logformat = log_format
        self.df_log = None
        self.rex = rex

    def LCS(self, seq1, seq2):
        lengths = [[0 for j in range(len(seq2)+1)] for i in range(len(seq1)+1)]
        # row 0 and column 0 are initialized to 0 already
        for i in range(len(seq1)):
            for j in range(len(seq2)):
                if seq1[i] == seq2[j]:
                    lengths[i+1][j+1] = lengths[i][j] + 1
                else:
                    lengths[i+1][j+1] = max(lengths[i+1][j], lengths[i][j+1])

        # read the substring out from the matrix
        result = []
        lenOfSeq1, lenOfSeq2 = len(seq1), len(seq2)
        while lenOfSeq1!=0 and lenOfSeq2 != 0:
            if lengths[lenOfSeq1][lenOfSeq2] == lengths[lenOfSeq1-1][lenOfSeq2]:
                lenOfSeq1 -= 1
            elif lengths[lenOfSeq1][lenOfSeq2] == lengths[lenOfSeq1][lenOfSeq2-1]:
                lenOfSeq2 -= 1
            else:
                assert seq1[lenOfSeq1-1] == seq2[lenOfSeq2-1]
                result.insert(0,seq1[lenOfSeq1-1])
                lenOfSeq1 -= 1
                lenOfSeq2 -= 1
        return result

    def SimpleLoopMatch(self, logClustL, seq):
        retLogClust = None

        for logClust in logClustL:
            if float(len(logClust.logTemplate)) < 0.5 * len(seq):
                continue

            #If the template is a subsequence of seq
            it = iter(seq)
            if all(token in seq or token == '*' for token in logClust.logTemplate):
                return logClust

        return retLogClust

    def PrefixTreeMatch(self, parentn, seq, idx):
        retLogClust = None
        length = len(seq)
        for i in xrange(idx, length):
            if seq[i] in parentn.childD:
                childn = parentn.childD[seq[i]]
                if (childn.logClust is not None):
                    constLM = [w for w in childn.logClust.logTemplate if w != '*']
                    if float(len(constLM)) >= self.tau * length:
                        return childn.logClust
                else:
                    return self.PrefixTreeMatch(childn, seq, i + 1)

        return retLogClust

    def LCSMatch(self, logClustL, seq):
        retLogClust = None

        maxLen = -1
        maxlcs = []
        maxClust = None
        set_seq = set(seq)
        size_seq = len(seq)
        for logClust in logClustL:
            set_template = set(logClust.logTemplate)
            if len(set_seq & set_template) < 0.5 * size_seq:
                continue
            lcs = self.LCS(seq, logClust.logTemplate)
            if len(lcs) > maxLen or (len(lcs) == maxLen and len(logClust.logTemplate) < len(maxClust.logTemplate)):
                maxLen = len(lcs)
                maxlcs = lcs
                maxClust = logClust

        # LCS should be large then tau * len(itself)
        if float(maxLen) >= self.tau * size_seq:
            retLogClust = maxClust

        return retLogClust

    def getTemplate(self, lcs, seq):
        retVal = []
        if not lcs:
            return retVal

        lcs = lcs[::-1]
        i = 0
        for token in seq:
            i += 1
            if token == lcs[-1]:
                retVal.append(token)
                lcs.pop()
            else:
                retVal.append('*')
            if not lcs:
                break
        if i < len(seq):
            retVal.append('*')
        return retVal

    def addSeqToPrefixTree(self, rootn, newCluster):
        parentn = rootn
        seq = newCluster.logTemplate
        seq = [w for w in seq if w != '*']

        for i in xrange(len(seq)):
            tokenInSeq = seq[i]
            # Match
            if tokenInSeq in parentn.childD:
                parentn.childD[tokenInSeq].templateNo += 1
            # Do not Match
            else:
                parentn.childD[tokenInSeq] = Node(token=tokenInSeq, templateNo=1)
            parentn = parentn.childD[tokenInSeq]

        if parentn.logClust is None:
            parentn.logClust = newCluster

    def removeSeqFromPrefixTree(self, rootn, newCluster):
        parentn = rootn
        seq = newCluster.logTemplate
        seq = [w for w in seq if w != '*']

        for tokenInSeq in seq:
            if tokenInSeq in parentn.childD:
                matchedNode = parentn.childD[tokenInSeq]
                if matchedNode.templateNo == 1:
                    del parentn.childD[tokenInSeq]
                    break
                else:
                    matchedNode.templateNo -= 1
                    parentn = matchedNode

    def outputResult(self, logClustL):

        templates = [0] * self.df_log.shape[0]
        ids = [0] * self.df_log.shape[0]
        df_event = []

        for logclust in logClustL:
            template_str = ' '.join(logclust.logTemplate)
            eid = hashlib.md5(template_str.encode('utf-8')).hexdigest()[0:8]
            for logid in logclust.logIDL:
                templates[logid - 1] = template_str
                ids[logid - 1] = eid
            df_event.append([eid, template_str, len(logclust.logIDL)])

        df_event = pd.DataFrame(df_event, columns=['EventId', 'EventTemplate', 'Occurrences'])

        self.df_log['EventId'] = ids
        self.df_log['EventTemplate'] = templates
        self.df_log.to_csv(os.path.join(self.savePath, self.logname + '_structured.csv'), index=False)
        df_event.to_csv(os.path.join(self.savePath, self.logname + '_templates.csv'), index=False)

    def printTree(self, node, dep):
        pStr = ''
        for i in xrange(dep):
            pStr += '\t'

        if node.token == '':
            pStr += 'Root'
        else:
            pStr += node.token
            if node.logClust is not None:
                pStr += '-->' + ' '.join(node.logClust.logTemplate)
        print pStr +' ('+ str(node.templateNo) + ')'

        for child in node.childD:
            self.printTree(node.childD[child], dep + 1)

    def parse(self, logname):
        starttime = datetime.now()
        print('Parsing file: ' + os.path.join(self.path, logname))
        self.logname = logname
        self.load_data()
        rootNode = Node()
        logCluL = []

        count = 0
        for idx, line in self.df_log.iterrows():
            logID = line['LineId']
            logmessageL = filter(lambda x: x != '', re.split(r'[\s=:,]', self.preprocess(line['Content'])))
            constLogMessL = [w for w in logmessageL if w != '*']

            #Find an existing matched log cluster
            matchCluster = self.PrefixTreeMatch(rootNode, constLogMessL, 0)

            if matchCluster is None:
                matchCluster = self.SimpleLoopMatch(logCluL, constLogMessL)

                if matchCluster is None:
                    matchCluster = self.LCSMatch(logCluL, logmessageL)

                    # Match no existing log cluster
                    if matchCluster is None:
                        newCluster = LCSObject(logTemplate=logmessageL, logIDL=[logID])
                        logCluL.append(newCluster)
                        self.addSeqToPrefixTree(rootNode, newCluster)
                    #Add the new log message to the existing cluster
                    else:
                        newTemplate = self.getTemplate(self.LCS(logmessageL, matchCluster.logTemplate),
                                                       matchCluster.logTemplate)
                        if ' '.join(newTemplate) != ' '.join(matchCluster.logTemplate):
                            self.removeSeqFromPrefixTree(rootNode, matchCluster)
                            matchCluster.logTemplate = newTemplate
                            self.addSeqToPrefixTree(rootNode, matchCluster)
            if matchCluster:
                matchCluster.logIDL.append(logID)
            count += 1
            if count % 1000 == 0 or count == len(self.df_log):
                print 'Processed {0:.1f}% of log lines.'.format(count * 100.0 / len(self.df_log))

        if not os.path.exists(self.savePath):
            os.makedirs(self.savePath)

        self.outputResult(logCluL)
        print('Parsing done. [Time taken: {!s}]'.format(datetime.now() - starttime))

    def load_data(self):
        headers, regex = self.generate_logformat_regex(self.logformat)
        self.df_log = self.log_to_dataframe(os.path.join(self.path, self.logname), regex, headers, self.logformat)

    def preprocess(self, line):
        for currentRex in self.rex:
            line = re.sub(currentRex, '*', line)
        return line

    def log_to_dataframe(self, log_file, regex, headers, logformat):
        """ Function to transform log file to dataframe
        """
        log_messages = []
        linecount = 0
        with open(log_file, 'r') as fin:
            for line in fin.readlines():
                line = re.sub(r'[^\x00-\x7F]+', '<NASCII>', line)
                try:
                    match = regex.search(line.strip())
                    message = [match.group(header) for header in headers]
                    log_messages.append(message)
                    linecount += 1
                except Exception as e:
                    pass
        logdf = pd.DataFrame(log_messages, columns=headers)
        logdf.insert(0, 'LineId', None)
        logdf['LineId'] = [i + 1 for i in range(linecount)]
        return logdf

    def generate_logformat_regex(self, logformat):
        """ Function to generate regular expression to split log messages
        """
        headers = []
        splitters = re.split(r'(<[^<>]+>)', logformat)
        regex = ''
        for k in range(len(splitters)):
            if k % 2 == 0:
                splitter = re.sub(' +', '\s+', splitters[k])
                regex += splitter
            else:
                header = splitters[k].strip('<').strip('>')
                regex += '(?P<%s>.*?)' % header
                headers.append(header)
        regex = re.compile('^' + regex + '$')
        return headers, regex
<commit_msg>Update Spell.py<commit_after>"""
Description : This file implements the Spell algorithm for log parsing
Author      : LogPAI team
License     : MIT
"""

import re
import os
import sys
import numpy as np
import pandas as pd
import hashlib
from datetime import datetime

class LCSObject:
    """ Class object to store a log group with the same template
    """
    def __init__(self, logTemplate='', logIDL=[]):
        self.logTemplate = logTemplate
        self.logIDL = logIDL

class Node:
    """ A node in prefix tree data structure
    """
    def __init__(self, token='', templateNo=0):
        self.logClust = None
        self.token = token
        self.templateNo = templateNo
        self.childD = dict()

class LogParser:
    """ LogParser class

    Attributes
    ----------
        path : the path of the input file
        logName : the file name of the input file
        savePath : the path of the output file
        tau : how much percentage of tokens matched to merge a log message
    """
    def __init__(self, indir='./', outdir='./result/', log_format=None, tau=0.5, rex=[]):
        self.path = indir
        self.logName = None
        self.savePath = outdir
        self.tau = tau
        self.logformat = log_format
        self.df_log = None
        self.rex = rex

    def LCS(self, seq1, seq2):
        lengths = [[0 for j in range(len(seq2)+1)] for i in range(len(seq1)+1)]
        # row 0 and column 0 are initialized to 0 already
        for i in range(len(seq1)):
            for j in range(len(seq2)):
                if seq1[i] == seq2[j]:
                    lengths[i+1][j+1] = lengths[i][j] + 1
                else:
                    lengths[i+1][j+1] = max(lengths[i+1][j], lengths[i][j+1])

        # read the substring out from the matrix
        result = []
        lenOfSeq1, lenOfSeq2 = len(seq1), len(seq2)
        while lenOfSeq1!=0 and lenOfSeq2 != 0:
            if lengths[lenOfSeq1][lenOfSeq2] == lengths[lenOfSeq1-1][lenOfSeq2]:
                lenOfSeq1 -= 1
            elif lengths[lenOfSeq1][lenOfSeq2] == lengths[lenOfSeq1][lenOfSeq2-1]:
                lenOfSeq2 -= 1
            else:
                assert seq1[lenOfSeq1-1] == seq2[lenOfSeq2-1]
                result.insert(0,seq1[lenOfSeq1-1])
                lenOfSeq1 -= 1
                lenOfSeq2 -= 1
        return result

    def SimpleLoopMatch(self, logClustL, seq):
        for logClust in logClustL:
            if float(len(logClust.logTemplate)) < 0.5 * len(seq):
                continue
            # Check the template is a subsequence of seq (we use set checking as a proxy here for speedup since
            # incorrect-ordering bad cases rarely occur in logs)
            token_set = set(seq)
            if all(token in token_set or token == '*' for token in logClust.logTemplate):
                return logClust
        return None

    def PrefixTreeMatch(self, parentn, seq, idx):
        retLogClust = None
        length = len(seq)
        for i in xrange(idx, length):
            if seq[i] in parentn.childD:
                childn = parentn.childD[seq[i]]
                if (childn.logClust is not None):
                    constLM = [w for w in childn.logClust.logTemplate if w != '*']
                    if float(len(constLM)) >= self.tau * length:
                        return childn.logClust
                else:
                    return self.PrefixTreeMatch(childn, seq, i + 1)

        return retLogClust

    def LCSMatch(self, logClustL, seq):
        retLogClust = None

        maxLen = -1
        maxlcs = []
        maxClust = None
        set_seq = set(seq)
        size_seq = len(seq)
        for logClust in logClustL:
            set_template = set(logClust.logTemplate)
            if len(set_seq & set_template) < 0.5 * size_seq:
                continue
            lcs = self.LCS(seq, logClust.logTemplate)
            if len(lcs) > maxLen or (len(lcs) == maxLen and len(logClust.logTemplate) < len(maxClust.logTemplate)):
                maxLen = len(lcs)
                maxlcs = lcs
                maxClust = logClust

        # LCS should be large then tau * len(itself)
        if float(maxLen) >= self.tau * size_seq:
            retLogClust = maxClust

        return retLogClust

    def getTemplate(self, lcs, seq):
        retVal = []
        if not lcs:
            return retVal

        lcs = lcs[::-1]
        i = 0
        for token in seq:
            i += 1
            if token == lcs[-1]:
                retVal.append(token)
                lcs.pop()
            else:
                retVal.append('*')
            if not lcs:
                break
        if i < len(seq):
            retVal.append('*')
        return retVal

    def addSeqToPrefixTree(self, rootn, newCluster):
        parentn = rootn
        seq = newCluster.logTemplate
        seq = [w for w in seq if w != '*']

        for i in xrange(len(seq)):
            tokenInSeq = seq[i]
            # Match
            if tokenInSeq in parentn.childD:
                parentn.childD[tokenInSeq].templateNo += 1
            # Do not Match
            else:
                parentn.childD[tokenInSeq] = Node(token=tokenInSeq, templateNo=1)
            parentn = parentn.childD[tokenInSeq]

        if parentn.logClust is None:
            parentn.logClust = newCluster

    def removeSeqFromPrefixTree(self, rootn, newCluster):
        parentn = rootn
        seq = newCluster.logTemplate
        seq = [w for w in seq if w != '*']

        for tokenInSeq in seq:
            if tokenInSeq in parentn.childD:
                matchedNode = parentn.childD[tokenInSeq]
                if matchedNode.templateNo == 1:
                    del parentn.childD[tokenInSeq]
                    break
                else:
                    matchedNode.templateNo -= 1
                    parentn = matchedNode

    def outputResult(self, logClustL):

        templates = [0] * self.df_log.shape[0]
        ids = [0] * self.df_log.shape[0]
        df_event = []

        for logclust in logClustL:
            template_str = ' '.join(logclust.logTemplate)
            eid = hashlib.md5(template_str.encode('utf-8')).hexdigest()[0:8]
            for logid in logclust.logIDL:
                templates[logid - 1] = template_str
                ids[logid - 1] = eid
            df_event.append([eid, template_str, len(logclust.logIDL)])

        df_event = pd.DataFrame(df_event, columns=['EventId', 'EventTemplate', 'Occurrences'])

        self.df_log['EventId'] = ids
        self.df_log['EventTemplate'] = templates
        self.df_log.to_csv(os.path.join(self.savePath, self.logname + '_structured.csv'), index=False)
        df_event.to_csv(os.path.join(self.savePath, self.logname + '_templates.csv'), index=False)

    def printTree(self, node, dep):
        pStr = ''
        for i in xrange(dep):
            pStr += '\t'

        if node.token == '':
            pStr += 'Root'
        else:
            pStr += node.token
            if node.logClust is not None:
                pStr += '-->' + ' '.join(node.logClust.logTemplate)
        print pStr +' ('+ str(node.templateNo) + ')'

        for child in node.childD:
            self.printTree(node.childD[child], dep + 1)

    def parse(self, logname):
        starttime = datetime.now()
        print('Parsing file: ' + os.path.join(self.path, logname))
        self.logname = logname
        self.load_data()
        rootNode = Node()
        logCluL = []

        count = 0
        for idx, line in self.df_log.iterrows():
            logID = line['LineId']
            logmessageL = filter(lambda x: x != '', re.split(r'[\s=:,]', self.preprocess(line['Content'])))
            constLogMessL = [w for w in logmessageL if w != '*']

            #Find an existing matched log cluster
            matchCluster = self.PrefixTreeMatch(rootNode, constLogMessL, 0)

            if matchCluster is None:
                matchCluster = self.SimpleLoopMatch(logCluL, constLogMessL)

                if matchCluster is None:
                    matchCluster = self.LCSMatch(logCluL, logmessageL)

                    # Match no existing log cluster
                    if matchCluster is None:
                        newCluster = LCSObject(logTemplate=logmessageL, logIDL=[logID])
                        logCluL.append(newCluster)
                        self.addSeqToPrefixTree(rootNode, newCluster)
                    #Add the new log message to the existing cluster
                    else:
                        newTemplate = self.getTemplate(self.LCS(logmessageL, matchCluster.logTemplate),
                                                       matchCluster.logTemplate)
                        if ' '.join(newTemplate) != ' '.join(matchCluster.logTemplate):
                            self.removeSeqFromPrefixTree(rootNode, matchCluster)
                            matchCluster.logTemplate = newTemplate
                            self.addSeqToPrefixTree(rootNode, matchCluster)
            if matchCluster:
                matchCluster.logIDL.append(logID)
            count += 1
            if count % 1000 == 0 or count == len(self.df_log):
                print 'Processed {0:.1f}% of log lines.'.format(count * 100.0 / len(self.df_log))

        if not os.path.exists(self.savePath):
            os.makedirs(self.savePath)

        self.outputResult(logCluL)
        print('Parsing done. [Time taken: {!s}]'.format(datetime.now() - starttime))

    def load_data(self):
        headers, regex = self.generate_logformat_regex(self.logformat)
        self.df_log = self.log_to_dataframe(os.path.join(self.path, self.logname), regex, headers, self.logformat)

    def preprocess(self, line):
        for currentRex in self.rex:
            line = re.sub(currentRex, '*', line)
        return line

    def log_to_dataframe(self, log_file, regex, headers, logformat):
        """ Function to transform log file to dataframe
        """
        log_messages = []
        linecount = 0
        with open(log_file, 'r') as fin:
            for line in fin.readlines():
                line = re.sub(r'[^\x00-\x7F]+', '<NASCII>', line)
                try:
                    match = regex.search(line.strip())
                    message = [match.group(header) for header in headers]
                    log_messages.append(message)
                    linecount += 1
                except Exception as e:
                    pass
        logdf = pd.DataFrame(log_messages, columns=headers)
        logdf.insert(0, 'LineId', None)
        logdf['LineId'] = [i + 1 for i in range(linecount)]
        return logdf

    def generate_logformat_regex(self, logformat):
        """ Function to generate regular expression to split log messages
        """
        headers = []
        splitters = re.split(r'(<[^<>]+>)', logformat)
        regex = ''
        for k in range(len(splitters)):
            if k % 2 == 0:
                splitter = re.sub(' +', '\s+', splitters[k])
                regex += splitter
            else:
                header = splitters[k].strip('<').strip('>')
                regex += '(?P<%s>.*?)' % header
                headers.append(header)
        regex = re.compile('^' + regex + '$')
        return headers, regex
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright (c) The Spyder Development Team
#
# Licensed under the terms of the MIT License
# (See LICENSE.txt for details)
# -----------------------------------------------------------------------------
"""Setup script for loghub."""

# Standard library imports
import ast
import os

# Third party imports
from setuptools import find_packages, setup

HERE = os.path.abspath(os.path.dirname(__file__))

def get_version(module='loghub'):
    """Get version."""
    with open(os.path.join(HERE, module, '__init__.py'), 'r') as f:
        data = f.read()
    lines = data.split('\n')
    for line in lines:
        if line.startswith('VERSION_INFO'):
            version_tuple = ast.literal_eval(line.split('=')[-1].strip())
            version = '.'.join(map(str, version_tuple))
            break
    return version

def get_description():
    """Get long description."""
    with open(os.path.join(HERE, 'README.rst'), 'r') as f:
        data = f.read()
    return data

REQUIREMENTS = ['jinja2']

setup(
    name='loghub',
    version=get_version(),
    keywords=["github changelog milestone"],
    url='https://github.com/spyder-ide/loghub',
    license='MIT',
    author='Carlos Cordoba',
    author_email='ccordoba12@gmail.com',
    maintainer='Gonzalo Pena-Castellanos',
    maintainer_email='goanpeca@gmail.com',
    description='Generate changelogs based on Github milestones or tags',
    long_description=get_description(),
    packages=find_packages(exclude=['contrib', 'docs', 'tests*']),
    package_data={'loghub.templates.content': ['*.txt']},
    install_requires=REQUIREMENTS,
    entry_points={
        'console_scripts': [
            'loghub = loghub.cli.main:main',
            'loghub-labels = loghub.cli.label_creator:main',
        ]
    },
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Operating System :: OS Independent',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6'
    ])
<commit_msg>Fix typo<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright (c) The Spyder Development Team
#
# Licensed under the terms of the MIT License
# (See LICENSE.txt for details)
# -----------------------------------------------------------------------------
"""Setup script for loghub."""

# Standard library imports
import ast
import os

# Third party imports
from setuptools import find_packages, setup

HERE = os.path.abspath(os.path.dirname(__file__))

def get_version(module='loghub'):
    """Get version."""
    with open(os.path.join(HERE, module, '__init__.py'), 'r') as f:
        data = f.read()
    lines = data.split('\n')
    for line in lines:
        if line.startswith('VERSION_INFO'):
            version_tuple = ast.literal_eval(line.split('=')[-1].strip())
            version = '.'.join(map(str, version_tuple))
            break
    return version

def get_description():
    """Get long description."""
    with open(os.path.join(HERE, 'README.rst'), 'r') as f:
        data = f.read()
    return data

REQUIREMENTS = ['jinja2']

setup(
    name='loghub',
    version=get_version(),
    keywords=["github changelog milestone"],
    url='https://github.com/spyder-ide/loghub',
    license='MIT',
    author='Carlos Cordoba',
    author_email='ccordoba12@gmail.com',
    maintainer='Gonzalo Pena-Castellanos',
    maintainer_email='goanpeca@gmail.com',
    description='Generate changelogs based on Github milestones or tags',
    long_description=get_description(),
    packages=find_packages(exclude=['contrib', 'docs', 'tests*']),
    package_data={'loghub.templates': ['*.txt']},
    install_requires=REQUIREMENTS,
    entry_points={
        'console_scripts': [
            'loghub = loghub.cli.main:main',
            'loghub-labels = loghub.cli.label_creator:main',
        ]
    },
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Operating System :: OS Independent',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6'
    ])
<|endoftext|>


<commit_before># coding: utf-8

#-------------------
# built in modules
#-------------------
import os
import sys
import html

#-------------------
# 3rd party modules
#-------------------
# https://pypi.python.org/pypi/openpyxl
import openpyxl

#-------------------
# My Script
#-------------------
from . import constants
from .midi import mididata
from .util import util
from .expressionmap import template
from .xlsx import xlsutil

INT_MAX = sys.maxsize

"""
Convert to xlsx file to Cubase *.expressionmap.
"""
class XLS2ExpressionMap:

    """
    ctor
    """
    def __init__( self, xlsxFileName, outputDir = '.' ):
        self.xlsxFileName = xlsxFileName
        self.book         = openpyxl.load_workbook( filename = xlsxFileName, read_only = True )
        self.outputDir    = outputDir

    """
    Generate articulation xml string from given sheet
    """
    def generateArticulation( self, sheet ):

        ret = template.ARTICULATION_HEADER

        for rowIndex in range( constants.START_ROW_INEDX, sheet.max_row ):

            name = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_ARTICULATION )

            if name == None:
                break

            artiType = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_ARTICULATION_TYPE )
            group    = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_GROUP )

            # Must be required values to generate
            if name == None or artiType == None or group == None:
                continue

            # "group" xml value is zero origin
            group = group - 1

            print( "[Articulation] {name}, Type={type}, Group={group}".format(
                name  = name,
                type  = artiType,
                group = group
            ))

            artiType = constants.ARTICULATION_TYPE.index( artiType ) # to integer format ( 0: Attribute 1: Direction).

            ret += template.ARTICULATION.format(
                uuid1 = util.createUUID(),
                type  = artiType,
                name  = html.escape( name ),
                group = group
            )

        ret += template.ARTICULATION_FOOTER
        return ret

    """
    Generate velocity xml string from given data
    """
    def generateMIDINote( self, note ):

        return template.MIDI_MESSAGE_IN_KEYSWITCH.format(
                    uuid1       = util.createUUID(),
                    midiMessage = 144,
                    data1       = note.noteNo,
                    data2       = note.velocity
        )

    """
    Generate MIDI CC xml string from given data
    """
    def generateCC( self, cc ):

        return template.MIDI_MESSAGE_IN_KEYSWITCH.format(
                    uuid1       = util.createUUID(),
                    midiMessage = 176,
                    data1       = cc.ccNo,
                    data2       = cc.ccValue
        )

    """
    Generate MIDI Program change message xml string from given data
    """
    def generatePC( self, pc ):
        return template.MIDI_MESSAGE_IN_KEYSWITCH.format(
                    uuid1       = util.createUUID(),
                    midiMessage = 192,
                    data1       = pc.lsb,
                    data2       = pc.msb
        )

    """
    Generate key switch xml string from given sheet
    """
    def generateKeySwitch( self, sheet ):

        ret = template.KEY_SWITCH_HEADER

        for rowIndex in range( constants.START_ROW_INEDX, sheet.max_row ):

            name = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_NAME )
            if name == None:
                break

            articulation    = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_ARTICULATION )
            color           = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_COLOR )
            group           = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_GROUP )

            if group == None:
                group = 0
            else:
                # "group" xml value is zero origin
                group = group - 1

            print( "[Slot] {name}".format( name = name ) )

            # Append articulation
            if len( articulation ) > 0:
                tmp = ""
                tmp += template.ARTICULATION_IN_SLOT_HEADER
                tmp += template.ARTICULATION_IN_SLOT.format(
                    uuid1 = util.createUUID(), name = articulation,
                    group = group
                )
                tmp += template.ARTICULATION_IN_SLOT_FOOTER
                articulation = tmp
            else:
                articulation = ""

            # Append MIDI Notes
            # * Multiple MIDI Note Supported
            # * Column name format:
            #   MIDI Note1 ... MIDI Note1+n
            #   Velocity1 ... Velocity1+n
            midiNoteList = []
            for i in range( 1, INT_MAX ):
                noteNo = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_MIDI_NOTE + str( i ) )
                vel    = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_MIDI_VELOCITY + str( i ) )
                if noteNo == None or vel == None:
                    break

                if noteNo in constants.NOTENUMBER:
                    noteNo = constants.NOTENUMBER.index( noteNo ) # to integer format (0-127)

                obj = mididata.MIDINote( noteNo, vel )
                if obj.valid():
                    midiNoteList.append( obj )

            # Append MIDI CC
            # * Multiple MIDI CC Supported
            # * Column name format:
            #   CC No1 ... CC No1+n
            #   CC Value1 ... CC Value1+n
            ccList = []
            for i in range( 1, INT_MAX ):
                ccNo    = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_MIDI_CC + str( i ) )
                ccValue = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_MIDI_CC_VALUE + str( i ) )
                if ccNo == None or ccValue == None:
                    break

                obj = mididata.MIDICc( ccNo, ccValue )
                if obj.valid():
                    ccList.append( obj )

            # Append MIDI PC
            # * Multiple MIDI Program Change Supported
            # * Column name format:
            #   PC LSB1 ... PC LSB1+n
            #   PC MSB1 ... PC MSB1+n (MSB not exist, MSB value will be 0 )
            programChangeList = []
            for i in range( 1, INT_MAX ):
                lsb = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_MIDI_PC_LSB + str( i ) )
                msb = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_MIDI_PC_MSB + str( i ) )

                if lsb == None:
                    break
                elif msb == None:
                    # msb is not set
                    msb = 0

                obj = mididata.MIDIPc( lsb, msb )
                if obj.valid():
                    programChangeList.append( obj )

            # Generate MIDI message xml node
            midiMessageXml = ""
            if len( midiNoteList ) > 0 or len( ccList ) > 0 or len( programChangeList ) > 0:

                midiMessageXml += template.MIDI_MESSAGE_IN_KEYSWITCH_HEADER

                # MIDI Note
                for n in midiNoteList:
                    midiMessageXml += self.generateMIDINote( n )

                # MIDI CC
                for cc in ccList:
                    midiMessageXml  += self.generateCC( cc )

                # MIDI PC
                for pc in programChangeList:
                    midiMessageXml += self.generatePC( pc )

                midiMessageXml += template.MIDI_MESSAGE_IN_KEYSWITCH_FOOTER
            else:
                midiMessageXml = template.EMPTY_MIDI_MESSAGE_IN_KEYSWITCH

            # Generate Keyswitch XML text
            ret += template.KEY_SWITCH.format(
                uuid1 = util.createUUID(),
                uuid2 = util.createUUID(),
                uuid3 = util.createUUID(),
                uuid4 = util.createUUID(),
                midimessage = midiMessageXml,
                name  = html.escape( name ),
                color = color,
                articulations = articulation
            )

        ret += template.SLOTS_FOOTER
        return ret

    """
    Convert to Expression map file
    """
    def convert( self ):
        for sheetName in self.book.sheetnames:

            xmlText = ''
            sheet   = self.book[ sheetName ]

            # Ignore sheet
            if sheetName == constants.LIST_DEFINITION_SHEETNAME:
                continue

            expressionMapName = sheetName
            outputFileName    = expressionMapName + ".expressionmap"
            xmlText  = template.XML_HEADER.format( name = expressionMapName )
            xmlText += self.generateArticulation( sheet )
            xmlText += self.generateKeySwitch( sheet )
            xmlText += template.XML_FOOTER

            xmlText = xmlText.encode( 'utf8' )

            fp = open( outputFileName, "wb" )
            fp.write( xmlText )
            fp.close()

<commit_msg>Bug fixes : output directory is not set<commit_after># coding: utf-8

#-------------------
# built in modules
#-------------------
import os
import sys
import html
from os import path

#-------------------
# 3rd party modules
#-------------------
# https://pypi.python.org/pypi/openpyxl
import openpyxl

#-------------------
# My Script
#-------------------
from . import constants
from .midi import mididata
from .util import util
from .expressionmap import template
from .xlsx import xlsutil

INT_MAX = sys.maxsize

"""
Convert to xlsx file to Cubase *.expressionmap.
"""
class XLS2ExpressionMap:

    """
    ctor
    """
    def __init__( self, xlsxFileName, outputDir = '.' ):
        self.xlsxFileName = xlsxFileName
        self.book         = openpyxl.load_workbook( filename = xlsxFileName, read_only = True )
        self.outputDir    = outputDir

    """
    Generate articulation xml string from given sheet
    """
    def generateArticulation( self, sheet ):

        ret = template.ARTICULATION_HEADER

        for rowIndex in range( constants.START_ROW_INEDX, sheet.max_row ):

            name = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_ARTICULATION )

            if name == None:
                break

            artiType = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_ARTICULATION_TYPE )
            group    = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_GROUP )

            # Must be required values to generate
            if name == None or artiType == None or group == None:
                continue

            # "group" xml value is zero origin
            group = group - 1

            print( "[Articulation] {name}, Type={type}, Group={group}".format(
                name  = name,
                type  = artiType,
                group = group
            ))

            artiType = constants.ARTICULATION_TYPE.index( artiType ) # to integer format ( 0: Attribute 1: Direction).

            ret += template.ARTICULATION.format(
                uuid1 = util.createUUID(),
                type  = artiType,
                name  = html.escape( name ),
                group = group
            )

        ret += template.ARTICULATION_FOOTER
        return ret

    """
    Generate velocity xml string from given data
    """
    def generateMIDINote( self, note ):

        return template.MIDI_MESSAGE_IN_KEYSWITCH.format(
                    uuid1       = util.createUUID(),
                    midiMessage = 144,
                    data1       = note.noteNo,
                    data2       = note.velocity
        )

    """
    Generate MIDI CC xml string from given data
    """
    def generateCC( self, cc ):

        return template.MIDI_MESSAGE_IN_KEYSWITCH.format(
                    uuid1       = util.createUUID(),
                    midiMessage = 176,
                    data1       = cc.ccNo,
                    data2       = cc.ccValue
        )

    """
    Generate MIDI Program change message xml string from given data
    """
    def generatePC( self, pc ):
        return template.MIDI_MESSAGE_IN_KEYSWITCH.format(
                    uuid1       = util.createUUID(),
                    midiMessage = 192,
                    data1       = pc.lsb,
                    data2       = pc.msb
        )

    """
    Generate key switch xml string from given sheet
    """
    def generateKeySwitch( self, sheet ):

        ret = template.KEY_SWITCH_HEADER

        for rowIndex in range( constants.START_ROW_INEDX, sheet.max_row ):

            name = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_NAME )
            if name == None:
                break

            articulation    = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_ARTICULATION )
            color           = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_COLOR )
            group           = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_GROUP )

            if group == None:
                group = 0
            else:
                # "group" xml value is zero origin
                group = group - 1

            print( "[Slot] {name}".format( name = name ) )

            # Append articulation
            if len( articulation ) > 0:
                tmp = ""
                tmp += template.ARTICULATION_IN_SLOT_HEADER
                tmp += template.ARTICULATION_IN_SLOT.format(
                    uuid1 = util.createUUID(), name = articulation,
                    group = group
                )
                tmp += template.ARTICULATION_IN_SLOT_FOOTER
                articulation = tmp
            else:
                articulation = ""

            # Append MIDI Notes
            # * Multiple MIDI Note Supported
            # * Column name format:
            #   MIDI Note1 ... MIDI Note1+n
            #   Velocity1 ... Velocity1+n
            midiNoteList = []
            for i in range( 1, INT_MAX ):
                noteNo = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_MIDI_NOTE + str( i ) )
                vel    = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_MIDI_VELOCITY + str( i ) )
                if noteNo == None or vel == None:
                    break

                if noteNo in constants.NOTENUMBER:
                    noteNo = constants.NOTENUMBER.index( noteNo ) # to integer format (0-127)

                obj = mididata.MIDINote( noteNo, vel )
                if obj.valid():
                    midiNoteList.append( obj )

            # Append MIDI CC
            # * Multiple MIDI CC Supported
            # * Column name format:
            #   CC No1 ... CC No1+n
            #   CC Value1 ... CC Value1+n
            ccList = []
            for i in range( 1, INT_MAX ):
                ccNo    = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_MIDI_CC + str( i ) )
                ccValue = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_MIDI_CC_VALUE + str( i ) )
                if ccNo == None or ccValue == None:
                    break

                obj = mididata.MIDICc( ccNo, ccValue )
                if obj.valid():
                    ccList.append( obj )

            # Append MIDI PC
            # * Multiple MIDI Program Change Supported
            # * Column name format:
            #   PC LSB1 ... PC LSB1+n
            #   PC MSB1 ... PC MSB1+n (MSB not exist, MSB value will be 0 )
            programChangeList = []
            for i in range( 1, INT_MAX ):
                lsb = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_MIDI_PC_LSB + str( i ) )
                msb = xlsutil.getValueFromColumnName( sheet, rowIndex, constants.COLUMN_MIDI_PC_MSB + str( i ) )

                if lsb == None:
                    break
                elif msb == None:
                    # msb is not set
                    msb = 0

                obj = mididata.MIDIPc( lsb, msb )
                if obj.valid():
                    programChangeList.append( obj )

            # Generate MIDI message xml node
            midiMessageXml = ""
            if len( midiNoteList ) > 0 or len( ccList ) > 0 or len( programChangeList ) > 0:

                midiMessageXml += template.MIDI_MESSAGE_IN_KEYSWITCH_HEADER

                # MIDI Note
                for n in midiNoteList:
                    midiMessageXml += self.generateMIDINote( n )

                # MIDI CC
                for cc in ccList:
                    midiMessageXml  += self.generateCC( cc )

                # MIDI PC
                for pc in programChangeList:
                    midiMessageXml += self.generatePC( pc )

                midiMessageXml += template.MIDI_MESSAGE_IN_KEYSWITCH_FOOTER
            else:
                midiMessageXml = template.EMPTY_MIDI_MESSAGE_IN_KEYSWITCH

            # Generate Keyswitch XML text
            ret += template.KEY_SWITCH.format(
                uuid1 = util.createUUID(),
                uuid2 = util.createUUID(),
                uuid3 = util.createUUID(),
                uuid4 = util.createUUID(),
                midimessage = midiMessageXml,
                name  = html.escape( name ),
                color = color,
                articulations = articulation
            )

        ret += template.SLOTS_FOOTER
        return ret

    """
    Convert to Expression map file
    """
    def convert( self ):
        for sheetName in self.book.sheetnames:

            xmlText = ''
            sheet   = self.book[ sheetName ]

            # Ignore sheet
            if sheetName == constants.LIST_DEFINITION_SHEETNAME:
                continue

            expressionMapName = sheetName
            outputFileName    = expressionMapName + ".expressionmap"
            outputFileName    = path.join( self.outputDir, outputFileName )
            xmlText  = template.XML_HEADER.format( name = expressionMapName )
            xmlText += self.generateArticulation( sheet )
            xmlText += self.generateKeySwitch( sheet )
            xmlText += template.XML_FOOTER

            xmlText = xmlText.encode( 'utf8' )

            fp = open( outputFileName, "wb" )
            fp.write( xmlText )
            fp.close()

<|endoftext|>


<commit_before>"""Reimplementation of the Mako template loader that supports dotted names."""

import os
import stat

try:
    import threading
except ImportError:
    import dummy_threading as threading

from pkg_resources import resource_filename
from mako.template import Template
from genshi.template import TemplateLoader
from paste.deploy.converters import asbool

import tg

class DottedTemplateLookup(object):
    """Mako template lookup emulation that supports
    zipped applications and dotted filenames.

    This is an emulation of the Mako template lookup that will handle
    get_template and support dotted names in Python path notation
    to support zipped eggs.

    This is necessary because Mako asserts that your project will always
    be installed in a zip-unsafe manner with all files somewhere on the
    hard drive.

    This is not the case when you want your application to be deployed
    in a single zip file (zip-safe). If you want to deploy in a zip
    file _and_ use the dotted template name notation then this class
    is necessary because it emulates files on the filesystem for the
    underlying Mako engine while they are in fact in your zip file.

    """

    def __init__(self, input_encoding, output_encoding,
            imports, default_filters, module_directory=None):

        self.input_encoding = input_encoding
        self.output_encoding = output_encoding
        self.imports = imports
        self.default_filters = default_filters
        # implement a cache for the loaded templates
        self.template_cache = dict()
        # implement a cache for the filename lookups
        self.template_filenames_cache = dict()
        self.module_directory = module_directory

        # a mutex to ensure thread safeness during template loading
        self._mutex = threading.Lock()

    def adjust_uri(self, uri, relativeto):
        """Adjust the given uri relative to a filename.

        This method is used by mako for filesystem based reasons.
        In dotted lookup land we don't adjust uri so we just return
        the value we are given without any change.

        """
        if uri.startswith('local:'):
            uri = tg.config['pylons.package'] + '.' + uri[6:]

        if '.' in uri:
            # We are in the DottedTemplateLookup system so dots in
            # names should be treated as a Python path. Since this
            # method is called by template inheritance we must
            # support dotted names also in the inheritance.
            result = tg.config['pylons.app_globals'].\
                dotted_filename_finder.get_dotted_filename(template_name=uri, template_extension='.mak')

            if not uri in self.template_filenames_cache:
                # feed our filename cache if needed.
                self.template_filenames_cache[uri] = result

        else:
            # no dot detected, just return plain name
            result = uri

        return result

    def __check(self, template):
        """private method used to verify if a template has changed
        since the last time it has been put in cache...

        This method being based on the mtime of a real file this should
        never be called on a zipped deployed application.

        This method is a ~copy/paste of the original caching system from
        the Mako lookup loader.

        """
        if template.filename is None:
            return template

        if not os.path.exists(template.filename):
            # remove from cache.
            self.template_cache.pop(template.filename, None)
            raise exceptions.TemplateLookupException(
                    "Cant locate template '%s'" % template.filename)

        elif template.module._modified_time < os.stat(
                template.filename)[stat.ST_MTIME]:

            # cache is too old, remove old template
            # from cache and reload.
            self.template_cache.pop(template.filename, None)
            return self.__load(template.filename)

        else:
            # cache is correct, use it.
            return template

    def __load(self, filename):
        """real loader function. copy paste from the mako template
        loader.

        """
        # make sure the template loading from filesystem is only done
        # one thread at a time to avoid bad clashes...
        self._mutex.acquire()
        try:
            try:
                # try returning from cache one more time in case
                # concurrent thread already loaded
                return self.template_cache[filename]

            except KeyError:
                # not in cache yet... we can continue normally
                pass

            try:
                self.template_cache[filename] = Template(
                    filename=filename,
                    module_directory=os.sep,
                    input_encoding=self.input_encoding,
                    output_encoding=self.output_encoding,
                    default_filters=self.default_filters,
                    imports=self.imports,
                    lookup=self)

                return self.template_cache[filename]

            except:
                self.template_cache.pop(filename, None)
                raise

        finally:
            # _always_ release the lock once done to avoid
            # "thread lock" effect
            self._mutex.release()

    def get_template(self, template_name):
        """this is the emulated method that must return a template
        instance based on a given template name
        """

        if not self.template_cache.has_key(template_name):
            # the template string is not yet loaded into the cache.
            # Do so now
            self.__load(template_name)

        if asbool(tg.config.get('templating.mako.reloadfromdisk', 'false')):
            # AUTO RELOADING will be activated only if user has
            # explicitly asked for it in the configuration
            # return the template, but first make sure it's not outdated
            # and if outdated, refresh the cache.
            return self.__check(self.template_cache[template_name])

        else:
            return self.template_cache[template_name]

class GenshiTemplateLoader(TemplateLoader):
    """Genshi template loader that supports
    zipped applications and dotted filenames as well as path names
    """

    def load(self, filename, relative_to=None, cls=None, encoding=None):
        """real loader function. copy paste from the mako template
        loader.
        """
        # TODO: get the template extension from the config!!
        if not filename.endswith('.html'):
            filename = tg.config['pylons.app_globals'
                    ].dotted_filename_finder.get_dotted_filename(
                            template_name=filename,
                            template_extension='.html')

        return TemplateLoader.load(self, filename,
                relative_to=relative_to, cls=cls, encoding=encoding)

<commit_msg>Pass our module_directory to Mako's Template, instead of os.sep (#2395)<commit_after>"""Reimplementation of the Mako template loader that supports dotted names."""

import os
import stat

try:
    import threading
except ImportError:
    import dummy_threading as threading

from pkg_resources import resource_filename
from mako.template import Template
from genshi.template import TemplateLoader
from paste.deploy.converters import asbool

import tg

class DottedTemplateLookup(object):
    """Mako template lookup emulation that supports
    zipped applications and dotted filenames.

    This is an emulation of the Mako template lookup that will handle
    get_template and support dotted names in Python path notation
    to support zipped eggs.

    This is necessary because Mako asserts that your project will always
    be installed in a zip-unsafe manner with all files somewhere on the
    hard drive.

    This is not the case when you want your application to be deployed
    in a single zip file (zip-safe). If you want to deploy in a zip
    file _and_ use the dotted template name notation then this class
    is necessary because it emulates files on the filesystem for the
    underlying Mako engine while they are in fact in your zip file.

    """

    def __init__(self, input_encoding, output_encoding,
            imports, default_filters, module_directory=None):

        self.input_encoding = input_encoding
        self.output_encoding = output_encoding
        self.imports = imports
        self.default_filters = default_filters
        # implement a cache for the loaded templates
        self.template_cache = dict()
        # implement a cache for the filename lookups
        self.template_filenames_cache = dict()
        self.module_directory = module_directory

        # a mutex to ensure thread safeness during template loading
        self._mutex = threading.Lock()

    def adjust_uri(self, uri, relativeto):
        """Adjust the given uri relative to a filename.

        This method is used by mako for filesystem based reasons.
        In dotted lookup land we don't adjust uri so we just return
        the value we are given without any change.

        """
        if uri.startswith('local:'):
            uri = tg.config['pylons.package'] + '.' + uri[6:]

        if '.' in uri:
            # We are in the DottedTemplateLookup system so dots in
            # names should be treated as a Python path. Since this
            # method is called by template inheritance we must
            # support dotted names also in the inheritance.
            result = tg.config['pylons.app_globals'].\
                dotted_filename_finder.get_dotted_filename(template_name=uri, template_extension='.mak')

            if not uri in self.template_filenames_cache:
                # feed our filename cache if needed.
                self.template_filenames_cache[uri] = result

        else:
            # no dot detected, just return plain name
            result = uri

        return result

    def __check(self, template):
        """private method used to verify if a template has changed
        since the last time it has been put in cache...

        This method being based on the mtime of a real file this should
        never be called on a zipped deployed application.

        This method is a ~copy/paste of the original caching system from
        the Mako lookup loader.

        """
        if template.filename is None:
            return template

        if not os.path.exists(template.filename):
            # remove from cache.
            self.template_cache.pop(template.filename, None)
            raise exceptions.TemplateLookupException(
                    "Cant locate template '%s'" % template.filename)

        elif template.module._modified_time < os.stat(
                template.filename)[stat.ST_MTIME]:

            # cache is too old, remove old template
            # from cache and reload.
            self.template_cache.pop(template.filename, None)
            return self.__load(template.filename)

        else:
            # cache is correct, use it.
            return template

    def __load(self, filename):
        """real loader function. copy paste from the mako template
        loader.

        """
        # make sure the template loading from filesystem is only done
        # one thread at a time to avoid bad clashes...
        self._mutex.acquire()
        try:
            try:
                # try returning from cache one more time in case
                # concurrent thread already loaded
                return self.template_cache[filename]

            except KeyError:
                # not in cache yet... we can continue normally
                pass

            try:
                self.template_cache[filename] = Template(
                    filename=filename,
                    module_directory=self.module_directory,
                    input_encoding=self.input_encoding,
                    output_encoding=self.output_encoding,
                    default_filters=self.default_filters,
                    imports=self.imports,
                    lookup=self)

                return self.template_cache[filename]

            except:
                self.template_cache.pop(filename, None)
                raise

        finally:
            # _always_ release the lock once done to avoid
            # "thread lock" effect
            self._mutex.release()

    def get_template(self, template_name):
        """this is the emulated method that must return a template
        instance based on a given template name
        """

        if not self.template_cache.has_key(template_name):
            # the template string is not yet loaded into the cache.
            # Do so now
            self.__load(template_name)

        if asbool(tg.config.get('templating.mako.reloadfromdisk', 'false')):
            # AUTO RELOADING will be activated only if user has
            # explicitly asked for it in the configuration
            # return the template, but first make sure it's not outdated
            # and if outdated, refresh the cache.
            return self.__check(self.template_cache[template_name])

        else:
            return self.template_cache[template_name]

class GenshiTemplateLoader(TemplateLoader):
    """Genshi template loader that supports
    zipped applications and dotted filenames as well as path names
    """

    def load(self, filename, relative_to=None, cls=None, encoding=None):
        """real loader function. copy paste from the mako template
        loader.
        """
        # TODO: get the template extension from the config!!
        if not filename.endswith('.html'):
            filename = tg.config['pylons.app_globals'
                    ].dotted_filename_finder.get_dotted_filename(
                            template_name=filename,
                            template_extension='.html')

        return TemplateLoader.load(self, filename,
                relative_to=relative_to, cls=cls, encoding=encoding)

<|endoftext|>