<commit_before>"HTML utilities suitable for global use."

import re
import string

from django.utils.safestring import SafeData, mark_safe
from django.utils.encoding import force_unicode
from django.utils.functional import allow_lazy
from django.utils.http import urlquote

# Configuration for urlize() function
LEADING_PUNCTUATION  = ['(', '<', '<']
TRAILING_PUNCTUATION = ['.', ',', ')', '>', '\n', '>']

# list of possible strings used for bullets in bulleted lists
DOTS = ['·', '*', '\xe2\x80\xa2', '•', '•', '•']

unencoded_ampersands_re = re.compile(r'&(?!(\w+|#\d+);)')
word_split_re = re.compile(r'(\s+)')
punctuation_re = re.compile('^(?P<lead>(?:%s)*)(?P<middle>.*?)(?P<trail>(?:%s)*)$' % \
    ('|'.join([re.escape(x) for x in LEADING_PUNCTUATION]),
    '|'.join([re.escape(x) for x in TRAILING_PUNCTUATION])))
simple_email_re = re.compile(r'^\S+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+$')
link_target_attribute_re = re.compile(r'(]*?)target=[^\s>]+')
html_gunk_re = re.compile(r'(?:
|<\/i>|<\/b>|<\/em>|<\/strong>|<\/?smallcaps>|<\/?uppercase>)', re.IGNORECASE)
hard_coded_bullets_re = re.compile(r'((?:
(?:%s).*?[a-zA-Z].*?
\s*)+)' % '|'.join([re.escape(x) for x in DOTS]), re.DOTALL)
trailing_empty_content_re = re.compile(r'(?:
(?: |\s|
)*?
\s*)+\Z')
del x # Temporary variable

def escape(html):
    "Return the given HTML with ampersands, quotes and carets encoded."
    return mark_safe(force_unicode(html).replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"').replace("'", '''))
escape = allow_lazy(escape, unicode)

def conditional_escape(html):
    """
    Similar to escape(), except that it doesn't operate on pre-escaped strings.
    """
    if isinstance(html, SafeData):
        return html
    else:
        return escape(html)

def linebreaks(value, autoescape=False):
    "Converts newlines into
 and
s"
    value = re.sub(r'\r\n|\r|\n', '\n', force_unicode(value)) # normalize newlines
    paras = re.split('\n{2,}', value)
    if autoescape:
        paras = [u'
%s
' % escape(p.strip()).replace('\n', '
') for p in paras]
    else:
        paras = [u'
%s
' % p.strip().replace('\n', '
') for p in paras]
    return u'\n\n'.join(paras)
linebreaks = allow_lazy(linebreaks, unicode)

def strip_tags(value):
    "Return the given HTML with all tags stripped."
    return re.sub(r'<[^>]*?>', '', force_unicode(value))
strip_tags = allow_lazy(strip_tags)

def strip_spaces_between_tags(value):
    "Return the given HTML with spaces between tags removed."
    return re.sub(r'>\s+<', '><', force_unicode(value))
strip_spaces_between_tags = allow_lazy(strip_spaces_between_tags, unicode)

def strip_entities(value):
    "Return the given HTML with all entities (&something;) stripped."
    return re.sub(r'&(?:\w+|#\d+);', '', force_unicode(value))
strip_entities = allow_lazy(strip_entities, unicode)

def fix_ampersands(value):
    "Return the given HTML with all unencoded ampersands encoded correctly."
    return unencoded_ampersands_re.sub('&', force_unicode(value))
fix_ampersands = allow_lazy(fix_ampersands, unicode)

def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False):
    """
    Convert any URLs in text into clickable links.

    Works on http://, https://, and www. links.  Links can have trailing
    punctuation (periods, commas, close-parens) and leading punctuation
    (opening parens) and it'll still do the right thing.

    If trim_url_limit is not None, the URLs in link text longer than this limit
    will truncated to trim_url_limit-3 characters and appended with an elipsis.

    If nofollow is True, the URLs in link text will get a rel="nofollow"
    attribute.
    """
    if autoescape:
        trim_url = lambda x, limit=trim_url_limit: conditional_escape(limit is not None and (len(x) > limit and ('%s...' % x[:max(0, limit - 3)])) or x)
    else:
        trim_url = lambda x, limit=trim_url_limit: limit is not None and (len(x) > limit and ('%s...' % x[:max(0, limit - 3)])) or x
    safe_input = isinstance(text, SafeData)
    words = word_split_re.split(force_unicode(text))
    nofollow_attr = nofollow and ' rel="nofollow"' or ''
    for i, word in enumerate(words):
        match = punctuation_re.match(word)
        if match:
            lead, middle, trail = match.groups()
            if safe_input:
                middle = mark_safe(middle)
            if middle.startswith('www.') or ('@' not in middle and not middle.startswith('http://') and \
                    len(middle) > 0 and middle[0] in string.letters + string.digits and \
                    (middle.endswith('.org') or middle.endswith('.net') or middle.endswith('.com'))):
                middle = '%s' % (
                        urlquote(middle, safe='/&=:;#?+'),  nofollow_attr,
                        trim_url(middle))
            if middle.startswith('http://') or middle.startswith('https://'):
                middle = '%s' % (
                        urlquote(middle, safe='/&=:;#?+'), nofollow_attr,
                        trim_url(middle))
            if '@' in middle and not middle.startswith('www.') and \
                    not ':' in middle and simple_email_re.match(middle):
                middle = '%s' % (middle, middle)
            if lead + middle + trail != word:
                words[i] = lead + middle + trail
            elif autoescape and not safe_input:
                words[i] = escape(word)
        elif safe_input:
            words[i] = mark_safe(word)
        elif autoescape:
            words[i] = escape(word)
    return u''.join(words)
urlize = allow_lazy(urlize, unicode)

def clean_html(text):
    """
    Clean the given HTML.  Specifically, do the following:
        * Convert  and  to  and .
        * Encode all ampersands correctly.
        * Remove all "target" attributes from  tags.
        * Remove extraneous HTML, such as presentational tags that open and
          immediately close and
.
        * Convert hard-coded bullets into HTML unordered lists.
        * Remove stuff like "

", but only if it's at the
          bottom of the text.
    """
    from django.utils.text import normalize_newlines
    text = normalize_newlines(force_unicode(text))
    text = re.sub(r'<(/?)\s*b\s*>', '<\\1strong>', text)
    text = re.sub(r'<(/?)\s*i\s*>', '<\\1em>', text)
    text = fix_ampersands(text)
    # Remove all target="" attributes from  tags.
    text = link_target_attribute_re.sub('\\1', text)
    # Trim stupid HTML such as
.
    text = html_gunk_re.sub('', text)
    # Convert hard-coded bullets into HTML unordered lists.
    def replace_p_tags(match):
        s = match.group().replace('
', '
')
        for d in DOTS:
            s = s.replace('
%s' % d, '
')
        return u'
\n%s\n
' % s
    text = hard_coded_bullets_re.sub(replace_p_tags, text)
    # Remove stuff like "

", but only if it's at the bottom
    # of the text.
    text = trailing_empty_content_re.sub('', text)
    return text
clean_html = allow_lazy(clean_html, unicode)
<commit_msg>Fixed #5657 -- Use string.ascii_letters instead of ascii.letters in the urlize filter to ensure consistent (and correct) results no matter what the server's locale setting might be. Thanks, Andrew Stoneman.<commit_after>"HTML utilities suitable for global use."

import re
import string

from django.utils.safestring import SafeData, mark_safe
from django.utils.encoding import force_unicode
from django.utils.functional import allow_lazy
from django.utils.http import urlquote

# Configuration for urlize() function
LEADING_PUNCTUATION  = ['(', '<', '<']
TRAILING_PUNCTUATION = ['.', ',', ')', '>', '\n', '>']

# list of possible strings used for bullets in bulleted lists
DOTS = ['·', '*', '\xe2\x80\xa2', '•', '•', '•']

unencoded_ampersands_re = re.compile(r'&(?!(\w+|#\d+);)')
word_split_re = re.compile(r'(\s+)')
punctuation_re = re.compile('^(?P<lead>(?:%s)*)(?P<middle>.*?)(?P<trail>(?:%s)*)$' % \
    ('|'.join([re.escape(x) for x in LEADING_PUNCTUATION]),
    '|'.join([re.escape(x) for x in TRAILING_PUNCTUATION])))
simple_email_re = re.compile(r'^\S+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+$')
link_target_attribute_re = re.compile(r'(]*?)target=[^\s>]+')
html_gunk_re = re.compile(r'(?:
|<\/i>|<\/b>|<\/em>|<\/strong>|<\/?smallcaps>|<\/?uppercase>)', re.IGNORECASE)
hard_coded_bullets_re = re.compile(r'((?:
(?:%s).*?[a-zA-Z].*?
\s*)+)' % '|'.join([re.escape(x) for x in DOTS]), re.DOTALL)
trailing_empty_content_re = re.compile(r'(?:
(?: |\s|
)*?
\s*)+\Z')
del x # Temporary variable

def escape(html):
    "Return the given HTML with ampersands, quotes and carets encoded."
    return mark_safe(force_unicode(html).replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"').replace("'", '''))
escape = allow_lazy(escape, unicode)

def conditional_escape(html):
    """
    Similar to escape(), except that it doesn't operate on pre-escaped strings.
    """
    if isinstance(html, SafeData):
        return html
    else:
        return escape(html)

def linebreaks(value, autoescape=False):
    "Converts newlines into
 and
s"
    value = re.sub(r'\r\n|\r|\n', '\n', force_unicode(value)) # normalize newlines
    paras = re.split('\n{2,}', value)
    if autoescape:
        paras = [u'
%s
' % escape(p.strip()).replace('\n', '
') for p in paras]
    else:
        paras = [u'
%s
' % p.strip().replace('\n', '
') for p in paras]
    return u'\n\n'.join(paras)
linebreaks = allow_lazy(linebreaks, unicode)

def strip_tags(value):
    "Return the given HTML with all tags stripped."
    return re.sub(r'<[^>]*?>', '', force_unicode(value))
strip_tags = allow_lazy(strip_tags)

def strip_spaces_between_tags(value):
    "Return the given HTML with spaces between tags removed."
    return re.sub(r'>\s+<', '><', force_unicode(value))
strip_spaces_between_tags = allow_lazy(strip_spaces_between_tags, unicode)

def strip_entities(value):
    "Return the given HTML with all entities (&something;) stripped."
    return re.sub(r'&(?:\w+|#\d+);', '', force_unicode(value))
strip_entities = allow_lazy(strip_entities, unicode)

def fix_ampersands(value):
    "Return the given HTML with all unencoded ampersands encoded correctly."
    return unencoded_ampersands_re.sub('&', force_unicode(value))
fix_ampersands = allow_lazy(fix_ampersands, unicode)

def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False):
    """
    Convert any URLs in text into clickable links.

    Works on http://, https://, and www. links.  Links can have trailing
    punctuation (periods, commas, close-parens) and leading punctuation
    (opening parens) and it'll still do the right thing.

    If trim_url_limit is not None, the URLs in link text longer than this limit
    will truncated to trim_url_limit-3 characters and appended with an elipsis.

    If nofollow is True, the URLs in link text will get a rel="nofollow"
    attribute.
    """
    if autoescape:
        trim_url = lambda x, limit=trim_url_limit: conditional_escape(limit is not None and (len(x) > limit and ('%s...' % x[:max(0, limit - 3)])) or x)
    else:
        trim_url = lambda x, limit=trim_url_limit: limit is not None and (len(x) > limit and ('%s...' % x[:max(0, limit - 3)])) or x
    safe_input = isinstance(text, SafeData)
    words = word_split_re.split(force_unicode(text))
    nofollow_attr = nofollow and ' rel="nofollow"' or ''
    for i, word in enumerate(words):
        match = punctuation_re.match(word)
        if match:
            lead, middle, trail = match.groups()
            if safe_input:
                middle = mark_safe(middle)
            if middle.startswith('www.') or ('@' not in middle and not middle.startswith('http://') and \
                    len(middle) > 0 and middle[0] in string.ascii_letters + string.digits and \
                    (middle.endswith('.org') or middle.endswith('.net') or middle.endswith('.com'))):
                middle = '%s' % (
                        urlquote(middle, safe='/&=:;#?+'),  nofollow_attr,
                        trim_url(middle))
            if middle.startswith('http://') or middle.startswith('https://'):
                middle = '%s' % (
                        urlquote(middle, safe='/&=:;#?+'), nofollow_attr,
                        trim_url(middle))
            if '@' in middle and not middle.startswith('www.') and \
                    not ':' in middle and simple_email_re.match(middle):
                middle = '%s' % (middle, middle)
            if lead + middle + trail != word:
                words[i] = lead + middle + trail
            elif autoescape and not safe_input:
                words[i] = escape(word)
        elif safe_input:
            words[i] = mark_safe(word)
        elif autoescape:
            words[i] = escape(word)
    return u''.join(words)
urlize = allow_lazy(urlize, unicode)

def clean_html(text):
    """
    Clean the given HTML.  Specifically, do the following:
        * Convert  and  to  and .
        * Encode all ampersands correctly.
        * Remove all "target" attributes from  tags.
        * Remove extraneous HTML, such as presentational tags that open and
          immediately close and
.
        * Convert hard-coded bullets into HTML unordered lists.
        * Remove stuff like "

", but only if it's at the
          bottom of the text.
    """
    from django.utils.text import normalize_newlines
    text = normalize_newlines(force_unicode(text))
    text = re.sub(r'<(/?)\s*b\s*>', '<\\1strong>', text)
    text = re.sub(r'<(/?)\s*i\s*>', '<\\1em>', text)
    text = fix_ampersands(text)
    # Remove all target="" attributes from  tags.
    text = link_target_attribute_re.sub('\\1', text)
    # Trim stupid HTML such as
.
    text = html_gunk_re.sub('', text)
    # Convert hard-coded bullets into HTML unordered lists.
    def replace_p_tags(match):
        s = match.group().replace('
', '
')
        for d in DOTS:
            s = s.replace('
%s' % d, '
')
        return u'
\n%s\n
' % s
    text = hard_coded_bullets_re.sub(replace_p_tags, text)
    # Remove stuff like "

", but only if it's at the bottom
    # of the text.
    text = trailing_empty_content_re.sub('', text)
    return text
clean_html = allow_lazy(clean_html, unicode)
<|endoftext|>


<commit_before>""" Utilities for modifying strings"""

import os
import warnings
import re
import functools

from pylearn2.datasets.exc import NoDataPathError

def preprocess(string):
    """
    Preprocesses a string, by replacing ${VARNAME} with
    os.environ['VARNAME']

    Parameters
    ----------
    string: the str object to preprocess

    Returns
    -------
    the preprocessed string
    """

    split = string.split('${')

    rval = [split[0]]

    for candidate in split[1:]:
        subsplit = candidate.split('}')

        if len(subsplit) < 2:
            raise ValueError('Open ${ not followed by } before ' \
                    + 'end of string or next ${ in "' \
                    + string + '"')

        varname = subsplit[0]

        if varname == 'PYLEARN2_TRAIN_FILE_NAME':
            warnings.warn("PYLEARN2_TRAIN_FILE_NAME is deprecated and may be "
                    "removed from the library on or after Oct 22, 2013. Switch"
                    " to PYLEARN2_TRAIN_FILE_FULL_STEM")

        try:
            val = os.environ[varname]
        except KeyError:
            if varname == 'PYLEARN2_DATA_PATH':
                raise NoDataPathError()
            if varname == 'PYLEARN2_VIEWER_COMMAND':
                raise EnvironmentVariableError(environment_variable_essay)

            raise ValueError('Unrecognized environment variable "' + varname
                    + '". Did you mean ' + match(varname, os.environ.keys())
                    + '?')

        rval.append(val)

        rval.append('}'.join(subsplit[1:]))

    rval = ''.join(rval)

    return rval

class EnvironmentVariableError(Exception):
    """ An exception raised when a required environment variable is not defined """

    def __init__(self, *args):
        super(EnvironmentVariableError,self).__init__(*args)

def find_number(s):
    """ s is a string
        returns None if there are no numbers in the string
        otherwise returns the range of characters occupied by the first
        number in the string """

    r = re.search('-?\d+[.e]?\d*',s)
    if r is not None:
        return r.span(0)
    return None

def tokenize_by_number(s):
    """ splits a string into a list of tokens
        each is either a string containing no numbers
        or a float """

    r = find_number(s)

    if r == None:
        return [ s ]
    else:
        tokens = []
        if r[0] > 0:
            tokens.append(s[0:r[0]])
        tokens.append( float(s[r[0]:r[1]]) )
        if r[1] < len(s):
            tokens.extend(tokenize_by_number(s[r[1]:]))
        return tokens
    assert False #line should be unreached

def number_aware_alphabetical_cmp(str1, str2):
    """ cmp function for sorting a list of strings by alphabetical order, but with
        numbers sorted numerically.

        i.e., foo1, foo2, foo10, foo11
        instead of foo1, foo10
    """

    def flatten_tokens(tokens):
        l = []
        for token in tokens:
            if isinstance(token, str):
                for char in token:
                    l.append(char)
            else:
                assert isinstance(token, float)
                l.append(token)
        return l

    seq1 = flatten_tokens(tokenize_by_number(str1))
    seq2 = flatten_tokens(tokenize_by_number(str2))

    l = min(len(seq1),len(seq2))

    i = 0

    while i < l:
        if seq1[i] < seq2[i]:
            return -1
        elif seq1[i] > seq2[i]:
            return 1
        i += 1

    if len(seq1) < len(seq2):
        return -1
    elif len(seq1) > len(seq2):
        return 1

    return 0

#key for sorting strings alphabetically with numbers
number_aware_alphabetical_key = functools.cmp_to_key(number_aware_alphabetical_cmp)

def match(wrong, candidates):
    """
        wrong: a mispelling
        candidates: a set of correct words

        returns a guess of which candidate is the right one

        This should be used with a small number of candidates and a high potential
        edit distance.
        ie, use it to correct a wrong filename in a directory, wrong class name
        in a module, etc. Don't use it to correct small typos of freeform natural
        language words.
    """

    assert len(candidates) > 0

    # Current implementation tries all candidates and outputs the one
    # with the min score
    # Could try to do something smarter

    def score(w1,w2):
        # Current implementation returns negative dot product of
        # the two words mapped into a feature space by mapping phi
        # w -> [ phi(w1), .1 phi(first letter of w), .1 phi(last letter of w) ]
        # Could try to do something smarter

        w1 = w1.lower()
        w2 = w2.lower()

        def phi(w):
            # Current feature mapping is to the vector of counts of
            # all letters and two-letter sequences
            # Could try to do something smarter
            rval = {}

            for i in xrange(len(w)):
                l = w[i]
                rval[l] = rval.get(l,0.) + 1.
                if i < len(w)-1:
                    b = w[i:i+2]
                    rval[b] = rval.get(b,0.) + 1.

            return rval

        d1 = phi(w1)
        d2 = phi(w2)

        def mul(d1, d2):
            rval = 0

            for key in set(d1).union(d2):
                rval += d1.get(key,0) * d2.get(key,0)

            return rval

        tot_score = mul(phi(w1),phi(w2)) / float(len(w1)*len(w2)) + \
            0.1 * mul(phi(w1[0:1]), phi(w2[0:1])) + \
            0.1 * mul(phi(w1[-1:]), phi(w2[-1:]))

        return  tot_score

    scored_candidates = [ (-score(wrong, candidate), candidate)
            for candidate in candidates ]

    scored_candidates.sort()

    return scored_candidates[0][1]

def censor_non_alphanum(s):
    """
    Returns s with all non-alphanumeric characters replaced with *
    """

    def censor(ch):
        if (ch >= 'A' and ch <= 'z') or (ch >= '0' and ch <= '9'):
            return ch
        return '*'

    return ''.join([censor(ch) for ch in s])

environment_variable_essay = """
PYLEARN2_VIEWER_COMMAND not defined. PLEASE READ THE FOLLOWING MESSAGE CAREFULLY
TO SET UP THIS ENVIRONMENT VARIABLE:

pylearn2 uses an external program to display images. Because different systems have different
image programs available, pylearn2 requires the user to specify what image viewer program to
use.

You need to choose an image viewer program that pylearn2 should use. Then tell pylearn2 to use
that image viewer program by defining your PYLEARN2_VIEWER_COMMAND environment variable.

You need to choose PYLEARN_VIEWER_COMMAND such that running

${PYLEARN2_VIEWER_COMMAND} image.png

in a command prompt on your machine will do the following:
    -open an image viewer in a new process.
    -not return until you have closed the image.

Acceptable commands include:
    gwenview
    eog --new-instance

This is assuming that you have gwenview or a version of eog that supports --new-instance
installed on your machine. If you don't, install one of those, or figure out a command
that has the above properties that is available from your setup.

On most linux setups, you can define your environment variable by adding this line to your
~/.bashrc file:

export PYLEARN2_VIEWER_COMMAND="eog --new-instance"

*** YOU MUST INCLUDE THE WORD "export". DO NOT JUST ASSIGN TO THE ENVIRONMENT VARIABLE ***
If you do not include the word "export", the environment variable will be set in your
bash shell, but will not be visible to processes that you launch from it, like the python
interpreter.

Don't forget that changes from your .bashrc file won't apply until you run

source ~/.bashrc

or open a new terminal window. If you're seeing this from an ipython notebook
you'll need to restart the ipython notebook, or maybe modify os.environ from
an ipython cell.
"""
<commit_msg>moved EnvironmentVariableError to utils.exc<commit_after>""" Utilities for modifying strings"""

import os
import warnings
import re
import functools

from pylearn2.datasets.exc import NoDataPathError
from pylearn2.utils.exc import EnvironmentVariableError

def preprocess(string):
    """
    Preprocesses a string, by replacing ${VARNAME} with
    os.environ['VARNAME']

    Parameters
    ----------
    string: the str object to preprocess

    Returns
    -------
    the preprocessed string
    """

    split = string.split('${')

    rval = [split[0]]

    for candidate in split[1:]:
        subsplit = candidate.split('}')

        if len(subsplit) < 2:
            raise ValueError('Open ${ not followed by } before ' \
                    + 'end of string or next ${ in "' \
                    + string + '"')

        varname = subsplit[0]

        if varname == 'PYLEARN2_TRAIN_FILE_NAME':
            warnings.warn("PYLEARN2_TRAIN_FILE_NAME is deprecated and may be "
                    "removed from the library on or after Oct 22, 2013. Switch"
                    " to PYLEARN2_TRAIN_FILE_FULL_STEM")

        try:
            val = os.environ[varname]
        except KeyError:
            if varname == 'PYLEARN2_DATA_PATH':
                raise NoDataPathError()
            if varname == 'PYLEARN2_VIEWER_COMMAND':
                raise EnvironmentVariableError(environment_variable_essay)

            raise ValueError('Unrecognized environment variable "' + varname
                    + '". Did you mean ' + match(varname, os.environ.keys())
                    + '?')

        rval.append(val)

        rval.append('}'.join(subsplit[1:]))

    rval = ''.join(rval)

    return rval

def find_number(s):
    """ s is a string
        returns None if there are no numbers in the string
        otherwise returns the range of characters occupied by the first
        number in the string """

    r = re.search('-?\d+[.e]?\d*',s)
    if r is not None:
        return r.span(0)
    return None

def tokenize_by_number(s):
    """ splits a string into a list of tokens
        each is either a string containing no numbers
        or a float """

    r = find_number(s)

    if r == None:
        return [ s ]
    else:
        tokens = []
        if r[0] > 0:
            tokens.append(s[0:r[0]])
        tokens.append( float(s[r[0]:r[1]]) )
        if r[1] < len(s):
            tokens.extend(tokenize_by_number(s[r[1]:]))
        return tokens
    assert False #line should be unreached

def number_aware_alphabetical_cmp(str1, str2):
    """ cmp function for sorting a list of strings by alphabetical order, but with
        numbers sorted numerically.

        i.e., foo1, foo2, foo10, foo11
        instead of foo1, foo10
    """

    def flatten_tokens(tokens):
        l = []
        for token in tokens:
            if isinstance(token, str):
                for char in token:
                    l.append(char)
            else:
                assert isinstance(token, float)
                l.append(token)
        return l

    seq1 = flatten_tokens(tokenize_by_number(str1))
    seq2 = flatten_tokens(tokenize_by_number(str2))

    l = min(len(seq1),len(seq2))

    i = 0

    while i < l:
        if seq1[i] < seq2[i]:
            return -1
        elif seq1[i] > seq2[i]:
            return 1
        i += 1

    if len(seq1) < len(seq2):
        return -1
    elif len(seq1) > len(seq2):
        return 1

    return 0

#key for sorting strings alphabetically with numbers
number_aware_alphabetical_key = functools.cmp_to_key(number_aware_alphabetical_cmp)

def match(wrong, candidates):
    """
        wrong: a mispelling
        candidates: a set of correct words

        returns a guess of which candidate is the right one

        This should be used with a small number of candidates and a high potential
        edit distance.
        ie, use it to correct a wrong filename in a directory, wrong class name
        in a module, etc. Don't use it to correct small typos of freeform natural
        language words.
    """

    assert len(candidates) > 0

    # Current implementation tries all candidates and outputs the one
    # with the min score
    # Could try to do something smarter

    def score(w1,w2):
        # Current implementation returns negative dot product of
        # the two words mapped into a feature space by mapping phi
        # w -> [ phi(w1), .1 phi(first letter of w), .1 phi(last letter of w) ]
        # Could try to do something smarter

        w1 = w1.lower()
        w2 = w2.lower()

        def phi(w):
            # Current feature mapping is to the vector of counts of
            # all letters and two-letter sequences
            # Could try to do something smarter
            rval = {}

            for i in xrange(len(w)):
                l = w[i]
                rval[l] = rval.get(l,0.) + 1.
                if i < len(w)-1:
                    b = w[i:i+2]
                    rval[b] = rval.get(b,0.) + 1.

            return rval

        d1 = phi(w1)
        d2 = phi(w2)

        def mul(d1, d2):
            rval = 0

            for key in set(d1).union(d2):
                rval += d1.get(key,0) * d2.get(key,0)

            return rval

        tot_score = mul(phi(w1),phi(w2)) / float(len(w1)*len(w2)) + \
            0.1 * mul(phi(w1[0:1]), phi(w2[0:1])) + \
            0.1 * mul(phi(w1[-1:]), phi(w2[-1:]))

        return  tot_score

    scored_candidates = [ (-score(wrong, candidate), candidate)
            for candidate in candidates ]

    scored_candidates.sort()

    return scored_candidates[0][1]

def censor_non_alphanum(s):
    """
    Returns s with all non-alphanumeric characters replaced with *
    """

    def censor(ch):
        if (ch >= 'A' and ch <= 'z') or (ch >= '0' and ch <= '9'):
            return ch
        return '*'

    return ''.join([censor(ch) for ch in s])

environment_variable_essay = """
PYLEARN2_VIEWER_COMMAND not defined. PLEASE READ THE FOLLOWING MESSAGE CAREFULLY
TO SET UP THIS ENVIRONMENT VARIABLE:

pylearn2 uses an external program to display images. Because different systems have different
image programs available, pylearn2 requires the user to specify what image viewer program to
use.

You need to choose an image viewer program that pylearn2 should use. Then tell pylearn2 to use
that image viewer program by defining your PYLEARN2_VIEWER_COMMAND environment variable.

You need to choose PYLEARN_VIEWER_COMMAND such that running

${PYLEARN2_VIEWER_COMMAND} image.png

in a command prompt on your machine will do the following:
    -open an image viewer in a new process.
    -not return until you have closed the image.

Acceptable commands include:
    gwenview
    eog --new-instance

This is assuming that you have gwenview or a version of eog that supports --new-instance
installed on your machine. If you don't, install one of those, or figure out a command
that has the above properties that is available from your setup.

On most linux setups, you can define your environment variable by adding this line to your
~/.bashrc file:

export PYLEARN2_VIEWER_COMMAND="eog --new-instance"

*** YOU MUST INCLUDE THE WORD "export". DO NOT JUST ASSIGN TO THE ENVIRONMENT VARIABLE ***
If you do not include the word "export", the environment variable will be set in your
bash shell, but will not be visible to processes that you launch from it, like the python
interpreter.

Don't forget that changes from your .bashrc file won't apply until you run

source ~/.bashrc

or open a new terminal window. If you're seeing this from an ipython notebook
you'll need to restart the ipython notebook, or maybe modify os.environ from
an ipython cell.
"""
<|endoftext|>


<commit_before>import unittest

import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import parameterize

@parameterize(
    {'axes': [1, 2], 'offsets': 0, 'sliced_shape': (4, 2, 1)},
    {'axes': [1, 2], 'offsets': [0, 1, 1], 'sliced_shape': (4, 2, 1)},
    {'axes': 1, 'offsets': 1, 'sliced_shape': (4, 2, 2)},
    {'axes': 1, 'offsets': [0, 1, 1], 'sliced_shape': (4, 2, 2)},
    {'axes': [], 'offsets': 0, 'new_axes': 0, 'sliced_shape': (1, 4, 3, 2)},
    {'axes': [], 'offsets': 0, 'new_axes': 2, 'sliced_shape': (4, 3, 1, 2)},
    {'axes': [], 'offsets': 0, 'new_axes': 3, 'sliced_shape': (4, 3, 2, 1)},
    {'slices': (1, -1, 0), 'sliced_shape': ()},
    {'slices': (1, -1), 'sliced_shape': (2,)},
    {'slices': (1, Ellipsis, -1), 'sliced_shape': (3,)},
    {'slices': (1, None, Ellipsis, None, -1), 'sliced_shape': (1, 3, 1)},
)
class TestGetItem(unittest.TestCase):

    def setUp(self):
        self.x_data = numpy.random.uniform(-1, 1, (4, 3, 2))
        self.shape = (4, 2, 1)
        self.gy_data = numpy.random.uniform(-1, 1, self.sliced_shape)

        if not hasattr(self, 'slices'):
            # Convert axes, offsets and shape to slices
            if isinstance(self.offsets, int):
                self.offsets = tuple([self.offsets] * len(self.shape))
            if isinstance(self.axes, int):
                self.axes = tuple([self.axes])

            self.slices = [slice(None)] * len(self.shape)
            for axis in self.axes:
                self.slices[axis] = slice(self.offsets[axis],
                                          self.offsets[axis]+self.shape[axis])

            if hasattr(self, 'new_axes'):
                self.slices.insert(self.new_axes, None)

            self.slices = tuple(self.slices)

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = functions.get_item(x, self.slices)
        self.assertEqual(y.data.dtype, numpy.float)
        numpy.testing.assert_equal(cuda.to_cpu(x_data)[self.slices],
                                   cuda.to_cpu(y.data))

    def test_forward_cpu(self):
        self.check_forward(self.x_data)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x_data))

    def check_backward(self, x_data, y_grad):
        gradient_check.check_backward(functions.GetItem(self.slices),
                                      (x_data,), y_grad)

    def test_backward_cpu(self):
        self.check_backward(self.x_data, self.gy_data)

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x_data),
                            cuda.to_gpu(self.gy_data))

testing.run_module(__name__, __file__)
<commit_msg>Make a test for invalid arguments<commit_after>import unittest

import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import parameterize
from chainer.utils import type_check

@parameterize(
    {'axes': [1, 2], 'offsets': 0, 'sliced_shape': (4, 2, 1)},
    {'axes': [1, 2], 'offsets': [0, 1, 1], 'sliced_shape': (4, 2, 1)},
    {'axes': 1, 'offsets': 1, 'sliced_shape': (4, 2, 2)},
    {'axes': 1, 'offsets': [0, 1, 1], 'sliced_shape': (4, 2, 2)},
    {'axes': [], 'offsets': 0, 'new_axes': 0, 'sliced_shape': (1, 4, 3, 2)},
    {'axes': [], 'offsets': 0, 'new_axes': 2, 'sliced_shape': (4, 3, 1, 2)},
    {'axes': [], 'offsets': 0, 'new_axes': 3, 'sliced_shape': (4, 3, 2, 1)},
    {'slices': (1, -1, 0), 'sliced_shape': ()},
    {'slices': (1, -1), 'sliced_shape': (2,)},
    {'slices': (1, Ellipsis, -1), 'sliced_shape': (3,)},
    {'slices': (1, None, Ellipsis, None, -1), 'sliced_shape': (1, 3, 1)},
)
class TestGetItem(unittest.TestCase):

    def setUp(self):
        self.x_data = numpy.random.uniform(-1, 1, (4, 3, 2))
        self.shape = (4, 2, 1)
        self.gy_data = numpy.random.uniform(-1, 1, self.sliced_shape)

        if not hasattr(self, 'slices'):
            # Convert axes, offsets and shape to slices
            if isinstance(self.offsets, int):
                self.offsets = tuple([self.offsets] * len(self.shape))
            if isinstance(self.axes, int):
                self.axes = tuple([self.axes])

            self.slices = [slice(None)] * len(self.shape)
            for axis in self.axes:
                self.slices[axis] = slice(self.offsets[axis],
                                          self.offsets[axis]+self.shape[axis])

            if hasattr(self, 'new_axes'):
                self.slices.insert(self.new_axes, None)

            self.slices = tuple(self.slices)

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = functions.get_item(x, self.slices)
        self.assertEqual(y.data.dtype, numpy.float)
        numpy.testing.assert_equal(cuda.to_cpu(x_data)[self.slices],
                                   cuda.to_cpu(y.data))

    def test_forward_cpu(self):
        self.check_forward(self.x_data)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x_data))

    def check_backward(self, x_data, y_grad):
        gradient_check.check_backward(functions.GetItem(self.slices),
                                      (x_data,), y_grad)

    def test_backward_cpu(self):
        self.check_backward(self.x_data, self.gy_data)

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x_data),
                            cuda.to_gpu(self.gy_data))

class TestInvalidGetItem(unittest.TestCase):

    def setUp(self):
        self.x_data = numpy.random.uniform(-1, 1, (4, 3, 2))

    def test_advanced_indexing(self):
        with self.assertRaises(ValueError):
            functions.get_item(self.x_data, ([0, 0, 0],))

    def test_multiple_ellipsis(self):
        with self.assertRaises(ValueError):
            functions.get_item(self.x_data, (Ellipsis, Ellipsis))

    def test_too_many_indices(self):
        with self.assertRaises(type_check.InvalidType):
            functions.get_item(self.x_data, (0, 0, 0, 0))

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>__author__ = "Warren"

from unittest import TestCase

import pandas as pd
from zipline.finance import trading
from zipline.utils.factory import create_simulation_parameters
from zipline.test_algorithms import TestAlgorithm
from zipline.finance.commission import PerShare

from gg.powerline.utils.data.data_generator import DataGeneratorEex
from gg.powerline.exchanges.eex_exchange import EexExchange

class TestEexAlgo(TestCase):
    """
    Tests the change in pnl and position for a simple EEX weekly algo.
    """
    @classmethod
    def setUpClass(cls):
        start = pd.Timestamp('2014-05-18', tz='Europe/Berlin').tz_convert(
            'UTC')
        end = pd.Timestamp('2015-05-22', tz='Europe/Berlin').tz_convert('UTC')
        exchange = EexExchange(start=start, end=end)

        env = exchange.env
        env.write_data(futures_data=exchange.asset_metadata)

        ident = '2013-05-20_F1B4'
        sid = \
            env.asset_finder.lookup_future_symbol(ident).sid

        cls.data, cls.pnl = DataGeneratorEex(identifier=ident,
                                             env=env).create_data()

        sim_params = create_simulation_parameters(start=cls.data.start,
                                                  end=cls.data.end)

        cls.algo = TestAlgorithm(sid=sid, amount=1, order_count=1,
                                 instant_fill=False,
                                 env=env,
                                 sim_params=sim_params,
                                 commission=PerShare(0))

        cls.results = cls.algo.run(cls.data)

    def test_algo_pnl(self):
        for dt, pnl in self.pnl.iterrows():
            self.assertEqual(self.results.pnl[dt], pnl[0])

    def test_algo_positions(self):
        expected_positions = pd.DataFrame([0, 1, 1, 0], index=self.pnl.index)
        for dt, amount in expected_positions.iterrows():
            if self.results.positions[dt]:
                actual_position = self.results.positions[dt][0]['amount']
            else:
                actual_position = 0

            self.assertEqual(actual_position, amount[0])

    @classmethod
    def tearDownClass(cls):
        cls.algo = None
        trading.environment = None
<commit_msg>make test pass<commit_after>__author__ = "Warren"

from unittest import TestCase

import pandas as pd
from zipline.finance import trading
from zipline.utils.factory import create_simulation_parameters
from zipline.test_algorithms import TestAlgorithm
from zipline.finance.commission import PerShare

from gg.powerline.utils.data.data_generator import DataGeneratorEex
from gg.powerline.exchanges.eex_exchange import EexExchange

class TestEexAlgo(TestCase):
    """
    Tests the change in pnl and position for a simple EEX weekly algo.
    """
    @classmethod
    def setUpClass(cls):
        start = pd.Timestamp('2014-05-18', tz='Europe/Berlin').tz_convert(
            'UTC')
        end = pd.Timestamp('2015-05-22', tz='Europe/Berlin').tz_convert('UTC')
        exchange = EexExchange(start=start, end=end)

        env = exchange.env
        env.write_data(futures_data=exchange.asset_metadata)

        ident = '2013-05-20_F1B4'
        sid = \
            env.asset_finder.lookup_future_symbol(ident).sid

        cls.data, cls.pnl = DataGeneratorEex(identifier=ident,
                                             env=env).create_data()

        sim_params = create_simulation_parameters(start=cls.data.start,
                                                  end=cls.data.end)

        cls.algo = TestAlgorithm(sid=sid, amount=1, order_count=1,
                                 instant_fill=True,
                                 env=env,
                                 sim_params=sim_params,
                                 commission=PerShare(0))

        cls.results = cls.algo.run(cls.data)

    def test_algo_pnl(self):
        for dt, pnl in self.pnl.iterrows():
            self.assertEqual(self.results.pnl[dt], pnl[0])

    def test_algo_positions(self):
        expected_positions = pd.DataFrame([0, 1, 1, 0], index=self.pnl.index)
        for dt, amount in expected_positions.iterrows():
            if self.results.positions[dt]:
                actual_position = self.results.positions[dt][0]['amount']
            else:
                actual_position = 0

            self.assertEqual(actual_position, amount[0])

    @classmethod
    def tearDownClass(cls):
        cls.algo = None
        trading.environment = None
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import os
import vtk
import numpy

from siconos.io.mechanics_io import Hdf5

## the best way to dump all data
#  $ h5dump toto.hdf5 > toto.txt

import h5py

import getopt

def usage():
    """
    {0} <hdf5 file>
    """.format(sys.argv[0])
    print '{0}: Usage'.format(sys.argv[0])
    print """
    {0} [--help] [--output_frequency=n] [--output_filename=]  <hdf5 file>
    """

try:
    opts, args = getopt.gnu_getopt(sys.argv[1:], '',
                                   ['output_frequency=',
                                    'output_filename=',
                                    'cf-scale='])
except getopt.GetoptError, err:
        sys.stderr.write('{0}\n'.format(str(err)))
        usage()
        exit(2)

min_time = None
max_time = None
scale_factor = 1
output_frequency=10
output_filename=None
for o, a in opts:

    if o == '--help':
        usage()
        exit(0)

    elif o == '--output_frequency':
        output_frequency = float(a)

    elif o == '--output_filename':
        out_filename=a

    elif o == '--cf-scale':
        scale_factor = float(a)

if len(args) > 0:
    io_filename = args[0]
    if output_filename == None:
        out_filename=''.join(io_filename.rsplit('.')[:-1])+'-filtered.hdf5'
else:
    usage()
    exit(1)

with Hdf5(io_filename=io_filename, mode='r') as io:
    with Hdf5(io_filename=out_filename, mode='w') as out:

        hdf1 = io._out
        hdf2 = out._out

        # copy /data/input
        hdf2.__delitem__('data/input')
        h5py.h5o.copy(hdf1.id, "data/input", hdf2.id, "data/input")
        # copy /data/nslaws
        hdf2.__delitem__('data/nslaws')
        h5py.h5o.copy(hdf1.id, "data/nslaws", hdf2.id, "data/nslaws")
        # copy /data/ref
        hdf2.__delitem__('data/ref')
        h5py.h5o.copy(hdf1.id, "data/ref", hdf2.id, "data/ref")

        print('***************************************************** ')
        print('************ Parsing simulation data ****************')
        print('***************************************************** ')

        def load():

            ispos_data = io.static_data()
            idpos_data = io.dynamic_data()

            icf_data = io.contact_forces_data()[:]

            isolv_data = io.solver_data()

            return ispos_data, idpos_data, icf_data, isolv_data

        spos_data, dpos_data, cf_data, solv_data = load()

        #print('io._data',io._data)
        #print('static position data : spos_data',spos_data)
        #print('spos_data.value',spos_data.value)
        #print('dynamic position data : dpos_data',dpos_data)
        print('dpos_data.value',dpos_data.value)
        print('cf_data',cf_data)
        #print('solv_data',solv_data)

        times = list(set(dpos_data[:, 0]))
        times.sort()
        print('len(times)',len(times))

        if (len(times) ==0 ):
            print('no results in the hdf5 file')
        else:

            print('Results for ',len(times),' steps in the hdf5 file')
            #ndyna = len(numpy.where(dpos_data[:, 0] == times[0])) does not work
            ndyna = len(dpos_data[:, 0])/len(times)
            print('ndyna =', ndyna)
            if len(spos_data) > 0:
                nstatic = len(numpy.where(spos_data[:, 0] == times[0]))
                nstatic = spos_data.shape[0]
            else:
                nstatic = 0
            print('nstatic =', nstatic)
            #    instances = set(dpos_data[:, 1])

        # filtering

        p=0
        current_line=0
        for k in range(len(times)):
            #print(times[k])
            if (k+1  < len(times) ):
                time_step=times[k+1]-times[k]
                #print('time_step',time_step)

            if (k%output_frequency==0):
                if k==0 :
                    print('filter for k',k,'at times', times[k], 'p', p)
                    out._dynamic_data.resize((p+1)*ndyna,0)
                    out._dynamic_data[p*ndyna:(p+1)*ndyna,:] = numpy.array(dpos_data[k*ndyna:(k+1)*ndyna,:])
                    #print('times',dpos_data[k*ndyna:(k+1)*ndyna,0])

                    out._static_data.resize((p+1)*nstatic,0)
                    out._static_data[p*nstatic:(p+1)*nstatic,:] = numpy.array(spos_data[0:nstatic,:])

                    out._solv_data.resize((p+1),0)
                    out._solv_data[p:(p+1),:] = numpy.array(solv_data[0:1,:])

                    id_f = numpy.where(abs(cf_data[:, 0] - times[k]) < time_step*1e-5)[0]
                    if len(id_f) == 0:
                        print('no contact data at time',times[k])
                    else:
                        #print('index of contact :', min(id_f), max(id_f))
                        out._cf_data.resize(max(id_f)+1,0)
                        out._cf_data[min(id_f):max(id_f),:] = numpy.array(cf_data[min(id_f):max(id_f),:])
                        current_line = max(id_f)
                else:
                    print('filter for k',k,'at times', times[k], 'p', p)
                    out._dynamic_data.resize((p+1)*ndyna,0)
                    #print( dpos_data[k*ndyna:(k+1)*ndyna,:])
                    #print('times',dpos_data[(k+1)*ndyna:(k+2)*ndyna,0])
                    out._dynamic_data[p*ndyna:(p+1)*ndyna,:] = dpos_data[(k+1)*ndyna:(k+2)*ndyna,:]

                    # out._static_data.resize((p+1)*nstatic,0)
                    # #print( dpos_data[k*nstatic:(k+1)*nstatic,:])
                    # out._static_data[p*nstatic:(p+1)*nstatic,:] = spos_data[k*nstatic:(k+1)*nstatic,:]

                    out._solv_data.resize((p+1),0)
                    out._solv_data[p:(p+1),:] = numpy.array(solv_data[k:k+1,:])

                    id_f = numpy.where(abs(cf_data[:, 0] - times[k]) < time_step*1e-5)[0]
                    if len(id_f) == 0:
                        print('no contact data at time',times[k])
                    else:
                        #print('index of contact :', min(id_f), max(id_f))
                        new_line = current_line+max(id_f)-min(id_f)+1
                        #print('new_line',new_line)
                        #print('current_line',current_line)
                        #print('size of contact data', max(id_f)-min(id_f)+1)
                        out._cf_data.resize(new_line,0)
                        #print('fill out._cf_data indices', current_line, new_line-1)
                        out._cf_data[current_line:new_line,:] = numpy.array(cf_data[min(id_f):max(id_f)+1,:])
                        current_line=new_line
                        #print('current_line',current_line)

                p = p+1
        #print(dpos_data)

        print(out._dynamic_data.shape)
        print(out._static_data.shape)
        print(out.static_data().value)
        print(out._cf_data.shape)
        print(out._solv_data.shape)
<commit_msg>[examples] move post processing script for hdf5<commit_after><|endoftext|>


<commit_before>from datetime import datetime, timedelta
from xml.dom import minidom
import logging

from celery.decorators import task
from celery.task.sets import subtask

from eve_proxy.exceptions import *
from eve_proxy.models import CachedDocument

from eve_api.api_exceptions import *
from eve_api.models import EVEPlayerCorporation, EVEPlayerCharacter, EVEPlayerCharacterRole, EVEPlayerCharacterSkill, EVESkill, EVEAccount
from eve_api.app_defines import *
from eve_api.utils import basic_xml_parse, basic_xml_parse_doc

@task()
def import_eve_character(character_id, api_key=None, user_id=None, callback=None, **kwargs):
    """
    Imports a character from the API, providing a API key will populate
    further details. Returns a single EVEPlayerCharacter object

    """

    log = import_eve_character.get_logger()
    try:
        pchar = import_eve_character_func(character_id, api_key, user_id, log)
    except APIAccessException, exc:
        log.error('Error importing character - flagging for retry')
        import_eve_character.retry(args=[character_id, api_key, user_id, callback], exc=exc, kwargs=kwargs)

    if not pchar:
        log.error('Error importing character %s' % character_id)
    else:
        if callback:
            subtask(callback).delay(character=pchar.id)
        else:
            return pchar

@task()
def import_eve_characters(character_list, api_key=None, user_id=None, callback=None, **kwargs):
    """
    Imports characters from the API, providing a API key will populate
    further details. Returns a list of EVEPlayerCharacter objects

    """

    log = import_eve_characters.get_logger()
    try:
        results = [import_eve_character_func(char, api_key, user_id, log) for char in character_list]
    except APIAccessException, exc:
        log.error('Error importing characters - flagging for retry')
        import_eve_characters.retry(args=[character_list, api_key, user_id, callback], exc=exc, kwargs=kwargs)
    if callback:
        subtask(callback).delay(characters=results)
    else:
        return results

def import_eve_character_func(character_id, api_key=None, user_id=None, logger=logging.getLogger(__name__)):

    try:
        char_doc = CachedDocument.objects.api_query('/eve/CharacterInfo.xml.aspx', params={'characterID': character_id}, no_cache=False)
    except DocumentRetrievalError, exc:
        logger.error('Error retrieving CharacterInfo.xml.aspx for Character ID %s - %s' % (character_id, exc))
        raise APIAccessException

    d = basic_xml_parse_doc(char_doc)['eveapi']
    if 'error' in d:
        logger.debug('EVE API Error enountered in API document')
        return

    values = d['result']
    pchar, created = EVEPlayerCharacter.objects.get_or_create(id=character_id)
    if not values['characterName'] == {}:
        pchar.name = values['characterName']
    else:
        pchar.name = ""
    pchar.security_status = values['securityStatus']

    corp, created = EVEPlayerCorporation.objects.get_or_create(id=values['corporationID'])
    from eve_api.tasks.corporation import import_corp_details
    if created or not corp.name or corp.api_last_updated < (datetime.utcnow() - timedelta(hours=12)):
        import_corp_details.delay(values['corporationID'])

    pchar.corporation = corp
    pchar.corporation_date = values['corporationDate']

    for v in API_RACES_CHOICES:
        val, race = v
        if race == values['race']:
            pchar.race = val
            break

    if api_key and user_id:
        auth_params = {'userID': user_id, 'apiKey': api_key, 'characterID': character_id }
        try:
            char_doc = CachedDocument.objects.api_query('/char/CharacterSheet.xml.aspx', params=auth_params, no_cache=False)
        except DocumentRetrievalError, exc:
            logger.error('Error retrieving CharacterSheet.xml.aspx for User ID %s, Character ID %s - %s' % (user_id, character_id, exc))
            raise APIAccessException

        doc = basic_xml_parse_doc(char_doc)['eveapi']
        if not 'error' in doc:

            values = doc['result']
            pchar.name = values['name']
            pchar.balance = values['balance']
            pchar.attrib_intelligence = values['attributes']['intelligence']
            pchar.attrib_charisma = values['attributes']['charisma']
            pchar.attrib_perception = values['attributes']['perception']
            pchar.attrib_willpower = values['attributes']['willpower']
            pchar.attrib_memory = values['attributes']['memory']

            # Process the character's skills
            pchar.total_sp = 0
            for skill in values.get('skills', None):
                skillobj, created = EVESkill.objects.get_or_create(id=skill['typeID'])
                charskillobj, created = EVEPlayerCharacterSkill.objects.get_or_create(skill=skillobj, character=pchar)
                if created or not charskillobj.level == int(skill['level']) or not charskillobj.skillpoints == int(skill['skillpoints']):
                    charskillobj.level = int(skill['level'])
                    charskillobj.skillpoints = int(skill['skillpoints'])
                    charskillobj.save()
                pchar.total_sp = pchar.total_sp + int(skill['skillpoints'])

            try:
                skillqueue = CachedDocument.objects.api_query('/char/SkillInTraining.xml.aspx', params=auth_params, no_cache=False)
            except DocumentRetrievalError, exc:
                logger.error('Error retrieving SkillInTraining.xml.aspx for User ID %s, Character ID %s - %s' % (user_id, character_id, exc))
            else:
                queuedoc = basic_xml_parse_doc(skillqueue)['eveapi']['result']
                EVEPlayerCharacterSkill.objects.filter(character=pchar).update(in_training=0)
                if int(queuedoc['skillInTraining']):
                    skillobj, created = EVESkill.objects.get_or_create(id=queuedoc['trainingTypeID'])
                    charskillobj = EVEPlayerCharacterSkill.objects.get_or_create(skill=skillobj, character=pchar)
                    charskillobj.in_training = queuedoc['trainingToLevel']
                    charskillobj.save()

            # Process the character's roles
            pchar.roles.clear()
            roles = values.get('corporationRoles', None)
            if roles and len(roles):
                for r in roles:
                    role, created = EVEPlayerCharacterRole.objects.get_or_create(roleid=r['roleID'], name=r['roleName'])
                    pchar.roles.add(role)

            if values['gender'] == 'Male':
                pchar.gender = API_GENDER_MALE
            else:
                pchar.gender = API_GENDER_FEMALE

    pchar.api_last_updated = datetime.utcnow()
    pchar.save()

    try:
        acc = EVEAccount.objects.get(api_user_id=user_id)
        if not pchar.id in acc.characters.all().values_list('id', flat=True):
            acc.characters.add(pchar)

        if pchar.director and acc.api_keytype == API_KEYTYPE_FULL:
            from eve_api.tasks.corporation import import_corp_members
            import_corp_members.delay(api_key=api_key, api_userid=user_id, character_id=pchar.id)
    except EVEAccount.DoesNotExist:
        pass

    return pchar
<commit_msg>Use the object, not the tuple<commit_after>from datetime import datetime, timedelta
from xml.dom import minidom
import logging

from celery.decorators import task
from celery.task.sets import subtask

from eve_proxy.exceptions import *
from eve_proxy.models import CachedDocument

from eve_api.api_exceptions import *
from eve_api.models import EVEPlayerCorporation, EVEPlayerCharacter, EVEPlayerCharacterRole, EVEPlayerCharacterSkill, EVESkill, EVEAccount
from eve_api.app_defines import *
from eve_api.utils import basic_xml_parse, basic_xml_parse_doc

@task()
def import_eve_character(character_id, api_key=None, user_id=None, callback=None, **kwargs):
    """
    Imports a character from the API, providing a API key will populate
    further details. Returns a single EVEPlayerCharacter object

    """

    log = import_eve_character.get_logger()
    try:
        pchar = import_eve_character_func(character_id, api_key, user_id, log)
    except APIAccessException, exc:
        log.error('Error importing character - flagging for retry')
        import_eve_character.retry(args=[character_id, api_key, user_id, callback], exc=exc, kwargs=kwargs)

    if not pchar:
        log.error('Error importing character %s' % character_id)
    else:
        if callback:
            subtask(callback).delay(character=pchar.id)
        else:
            return pchar

@task()
def import_eve_characters(character_list, api_key=None, user_id=None, callback=None, **kwargs):
    """
    Imports characters from the API, providing a API key will populate
    further details. Returns a list of EVEPlayerCharacter objects

    """

    log = import_eve_characters.get_logger()
    try:
        results = [import_eve_character_func(char, api_key, user_id, log) for char in character_list]
    except APIAccessException, exc:
        log.error('Error importing characters - flagging for retry')
        import_eve_characters.retry(args=[character_list, api_key, user_id, callback], exc=exc, kwargs=kwargs)
    if callback:
        subtask(callback).delay(characters=results)
    else:
        return results

def import_eve_character_func(character_id, api_key=None, user_id=None, logger=logging.getLogger(__name__)):

    try:
        char_doc = CachedDocument.objects.api_query('/eve/CharacterInfo.xml.aspx', params={'characterID': character_id}, no_cache=False)
    except DocumentRetrievalError, exc:
        logger.error('Error retrieving CharacterInfo.xml.aspx for Character ID %s - %s' % (character_id, exc))
        raise APIAccessException

    d = basic_xml_parse_doc(char_doc)['eveapi']
    if 'error' in d:
        logger.debug('EVE API Error enountered in API document')
        return

    values = d['result']
    pchar, created = EVEPlayerCharacter.objects.get_or_create(id=character_id)
    if not values['characterName'] == {}:
        pchar.name = values['characterName']
    else:
        pchar.name = ""
    pchar.security_status = values['securityStatus']

    corp, created = EVEPlayerCorporation.objects.get_or_create(id=values['corporationID'])
    from eve_api.tasks.corporation import import_corp_details
    if created or not corp.name or corp.api_last_updated < (datetime.utcnow() - timedelta(hours=12)):
        import_corp_details.delay(values['corporationID'])

    pchar.corporation = corp
    pchar.corporation_date = values['corporationDate']

    for v in API_RACES_CHOICES:
        val, race = v
        if race == values['race']:
            pchar.race = val
            break

    if api_key and user_id:
        auth_params = {'userID': user_id, 'apiKey': api_key, 'characterID': character_id }
        try:
            char_doc = CachedDocument.objects.api_query('/char/CharacterSheet.xml.aspx', params=auth_params, no_cache=False)
        except DocumentRetrievalError, exc:
            logger.error('Error retrieving CharacterSheet.xml.aspx for User ID %s, Character ID %s - %s' % (user_id, character_id, exc))
            raise APIAccessException

        doc = basic_xml_parse_doc(char_doc)['eveapi']
        if not 'error' in doc:

            values = doc['result']
            pchar.name = values['name']
            pchar.balance = values['balance']
            pchar.attrib_intelligence = values['attributes']['intelligence']
            pchar.attrib_charisma = values['attributes']['charisma']
            pchar.attrib_perception = values['attributes']['perception']
            pchar.attrib_willpower = values['attributes']['willpower']
            pchar.attrib_memory = values['attributes']['memory']

            # Process the character's skills
            pchar.total_sp = 0
            for skill in values.get('skills', None):
                skillobj, created = EVESkill.objects.get_or_create(id=skill['typeID'])
                charskillobj, created = EVEPlayerCharacterSkill.objects.get_or_create(skill=skillobj, character=pchar)
                if created or not charskillobj.level == int(skill['level']) or not charskillobj.skillpoints == int(skill['skillpoints']):
                    charskillobj.level = int(skill['level'])
                    charskillobj.skillpoints = int(skill['skillpoints'])
                    charskillobj.save()
                pchar.total_sp = pchar.total_sp + int(skill['skillpoints'])

            try:
                skillqueue = CachedDocument.objects.api_query('/char/SkillInTraining.xml.aspx', params=auth_params, no_cache=False)
            except DocumentRetrievalError, exc:
                logger.error('Error retrieving SkillInTraining.xml.aspx for User ID %s, Character ID %s - %s' % (user_id, character_id, exc))
            else:
                queuedoc = basic_xml_parse_doc(skillqueue)['eveapi']['result']
                EVEPlayerCharacterSkill.objects.filter(character=pchar).update(in_training=0)
                if int(queuedoc['skillInTraining']):
                    skillobj, created = EVESkill.objects.get_or_create(id=queuedoc['trainingTypeID'])
                    charskillobj, created = EVEPlayerCharacterSkill.objects.get_or_create(skill=skillobj, character=pchar)
                    charskillobj.in_training = queuedoc['trainingToLevel']
                    charskillobj.save()

            # Process the character's roles
            pchar.roles.clear()
            roles = values.get('corporationRoles', None)
            if roles and len(roles):
                for r in roles:
                    role, created = EVEPlayerCharacterRole.objects.get_or_create(roleid=r['roleID'], name=r['roleName'])
                    pchar.roles.add(role)

            if values['gender'] == 'Male':
                pchar.gender = API_GENDER_MALE
            else:
                pchar.gender = API_GENDER_FEMALE

    pchar.api_last_updated = datetime.utcnow()
    pchar.save()

    try:
        acc = EVEAccount.objects.get(api_user_id=user_id)
        if not pchar.id in acc.characters.all().values_list('id', flat=True):
            acc.characters.add(pchar)

        if pchar.director and acc.api_keytype == API_KEYTYPE_FULL:
            from eve_api.tasks.corporation import import_corp_members
            import_corp_members.delay(api_key=api_key, api_userid=user_id, character_id=pchar.id)
    except EVEAccount.DoesNotExist:
        pass

    return pchar
<|endoftext|>


<commit_before>"""Utility functions to expand configuration directives or special values
(such glob patterns).

We can split the process of interpreting configuration files into 2 steps:

1. The parsing the file contents from strings to value objects
   that can be understand by Python (for example a string with a comma
   separated list of keywords into an actual Python list of strings).

2. The expansion (or post-processing) of these values according to the
   semantics ``setuptools`` assign to them (for example a configuration field
   with the ``file:`` directive should be expanded from a list of file paths to
   a single string with the contents of those files concatenated)

This module focus on the second step, and therefore allow sharing the expansion
functions among several configuration file formats.
"""
import ast
import importlib
import io
import os
import sys
from glob import iglob
from configparser import ConfigParser
from importlib.machinery import ModuleSpec
from itertools import chain
from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union, cast
from types import ModuleType

from distutils.errors import DistutilsOptionError

chain_iter = chain.from_iterable
_Path = Union[str, os.PathLike]

class StaticModule:
    """Proxy to a module object that avoids executing arbitrary code."""

    def __init__(self, name: str, spec: ModuleSpec):
        with open(spec.origin) as strm:  # type: ignore
            src = strm.read()
        module = ast.parse(src)
        vars(self).update(locals())
        del self.self

    def __getattr__(self, attr):
        """Attempt to load an attribute "statically", via :func:`ast.literal_eval`."""
        try:
            assignment_expressions = (
                statement
                for statement in self.module.body
                if isinstance(statement, ast.Assign)
            )
            expressions_with_target = (
                (statement, target)
                for statement in assignment_expressions
                for target in statement.targets
            )
            matching_values = (
                statement.value
                for statement, target in expressions_with_target
                if isinstance(target, ast.Name) and target.id == attr
            )
            return next(ast.literal_eval(value) for value in matching_values)
        except Exception as e:
            raise AttributeError(f"{self.name} has no attribute {attr}") from e

def glob_relative(
    patterns: Iterable[str], root_dir: Optional[_Path] = None
) -> List[str]:
    """Expand the list of glob patterns, but preserving relative paths.

    :param list[str] patterns: List of glob patterns
    :param str root_dir: Path to which globs should be relative
                         (current directory by default)
    :rtype: list
    """
    glob_characters = {'*', '?', '[', ']', '{', '}'}
    expanded_values = []
    root_dir = root_dir or os.getcwd()
    for value in patterns:

        # Has globby characters?
        if any(char in value for char in glob_characters):
            # then expand the glob pattern while keeping paths *relative*:
            glob_path = os.path.abspath(os.path.join(root_dir, value))
            expanded_values.extend(sorted(
                os.path.relpath(path, root_dir).replace(os.sep, "/")
                for path in iglob(glob_path, recursive=True)))

        else:
            # take the value as-is
            path = os.path.relpath(value, root_dir).replace(os.sep, "/")
            expanded_values.append(path)

    return expanded_values

def read_files(filepaths: Union[str, bytes, Iterable[_Path]], root_dir=None) -> str:
    """Return the content of the files concatenated using ``\n`` as str

    This function is sandboxed and won't reach anything outside ``root_dir``

    (By default ``root_dir`` is the current directory).
    """
    if isinstance(filepaths, (str, bytes)):
        filepaths = [filepaths]  # type: ignore

    root_dir = os.path.abspath(root_dir or os.getcwd())
    _filepaths = (os.path.join(root_dir, path) for path in filepaths)
    return '\n'.join(
        _read_file(path)
        for path in _filepaths
        if _assert_local(path, root_dir) and os.path.isfile(path)
    )

def _read_file(filepath: Union[bytes, _Path]) -> str:
    with io.open(filepath, encoding='utf-8') as f:
        return f.read()

def _assert_local(filepath: _Path, root_dir: str):
    if not os.path.abspath(filepath).startswith(root_dir):
        msg = f"Cannot access {filepath!r} (or anything outside {root_dir!r})"
        raise DistutilsOptionError(msg)

    return True

def read_attr(
    attr_desc: str,
    package_dir: Optional[dict] = None,
    root_dir: Optional[_Path] = None
):
    """Reads the value of an attribute from a module.

    This function will try to read the attributed statically first
    (via :func:`ast.literal_eval`), and only evaluate the module if it fails.

    Examples:
        read_attr("package.attr")
        read_attr("package.module.attr")

    :param str attr_desc: Dot-separated string describing how to reach the
        attribute (see examples above)
    :param dict[str, str] package_dir: Mapping of package names to their
        location in disk (represented by paths relative to ``root_dir``).
    :param str root_dir: Path to directory containing all the packages in
        ``package_dir`` (current directory by default).
    :rtype: str
    """
    root_dir = root_dir or os.getcwd()
    attrs_path = attr_desc.strip().split('.')
    attr_name = attrs_path.pop()
    module_name = '.'.join(attrs_path)
    module_name = module_name or '__init__'
    _parent_path, path, module_name = _find_module(module_name, package_dir, root_dir)
    spec = _find_spec(module_name, path)

    try:
        return getattr(StaticModule(module_name, spec), attr_name)
    except Exception:
        # fallback to evaluate module
        module = _load_spec(spec, module_name)
        return getattr(module, attr_name)

def _find_spec(module_name: str, module_path: Optional[_Path]) -> ModuleSpec:
    spec = importlib.util.spec_from_file_location(module_name, module_path)
    spec = spec or importlib.util.find_spec(module_name)

    if spec is None:
        raise ModuleNotFoundError(module_name)

    return spec

def _load_spec(spec: ModuleSpec, module_name: str) -> ModuleType:
    name = getattr(spec, "__name__", module_name)
    if name in sys.modules:
        return sys.modules[name]
    module = importlib.util.module_from_spec(spec)
    sys.modules[name] = module  # cache (it also ensures `==` works on loaded items)
    spec.loader.exec_module(module)  # type: ignore
    return module

def _find_module(
    module_name: str, package_dir: Optional[dict], root_dir: _Path
) -> Tuple[_Path, Optional[str], str]:
    """Given a module (that could normally be imported by ``module_name``
    after the build is complete), find the path to the parent directory where
    it is contained and the canonical name that could be used to import it
    considering the ``package_dir`` in the build configuration and ``root_dir``
    """
    parent_path = root_dir
    module_parts = module_name.split('.')
    if package_dir:
        if module_parts[0] in package_dir:
            # A custom path was specified for the module we want to import
            custom_path = package_dir[module_parts[0]]
            parts = custom_path.rsplit('/', 1)
            if len(parts) > 1:
                parent_path = os.path.join(root_dir, parts[0])
                parent_module = parts[1]
            else:
                parent_module = custom_path
            module_name = ".".join([parent_module, *module_parts[1:]])
        elif '' in package_dir:
            # A custom parent directory was specified for all root modules
            parent_path = os.path.join(root_dir, package_dir[''])

    path_start = os.path.join(parent_path, *module_name.split("."))
    candidates = chain(
        (f"{path_start}.py", os.path.join(path_start, "__init__.py")),
        iglob(f"{path_start}.*")
    )
    module_path = next((x for x in candidates if os.path.isfile(x)), None)
    return parent_path, module_path, module_name

def resolve_class(
    qualified_class_name: str,
    package_dir: Optional[dict] = None,
    root_dir: Optional[_Path] = None
) -> Callable:
    """Given a qualified class name, return the associated class object"""
    root_dir = root_dir or os.getcwd()
    idx = qualified_class_name.rfind('.')
    class_name = qualified_class_name[idx + 1 :]
    pkg_name = qualified_class_name[:idx]

    _parent_path, path, module_name = _find_module(pkg_name, package_dir, root_dir)
    module = _load_spec(_find_spec(module_name, path), module_name)
    return getattr(module, class_name)

def cmdclass(
    values: Dict[str, str],
    package_dir: Optional[dict] = None,
    root_dir: Optional[_Path] = None
) -> Dict[str, Callable]:
    """Given a dictionary mapping command names to strings for qualified class
    names, apply :func:`resolve_class` to the dict values.
    """
    return {k: resolve_class(v, package_dir, root_dir) for k, v in values.items()}

def find_packages(
    *, namespaces=True, root_dir: Optional[_Path] = None, **kwargs
) -> List[str]:
    """Works similarly to :func:`setuptools.find_packages`, but with all
    arguments given as keyword arguments. Moreover, ``where`` can be given
    as a list (the results will be simply concatenated).

    When the additional keyword argument ``namespaces`` is ``True``, it will
    behave like :func:`setuptools.find_namespace_packages`` (i.e. include
    implicit namespaces as per :pep:`420`).

    :rtype: list
    """

    if namespaces:
        from setuptools import PEP420PackageFinder as PackageFinder
    else:
        from setuptools import PackageFinder  # type: ignore

    root_dir = root_dir or "."
    where = kwargs.pop('where', ['.'])
    if isinstance(where, str):
        where = [where]
    target = [_nest_path(root_dir, path) for path in where]
    return list(chain_iter(PackageFinder.find(x, **kwargs) for x in target))

def _nest_path(parent: _Path, path: _Path) -> str:
    path = parent if path == "." else os.path.join(parent, path)
    return os.path.normpath(path)

def version(value: Union[Callable, Iterable[Union[str, int]], str]) -> str:
    """When getting the version directly from an attribute,
    it should be normalised to string.
    """
    if callable(value):
        value = value()

    value = cast(Iterable[Union[str, int]], value)

    if not isinstance(value, str):
        if hasattr(value, '__iter__'):
            value = '.'.join(map(str, value))
        else:
            value = '%s' % value

    return value

def canonic_package_data(package_data: dict) -> dict:
    if "*" in package_data:
        package_data[""] = package_data.pop("*")
    return package_data

def canonic_data_files(
    data_files: Union[list, dict], root_dir: Optional[_Path] = None
) -> List[Tuple[str, List[str]]]:
    """For compatibility with ``setup.py``, ``data_files`` should be a list
    of pairs instead of a dict.

    This function also expands glob patterns.
    """
    if isinstance(data_files, list):
        return data_files

    return [
        (dest, glob_relative(patterns, root_dir))
        for dest, patterns in data_files.items()
    ]

def entry_points(text: str, text_source="entry-points") -> Dict[str, dict]:
    """Given the contents of entry-points file,
    process it into a 2-level dictionary (``dict[str, dict[str, str]]``).
    The first level keys are entry-point groups, the second level keys are
    entry-point names, and the second level values are references to objects
    (that correspond to the entry-point value).
    """
    parser = ConfigParser(default_section=None, delimiters=("=",))  # type: ignore
    parser.optionxform = str  # case sensitive
    parser.read_string(text, text_source)
    groups = {k: dict(v.items()) for k, v in parser.items()}
    groups.pop(parser.default_section, None)
    return groups
<commit_msg>Import package finders directly from discovery module<commit_after>"""Utility functions to expand configuration directives or special values
(such glob patterns).

We can split the process of interpreting configuration files into 2 steps:

1. The parsing the file contents from strings to value objects
   that can be understand by Python (for example a string with a comma
   separated list of keywords into an actual Python list of strings).

2. The expansion (or post-processing) of these values according to the
   semantics ``setuptools`` assign to them (for example a configuration field
   with the ``file:`` directive should be expanded from a list of file paths to
   a single string with the contents of those files concatenated)

This module focus on the second step, and therefore allow sharing the expansion
functions among several configuration file formats.
"""
import ast
import importlib
import io
import os
import sys
from glob import iglob
from configparser import ConfigParser
from importlib.machinery import ModuleSpec
from itertools import chain
from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union, cast
from types import ModuleType

from distutils.errors import DistutilsOptionError

chain_iter = chain.from_iterable
_Path = Union[str, os.PathLike]

class StaticModule:
    """Proxy to a module object that avoids executing arbitrary code."""

    def __init__(self, name: str, spec: ModuleSpec):
        with open(spec.origin) as strm:  # type: ignore
            src = strm.read()
        module = ast.parse(src)
        vars(self).update(locals())
        del self.self

    def __getattr__(self, attr):
        """Attempt to load an attribute "statically", via :func:`ast.literal_eval`."""
        try:
            assignment_expressions = (
                statement
                for statement in self.module.body
                if isinstance(statement, ast.Assign)
            )
            expressions_with_target = (
                (statement, target)
                for statement in assignment_expressions
                for target in statement.targets
            )
            matching_values = (
                statement.value
                for statement, target in expressions_with_target
                if isinstance(target, ast.Name) and target.id == attr
            )
            return next(ast.literal_eval(value) for value in matching_values)
        except Exception as e:
            raise AttributeError(f"{self.name} has no attribute {attr}") from e

def glob_relative(
    patterns: Iterable[str], root_dir: Optional[_Path] = None
) -> List[str]:
    """Expand the list of glob patterns, but preserving relative paths.

    :param list[str] patterns: List of glob patterns
    :param str root_dir: Path to which globs should be relative
                         (current directory by default)
    :rtype: list
    """
    glob_characters = {'*', '?', '[', ']', '{', '}'}
    expanded_values = []
    root_dir = root_dir or os.getcwd()
    for value in patterns:

        # Has globby characters?
        if any(char in value for char in glob_characters):
            # then expand the glob pattern while keeping paths *relative*:
            glob_path = os.path.abspath(os.path.join(root_dir, value))
            expanded_values.extend(sorted(
                os.path.relpath(path, root_dir).replace(os.sep, "/")
                for path in iglob(glob_path, recursive=True)))

        else:
            # take the value as-is
            path = os.path.relpath(value, root_dir).replace(os.sep, "/")
            expanded_values.append(path)

    return expanded_values

def read_files(filepaths: Union[str, bytes, Iterable[_Path]], root_dir=None) -> str:
    """Return the content of the files concatenated using ``\n`` as str

    This function is sandboxed and won't reach anything outside ``root_dir``

    (By default ``root_dir`` is the current directory).
    """
    if isinstance(filepaths, (str, bytes)):
        filepaths = [filepaths]  # type: ignore

    root_dir = os.path.abspath(root_dir or os.getcwd())
    _filepaths = (os.path.join(root_dir, path) for path in filepaths)
    return '\n'.join(
        _read_file(path)
        for path in _filepaths
        if _assert_local(path, root_dir) and os.path.isfile(path)
    )

def _read_file(filepath: Union[bytes, _Path]) -> str:
    with io.open(filepath, encoding='utf-8') as f:
        return f.read()

def _assert_local(filepath: _Path, root_dir: str):
    if not os.path.abspath(filepath).startswith(root_dir):
        msg = f"Cannot access {filepath!r} (or anything outside {root_dir!r})"
        raise DistutilsOptionError(msg)

    return True

def read_attr(
    attr_desc: str,
    package_dir: Optional[dict] = None,
    root_dir: Optional[_Path] = None
):
    """Reads the value of an attribute from a module.

    This function will try to read the attributed statically first
    (via :func:`ast.literal_eval`), and only evaluate the module if it fails.

    Examples:
        read_attr("package.attr")
        read_attr("package.module.attr")

    :param str attr_desc: Dot-separated string describing how to reach the
        attribute (see examples above)
    :param dict[str, str] package_dir: Mapping of package names to their
        location in disk (represented by paths relative to ``root_dir``).
    :param str root_dir: Path to directory containing all the packages in
        ``package_dir`` (current directory by default).
    :rtype: str
    """
    root_dir = root_dir or os.getcwd()
    attrs_path = attr_desc.strip().split('.')
    attr_name = attrs_path.pop()
    module_name = '.'.join(attrs_path)
    module_name = module_name or '__init__'
    _parent_path, path, module_name = _find_module(module_name, package_dir, root_dir)
    spec = _find_spec(module_name, path)

    try:
        return getattr(StaticModule(module_name, spec), attr_name)
    except Exception:
        # fallback to evaluate module
        module = _load_spec(spec, module_name)
        return getattr(module, attr_name)

def _find_spec(module_name: str, module_path: Optional[_Path]) -> ModuleSpec:
    spec = importlib.util.spec_from_file_location(module_name, module_path)
    spec = spec or importlib.util.find_spec(module_name)

    if spec is None:
        raise ModuleNotFoundError(module_name)

    return spec

def _load_spec(spec: ModuleSpec, module_name: str) -> ModuleType:
    name = getattr(spec, "__name__", module_name)
    if name in sys.modules:
        return sys.modules[name]
    module = importlib.util.module_from_spec(spec)
    sys.modules[name] = module  # cache (it also ensures `==` works on loaded items)
    spec.loader.exec_module(module)  # type: ignore
    return module

def _find_module(
    module_name: str, package_dir: Optional[dict], root_dir: _Path
) -> Tuple[_Path, Optional[str], str]:
    """Given a module (that could normally be imported by ``module_name``
    after the build is complete), find the path to the parent directory where
    it is contained and the canonical name that could be used to import it
    considering the ``package_dir`` in the build configuration and ``root_dir``
    """
    parent_path = root_dir
    module_parts = module_name.split('.')
    if package_dir:
        if module_parts[0] in package_dir:
            # A custom path was specified for the module we want to import
            custom_path = package_dir[module_parts[0]]
            parts = custom_path.rsplit('/', 1)
            if len(parts) > 1:
                parent_path = os.path.join(root_dir, parts[0])
                parent_module = parts[1]
            else:
                parent_module = custom_path
            module_name = ".".join([parent_module, *module_parts[1:]])
        elif '' in package_dir:
            # A custom parent directory was specified for all root modules
            parent_path = os.path.join(root_dir, package_dir[''])

    path_start = os.path.join(parent_path, *module_name.split("."))
    candidates = chain(
        (f"{path_start}.py", os.path.join(path_start, "__init__.py")),
        iglob(f"{path_start}.*")
    )
    module_path = next((x for x in candidates if os.path.isfile(x)), None)
    return parent_path, module_path, module_name

def resolve_class(
    qualified_class_name: str,
    package_dir: Optional[dict] = None,
    root_dir: Optional[_Path] = None
) -> Callable:
    """Given a qualified class name, return the associated class object"""
    root_dir = root_dir or os.getcwd()
    idx = qualified_class_name.rfind('.')
    class_name = qualified_class_name[idx + 1 :]
    pkg_name = qualified_class_name[:idx]

    _parent_path, path, module_name = _find_module(pkg_name, package_dir, root_dir)
    module = _load_spec(_find_spec(module_name, path), module_name)
    return getattr(module, class_name)

def cmdclass(
    values: Dict[str, str],
    package_dir: Optional[dict] = None,
    root_dir: Optional[_Path] = None
) -> Dict[str, Callable]:
    """Given a dictionary mapping command names to strings for qualified class
    names, apply :func:`resolve_class` to the dict values.
    """
    return {k: resolve_class(v, package_dir, root_dir) for k, v in values.items()}

def find_packages(
    *, namespaces=True, root_dir: Optional[_Path] = None, **kwargs
) -> List[str]:
    """Works similarly to :func:`setuptools.find_packages`, but with all
    arguments given as keyword arguments. Moreover, ``where`` can be given
    as a list (the results will be simply concatenated).

    When the additional keyword argument ``namespaces`` is ``True``, it will
    behave like :func:`setuptools.find_namespace_packages`` (i.e. include
    implicit namespaces as per :pep:`420`).

    :rtype: list
    """

    if namespaces:
        from setuptools.discovery import PEP420PackageFinder as PackageFinder
    else:
        from setuptools.discovery import PackageFinder  # type: ignore

    root_dir = root_dir or "."
    where = kwargs.pop('where', ['.'])
    if isinstance(where, str):
        where = [where]
    target = [_nest_path(root_dir, path) for path in where]
    return list(chain_iter(PackageFinder.find(x, **kwargs) for x in target))

def _nest_path(parent: _Path, path: _Path) -> str:
    path = parent if path == "." else os.path.join(parent, path)
    return os.path.normpath(path)

def version(value: Union[Callable, Iterable[Union[str, int]], str]) -> str:
    """When getting the version directly from an attribute,
    it should be normalised to string.
    """
    if callable(value):
        value = value()

    value = cast(Iterable[Union[str, int]], value)

    if not isinstance(value, str):
        if hasattr(value, '__iter__'):
            value = '.'.join(map(str, value))
        else:
            value = '%s' % value

    return value

def canonic_package_data(package_data: dict) -> dict:
    if "*" in package_data:
        package_data[""] = package_data.pop("*")
    return package_data

def canonic_data_files(
    data_files: Union[list, dict], root_dir: Optional[_Path] = None
) -> List[Tuple[str, List[str]]]:
    """For compatibility with ``setup.py``, ``data_files`` should be a list
    of pairs instead of a dict.

    This function also expands glob patterns.
    """
    if isinstance(data_files, list):
        return data_files

    return [
        (dest, glob_relative(patterns, root_dir))
        for dest, patterns in data_files.items()
    ]

def entry_points(text: str, text_source="entry-points") -> Dict[str, dict]:
    """Given the contents of entry-points file,
    process it into a 2-level dictionary (``dict[str, dict[str, str]]``).
    The first level keys are entry-point groups, the second level keys are
    entry-point names, and the second level values are references to objects
    (that correspond to the entry-point value).
    """
    parser = ConfigParser(default_section=None, delimiters=("=",))  # type: ignore
    parser.optionxform = str  # case sensitive
    parser.read_string(text, text_source)
    groups = {k: dict(v.items()) for k, v in parser.items()}
    groups.pop(parser.default_section, None)
    return groups
<|endoftext|>