<commit_before>"""This module provides methods for loading XML files, as well checking
for potential errors that may result from reading XML data.
"""
from lxml import etree
from lib.custom_data.character_data import *
from lib.custom_data.settings_data import *

# ============================================================================
# Loading From File
# ============================================================================
def load_xml_doc_as_object(xml_path, schema_path):
    """Return an object containing the contents of an XML document as
    attributes.

    Args:
        xml_path (String): The file path to a valid XML document.
        schema_path (String): The file path to an XML Schema that will
            be used to verify the XML document.

    Returns:
        An instance of a class with the same name as the XML document's
        root element.
        None is returned instead if there were any errors reading the
        specified files.
    """
    root_element = load_xml_from_file(xml_path, schema_path)
    if root_element is None:
        return None
    else:
        return convert_element_to_object(root_element)

def load_xml_from_file(xml_path, schema_path):
    """Retrieve the entire contents of an XML document.

    Args:
        xml_path (String): The file path to a valid XML document.
        schema_path (String): The file path to an XML Schema that will
            be used to verify the XML document.

    Returns:
        The root XML element of the specified document.
        If the document was deemed invalid according to the specified
        XML Schema, None is returned instead.
        None is also returned if either of the file paths are faulty, or
        if either file had trouble being parsed.
    """
    parser = etree.XMLParser(remove_blank_text=True)

    try:
        xml_doc = etree.parse(xml_path, parser)
        schema_doc = etree.parse(schema_path, parser)
    except:
        return None

    schema = etree.XMLSchema(schema_doc)

    if schema.validate(xml_doc):
        return xml_doc.getroot()
    else:
        return None

def convert_element_to_object(element):
    """Load all of an XML element's data into an object.

    Args:
        element (lxml.XML): An XML element.

    Returns:
        An instance of a class with the same name as element's tag,
        containing all of element's attributes, text, and child
        elements.
    """
    data_object = instantiate_from_string(element.tag)

    for attr_name, attr_value in object_attributes(data_object):
        if is_list_of_text_data(element, attr_name):
            loaded_data = load_xml_text_data(element, attr_name)
        elif type(attr_value) is dict:
            loaded_data = load_attribute_dict(element, attr_name)
        elif type(attr_value) is list:
            loaded_data = load_child_objects(element, attr_name)
        elif is_element_attribute(element, attr_name):
            loaded_data = load_element_attribute(element, attr_name)
        else:   # Optional attribute was omitted.
            loaded_data = attr_value

        setattr(data_object, attr_name, loaded_data)

    return data_object

def instantiate_from_string(class_name):
    """Return an instance of the specified class.

    Args:
        class_name (String): The name of a class included in this
            module's namespace. It will automatically be converted into
            CamelCase to match the class naming convention.
    """
    class_name = convert_underscore_to_camel_case(class_name)
    return globals()[class_name]()

def convert_underscore_to_camel_case(text):
    """Return text converted into CamelCase.

    Args:
        text (String): A string with words separated by underscores.
    """
    words = text.split('_')
    words = [word.capitalize() for word in words]
    return ''.join(words)

def object_attributes(obj):
    """Return a dictionary containing the names and values of all
        attributes within an object.

    Args:
        obj (Object): An instance of some class.
    """
    return obj.__dict__.items()

def is_list_of_text_data(parent_element, list_name):
    """Return a Boolean indicating whether the specified XML element
    contains one or more child elements of the same tag and containing
    text data.

    Args:
        parent_element (Element): An XML element containing all of the
            child text elements.
        list_name (String): An appropriate list name for the child
            elements.  For example, if the child elements have the tag
            'box', then 'boxes' should be passed for this parameter.
    """
    list_item_name = get_singular_from_plural(list_name)

    if parent_element.find(list_item_name) is None:
        return False

    return parent_element.find(list_item_name).text is not None

def load_xml_text_data(parent_element, text_list_name):
    """Load child elements containing text from a parent XML element.

    Args:
        list_name (String): An appropriate list name for the child
            elements. For example, if the child elements have the tag
            'box', then 'boxes' should be passed for this parameter.
        parent_element (Element): An XML element containing all of the
            child text elements.
    """
    data_from_text = []
    list_item_name = get_singular_from_plural(text_list_name)

    for text_element in parent_element.findall(list_item_name):
        new_data = convert_to_int_if_numeric(text_element.text)
        data_from_text.append(new_data)

    return data_from_text

def convert_to_int_if_numeric(original_string):
    """Cast a numeric string into an integer.

    Args:
        original_string (String): A text string.

    Returns:
        An integer containing the contents of original_string.
        If original_string is not numeric, original_string is returned
        unmodified as a String.
    """
    if original_string.isdigit():
        return int(original_string)
    else:
        return original_string

def load_attribute_dict(parent_element, tag_name):
    """Load all of the attribute names and values within an XML element
    into a dict.

    Args:
        parent_element (Element): The parent XML Element containing the
            desired Element as a child node.
        tag_name (String): The tag name of the desired XML element.
    """
    loaded_element = parent_element.find(tag_name)
    attribute_dict = {attr[0] : convert_to_int_if_numeric(attr[1])
                      for attr in loaded_element.items()}
    return attribute_dict

def load_child_objects(parent_element, child_list_name):
    """Load child elements of the same tag name from a parent XML
    element.

    Args:
        parent_element (Element): The XML element containing all of the
            child elements.
        child_list_name (String): An appropriate name for a list
            containing the child elements. For example, if the child
            elements' tag is 'box', 'boxes' should be passed for this
            parameter.
    """
    children = []
    child_name = get_singular_from_plural(child_list_name)

    for child_element in parent_element.findall(child_name):
        child_object = convert_element_to_object(child_element)
        children.append(child_object)

    return children

def get_singular_from_plural(plural):
    """Convert a plural word into its singular form.

    Args:
        plural (String): A plural. For example, 'cats', 'potatoes',
            'knives'.
    """
    if plural[-3:] in ['oes', 'xes']:
        return plural[:-2]
    elif plural[-3:] == 'ves':
        return plural[:-3] + 'fe'
    elif plural[-3:] == 'ies':
        return plural[:-3] + 'y'
    else:
        return plural[:-1]

def is_element_attribute(element, attribute_name):
    """Return a Boolean indicating whether the specified element has an
    attribute of the specified name.

    Args:
        element (Element): An XML element that will be searched.
        attribute_name (String): The name of the attribute to find.
    """
    return element.get(attribute_name) is not None

def load_element_attribute(element, attribute_name):
    """Return an attribute from an XML element.

    Args:
        element (Element): An XML element containing the desired
            attribute.
        attribute_name (String): The name of the attribute to load.
    """
    attribute_value = element.get(attribute_name)
    return convert_to_int_if_numeric(attribute_value)

# ============================================================================
# Saving To File
# ============================================================================
def save_object_as_xml_doc(data_object, doc_filepath):
    """Save the attributes of an object to an XML document.

    Args:
        data_object (Object): An instance of some class.
        doc_filepath (String): The file path to an XML document. If the
            file doesn't exist, it will be created.
    """
    root = convert_object_to_element(data_object)
    xml_tree = etree.ElementTree(root)
    xml_tree.write(doc_filepath, pretty_print=True)

def convert_object_to_element(data_object):
    """Convert an object into an XML Element.

    Args:
        data_object (object): An instance of some class.

    Returns:
        An XML Element whose tag name is the same as data_object's class
        name, but in underscore_case. Its attributes and child Elements
        will contain data from all of data_object's attributes.
    """
    class_name = type(data_object).__name__
    element_name = convert_camel_case_to_underscore(class_name)
    new_element = etree.Element(element_name)

    for attribute_name, attribute_value in object_attributes(data_object):
        if type(attribute_value) is dict:
            dict_element = convert_dict_to_element(attribute_value,
                                                   attribute_name)
            new_element.append(dict_element)
        else:
            new_element.set(attribute_name, str(attribute_value))

    return new_element

def convert_camel_case_to_underscore(text):
    """Return text written in CamelCase converted to underscore_case.

    Args:
        text (String): A text string written in CamelCase.
    """
    words = []
    last_capital_index = 0

    for current_letter_index in xrange(1, len(text)):
        if text[current_letter_index].isupper():
            words.append(text[last_capital_index:current_letter_index])
            last_capital_index = current_letter_index
        elif current_letter_index == len(text) - 1:
            words.append(text[last_capital_index:])

    return '_'.join(words).lower()

def convert_dict_to_element(dictionary, element_name):
    """Convert a dict into an XML element.

    Args:
        dictionary (dict): A dict with one or more key-value pairs.
        element_name (String): The name given to the XML Element's tag.

    Returns:
        An Element with element_name as its tag and the key-value pairs
        in dictionary as its attributes.
    """
    new_element = etree.Element(element_name)

    for key, value in dictionary.items():
        new_element.set(key, str(value))

    return new_element
<commit_msg>Allow loading of singular complex XML elements<commit_after>"""This module provides methods for loading XML files, as well checking
for potential errors that may result from reading XML data.
"""
from lxml import etree
from lib.custom_data.character_data import *
from lib.custom_data.settings_data import *

# ============================================================================
# Loading From File
# ============================================================================
def load_xml_doc_as_object(xml_path, schema_path):
    """Return an object containing the contents of an XML document as
    attributes.

    Args:
        xml_path (String): The file path to a valid XML document.
        schema_path (String): The file path to an XML Schema that will
            be used to verify the XML document.

    Returns:
        An instance of a class with the same name as the XML document's
        root element.
        None is returned instead if there were any errors reading the
        specified files.
    """
    root_element = load_xml_from_file(xml_path, schema_path)
    if root_element is None:
        return None
    else:
        return convert_element_to_object(root_element)

def load_xml_from_file(xml_path, schema_path):
    """Retrieve the entire contents of an XML document.

    Args:
        xml_path (String): The file path to a valid XML document.
        schema_path (String): The file path to an XML Schema that will
            be used to verify the XML document.

    Returns:
        The root XML element of the specified document.
        If the document was deemed invalid according to the specified
        XML Schema, None is returned instead.
        None is also returned if either of the file paths are faulty, or
        if either file had trouble being parsed.
    """
    parser = etree.XMLParser(remove_blank_text=True)

    try:
        xml_doc = etree.parse(xml_path, parser)
        schema_doc = etree.parse(schema_path, parser)
    except:
        return None

    schema = etree.XMLSchema(schema_doc)

    if schema.validate(xml_doc):
        return xml_doc.getroot()
    else:
        return None

def convert_element_to_object(element):
    """Load all of an XML element's data into an object.

    Args:
        element (lxml.XML): An XML element.

    Returns:
        An instance of a class with the same name as element's tag,
        containing all of element's attributes, text, and child
        elements.
    """
    data_object = instantiate_from_string(element.tag)

    for attr_name, attr_value in object_attributes(data_object):
        if is_list_of_text_data(element, attr_name):
            loaded_data = load_xml_text_data(element, attr_name)
        elif attr_value is None:    # A complex element that appears once.
            loaded_data = convert_element_to_object(element.find(attr_name))
        elif type(attr_value) is dict:
            loaded_data = load_attribute_dict(element, attr_name)
        elif type(attr_value) is list:
            loaded_data = load_child_objects(element, attr_name)
        elif is_element_attribute(element, attr_name):
            loaded_data = load_element_attribute(element, attr_name)
        else:   # Optional attribute was omitted.
            loaded_data = attr_value

        setattr(data_object, attr_name, loaded_data)

    return data_object

def instantiate_from_string(class_name):
    """Return an instance of the specified class.

    Args:
        class_name (String): The name of a class included in this
            module's namespace. It will automatically be converted into
            CamelCase to match the class naming convention.
    """
    class_name = convert_underscore_to_camel_case(class_name)
    return globals()[class_name]()

def convert_underscore_to_camel_case(text):
    """Return text converted into CamelCase.

    Args:
        text (String): A string with words separated by underscores.
    """
    words = text.split('_')
    words = [word.capitalize() for word in words]
    return ''.join(words)

def object_attributes(obj):
    """Return a dictionary containing the names and values of all
        attributes within an object.

    Args:
        obj (Object): An instance of some class.
    """
    return obj.__dict__.items()

def is_list_of_text_data(parent_element, list_name):
    """Return a Boolean indicating whether the specified XML element
    contains one or more child elements of the same tag and containing
    text data.

    Args:
        parent_element (Element): An XML element containing all of the
            child text elements.
        list_name (String): An appropriate list name for the child
            elements.  For example, if the child elements have the tag
            'box', then 'boxes' should be passed for this parameter.
    """
    list_item_name = get_singular_from_plural(list_name)

    if parent_element.find(list_item_name) is None:
        return False

    return parent_element.find(list_item_name).text is not None

def load_xml_text_data(parent_element, text_list_name):
    """Load child elements containing text from a parent XML element.

    Args:
        list_name (String): An appropriate list name for the child
            elements. For example, if the child elements have the tag
            'box', then 'boxes' should be passed for this parameter.
        parent_element (Element): An XML element containing all of the
            child text elements.
    """
    data_from_text = []
    list_item_name = get_singular_from_plural(text_list_name)

    for text_element in parent_element.findall(list_item_name):
        new_data = convert_to_int_if_numeric(text_element.text)
        data_from_text.append(new_data)

    return data_from_text

def convert_to_int_if_numeric(original_string):
    """Cast a numeric string into an integer.

    Args:
        original_string (String): A text string.

    Returns:
        An integer containing the contents of original_string.
        If original_string is not numeric, original_string is returned
        unmodified as a String.
    """
    if original_string.isdigit():
        return int(original_string)
    else:
        return original_string

def load_attribute_dict(parent_element, tag_name):
    """Load all of the attribute names and values within an XML element
    into a dict.

    Args:
        parent_element (Element): The parent XML Element containing the
            desired Element as a child node.
        tag_name (String): The tag name of the desired XML element.
    """
    loaded_element = parent_element.find(tag_name)
    attribute_dict = {attr[0] : convert_to_int_if_numeric(attr[1])
                      for attr in loaded_element.items()}
    return attribute_dict

def load_child_objects(parent_element, child_list_name):
    """Load child elements of the same tag name from a parent XML
    element.

    Args:
        parent_element (Element): The XML element containing all of the
            child elements.
        child_list_name (String): An appropriate name for a list
            containing the child elements. For example, if the child
            elements' tag is 'box', 'boxes' should be passed for this
            parameter.
    """
    children = []
    child_name = get_singular_from_plural(child_list_name)

    for child_element in parent_element.findall(child_name):
        child_object = convert_element_to_object(child_element)
        children.append(child_object)

    return children

def get_singular_from_plural(plural):
    """Convert a plural word into its singular form.

    Args:
        plural (String): A plural. For example, 'cats', 'potatoes',
            'knives'.
    """
    if plural[-3:] in ['oes', 'xes']:
        return plural[:-2]
    elif plural[-3:] == 'ves':
        return plural[:-3] + 'fe'
    elif plural[-3:] == 'ies':
        return plural[:-3] + 'y'
    else:
        return plural[:-1]

def is_element_attribute(element, attribute_name):
    """Return a Boolean indicating whether the specified element has an
    attribute of the specified name.

    Args:
        element (Element): An XML element that will be searched.
        attribute_name (String): The name of the attribute to find.
    """
    return element.get(attribute_name) is not None

def load_element_attribute(element, attribute_name):
    """Return an attribute from an XML element.

    Args:
        element (Element): An XML element containing the desired
            attribute.
        attribute_name (String): The name of the attribute to load.
    """
    attribute_value = element.get(attribute_name)
    return convert_to_int_if_numeric(attribute_value)

# ============================================================================
# Saving To File
# ============================================================================
def save_object_as_xml_doc(data_object, doc_filepath):
    """Save the attributes of an object to an XML document.

    Args:
        data_object (Object): An instance of some class.
        doc_filepath (String): The file path to an XML document. If the
            file doesn't exist, it will be created.
    """
    root = convert_object_to_element(data_object)
    xml_tree = etree.ElementTree(root)
    xml_tree.write(doc_filepath, pretty_print=True)

def convert_object_to_element(data_object):
    """Convert an object into an XML Element.

    Args:
        data_object (object): An instance of some class.

    Returns:
        An XML Element whose tag name is the same as data_object's class
        name, but in underscore_case. Its attributes and child Elements
        will contain data from all of data_object's attributes.
    """
    class_name = type(data_object).__name__
    element_name = convert_camel_case_to_underscore(class_name)
    new_element = etree.Element(element_name)

    for attribute_name, attribute_value in object_attributes(data_object):
        if type(attribute_value) is dict:
            dict_element = convert_dict_to_element(attribute_value,
                                                   attribute_name)
            new_element.append(dict_element)
        else:
            new_element.set(attribute_name, str(attribute_value))

    return new_element

def convert_camel_case_to_underscore(text):
    """Return text written in CamelCase converted to underscore_case.

    Args:
        text (String): A text string written in CamelCase.
    """
    words = []
    last_capital_index = 0

    for current_letter_index in xrange(1, len(text)):
        if text[current_letter_index].isupper():
            words.append(text[last_capital_index:current_letter_index])
            last_capital_index = current_letter_index
        elif current_letter_index == len(text) - 1:
            words.append(text[last_capital_index:])

    return '_'.join(words).lower()

def convert_dict_to_element(dictionary, element_name):
    """Convert a dict into an XML element.

    Args:
        dictionary (dict): A dict with one or more key-value pairs.
        element_name (String): The name given to the XML Element's tag.

    Returns:
        An Element with element_name as its tag and the key-value pairs
        in dictionary as its attributes.
    """
    new_element = etree.Element(element_name)

    for key, value in dictionary.items():
        new_element.set(key, str(value))

    return new_element
<|endoftext|>


<commit_before>""" A universal module with functions / classes without dependencies. """
import sys
import contextlib
import functools
import tokenizer as tokenize

from jedi._compatibility import next, reraise
from jedi import debug
from jedi import settings

class MultiLevelStopIteration(Exception):
    """
    StopIteration's get catched pretty easy by for loops, let errors propagate.
    """
    pass

class UncaughtAttributeError(Exception):
    """
    Important, because `__getattr__` and `hasattr` catch AttributeErrors
    implicitly. This is really evil (mainly because of `__getattr__`).
    `hasattr` in Python 2 is even more evil, because it catches ALL exceptions.
    Therefore this class originally had to be derived from `BaseException`
    instead of `Exception`.  But because I removed relevant `hasattr` from
    the code base, we can now switch back to `Exception`.

    :param base: return values of sys.exc_info().
    """

def rethrow_uncaught(func):
    """
    Re-throw uncaught `AttributeError`.

    Usage:  Put ``@rethrow_uncaught`` in front of the function
    which does **not** suppose to raise `AttributeError`.

    AttributeError is easily get caught by `hasattr` and another
    ``except AttributeError`` clause.  This becomes problem when you use
    a lot of "dynamic" attributes (e.g., using ``@property``) because you
    can't distinguish if the property does not exist for real or some code
    inside of the "dynamic" attribute through that error.  In a well
    written code, such error should not exist but getting there is very
    difficult.  This decorator is to help us getting there by changing
    `AttributeError` to `UncaughtAttributeError` to avoid unexpected catch.
    This helps us noticing bugs earlier and facilitates debugging.

    .. note:: Treating StopIteration here is easy.
              Add that feature when needed.
    """
    @functools.wraps(func)
    def wrapper(*args, **kwds):
        try:
            return func(*args, **kwds)
        except AttributeError:
            exc_info = sys.exc_info()
            reraise(UncaughtAttributeError(exc_info[1]), exc_info[2])
    return wrapper

class PushBackIterator(object):
    def __init__(self, iterator):
        self.pushes = []
        self.iterator = iterator

    def push_back(self, value):
        self.pushes.append(value)

    def __iter__(self):
        return self

    def next(self):
        """ Python 2 Compatibility """
        return self.__next__()

    def __next__(self):
        if self.pushes:
            return self.pushes.pop()
        else:
            return next(self.iterator)

class NoErrorTokenizer(object):
    def __init__(self, readline, offset=(0, 0), stop_on_scope=False):
        self.readline = readline
        self.gen = PushBackIterator(tokenize.generate_tokens(readline))
        self.offset = offset
        self.stop_on_scope = stop_on_scope
        self.first_scope = False
        self.closed = False
        self.first = True

    def push_last_back(self):
        self.gen.push_back(self.current)

    def next(self):
        """ Python 2 Compatibility """
        return self.__next__()

    def __next__(self):
        if self.closed:
            raise MultiLevelStopIteration()
        try:
            self.current = next(self.gen)
        except tokenize.TokenError:
            # We just ignore this error, I try to handle it earlier - as
            # good as possible
            debug.warning('parentheses not closed error')
            return self.__next__()
        except IndentationError:
            # This is an error, that tokenize may produce, because the code
            # is not indented as it should. Here it just ignores this line
            # and restarts the parser.
            # (This is a rather unlikely error message, for normal code,
            # tokenize seems to be pretty tolerant)
            debug.warning('indentation error on line %s, ignoring it' %
                                                        self.current[2][0])
            # add the starting line of the last position
            self.offset = (self.offset[0] + self.current[2][0],
                           self.current[2][1])
            self.gen = PushBackIterator(tokenize.generate_tokens(
                                                                self.readline))
            return self.__next__()

        c = list(self.current)

        # stop if a new class or definition is started at position zero.
        breaks = ['def', 'class', '@']
        if self.stop_on_scope and c[1] in breaks and c[2][1] == 0:
            if self.first_scope:
                self.closed = True
                raise MultiLevelStopIteration()
            elif c[1] != '@':
                self.first_scope = True

        if self.first:
            c[2] = self.offset[0] + c[2][0], self.offset[1] + c[2][1]
            c[3] = self.offset[0] + c[3][0], self.offset[1] + c[3][1]
            self.first = False
        else:
            c[2] = self.offset[0] + c[2][0], c[2][1]
            c[3] = self.offset[0] + c[3][0], c[3][1]
        return c

@contextlib.contextmanager
def scale_speed_settings(factor):
    a = settings.max_executions
    b = settings.max_until_execution_unique
    settings.max_executions *= factor
    settings.max_until_execution_unique *= factor
    yield
    settings.max_executions = a
    settings.max_until_execution_unique = b

def indent_block(text, indention='    '):
    """ This function indents a text block with a default of four spaces """
    temp = ''
    while text and text[-1] == '\n':
        temp += text[-1]
        text = text[:-1]
    lines = text.split('\n')
    return '\n'.join(map(lambda s: indention + s, lines)) + temp

@contextlib.contextmanager
def ignored(*exceptions):
    """Context manager that ignores all of the specified exceptions. This will
    be in the standard library starting with Python 3.4."""
    try:
        yield
    except exceptions:
        pass
<commit_msg>removed more unused code<commit_after>""" A universal module with functions / classes without dependencies. """
import sys
import contextlib
import functools
import tokenizer as tokenize

from jedi._compatibility import next, reraise
from jedi import settings

class MultiLevelStopIteration(Exception):
    """
    StopIteration's get catched pretty easy by for loops, let errors propagate.
    """
    pass

class UncaughtAttributeError(Exception):
    """
    Important, because `__getattr__` and `hasattr` catch AttributeErrors
    implicitly. This is really evil (mainly because of `__getattr__`).
    `hasattr` in Python 2 is even more evil, because it catches ALL exceptions.
    Therefore this class originally had to be derived from `BaseException`
    instead of `Exception`.  But because I removed relevant `hasattr` from
    the code base, we can now switch back to `Exception`.

    :param base: return values of sys.exc_info().
    """

def rethrow_uncaught(func):
    """
    Re-throw uncaught `AttributeError`.

    Usage:  Put ``@rethrow_uncaught`` in front of the function
    which does **not** suppose to raise `AttributeError`.

    AttributeError is easily get caught by `hasattr` and another
    ``except AttributeError`` clause.  This becomes problem when you use
    a lot of "dynamic" attributes (e.g., using ``@property``) because you
    can't distinguish if the property does not exist for real or some code
    inside of the "dynamic" attribute through that error.  In a well
    written code, such error should not exist but getting there is very
    difficult.  This decorator is to help us getting there by changing
    `AttributeError` to `UncaughtAttributeError` to avoid unexpected catch.
    This helps us noticing bugs earlier and facilitates debugging.

    .. note:: Treating StopIteration here is easy.
              Add that feature when needed.
    """
    @functools.wraps(func)
    def wrapper(*args, **kwds):
        try:
            return func(*args, **kwds)
        except AttributeError:
            exc_info = sys.exc_info()
            reraise(UncaughtAttributeError(exc_info[1]), exc_info[2])
    return wrapper

class PushBackIterator(object):
    def __init__(self, iterator):
        self.pushes = []
        self.iterator = iterator

    def push_back(self, value):
        self.pushes.append(value)

    def __iter__(self):
        return self

    def next(self):
        """ Python 2 Compatibility """
        return self.__next__()

    def __next__(self):
        if self.pushes:
            return self.pushes.pop()
        else:
            return next(self.iterator)

class NoErrorTokenizer(object):
    def __init__(self, readline, offset=(0, 0), stop_on_scope=False):
        self.readline = readline
        self.gen = PushBackIterator(tokenize.generate_tokens(readline))
        self.offset = offset
        self.stop_on_scope = stop_on_scope
        self.first_scope = False
        self.closed = False
        self.first = True

    def push_last_back(self):
        self.gen.push_back(self.current)

    def next(self):
        """ Python 2 Compatibility """
        return self.__next__()

    def __next__(self):
        if self.closed:
            raise MultiLevelStopIteration()
        self.current = next(self.gen)
        c = list(self.current)

        # stop if a new class or definition is started at position zero.
        breaks = ['def', 'class', '@']
        if self.stop_on_scope and c[1] in breaks and c[2][1] == 0:
            if self.first_scope:
                self.closed = True
                raise MultiLevelStopIteration()
            elif c[1] != '@':
                self.first_scope = True

        if self.first:
            c[2] = self.offset[0] + c[2][0], self.offset[1] + c[2][1]
            c[3] = self.offset[0] + c[3][0], self.offset[1] + c[3][1]
            self.first = False
        else:
            c[2] = self.offset[0] + c[2][0], c[2][1]
            c[3] = self.offset[0] + c[3][0], c[3][1]
        return c

@contextlib.contextmanager
def scale_speed_settings(factor):
    a = settings.max_executions
    b = settings.max_until_execution_unique
    settings.max_executions *= factor
    settings.max_until_execution_unique *= factor
    yield
    settings.max_executions = a
    settings.max_until_execution_unique = b

def indent_block(text, indention='    '):
    """ This function indents a text block with a default of four spaces """
    temp = ''
    while text and text[-1] == '\n':
        temp += text[-1]
        text = text[:-1]
    lines = text.split('\n')
    return '\n'.join(map(lambda s: indention + s, lines)) + temp

@contextlib.contextmanager
def ignored(*exceptions):
    """Context manager that ignores all of the specified exceptions. This will
    be in the standard library starting with Python 3.4."""
    try:
        yield
    except exceptions:
        pass
<|endoftext|>


<commit_before>from json import load

from django_archive import __version__

from .base import BaseArchiveTestCase
from .sample.models import Sample

class ArchiveTestCase(BaseArchiveTestCase):
    """
    Test that the archive command includes correct data in the archive
    """

    def setUp(self):
        Sample().save()
        super().setUp()

    def test_data(self):
        """
        Confirm that the model was archived
        """
        with self.tarfile.extractfile('data.json') as fileobj:
            data = load(fileobj)
            self.assertEqual(len(data), 1)
            self.assertEqual(data[0]['model'], 'sample.sample')

    def test_meta(self):
        """
        Confirm that meta information is present
        """
        with self.tarfile.extractfile('meta.json') as fileobj:
            data = load(fileobj)
            self.assertEqual(data['version'], __version__)
<commit_msg>Update test to ensure attached files are present in archives.<commit_after>from json import load

from django.core.files.base import ContentFile
from django_archive import __version__

from .base import BaseArchiveTestCase
from .sample.models import Sample

class ArchiveTestCase(BaseArchiveTestCase):
    """
    Test that the archive command includes correct data in the archive
    """

    _ATTACHMENT_FILENAME = 'sample.txt'
    _ATTACHMENT_CONTENT = b'sample'

    def setUp(self):
        sample = Sample()
        sample.attachment.save(
            self._ATTACHMENT_FILENAME,
            ContentFile(self._ATTACHMENT_CONTENT),
        )
        super().setUp()

    def test_data(self):
        """
        Confirm that the model and attached files were archived
        """
        with self.tarfile.extractfile('data.json') as fileobj:
            data = load(fileobj)
        self.assertEqual(len(data), 1)
        self.assertEqual(data[0]['model'], 'sample.sample')
        with self.tarfile.extractfile(data[0]['fields']['attachment']) as fileobj:
            content = fileobj.read()
        self.assertEqual(content, self._ATTACHMENT_CONTENT)

    def test_meta(self):
        """
        Confirm that meta information is present
        """
        with self.tarfile.extractfile('meta.json') as fileobj:
            data = load(fileobj)
            self.assertEqual(data['version'], __version__)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import os.path

import mock

import pafy

import pytest

import vcr

from mopidy_youtube import backend
from mopidy_youtube.backend import YouTubeLibraryProvider

@pytest.yield_fixture
def pafy_mock():
    patcher = mock.patch.object(backend, 'pafy', spec=pafy)
    yield patcher.start()
    patcher.stop()

@pytest.fixture
def pafy_mock_with_video(pafy_mock):
    video_mock = pafy_mock.new.return_value
    video_mock.bigthumb = 'big thumb'
    video_mock.bigthumbhd = 'big thumb in hd'
    video_mock.getbestaudio.return_value.url = 'http://example.com/'
    video_mock.length = 2000
    video_mock.title = 'a title'
    video_mock.videoid = 'a video id'

    return pafy_mock

here = os.path.abspath(os.path.dirname(__file__))

my_vcr = vcr.VCR(
    serializer='yaml',
    cassette_library_dir=os.path.join(here, 'fixtures'),
    record_mode='once',
    match_on=['method', 'scheme', 'host', 'port', 'path', 'query'],
)

@my_vcr.use_cassette('youtube_playlist_resolve.yaml')
def test_playlist_resolver(pafy_mock_with_video):
    videos = backend.resolve_playlist('PLOxORm4jpOQfMU7bpfGCzDyLropIYEHuR')

    assert len(videos) == 108

@my_vcr.use_cassette('youtube_search.yaml')
def test_search_yt(pafy_mock_with_video):
    videos = backend.search_youtube('chvrches')

    assert len(videos) == 15

@my_vcr.use_cassette('resolve_track.yaml')
def test_resolve_track(pafy_mock_with_video):
    video = backend.resolve_track('C0DPdy98e4c')

    assert video

@my_vcr.use_cassette('resolve_track_failed.yaml')
def test_resolve_track_failed(pafy_mock):
    pafy_mock.new.side_effect = Exception('Removed')

    video = backend.resolve_track('unknown')

    assert not video

@my_vcr.use_cassette('resolve_track_stream.yaml')
def test_resolve_track_stream(pafy_mock_with_video):
    video = backend.resolve_track('C0DPdy98e4c', stream=True)

    assert video

@my_vcr.use_cassette('resolve_video_track_stream.yaml')
def test_resolve_video_track_stream(pafy_mock_with_video):
    video = backend.resolve_track('youtube:video/a title.a video id',
                                  stream=True)

    assert video
    assert video.uri == "http://example.com/"

@my_vcr.use_cassette('lookup_video_uri.yaml')
def test_lookup_video_uri(caplog):
    provider = YouTubeLibraryProvider(mock.PropertyMock())

    track = provider.lookup(backend.video_uri_prefix +
                            '/a title.C0DPdy98e4c')

    assert 'Need 11 character video id or the URL of the video.' \
           not in caplog.text()

    assert track

    assert track[0].uri == backend.video_uri_prefix + \
        '/TEST VIDEO.C0DPdy98e4c'
<commit_msg>add 'fake-api-key' to tests; fix caplog.text()<commit_after>from __future__ import unicode_literals

import os.path

import mock

import pafy

import pytest

import vcr

from mopidy_youtube import backend
from mopidy_youtube.backend import YouTubeLibraryProvider

@pytest.yield_fixture
def pafy_mock():
    patcher = mock.patch.object(backend, 'pafy', spec=pafy)
    yield patcher.start()
    patcher.stop()

@pytest.fixture
def pafy_mock_with_video(pafy_mock):
    video_mock = pafy_mock.new.return_value
    video_mock.bigthumb = 'big thumb'
    video_mock.bigthumbhd = 'big thumb in hd'
    video_mock.getbestaudio.return_value.url = 'http://example.com/'
    video_mock.length = 2000
    video_mock.title = 'a title'
    video_mock.videoid = 'a video id'

    return pafy_mock

here = os.path.abspath(os.path.dirname(__file__))

my_vcr = vcr.VCR(
    serializer='yaml',
    cassette_library_dir=os.path.join(here, 'fixtures'),
    record_mode='once',
    match_on=['method', 'scheme', 'host', 'port', 'path', 'query'],
)

@my_vcr.use_cassette('youtube_playlist_resolve.yaml')
def test_playlist_resolver(pafy_mock_with_video):
    videos = backend.resolve_playlist('PLOxORm4jpOQfMU7bpfGCzDyLropIYEHuR', 'fake-api-key')

    assert len(videos) == 108

@my_vcr.use_cassette('youtube_search.yaml')
def test_search_yt(pafy_mock_with_video):
    videos = backend.search_youtube('chvrches', 'fake-api-key')

    assert len(videos) == 15

@my_vcr.use_cassette('resolve_track.yaml')
def test_resolve_track(pafy_mock_with_video):
    video = backend.resolve_track('C0DPdy98e4c')

    assert video

@my_vcr.use_cassette('resolve_track_failed.yaml')
def test_resolve_track_failed(pafy_mock):
    pafy_mock.new.side_effect = Exception('Removed')

    video = backend.resolve_track('unknown')

    assert not video

@my_vcr.use_cassette('resolve_track_stream.yaml')
def test_resolve_track_stream(pafy_mock_with_video):
    video = backend.resolve_track('C0DPdy98e4c', stream=True)

    assert video

@my_vcr.use_cassette('resolve_video_track_stream.yaml')
def test_resolve_video_track_stream(pafy_mock_with_video):
    video = backend.resolve_track('youtube:video/a title.a video id',
                                  stream=True)

    assert video
    assert video.uri == "http://example.com/"

@my_vcr.use_cassette('lookup_video_uri.yaml')
def test_lookup_video_uri(caplog):
    provider = YouTubeLibraryProvider(mock.PropertyMock())

    track = provider.lookup(backend.video_uri_prefix +
                            '/a title.C0DPdy98e4c')

    assert 'Need 11 character video id or the URL of the video.' \
           not in caplog.text

    assert track

    assert track[0].uri == backend.video_uri_prefix + \
        '/TEST VIDEO.C0DPdy98e4c'
<|endoftext|>


<commit_before># -*- coding: utf-8  -*-
#
# Copyright (C) 2012-2013 Ben Kurtovic <ben.kurtovic@verizon.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import unicode_literals
import unittest

from mwparserfromhell.nodes import (Argument, Comment, Heading, HTMLEntity,
                                    Tag, Template, Text, Wikilink)
from mwparserfromhell.nodes.extras import Attribute, Parameter
from mwparserfromhell.parser import tokens
from mwparserfromhell.parser.builder import Builder
from mwparserfromhell.smart_list import SmartList
from mwparserfromhell.wikicode import Wikicode

from ._test_tree_equality import TreeEqualityTestCase

wrap = lambda L: Wikicode(SmartList(L))

class TestBuilder(TreeEqualityTestCase):
    """Tests for the builder, which turns tokens into Wikicode objects."""

    def setUp(self):
        self.builder = Builder()

    def test_text(self):
        """tests for building Text nodes"""
        tests = [
            ([tokens.Text(text="foobar")], wrap([Text("foobar")])),
            ([tokens.Text(text="fóóbar")], wrap([Text("fóóbar")])),
            ([tokens.Text(text="spam"), tokens.Text(text="eggs")],
             wrap([Text("spam"), Text("eggs")])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_template(self):
        """tests for building Template nodes"""
        tests = [
            ([tokens.TemplateOpen(), tokens.Text(text="foobar"),
              tokens.TemplateClose()],
             wrap([Template(wrap([Text("foobar")]))])),

            ([tokens.TemplateOpen(), tokens.Text(text="spam"),
              tokens.Text(text="eggs"), tokens.TemplateClose()],
             wrap([Template(wrap([Text("spam"), Text("eggs")]))])),

            ([tokens.TemplateOpen(), tokens.Text(text="foo"),
              tokens.TemplateParamSeparator(), tokens.Text(text="bar"),
              tokens.TemplateClose()],
             wrap([Template(wrap([Text("foo")]), params=[
                 Parameter(wrap([Text("1")]), wrap([Text("bar")]),
                           showkey=False)])])),

            ([tokens.TemplateOpen(), tokens.Text(text="foo"),
              tokens.TemplateParamSeparator(), tokens.Text(text="bar"),
              tokens.TemplateParamEquals(), tokens.Text(text="baz"),
              tokens.TemplateClose()],
             wrap([Template(wrap([Text("foo")]), params=[
                 Parameter(wrap([Text("bar")]), wrap([Text("baz")]))])])),

            ([tokens.TemplateOpen(), tokens.Text(text="foo"),
              tokens.TemplateParamSeparator(), tokens.Text(text="bar"),
              tokens.TemplateParamEquals(), tokens.Text(text="baz"),
              tokens.TemplateParamSeparator(), tokens.Text(text="biz"),
              tokens.TemplateParamSeparator(), tokens.Text(text="buzz"),
              tokens.TemplateParamSeparator(), tokens.Text(text="3"),
              tokens.TemplateParamEquals(), tokens.Text(text="buff"),
              tokens.TemplateParamSeparator(), tokens.Text(text="baff"),
              tokens.TemplateClose()],
             wrap([Template(wrap([Text("foo")]), params=[
                 Parameter(wrap([Text("bar")]), wrap([Text("baz")])),
                 Parameter(wrap([Text("1")]), wrap([Text("biz")]),
                           showkey=False),
                 Parameter(wrap([Text("2")]), wrap([Text("buzz")]),
                           showkey=False),
                 Parameter(wrap([Text("3")]), wrap([Text("buff")])),
                 Parameter(wrap([Text("3")]), wrap([Text("baff")]),
                           showkey=False)])])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_argument(self):
        """tests for building Argument nodes"""
        tests = [
            ([tokens.ArgumentOpen(), tokens.Text(text="foobar"),
              tokens.ArgumentClose()],
             wrap([Argument(wrap([Text("foobar")]))])),

            ([tokens.ArgumentOpen(), tokens.Text(text="spam"),
              tokens.Text(text="eggs"), tokens.ArgumentClose()],
             wrap([Argument(wrap([Text("spam"), Text("eggs")]))])),

            ([tokens.ArgumentOpen(), tokens.Text(text="foo"),
              tokens.ArgumentSeparator(), tokens.Text(text="bar"),
              tokens.ArgumentClose()],
             wrap([Argument(wrap([Text("foo")]), wrap([Text("bar")]))])),

            ([tokens.ArgumentOpen(), tokens.Text(text="foo"),
              tokens.Text(text="bar"), tokens.ArgumentSeparator(),
              tokens.Text(text="baz"), tokens.Text(text="biz"),
              tokens.ArgumentClose()],
             wrap([Argument(wrap([Text("foo"), Text("bar")]),
                            wrap([Text("baz"), Text("biz")]))])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_wikilink(self):
        """tests for building Wikilink nodes"""
        tests = [
            ([tokens.WikilinkOpen(), tokens.Text(text="foobar"),
              tokens.WikilinkClose()],
             wrap([Wikilink(wrap([Text("foobar")]))])),

            ([tokens.WikilinkOpen(), tokens.Text(text="spam"),
              tokens.Text(text="eggs"), tokens.WikilinkClose()],
             wrap([Wikilink(wrap([Text("spam"), Text("eggs")]))])),

            ([tokens.WikilinkOpen(), tokens.Text(text="foo"),
              tokens.WikilinkSeparator(), tokens.Text(text="bar"),
              tokens.WikilinkClose()],
             wrap([Wikilink(wrap([Text("foo")]), wrap([Text("bar")]))])),

            ([tokens.WikilinkOpen(), tokens.Text(text="foo"),
              tokens.Text(text="bar"), tokens.WikilinkSeparator(),
              tokens.Text(text="baz"), tokens.Text(text="biz"),
              tokens.WikilinkClose()],
             wrap([Wikilink(wrap([Text("foo"), Text("bar")]),
                            wrap([Text("baz"), Text("biz")]))])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_html_entity(self):
        """tests for building HTMLEntity nodes"""
        pass

    def test_heading(self):
        """tests for building Heading nodes"""
        pass

    def test_comment(self):
        """tests for building Comment nodes"""
        pass

    @unittest.skip("holding this until feature/html_tags is ready")
    def test_tag(self):
        """tests for building Tag nodes"""
        pass

if __name__ == "__main__":
    unittest.main(verbosity=2)
<commit_msg>test_html_entity()<commit_after># -*- coding: utf-8  -*-
#
# Copyright (C) 2012-2013 Ben Kurtovic <ben.kurtovic@verizon.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import unicode_literals
import unittest

from mwparserfromhell.nodes import (Argument, Comment, Heading, HTMLEntity,
                                    Tag, Template, Text, Wikilink)
from mwparserfromhell.nodes.extras import Attribute, Parameter
from mwparserfromhell.parser import tokens
from mwparserfromhell.parser.builder import Builder
from mwparserfromhell.smart_list import SmartList
from mwparserfromhell.wikicode import Wikicode

from ._test_tree_equality import TreeEqualityTestCase

wrap = lambda L: Wikicode(SmartList(L))

class TestBuilder(TreeEqualityTestCase):
    """Tests for the builder, which turns tokens into Wikicode objects."""

    def setUp(self):
        self.builder = Builder()

    def test_text(self):
        """tests for building Text nodes"""
        tests = [
            ([tokens.Text(text="foobar")], wrap([Text("foobar")])),
            ([tokens.Text(text="fóóbar")], wrap([Text("fóóbar")])),
            ([tokens.Text(text="spam"), tokens.Text(text="eggs")],
             wrap([Text("spam"), Text("eggs")])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_template(self):
        """tests for building Template nodes"""
        tests = [
            ([tokens.TemplateOpen(), tokens.Text(text="foobar"),
              tokens.TemplateClose()],
             wrap([Template(wrap([Text("foobar")]))])),

            ([tokens.TemplateOpen(), tokens.Text(text="spam"),
              tokens.Text(text="eggs"), tokens.TemplateClose()],
             wrap([Template(wrap([Text("spam"), Text("eggs")]))])),

            ([tokens.TemplateOpen(), tokens.Text(text="foo"),
              tokens.TemplateParamSeparator(), tokens.Text(text="bar"),
              tokens.TemplateClose()],
             wrap([Template(wrap([Text("foo")]), params=[
                 Parameter(wrap([Text("1")]), wrap([Text("bar")]),
                           showkey=False)])])),

            ([tokens.TemplateOpen(), tokens.Text(text="foo"),
              tokens.TemplateParamSeparator(), tokens.Text(text="bar"),
              tokens.TemplateParamEquals(), tokens.Text(text="baz"),
              tokens.TemplateClose()],
             wrap([Template(wrap([Text("foo")]), params=[
                 Parameter(wrap([Text("bar")]), wrap([Text("baz")]))])])),

            ([tokens.TemplateOpen(), tokens.Text(text="foo"),
              tokens.TemplateParamSeparator(), tokens.Text(text="bar"),
              tokens.TemplateParamEquals(), tokens.Text(text="baz"),
              tokens.TemplateParamSeparator(), tokens.Text(text="biz"),
              tokens.TemplateParamSeparator(), tokens.Text(text="buzz"),
              tokens.TemplateParamSeparator(), tokens.Text(text="3"),
              tokens.TemplateParamEquals(), tokens.Text(text="buff"),
              tokens.TemplateParamSeparator(), tokens.Text(text="baff"),
              tokens.TemplateClose()],
             wrap([Template(wrap([Text("foo")]), params=[
                 Parameter(wrap([Text("bar")]), wrap([Text("baz")])),
                 Parameter(wrap([Text("1")]), wrap([Text("biz")]),
                           showkey=False),
                 Parameter(wrap([Text("2")]), wrap([Text("buzz")]),
                           showkey=False),
                 Parameter(wrap([Text("3")]), wrap([Text("buff")])),
                 Parameter(wrap([Text("3")]), wrap([Text("baff")]),
                           showkey=False)])])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_argument(self):
        """tests for building Argument nodes"""
        tests = [
            ([tokens.ArgumentOpen(), tokens.Text(text="foobar"),
              tokens.ArgumentClose()],
             wrap([Argument(wrap([Text("foobar")]))])),

            ([tokens.ArgumentOpen(), tokens.Text(text="spam"),
              tokens.Text(text="eggs"), tokens.ArgumentClose()],
             wrap([Argument(wrap([Text("spam"), Text("eggs")]))])),

            ([tokens.ArgumentOpen(), tokens.Text(text="foo"),
              tokens.ArgumentSeparator(), tokens.Text(text="bar"),
              tokens.ArgumentClose()],
             wrap([Argument(wrap([Text("foo")]), wrap([Text("bar")]))])),

            ([tokens.ArgumentOpen(), tokens.Text(text="foo"),
              tokens.Text(text="bar"), tokens.ArgumentSeparator(),
              tokens.Text(text="baz"), tokens.Text(text="biz"),
              tokens.ArgumentClose()],
             wrap([Argument(wrap([Text("foo"), Text("bar")]),
                            wrap([Text("baz"), Text("biz")]))])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_wikilink(self):
        """tests for building Wikilink nodes"""
        tests = [
            ([tokens.WikilinkOpen(), tokens.Text(text="foobar"),
              tokens.WikilinkClose()],
             wrap([Wikilink(wrap([Text("foobar")]))])),

            ([tokens.WikilinkOpen(), tokens.Text(text="spam"),
              tokens.Text(text="eggs"), tokens.WikilinkClose()],
             wrap([Wikilink(wrap([Text("spam"), Text("eggs")]))])),

            ([tokens.WikilinkOpen(), tokens.Text(text="foo"),
              tokens.WikilinkSeparator(), tokens.Text(text="bar"),
              tokens.WikilinkClose()],
             wrap([Wikilink(wrap([Text("foo")]), wrap([Text("bar")]))])),

            ([tokens.WikilinkOpen(), tokens.Text(text="foo"),
              tokens.Text(text="bar"), tokens.WikilinkSeparator(),
              tokens.Text(text="baz"), tokens.Text(text="biz"),
              tokens.WikilinkClose()],
             wrap([Wikilink(wrap([Text("foo"), Text("bar")]),
                            wrap([Text("baz"), Text("biz")]))])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_html_entity(self):
        """tests for building HTMLEntity nodes"""
        tests = [
            ([tokens.HTMLEntityStart(), tokens.Text(text="nbsp"),
              tokens.HTMLEntityEnd()],
             wrap([HTMLEntity("nbsp", named=True, hexadecimal=False)])),

            ([tokens.HTMLEntityStart(), tokens.HTMLEntityNumeric(),
              tokens.Text(text="107"), tokens.HTMLEntityEnd()],
             wrap([HTMLEntity("107", named=False, hexadecimal=False)])),

            ([tokens.HTMLEntityStart(), tokens.HTMLEntityNumeric(),
              tokens.HTMLEntityHex(char="X"), tokens.Text(text="6B"),
              tokens.HTMLEntityEnd()],
             wrap([HTMLEntity("6B", named=False, hexadecimal=True,
                              hex_char="X")])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_heading(self):
        """tests for building Heading nodes"""
        pass

    def test_comment(self):
        """tests for building Comment nodes"""
        pass

    @unittest.skip("holding this until feature/html_tags is ready")
    def test_tag(self):
        """tests for building Tag nodes"""
        pass

if __name__ == "__main__":
    unittest.main(verbosity=2)
<|endoftext|>


<commit_before>import os
import logging

import nose.tools

import angr

from angr.analyses.cfg_fast import Segment, SegmentList

l = logging.getLogger("angr.tests.test_cfgfast")

test_location = str(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../binaries/tests'))

def cfg_fast(arch, binary_path, func_addrs):
    """
    Generate a CFGFast analysis result on the given binary, and test if all specified functions are found

    :param arch: the architecture, will be prepended to `binary_path`
    :param binary_path: path to the binary under the architecture directory
    :param func_addrs: A collection of function addresses that should be recovered
    :return: None
    """

    path = os.path.join(test_location, arch, binary_path)
    proj = angr.Project(path, load_options={'auto_load_libs': False})

    cfg = proj.analyses.CFGFast()
    nose.tools.assert_true(set(cfg.kb.functions.keys()).issuperset(func_addrs))

    # Segment only
    cfg = proj.analyses.CFGFast(force_segment=True)
    nose.tools.assert_true(set(cfg.kb.functions.keys()).issuperset(func_addrs))

def test_cfg_0():
    filename = 'cfg_0'
    functions = {
        'x86_64': {
            0x400410,
            0x400420,
            0x400430,
            0x400440,
            0x400470,
            0x40052c,
            0x40053c,
        }
    }
    arches = functions.keys()

    for arch in arches:
        yield cfg_fast, arch, filename, functions[arch]

def test_cfg_0_pe():
    filename = 'cfg_0_pe'
    functions = {
        'x86_64': {
            # 0x40150a,  # currently angr identifies 0x40150e due to the way _func_addrs_from_prologues() is
                         # implemented. this issue can be resolved with a properly implemented approach like Byte-Weight
            0x4014f0,
        }
    }
    arches = functions.keys()

    for arch in arches:
        yield cfg_fast, arch, filename, functions[arch]

def test_segment_list_0():
    seg_list = SegmentList()
    seg_list.occupy(0, 1, "code")
    seg_list.occupy(2, 3, "code")

    nose.tools.assert_equal(len(seg_list), 2)
    nose.tools.assert_equal(seg_list._list[0].end, 1)
    nose.tools.assert_equal(seg_list._list[1].end, 5)
    nose.tools.assert_equal(seg_list.is_occupied(4), True)
    nose.tools.assert_equal(seg_list.is_occupied(5), False)

def test_segment_list_1():
    seg_list = SegmentList()

    # They should be merged
    seg_list.occupy(0, 1, "code")
    seg_list.occupy(1, 2, "code")

    nose.tools.assert_equal(len(seg_list), 1)
    nose.tools.assert_equal(seg_list._list[0].start, 0)
    nose.tools.assert_equal(seg_list._list[0].end, 3)

def test_segment_list_2():
    seg_list = SegmentList()

    # They should not be merged
    seg_list.occupy(0, 1, "code")
    seg_list.occupy(1, 2, "data")

    nose.tools.assert_equal(len(seg_list), 2)
    nose.tools.assert_equal(seg_list._list[0].start, 0)
    nose.tools.assert_equal(seg_list._list[0].end, 1)
    nose.tools.assert_equal(seg_list._list[1].start, 1)
    nose.tools.assert_equal(seg_list._list[1].end, 3)

def test_segment_list_3():
    seg_list = SegmentList()

    # They should be merged, and create three different segments
    seg_list.occupy(0, 5, "code")
    seg_list.occupy(5, 5, "code")
    seg_list.occupy(1, 2, "data")

    nose.tools.assert_equal(len(seg_list), 3)

    nose.tools.assert_equal(seg_list._list[0].start, 0)
    nose.tools.assert_equal(seg_list._list[0].end, 1)
    nose.tools.assert_equal(seg_list._list[0].sort, "code")

    nose.tools.assert_equal(seg_list._list[1].start, 1)
    nose.tools.assert_equal(seg_list._list[1].end, 3)
    nose.tools.assert_equal(seg_list._list[1].sort, "data")

    nose.tools.assert_equal(seg_list._list[2].start, 3)
    nose.tools.assert_equal(seg_list._list[2].end, 10)
    nose.tools.assert_equal(seg_list._list[2].sort, "code")

def test_segment_list_4():
    seg_list = SegmentList()

    seg_list.occupy(5, 5, "code")
    seg_list.occupy(4, 1, "code")
    seg_list.occupy(2, 2, "code")

    nose.tools.assert_equal(len(seg_list), 1)
    nose.tools.assert_equal(seg_list._list[0].start, 2)
    nose.tools.assert_equal(seg_list._list[0].end, 10)

def test_segment_list_5():
    seg_list = SegmentList()

    seg_list.occupy(5, 5, "data")
    seg_list.occupy(4, 1, "code")
    seg_list.occupy(2, 2, "data")

    nose.tools.assert_equal(len(seg_list), 3)
    nose.tools.assert_equal(seg_list._list[0].start, 2)
    nose.tools.assert_equal(seg_list._list[2].end, 10)

    seg_list.occupy(3, 2, "data")

    nose.tools.assert_equal(len(seg_list), 1)
    nose.tools.assert_equal(seg_list._list[0].start, 2)
    nose.tools.assert_equal(seg_list._list[0].end, 10)

def main():

    g = globals()
    segmentlist_tests = [ v for k, v in g.iteritems() if k.startswith("test_segment_list_") and hasattr(v, "__call__")]

    for func in segmentlist_tests:
        func()

    for func, arch, filename, functions in test_cfg_0():
        func(arch, filename, functions)

    for func, arch, filename, functions in test_cfg_0_pe():
        func(arch, filename, functions)

if __name__ == "__main__":
    main()
<commit_msg>Add a test case for CFGFast on x86_64 fauxware<commit_after>import os
import logging

import nose.tools

import angr

from angr.analyses.cfg_fast import Segment, SegmentList

l = logging.getLogger("angr.tests.test_cfgfast")

test_location = str(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../binaries/tests'))

def cfg_fast(arch, binary_path, func_addrs):
    """
    Generate a CFGFast analysis result on the given binary, and test if all specified functions are found

    :param arch: the architecture, will be prepended to `binary_path`
    :param binary_path: path to the binary under the architecture directory
    :param func_addrs: A collection of function addresses that should be recovered
    :return: None
    """

    path = os.path.join(test_location, arch, binary_path)
    proj = angr.Project(path, load_options={'auto_load_libs': False})

    cfg = proj.analyses.CFGFast()
    nose.tools.assert_true(set(cfg.kb.functions.keys()).issuperset(func_addrs))

    # Segment only
    cfg = proj.analyses.CFGFast(force_segment=True)
    nose.tools.assert_true(set(cfg.kb.functions.keys()).issuperset(func_addrs))

def test_cfg_0():
    filename = 'cfg_0'
    functions = {
        'x86_64': {
            0x400410,
            0x400420,
            0x400430,
            0x400440,
            0x400470,
            0x40052c,
            0x40053c,
        }
    }
    arches = functions.keys()

    for arch in arches:
        yield cfg_fast, arch, filename, functions[arch]

def test_cfg_0_pe():
    filename = 'cfg_0_pe'
    functions = {
        'x86_64': {
            # 0x40150a,  # currently angr identifies 0x40150e due to the way _func_addrs_from_prologues() is
                         # implemented. this issue can be resolved with a properly implemented approach like Byte-Weight
            0x4014f0,
        }
    }
    arches = functions.keys()

    for arch in arches:
        yield cfg_fast, arch, filename, functions[arch]

def test_fauxware():
    filename = "fauxware"
    functions = {
        'x86_64': {
            0x4004e0,
            0x400510,
            0x400520,
            0x400530,
            0x400540,
            0x400550,
            0x400560,
            0x400570,  # .plt._exit
            0x400580,  # _start
            0x4005ac,
            0x4005d0,
            0x400640,
            0x400664,
            0x4006ed,
            0x4006fd,
            0x40071d,  # main
            0x4007e0,
            0x400870,
            0x400880,
            0x4008b8,
        }
    }

    arches = functions.keys()

    for arch in arches:
        yield cfg_fast, arch, filename, functions[arch]

def test_segment_list_0():
    seg_list = SegmentList()
    seg_list.occupy(0, 1, "code")
    seg_list.occupy(2, 3, "code")

    nose.tools.assert_equal(len(seg_list), 2)
    nose.tools.assert_equal(seg_list._list[0].end, 1)
    nose.tools.assert_equal(seg_list._list[1].end, 5)
    nose.tools.assert_equal(seg_list.is_occupied(4), True)
    nose.tools.assert_equal(seg_list.is_occupied(5), False)

def test_segment_list_1():
    seg_list = SegmentList()

    # They should be merged
    seg_list.occupy(0, 1, "code")
    seg_list.occupy(1, 2, "code")

    nose.tools.assert_equal(len(seg_list), 1)
    nose.tools.assert_equal(seg_list._list[0].start, 0)
    nose.tools.assert_equal(seg_list._list[0].end, 3)

def test_segment_list_2():
    seg_list = SegmentList()

    # They should not be merged
    seg_list.occupy(0, 1, "code")
    seg_list.occupy(1, 2, "data")

    nose.tools.assert_equal(len(seg_list), 2)
    nose.tools.assert_equal(seg_list._list[0].start, 0)
    nose.tools.assert_equal(seg_list._list[0].end, 1)
    nose.tools.assert_equal(seg_list._list[1].start, 1)
    nose.tools.assert_equal(seg_list._list[1].end, 3)

def test_segment_list_3():
    seg_list = SegmentList()

    # They should be merged, and create three different segments
    seg_list.occupy(0, 5, "code")
    seg_list.occupy(5, 5, "code")
    seg_list.occupy(1, 2, "data")

    nose.tools.assert_equal(len(seg_list), 3)

    nose.tools.assert_equal(seg_list._list[0].start, 0)
    nose.tools.assert_equal(seg_list._list[0].end, 1)
    nose.tools.assert_equal(seg_list._list[0].sort, "code")

    nose.tools.assert_equal(seg_list._list[1].start, 1)
    nose.tools.assert_equal(seg_list._list[1].end, 3)
    nose.tools.assert_equal(seg_list._list[1].sort, "data")

    nose.tools.assert_equal(seg_list._list[2].start, 3)
    nose.tools.assert_equal(seg_list._list[2].end, 10)
    nose.tools.assert_equal(seg_list._list[2].sort, "code")

def test_segment_list_4():
    seg_list = SegmentList()

    seg_list.occupy(5, 5, "code")
    seg_list.occupy(4, 1, "code")
    seg_list.occupy(2, 2, "code")

    nose.tools.assert_equal(len(seg_list), 1)
    nose.tools.assert_equal(seg_list._list[0].start, 2)
    nose.tools.assert_equal(seg_list._list[0].end, 10)

def test_segment_list_5():
    seg_list = SegmentList()

    seg_list.occupy(5, 5, "data")
    seg_list.occupy(4, 1, "code")
    seg_list.occupy(2, 2, "data")

    nose.tools.assert_equal(len(seg_list), 3)
    nose.tools.assert_equal(seg_list._list[0].start, 2)
    nose.tools.assert_equal(seg_list._list[2].end, 10)

    seg_list.occupy(3, 2, "data")

    nose.tools.assert_equal(len(seg_list), 1)
    nose.tools.assert_equal(seg_list._list[0].start, 2)
    nose.tools.assert_equal(seg_list._list[0].end, 10)

def main():

    g = globals()
    segmentlist_tests = [ v for k, v in g.iteritems() if k.startswith("test_segment_list_") and hasattr(v, "__call__")]

    for func in segmentlist_tests:
        func()

    for func, arch, filename, functions in test_cfg_0():
        func(arch, filename, functions)

    for func, arch, filename, functions in test_cfg_0_pe():
        func(arch, filename, functions)

    for func, arch, filename, functions in test_fauxware():
        func(arch, filename, functions)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import unittest

from csvkit.cleanup import *

class TestNormalizeType(unittest.TestCase):
    def test_fix_rows(self):
        """Test to ensure that row merging yields correct results"""
        start = [['1', '2', '3',],
                 [''],
                 ['abc'],
                 ['4', '5']
                ]
        fixed = join_rows(start)
        self.assertEqual(4,len(fixed))
        self.assertEqual(start[0][0],fixed[0])
        self.assertEqual(start[0][1],fixed[1])
        self.assertEqual("\n".join([start[0][-1], start[1][0], start[2][0], start[3][0]]),fixed[2])
        self.assertEqual(start[3][1],fixed[3])

    def test_fix_length_errors_basic(self):
        expected_length = 4
        errs = [LengthMismatch(1,['alpha','beta','gam'],expected_length)]
        errs.append(LengthMismatch(2,['ma','delta'],expected_length))
        fixed = fix_length_errors(errs,expected_length)
        self.assertEqual(1,len(fixed))
        fixed = fixed[0]
        self.assertEqual('alpha',fixed[0])
        self.assertEqual('beta',fixed[1])
        self.assertEqual('gam\nma',fixed[2])
        self.assertEqual('delta',fixed[3])

    def test_extract_joinable_row_errors(self):
        e1 = LengthMismatch(1,['foo', 'bar', 'baz'], 10)
        e2 = LengthMismatch(2,['foo', 'bar', 'baz'], 10)
        e3 = LengthMismatch(3,['foo', 'bar', 'baz'], 10)
        errs = [e1, e2, e3]
        joinable = extract_joinable_row_errors(errs)
        self.assertEqual(3,len(joinable))
        for e, j in zip(errs, joinable):
            self.assertTrue(e is j)

    def test_extract_joinable_row_errors_2(self):
        e1 = LengthMismatch(1,['foo', 'bar', 'baz'], 10)
        e2 = CSVTestException(2,['foo', 'bar', 'baz'], "A throwaway message.")
        e3 = LengthMismatch(3,['foo', 'bar', 'baz'], 10)
        errs = [e1, e2, e3]
        joinable = extract_joinable_row_errors(errs)
        self.assertEqual(1,len(joinable))
        self.assertTrue(iter(joinable).next() is e3)

    def test_extract_joinable_row_errors_3(self):
        e1 = CSVTestException(1,['foo', 'bar', 'baz'], "A throwaway message.")
        e2 = LengthMismatch(2,['foo', 'bar', 'baz'], 10)
        e3 = LengthMismatch(3,['foo', 'bar', 'baz'], 10)
        errs = [e1, e2, e3]
        joinable = extract_joinable_row_errors(errs)
        self.assertEqual(2,len(joinable))
        self.assertTrue(iter(joinable).next() is e2)
        self.assertTrue(iter(joinable).next() is e3)

    def test_extract_joinable_row_errors_4(self):
        e1 = CSVTestException(1,['foo', 'bar', 'baz'], "A throwaway message.")
        e2 = LengthMismatch(2,['foo', 'bar', 'baz'], 10)
        e3 = LengthMismatch(4,['foo', 'bar', 'baz'], 10)
        errs = [e1, e2, e3]
        joinable = extract_joinable_row_errors(errs)
        self.assertEqual(1,len(joinable))
        self.assertTrue(iter(joinable).next() is e3)

    def test_real_world_join_fail(self):
        start = [['168772', '1102', '$0.23 TO $0.72', 'HOUR', '1.5%'],
        ['GROSS', '1.5% '],
        ['GROSS', '430938']]
        fixed = join_rows(start)
        self.assertEqual(7,len(fixed))
        self.assertEqual(start[0][0],fixed[0])
        self.assertEqual(start[0][1],fixed[1])
        self.assertEqual(start[0][2],fixed[2])
        self.assertEqual(start[0][3],fixed[3])
        expected4 = "\n".join([start[0][-1], start[1][0]])
        self.assertEqual(expected4,fixed[4])
        expected5 = "\n".join([start[1][1],start[2][0]])
        self.assertEqual(expected5,fixed[5])
        self.assertEqual(start[2][1],fixed[6])

<commit_msg>fix test_extract_joinable_row_errors_3<commit_after>import unittest

from csvkit.cleanup import *

class TestNormalizeType(unittest.TestCase):
    def test_fix_rows(self):
        """Test to ensure that row merging yields correct results"""
        start = [['1', '2', '3',],
                 [''],
                 ['abc'],
                 ['4', '5']
                ]
        fixed = join_rows(start)
        self.assertEqual(4,len(fixed))
        self.assertEqual(start[0][0],fixed[0])
        self.assertEqual(start[0][1],fixed[1])
        self.assertEqual("\n".join([start[0][-1], start[1][0], start[2][0], start[3][0]]),fixed[2])
        self.assertEqual(start[3][1],fixed[3])

    def test_fix_length_errors_basic(self):
        expected_length = 4
        errs = [LengthMismatch(1,['alpha','beta','gam'],expected_length)]
        errs.append(LengthMismatch(2,['ma','delta'],expected_length))
        fixed = fix_length_errors(errs,expected_length)
        self.assertEqual(1,len(fixed))
        fixed = fixed[0]
        self.assertEqual('alpha',fixed[0])
        self.assertEqual('beta',fixed[1])
        self.assertEqual('gam\nma',fixed[2])
        self.assertEqual('delta',fixed[3])

    def test_extract_joinable_row_errors(self):
        e1 = LengthMismatch(1,['foo', 'bar', 'baz'], 10)
        e2 = LengthMismatch(2,['foo', 'bar', 'baz'], 10)
        e3 = LengthMismatch(3,['foo', 'bar', 'baz'], 10)
        errs = [e1, e2, e3]
        joinable = extract_joinable_row_errors(errs)
        self.assertEqual(3,len(joinable))
        for e, j in zip(errs, joinable):
            self.assertTrue(e is j)

    def test_extract_joinable_row_errors_2(self):
        e1 = LengthMismatch(1,['foo', 'bar', 'baz'], 10)
        e2 = CSVTestException(2,['foo', 'bar', 'baz'], "A throwaway message.")
        e3 = LengthMismatch(3,['foo', 'bar', 'baz'], 10)
        errs = [e1, e2, e3]
        joinable = extract_joinable_row_errors(errs)
        self.assertEqual(1,len(joinable))
        self.assertTrue(iter(joinable).next() is e3)

    def test_extract_joinable_row_errors_3(self):
        e1 = CSVTestException(1,['foo', 'bar', 'baz'], "A throwaway message.")
        e2 = LengthMismatch(2,['foo', 'bar', 'baz'], 10)
        e3 = LengthMismatch(3,['foo', 'bar', 'baz'], 10)
        errs = [e1, e2, e3]
        joinable = extract_joinable_row_errors(errs)
        self.assertEqual(2,len(joinable))
        joinable = list(joinable)
        self.assertTrue(joinable[0] is e2)
        self.assertTrue(joinable[1] is e3)

    def test_extract_joinable_row_errors_4(self):
        e1 = CSVTestException(1,['foo', 'bar', 'baz'], "A throwaway message.")
        e2 = LengthMismatch(2,['foo', 'bar', 'baz'], 10)
        e3 = LengthMismatch(4,['foo', 'bar', 'baz'], 10)
        errs = [e1, e2, e3]
        joinable = extract_joinable_row_errors(errs)
        self.assertEqual(1,len(joinable))
        self.assertTrue(iter(joinable).next() is e3)

    def test_real_world_join_fail(self):
        start = [['168772', '1102', '$0.23 TO $0.72', 'HOUR', '1.5%'],
        ['GROSS', '1.5% '],
        ['GROSS', '430938']]
        fixed = join_rows(start)
        self.assertEqual(7,len(fixed))
        self.assertEqual(start[0][0],fixed[0])
        self.assertEqual(start[0][1],fixed[1])
        self.assertEqual(start[0][2],fixed[2])
        self.assertEqual(start[0][3],fixed[3])
        expected4 = "\n".join([start[0][-1], start[1][0]])
        self.assertEqual(expected4,fixed[4])
        expected5 = "\n".join([start[1][1],start[2][0]])
        self.assertEqual(expected5,fixed[5])
        self.assertEqual(start[2][1],fixed[6])

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    Command line test
    ~~~~~~~~~~~~~~~~~

    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

from __future__ import print_function

import io
import os
import sys
import tempfile
import unittest

from pygments import highlight
from pygments.util import StringIO, BytesIO
from pygments.cmdline import main as cmdline_main

import support

TESTFILE, TESTDIR = support.location(__file__)

def run_cmdline(*args):
    saved_stdout = sys.stdout
    saved_stderr = sys.stderr
    if sys.version_info > (3,):
        stdout_buffer = BytesIO()
        stderr_buffer = BytesIO()
        new_stdout = sys.stdout = io.TextIOWrapper(stdout_buffer, 'utf-8')
        new_stderr = sys.stderr = io.TextIOWrapper(stderr_buffer, 'utf-8')
    else:
        stdout_buffer = new_stdout = sys.stdout = StringIO()
        stderr_buffer = new_stderr = sys.stderr = StringIO()
    try:
        ret = cmdline_main(["pygmentize"] + list(args))
    finally:
        sys.stdout = saved_stdout
        sys.stderr = saved_stderr
    new_stdout.flush()
    new_stderr.flush()
    out, err = stdout_buffer.getvalue().decode('utf-8'), \
        stderr_buffer.getvalue().decode('utf-8')
    return (ret, out, err)

def run_cmdline_with_closed_stdout(*args):
    saved_stdout, saved_stderr = sys.stdout, sys.stderr
    sys.stdout, sys.stderr = StringIO(), StringIO()
    sys.stdout.buffer = sys.stdout
    sys.stdout.close()
    try:
        ret = cmdline_main(['pygmentize'] + list(args))
        err = sys.stderr.getvalue()
    finally:
        sys.stdout, sys.stderr = saved_stdout, saved_stderr
    return ret, err

class CmdLineTest(unittest.TestCase):

    def test_L_opt(self):
        c, o, e = run_cmdline("-L")
        self.assertEqual(c, 0)
        self.assertTrue("Lexers" in o and "Formatters" in o and
                        "Filters" in o and "Styles" in o)
        c, o, e = run_cmdline("-L", "lexer")
        self.assertEqual(c, 0)
        self.assertTrue("Lexers" in o and "Formatters" not in o)
        c, o, e = run_cmdline("-L", "lexers")
        self.assertEqual(c, 0)

    def test_O_opt(self):
        filename = TESTFILE
        c, o, e = run_cmdline("-Ofull=1,linenos=true,foo=bar",
                              "-fhtml", filename)
        self.assertEqual(c, 0)
        self.assertTrue("<html" in o)
        self.assertTrue('class="linenos"' in o)

    def test_P_opt(self):
        filename = TESTFILE
        c, o, e = run_cmdline("-Pfull", "-Ptitle=foo, bar=baz=,",
                              "-fhtml", filename)
        self.assertEqual(c, 0)
        self.assertTrue("<title>foo, bar=baz=,</title>" in o)

    def test_F_opt(self):
        filename = TESTFILE
        c, o, e = run_cmdline("-Fhighlight:tokentype=Name.Blubb,"
                              "names=TESTFILE filename",
                              "-fhtml", filename)
        self.assertEqual(c, 0)
        self.assertTrue('<span class="n-Blubb' in o)

    def test_H_opt(self):
        c, o, e = run_cmdline("-H", "formatter", "html")
        self.assertEqual(c, 0)
        self.assertTrue('HTML' in o)

    def test_S_opt(self):
        c, o, e = run_cmdline("-S", "default", "-f", "html", "-O", "linenos=1")
        self.assertEqual(c, 0)

    def test_invalid_opts(self):
        for opts in [("-L", "-lpy"), ("-L", "-fhtml"), ("-L", "-Ox"),
                     ("-a",), ("-Sst", "-lpy"), ("-H",),
                     ("-H", "formatter")]:
            self.assertTrue(run_cmdline(*opts)[0] == 2)

    def test_normal(self):
        # test that cmdline gives the same output as library api
        from pygments.lexers import PythonLexer
        from pygments.formatters import HtmlFormatter
        filename = TESTFILE
        with open(filename, 'rb') as fp:
            code = fp.read()

        output = highlight(code, PythonLexer(), HtmlFormatter())

        c, o, e = run_cmdline("-lpython", "-fhtml", filename)

        self.assertEqual(o, output)
        self.assertEqual(e, "")
        self.assertEqual(c, 0)

    def test_outfile(self):
        # test that output file works with and without encoding
        fd, name = tempfile.mkstemp()
        os.close(fd)
        for opts in [['-fhtml', '-o', name, TESTFILE],
                     ['-flatex', '-o', name, TESTFILE],
                     ['-fhtml', '-o', name, '-O', 'encoding=utf-8', TESTFILE]]:
            try:
                self.assertEqual(run_cmdline(*opts)[0], 0)
            finally:
                os.unlink(name)

    def check_failure(self, *cmdline):
        c, o, e = run_cmdline(*cmdline)
        self.assertEqual(c, 1)
        self.assertEqual(o, '')
        return e

    def test_errors(self):
        # input file not found
        e = self.check_failure('-lpython', 'nonexistent.py')
        self.assertIn('Error: cannot read infile', e)
        self.assertIn('nonexistent.py', e)

        # lexer not found
        e = self.check_failure('-lfooo', TESTFILE)
        self.assertIn('Error: no lexer for alias', e)

        # formatter not found
        e = self.check_failure('-lpython', '-ffoo', TESTFILE)
        self.assertIn('Error: no formatter found for name', e)

        # output file not writable
        e = self.check_failure('-o', os.path.join('nonexistent', 'dir', 'out.html'),
                               '-lpython', TESTFILE)
        self.assertIn('Error: cannot open outfile', e)
        self.assertIn('out.html', e)

        # unknown filter
        e = self.check_failure('-F', 'foo', TESTFILE)
        self.assertIn('Error: filter \'foo\' not found', e)

    def test_exception(self):
        # unexpected exception while highlighting
        # (we can force that by closing stdout)
        c, e = run_cmdline_with_closed_stdout('-lpython', TESTFILE)
        self.assertEqual(c, 1)
        self.assertIn('*** Error while highlighting:', e)
<commit_msg>Fix use of test methods not in 2.6.<commit_after># -*- coding: utf-8 -*-
"""
    Command line test
    ~~~~~~~~~~~~~~~~~

    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

from __future__ import print_function

import io
import os
import sys
import tempfile
import unittest

from pygments import highlight
from pygments.util import StringIO, BytesIO
from pygments.cmdline import main as cmdline_main

import support

TESTFILE, TESTDIR = support.location(__file__)

def run_cmdline(*args):
    saved_stdout = sys.stdout
    saved_stderr = sys.stderr
    if sys.version_info > (3,):
        stdout_buffer = BytesIO()
        stderr_buffer = BytesIO()
        new_stdout = sys.stdout = io.TextIOWrapper(stdout_buffer, 'utf-8')
        new_stderr = sys.stderr = io.TextIOWrapper(stderr_buffer, 'utf-8')
    else:
        stdout_buffer = new_stdout = sys.stdout = StringIO()
        stderr_buffer = new_stderr = sys.stderr = StringIO()
    try:
        ret = cmdline_main(["pygmentize"] + list(args))
    finally:
        sys.stdout = saved_stdout
        sys.stderr = saved_stderr
    new_stdout.flush()
    new_stderr.flush()
    out, err = stdout_buffer.getvalue().decode('utf-8'), \
        stderr_buffer.getvalue().decode('utf-8')
    return (ret, out, err)

def run_cmdline_with_closed_stdout(*args):
    saved_stdout, saved_stderr = sys.stdout, sys.stderr
    sys.stdout, sys.stderr = StringIO(), StringIO()
    sys.stdout.buffer = sys.stdout
    sys.stdout.close()
    try:
        ret = cmdline_main(['pygmentize'] + list(args))
        err = sys.stderr.getvalue()
    finally:
        sys.stdout, sys.stderr = saved_stdout, saved_stderr
    return ret, err

class CmdLineTest(unittest.TestCase):

    def test_L_opt(self):
        c, o, e = run_cmdline("-L")
        self.assertEqual(c, 0)
        self.assertTrue("Lexers" in o and "Formatters" in o and
                        "Filters" in o and "Styles" in o)
        c, o, e = run_cmdline("-L", "lexer")
        self.assertEqual(c, 0)
        self.assertTrue("Lexers" in o and "Formatters" not in o)
        c, o, e = run_cmdline("-L", "lexers")
        self.assertEqual(c, 0)

    def test_O_opt(self):
        filename = TESTFILE
        c, o, e = run_cmdline("-Ofull=1,linenos=true,foo=bar",
                              "-fhtml", filename)
        self.assertEqual(c, 0)
        self.assertTrue("<html" in o)
        self.assertTrue('class="linenos"' in o)

    def test_P_opt(self):
        filename = TESTFILE
        c, o, e = run_cmdline("-Pfull", "-Ptitle=foo, bar=baz=,",
                              "-fhtml", filename)
        self.assertEqual(c, 0)
        self.assertTrue("<title>foo, bar=baz=,</title>" in o)

    def test_F_opt(self):
        filename = TESTFILE
        c, o, e = run_cmdline("-Fhighlight:tokentype=Name.Blubb,"
                              "names=TESTFILE filename",
                              "-fhtml", filename)
        self.assertEqual(c, 0)
        self.assertTrue('<span class="n-Blubb' in o)

    def test_H_opt(self):
        c, o, e = run_cmdline("-H", "formatter", "html")
        self.assertEqual(c, 0)
        self.assertTrue('HTML' in o)

    def test_S_opt(self):
        c, o, e = run_cmdline("-S", "default", "-f", "html", "-O", "linenos=1")
        self.assertEqual(c, 0)

    def test_invalid_opts(self):
        for opts in [("-L", "-lpy"), ("-L", "-fhtml"), ("-L", "-Ox"),
                     ("-a",), ("-Sst", "-lpy"), ("-H",),
                     ("-H", "formatter")]:
            self.assertTrue(run_cmdline(*opts)[0] == 2)

    def test_normal(self):
        # test that cmdline gives the same output as library api
        from pygments.lexers import PythonLexer
        from pygments.formatters import HtmlFormatter
        filename = TESTFILE
        with open(filename, 'rb') as fp:
            code = fp.read()

        output = highlight(code, PythonLexer(), HtmlFormatter())

        c, o, e = run_cmdline("-lpython", "-fhtml", filename)

        self.assertEqual(o, output)
        self.assertEqual(e, "")
        self.assertEqual(c, 0)

    def test_outfile(self):
        # test that output file works with and without encoding
        fd, name = tempfile.mkstemp()
        os.close(fd)
        for opts in [['-fhtml', '-o', name, TESTFILE],
                     ['-flatex', '-o', name, TESTFILE],
                     ['-fhtml', '-o', name, '-O', 'encoding=utf-8', TESTFILE]]:
            try:
                self.assertEqual(run_cmdline(*opts)[0], 0)
            finally:
                os.unlink(name)

    def check_failure(self, *cmdline):
        c, o, e = run_cmdline(*cmdline)
        self.assertEqual(c, 1)
        self.assertEqual(o, '')
        return e

    def test_errors(self):
        # input file not found
        e = self.check_failure('-lpython', 'nonexistent.py')
        self.assertTrue('Error: cannot read infile' in e)
        self.assertTrue('nonexistent.py' in e)

        # lexer not found
        e = self.check_failure('-lfooo', TESTFILE)
        self.assertTrue('Error: no lexer for alias' in e)

        # formatter not found
        e = self.check_failure('-lpython', '-ffoo', TESTFILE)
        self.assertTrue('Error: no formatter found for name' in e)

        # output file not writable
        e = self.check_failure('-o', os.path.join('nonexistent', 'dir', 'out.html'),
                               '-lpython', TESTFILE)
        self.assertTrue('Error: cannot open outfile' in e)
        self.assertTrue('out.html' in e)

        # unknown filter
        e = self.check_failure('-F', 'foo', TESTFILE)
        self.assertTrue('Error: filter \'foo\' not found' in e)

    def test_exception(self):
        # unexpected exception while highlighting
        # (we can force that by closing stdout)
        c, e = run_cmdline_with_closed_stdout('-lpython', TESTFILE)
        self.assertEqual(c, 1)
        self.assertTrue('*** Error while highlighting:' in e)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# -----------------------------------------------------------------------------
# Copyright (c) 2017 The Regents of the University of California
#
# This file is part of kevlar (http://github.com/standage/kevlar) and is
# licensed under the MIT license: see LICENSE.
# -----------------------------------------------------------------------------

import pytest
import sys
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO
import khmer
import kevlar

# @pytest.mark.parametrize('filename,creads,ckmers,ukmers,kmerinst', [
#     ('tests/data/trio1/novel_1_1,2.txt', 18, 684, 13, 171),
#     ('tests/data/trio1/novel_2_1,2.txt', 53, 2014, 39, 540),
#     ('tests/data/trio1/novel_3_1,2.txt', 178, 6764, 424, 5782),
#     ('tests/data/trio1/novel_4_1,2.txt', 16, 608, 11, 99),
#     ('tests/data/trio1/novel_5_3,4.txt', 16, 553, 9, 109),
#     ('tests/data/trio1/novel_6_5,6.txt', 9, 336, 2, 9),
# ])
# def test_load_input(filename, creads, ckmers, ukmers, kmerinst):
#     ng = khmer.Nodegraph(13, 3e5 / 4, 4)
#     vs = kevlar.VariantSet()
#     with open(filename, 'r') as infile:
#         nreads, nkmers = kevlar.collect.load_input(infile, ng, vs)
#     assert nreads == creads
#     assert nkmers == ckmers
#     assert vs.nkmers == ukmers
#     assert vs.nkmerinst == kmerinst

def test_load_all_inputs():
    cg = khmer.Countgraph(13, 2e6 / 4, 4)
    vs = kevlar.VariantSet()
    minabund = 8
    maxfpr = 0.2
    log = StringIO()

    batches = [3, 4, 5, 7, 8]
    filepattern = 'tests/data/trio1/novel_1_1,2_batch{:d}.txt'
    infilenames = [filepattern.format(batch) for batch in batches]

    kevlar.collect.load_all_inputs(infilenames, cg, vs, minabund, maxfpr, log)
    assert vs.nkmers == 6
    assert vs.nkmerinst == 49
    assert vs.nreads == 10

def test_load_all_inputs_alpha():
    cg = khmer.Countgraph(19, 4e4 / 4, 4)
    vs = kevlar.VariantSet()
    minabund = 8
    maxfpr = 0.2
    log = StringIO()
    infilenames = ['tests/data/collect.alpha.txt']

    kevlar.collect.load_all_inputs(infilenames, cg, vs, minabund, maxfpr, log)
    assert 'CAGGCCAGGGATCGCCGTG' not in vs.kmers and \
           kevlar.revcom('CAGGCCAGGGATCGCCGTG') not in vs.kmers

    kmers = ['TAGGGGCGTGACTTAATAA', 'AGGGGCGTGACTTAATAAG',
             'GGGGCGTGACTTAATAAGG', 'GGGCGTGACTTAATAAGGT']
    for kmer in kmers:
        assert kmer in vs.kmers or kevlar.revcom(kmer) in vs.kmers
<commit_msg>Added test for duplicated reads<commit_after>#!/usr/bin/env python
#
# -----------------------------------------------------------------------------
# Copyright (c) 2017 The Regents of the University of California
#
# This file is part of kevlar (http://github.com/standage/kevlar) and is
# licensed under the MIT license: see LICENSE.
# -----------------------------------------------------------------------------

import glob
import pytest
import sys
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO
import khmer
import kevlar

# @pytest.mark.parametrize('filename,creads,ckmers,ukmers,kmerinst', [
#     ('tests/data/trio1/novel_1_1,2.txt', 18, 684, 13, 171),
#     ('tests/data/trio1/novel_2_1,2.txt', 53, 2014, 39, 540),
#     ('tests/data/trio1/novel_3_1,2.txt', 178, 6764, 424, 5782),
#     ('tests/data/trio1/novel_4_1,2.txt', 16, 608, 11, 99),
#     ('tests/data/trio1/novel_5_3,4.txt', 16, 553, 9, 109),
#     ('tests/data/trio1/novel_6_5,6.txt', 9, 336, 2, 9),
# ])
# def test_load_input(filename, creads, ckmers, ukmers, kmerinst):
#     ng = khmer.Nodegraph(13, 3e5 / 4, 4)
#     vs = kevlar.VariantSet()
#     with open(filename, 'r') as infile:
#         nreads, nkmers = kevlar.collect.load_input(infile, ng, vs)
#     assert nreads == creads
#     assert nkmers == ckmers
#     assert vs.nkmers == ukmers
#     assert vs.nkmerinst == kmerinst

def test_load_all_inputs():
    cg = khmer.Countgraph(13, 2e6 / 4, 4)
    vs = kevlar.VariantSet()
    minabund = 8
    maxfpr = 0.2
    log = StringIO()

    batches = [3, 4, 5, 7, 8]
    filepattern = 'tests/data/trio1/novel_1_1,2_batch{:d}.txt'
    infilenames = [filepattern.format(batch) for batch in batches]

    kevlar.collect.load_all_inputs(infilenames, cg, vs, minabund, maxfpr, log)
    assert vs.nkmers == 6
    assert vs.nkmerinst == 49
    assert vs.nreads == 10

def test_load_all_inputs_alpha():
    cg = khmer.Countgraph(19, 4e4 / 4, 4)
    vs = kevlar.VariantSet()
    minabund = 8
    maxfpr = 0.2
    log = StringIO()
    infilenames = ['tests/data/collect.alpha.txt']

    kevlar.collect.load_all_inputs(infilenames, cg, vs, minabund, maxfpr, log)
    assert 'CAGGCCAGGGATCGCCGTG' not in vs.kmers and \
           kevlar.revcom('CAGGCCAGGGATCGCCGTG') not in vs.kmers

    kmers = ['TAGGGGCGTGACTTAATAA', 'AGGGGCGTGACTTAATAAG',
             'GGGGCGTGACTTAATAAGG', 'GGGCGTGACTTAATAAGGT']
    for kmer in kmers:
        assert kmer in vs.kmers or kevlar.revcom(kmer) in vs.kmers

def test_load_all_inputs_beta():
    cg = khmer.Countgraph(19, 4e4 / 4, 4)
    vs = kevlar.VariantSet()
    minabund = 8
    maxfpr = 0.2
    log = StringIO()
    infilenames = glob.glob('tests/data/collect.beta.?.txt')

    kevlar.collect.load_all_inputs(infilenames, cg, vs, minabund, maxfpr, log)
    assert vs.nreads == 8

    readseq = 'TTAACTCTAGATTAGGGGCGTGACTTAATAAGGTGTGGGCCTAAGCGTCT'
    for kmer in cg.get_kmers(readseq):
        assert cg.get(kmer) == 8
<|endoftext|>


<commit_before>import logging
from collections import namedtuple
from threading import local

LIBRARY_PATH = None
API_THREAD_LOCAL = local()
LOGGER = logging.getLogger('steampak.libsteam')

# Function argument which will contain additional function result.
# Expects ctypes type as a parameter.
ResultArg = namedtuple('ResultArg', ['ctype'])

class _EnumBase(object):
    """Enumeration base class."""

    aliases = {}

    @classmethod
    def get_alias(cls, item_id):
        """Returns item alias by ID.

        :param int item_id: Item ID.
        :rtype: str
        """
        return cls.aliases.get(item_id)

class _ApiResourceBase:
    """Base class for Steam API classes describing various resources
    (friends, stats, music, etc.).

    """

    def __init__(self, iface=None):  # todo remove none
        self.iface = iface

class FriendFilter(object):
    """Filters to be provided to functions returning friends.
    Can be combined using `|`.

    """
    NONE = 0x00
    BLOCKED = 0x01
    FRIENDSHIP_REQUESTED = 0x02
    IMMEDIATE = 0x04  # "REGULAR" FRIEND
    CLAN_MEMBER = 0x08
    ON_GAME_SERVER = 0x10
    HAS_PLAYED_WITH = 0x20  # NOT CURRENTLY USED
    FRIEND_OF_FRIEND = 0x40  # NOT CURRENTLY USED
    REQUESTING_FRIENDSHIP = 0x80
    REQUESTING_INFO = 0x100
    IGNORED = 0x200
    IGNORED_FRIEND = 0x400
    SUGGESTED = 0x800
    ALL = 0xFFFF
<commit_msg>Additional attributes for _ApiResourceBase.<commit_after>from collections import namedtuple

if False:  # pragma: nocover
    from ._wrapper import Client

# Function argument which will contain additional function result.
# Expects ctypes type as a parameter.
ResultArg = namedtuple('ResultArg', ['ctype'])

class _EnumBase:
    """Enumeration base class."""

    aliases = {}

    @classmethod
    def get_alias(cls, item_id):
        """Returns item alias by ID.

        :param int item_id: Item ID.
        :rtype: str
        """
        return cls.aliases.get(item_id)

class _ApiResourceBase:
    """Base class for Steam API classes describing various resources
    (friends, stats, music, etc.).

    """

    _iface = None
    _client = None  # type: Client

    def _contribute_internals(self, to, *, iface=None, client=None):
        to._iface = iface or self._iface
        to._client = client or self._client

class FriendFilter:
    """Filters to be provided to functions returning friends.
    Can be combined using `|`.

    """
    NONE = 0x00
    BLOCKED = 0x01
    FRIENDSHIP_REQUESTED = 0x02
    IMMEDIATE = 0x04  # "REGULAR" FRIEND
    CLAN_MEMBER = 0x08
    ON_GAME_SERVER = 0x10
    HAS_PLAYED_WITH = 0x20  # NOT CURRENTLY USED
    FRIEND_OF_FRIEND = 0x40  # NOT CURRENTLY USED
    REQUESTING_FRIENDSHIP = 0x80
    REQUESTING_INFO = 0x100
    IGNORED = 0x200
    IGNORED_FRIEND = 0x400
    SUGGESTED = 0x800
    CHAT_MEMBER = 0x1000
    ALL = 0xFFFF
<|endoftext|>


<commit_before>"""
Some Jupyter contrib extensions
"""

__version__ = '0.0.4'

from .py2to3exporter import Python2to3Exporter
from .mdoutputexporter import MarkdownOutputExporter
<commit_msg>Bump version to 0.0.5 to placate PyPI<commit_after>"""
Some Jupyter contrib extensions
"""

__version__ = '0.0.5'

from .py2to3exporter import Python2to3Exporter
from .mdoutputexporter import MarkdownOutputExporter
<|endoftext|>