<commit_before>#!/usr/bin/python
from types import StringTypes
from lxml import etree
from StringIO import StringIO
from sfa.util.faults import InvalidXML
from sfa.rspecs.elements.element import Element

# helper functions to help build xpaths
class XpathFilter:
    @staticmethod

    def filter_value(key, value):
        xpath = ""
        if isinstance(value, str):
            if '*' in value:
                value = value.replace('*', '')
                xpath = 'contains(%s, "%s")' % (key, value)
            else:
                xpath = '%s="%s"' % (key, value)
        return xpath

    @staticmethod
    def xpath(filter={}):
        xpath = ""
        if filter:
            filter_list = []
            for (key, value) in filter.items():
                if key == 'text':
                    key = 'text()'
                else:
                    key = '@'+key
                if isinstance(value, str):
                    filter_list.append(XpathFilter.filter_value(key, value))
                elif isinstance(value, list):
                    stmt = ' or '.join([XpathFilter.filter_value(key, str(val)) for val in value])
                    filter_list.append(stmt)
            if filter_list:
                xpath = ' and '.join(filter_list)
                xpath = '[' + xpath + ']'
        return xpath

# a wrapper class around lxml.etree._Element
# the reason why we need this one is because of the limitations
# we've found in xpath to address documents with multiple namespaces defined
# in a nutshell, we deal with xml documents that have
# a default namespace defined (xmlns="http://default.com/") and specific prefixes defined
# (xmlns:foo="http://foo.com")
# according to the documentation instead of writing
# element.xpath ( "//node/foo:subnode" )
# we'd then need to write xpaths like
# element.xpath ( "//{http://default.com/}node/{http://foo.com}subnode" )
# which is a real pain..
# So just so we can keep some reasonable programming style we need to manage the
# namespace map that goes with the _Element (its internal .nsmap being unmutable)

class XmlElement:
    def __init__(self, element, namespaces):
        self.element = element
        self.namespaces = namespaces

    # redefine as few methods as possible
    def xpath(self, xpath, namespaces=None):
        if not namespaces:
            namespaces = self.namespaces
        elems = self.element.xpath(xpath, namespaces=namespaces)
        return [XmlElement(elem, namespaces) for elem in elems]

    def add_element(self, tagname, **kwds):
        element = etree.SubElement(self.element, tagname, **kwds)
        return XmlElement(element, self.namespaces)

    def append(self, elem):
        if isinstance(elem, XmlElement):
            self.element.append(elem.element)
        else:
            self.element.append(elem)

    def getparent(self):
        return XmlElement(self.element.getparent(), self.namespaces)

    def get_instance(self, instance_class=None, fields=[]):
        """
        Returns an instance (dict) of this xml element. The instance
        holds a reference to this xml element.
        """
        if not instance_class:
            instance_class = Element
        if not fields and hasattr(instance_class, 'fields'):
            fields = instance_class.fields

        if not fields:
            instance = instance_class(self.attrib, self)
        else:
            instance = instance_class({}, self)
            for field in fields:
                if field in self.attrib:
                   instance[field] = self.attrib[field]
        return instance

    def add_instance(self, name, instance, fields=[]):
        """
        Adds the specifed instance(s) as a child element of this xml
        element.
        """
        if not fields and hasattr(instance, 'keys'):
            fields = instance.keys()
        elem = self.add_element(name)
        for field in fields:
            if field in instance and instance[field]:
                elem.set(field, unicode(instance[field]))
        return elem

    def remove_elements(self, name):
        """
        Removes all occurences of an element from the tree. Start at
        specified root_node if specified, otherwise start at tree's root.
        """

        if not element_name.startswith('//'):
            element_name = '//' + element_name
        elements = self.element.xpath('%s ' % name, namespaces=self.namespaces)
        for element in elements:
            parent = element.getparent()
            parent.remove(element)

    def delete(self):
        parent = self.getparent()
        parent.remove(self)

    def remove(self, element):
        if isinstance(element, XmlElement):
            self.element.remove(element.element)
        else:
            self.element.remove(element)

    def set_text(self, text):
        self.element.text = text

    # Element does not have unset ?!?
    def unset(self, key):
        del self.element.attrib[key]

    def toxml(self):
        return etree.tostring(self.element, encoding='UTF-8', pretty_print=True)

    def __str__(self):
        return self.toxml()

    ### other method calls or attribute access like .text or .tag or .get
    # are redirected on self.element
    def __getattr__ (self, name):
        if not hasattr(self.element, name):
            raise AttributeError, name
        return getattr(self.element, name)

class XML:

    def __init__(self, xml=None, namespaces=None):
        self.root = None
        self.namespaces = namespaces
        self.default_namespace = None
        self.schema = None
        if isinstance(xml, basestring):
            self.parse_xml(xml)
        if isinstance(xml, XmlElement):
            self.root = xml
            self.namespaces = xml.namespaces
        elif isinstance(xml, etree._ElementTree) or isinstance(xml, etree._Element):
            self.parse_xml(etree.tostring(xml))

    def parse_xml(self, xml):
        """
        parse rspec into etree
        """
        parser = etree.XMLParser(remove_blank_text=True)
        try:
            tree = etree.parse(xml, parser)
        except IOError:
            # 'rspec' file doesnt exist. 'rspec' is proably an xml string
            try:
                tree = etree.parse(StringIO(xml), parser)
            except Exception, e:
                raise InvalidXML(str(e))
        root = tree.getroot()
        self.namespaces = dict(root.nsmap)
        # set namespaces map
        if 'default' not in self.namespaces and None in self.namespaces:
            # If the 'None' exist, then it's pointing to the default namespace. This makes
            # it hard for us to write xpath queries for the default naemspace because lxml
            # wont understand a None prefix. We will just associate the default namespeace
            # with a key named 'default'.
            self.namespaces['default'] = self.namespaces.pop(None)

        else:
            self.namespaces['default'] = 'default'

        self.root = XmlElement(root, self.namespaces)
        # set schema
        for key in self.root.attrib.keys():
            if key.endswith('schemaLocation'):
                # schema location should be at the end of the list
                schema_parts  = self.root.attrib[key].split(' ')
                self.schema = schema_parts[1]
                namespace, schema  = schema_parts[0], schema_parts[1]
                break

    def parse_dict(self, d, root_tag_name='xml', element = None):
        if element is None:
            if self.root is None:
                self.parse_xml('<%s/>' % root_tag_name)
            element = self.root

        if 'text' in d:
            text = d.pop('text')
            element.text = text

        # handle repeating fields
        for (key, value) in d.items():
            if isinstance(value, list):
                value = d.pop(key)
                for val in value:
                    if isinstance(val, dict):
                        child_element = etree.SubElement(element, key)
                        self.parse_dict(val, key, child_element)
                    elif isinstance(val, basestring):
                        child_element = etree.SubElement(element, key).text = val

            elif isinstance(value, int):
                d[key] = unicode(d[key])
            elif value is None:
                d.pop(key)

        # element.attrib.update will explode if DateTimes are in the
        # dcitionary.
        d=d.copy()
        # looks like iteritems won't stand side-effects
        for k in d.keys():
            if not isinstance(d[k],StringTypes):
                del d[k]

        element.attrib.update(d)

    def validate(self, schema):
        """
        Validate against rng schema
        """
        relaxng_doc = etree.parse(schema)
        relaxng = etree.RelaxNG(relaxng_doc)
        if not relaxng(self.root):
            error = relaxng.error_log.last_error
            message = "%s (line %s)" % (error.message, error.line)
            raise InvalidXML(message)
        return True

    def xpath(self, xpath, namespaces=None):
        if not namespaces:
            namespaces = self.namespaces
        return self.root.xpath(xpath, namespaces=namespaces)

    def set(self, key, value):
        return self.root.set(key, value)

    def remove_attribute(self, name, element=None):
        if not element:
            element = self.root
        element.remove_attribute(name)

    def add_element(self, *args, **kwds):
        """
        Wrapper around etree.SubElement(). Adds an element to
        specified parent node. Adds element to root node is parent is
        not specified.
        """
        return self.root.add_element(*args, **kwds)

    def remove_elements(self, name, element = None):
        """
        Removes all occurences of an element from the tree. Start at
        specified root_node if specified, otherwise start at tree's root.
        """
        if not element:
            element = self.root

        element.remove_elements(name)

    def add_instance(self, *args, **kwds):
        return self.root.add_instance(*args, **kwds)

    def get_instance(self, *args, **kwds):
        return self.root.get_instnace(*args, **kwds)

    def get_element_attributes(self, elem=None, depth=0):
        if elem == None:
            elem = self.root
        if not hasattr(elem, 'attrib'):
            # this is probably not an element node with attribute. could be just and an
            # attribute, return it
            return elem
        attrs = dict(elem.attrib)
        attrs['text'] = str(elem.text).strip()
        attrs['parent'] = elem.getparent()
        if isinstance(depth, int) and depth > 0:
            for child_elem in list(elem):
                key = str(child_elem.tag)
                if key not in attrs:
                    attrs[key] = [self.get_element_attributes(child_elem, depth-1)]
                else:
                    attrs[key].append(self.get_element_attributes(child_elem, depth-1))
        else:
            attrs['child_nodes'] = list(elem)
        return attrs

    def append(self, elem):
        return self.root.append(elem)

    def iterchildren(self):
        return self.root.iterchildren()

    def merge(self, in_xml):
        pass

    def __str__(self):
        return self.toxml()

    def toxml(self):
        return etree.tostring(self.root.element, encoding='UTF-8', pretty_print=True)

    # XXX smbaker, for record.load_from_string
    def todict(self, elem=None):
        if elem is None:
            elem = self.root
        d = {}
        d.update(elem.attrib)
        d['text'] = elem.text
        for child in elem.iterchildren():
            if child.tag not in d:
                d[child.tag] = []
            d[child.tag].append(self.todict(child))

        if len(d)==1 and ("text" in d):
            d = d["text"]

        return d

    def save(self, filename):
        f = open(filename, 'w')
        f.write(self.toxml())
        f.close()

# no RSpec in scope
#if __name__ == '__main__':
#    rspec = RSpec('/tmp/resources.rspec')
#    print rspec

<commit_msg>fix typeerror when calling xml.parse_dict<commit_after>#!/usr/bin/python
from types import StringTypes
from lxml import etree
from StringIO import StringIO
from sfa.util.faults import InvalidXML
from sfa.rspecs.elements.element import Element

# helper functions to help build xpaths
class XpathFilter:
    @staticmethod

    def filter_value(key, value):
        xpath = ""
        if isinstance(value, str):
            if '*' in value:
                value = value.replace('*', '')
                xpath = 'contains(%s, "%s")' % (key, value)
            else:
                xpath = '%s="%s"' % (key, value)
        return xpath

    @staticmethod
    def xpath(filter={}):
        xpath = ""
        if filter:
            filter_list = []
            for (key, value) in filter.items():
                if key == 'text':
                    key = 'text()'
                else:
                    key = '@'+key
                if isinstance(value, str):
                    filter_list.append(XpathFilter.filter_value(key, value))
                elif isinstance(value, list):
                    stmt = ' or '.join([XpathFilter.filter_value(key, str(val)) for val in value])
                    filter_list.append(stmt)
            if filter_list:
                xpath = ' and '.join(filter_list)
                xpath = '[' + xpath + ']'
        return xpath

# a wrapper class around lxml.etree._Element
# the reason why we need this one is because of the limitations
# we've found in xpath to address documents with multiple namespaces defined
# in a nutshell, we deal with xml documents that have
# a default namespace defined (xmlns="http://default.com/") and specific prefixes defined
# (xmlns:foo="http://foo.com")
# according to the documentation instead of writing
# element.xpath ( "//node/foo:subnode" )
# we'd then need to write xpaths like
# element.xpath ( "//{http://default.com/}node/{http://foo.com}subnode" )
# which is a real pain..
# So just so we can keep some reasonable programming style we need to manage the
# namespace map that goes with the _Element (its internal .nsmap being unmutable)

class XmlElement:
    def __init__(self, element, namespaces):
        self.element = element
        self.namespaces = namespaces

    # redefine as few methods as possible
    def xpath(self, xpath, namespaces=None):
        if not namespaces:
            namespaces = self.namespaces
        elems = self.element.xpath(xpath, namespaces=namespaces)
        return [XmlElement(elem, namespaces) for elem in elems]

    def add_element(self, tagname, **kwds):
        element = etree.SubElement(self.element, tagname, **kwds)
        return XmlElement(element, self.namespaces)

    def append(self, elem):
        if isinstance(elem, XmlElement):
            self.element.append(elem.element)
        else:
            self.element.append(elem)

    def getparent(self):
        return XmlElement(self.element.getparent(), self.namespaces)

    def get_instance(self, instance_class=None, fields=[]):
        """
        Returns an instance (dict) of this xml element. The instance
        holds a reference to this xml element.
        """
        if not instance_class:
            instance_class = Element
        if not fields and hasattr(instance_class, 'fields'):
            fields = instance_class.fields

        if not fields:
            instance = instance_class(self.attrib, self)
        else:
            instance = instance_class({}, self)
            for field in fields:
                if field in self.attrib:
                   instance[field] = self.attrib[field]
        return instance

    def add_instance(self, name, instance, fields=[]):
        """
        Adds the specifed instance(s) as a child element of this xml
        element.
        """
        if not fields and hasattr(instance, 'keys'):
            fields = instance.keys()
        elem = self.add_element(name)
        for field in fields:
            if field in instance and instance[field]:
                elem.set(field, unicode(instance[field]))
        return elem

    def remove_elements(self, name):
        """
        Removes all occurences of an element from the tree. Start at
        specified root_node if specified, otherwise start at tree's root.
        """

        if not element_name.startswith('//'):
            element_name = '//' + element_name
        elements = self.element.xpath('%s ' % name, namespaces=self.namespaces)
        for element in elements:
            parent = element.getparent()
            parent.remove(element)

    def delete(self):
        parent = self.getparent()
        parent.remove(self)

    def remove(self, element):
        if isinstance(element, XmlElement):
            self.element.remove(element.element)
        else:
            self.element.remove(element)

    def set_text(self, text):
        self.element.text = text

    # Element does not have unset ?!?
    def unset(self, key):
        del self.element.attrib[key]

    def toxml(self):
        return etree.tostring(self.element, encoding='UTF-8', pretty_print=True)

    def __str__(self):
        return self.toxml()

    ### other method calls or attribute access like .text or .tag or .get
    # are redirected on self.element
    def __getattr__ (self, name):
        if not hasattr(self.element, name):
            raise AttributeError, name
        return getattr(self.element, name)

class XML:

    def __init__(self, xml=None, namespaces=None):
        self.root = None
        self.namespaces = namespaces
        self.default_namespace = None
        self.schema = None
        if isinstance(xml, basestring):
            self.parse_xml(xml)
        if isinstance(xml, XmlElement):
            self.root = xml
            self.namespaces = xml.namespaces
        elif isinstance(xml, etree._ElementTree) or isinstance(xml, etree._Element):
            self.parse_xml(etree.tostring(xml))

    def parse_xml(self, xml):
        """
        parse rspec into etree
        """
        parser = etree.XMLParser(remove_blank_text=True)
        try:
            tree = etree.parse(xml, parser)
        except IOError:
            # 'rspec' file doesnt exist. 'rspec' is proably an xml string
            try:
                tree = etree.parse(StringIO(xml), parser)
            except Exception, e:
                raise InvalidXML(str(e))
        root = tree.getroot()
        self.namespaces = dict(root.nsmap)
        # set namespaces map
        if 'default' not in self.namespaces and None in self.namespaces:
            # If the 'None' exist, then it's pointing to the default namespace. This makes
            # it hard for us to write xpath queries for the default naemspace because lxml
            # wont understand a None prefix. We will just associate the default namespeace
            # with a key named 'default'.
            self.namespaces['default'] = self.namespaces.pop(None)

        else:
            self.namespaces['default'] = 'default'

        self.root = XmlElement(root, self.namespaces)
        # set schema
        for key in self.root.attrib.keys():
            if key.endswith('schemaLocation'):
                # schema location should be at the end of the list
                schema_parts  = self.root.attrib[key].split(' ')
                self.schema = schema_parts[1]
                namespace, schema  = schema_parts[0], schema_parts[1]
                break

    def parse_dict(self, d, root_tag_name='xml', element = None):
        if element is None:
            if self.root is None:
                self.parse_xml('<%s/>' % root_tag_name)
            element = self.root.element

        if 'text' in d:
            text = d.pop('text')
            element.text = text

        # handle repeating fields
        for (key, value) in d.items():
            if isinstance(value, list):
                value = d.pop(key)
                for val in value:
                    if isinstance(val, dict):
                        child_element = etree.SubElement(element, key)
                        self.parse_dict(val, key, child_element)
                    elif isinstance(val, basestring):
                        child_element = etree.SubElement(element, key).text = val

            elif isinstance(value, int):
                d[key] = unicode(d[key])
            elif value is None:
                d.pop(key)

        # element.attrib.update will explode if DateTimes are in the
        # dcitionary.
        d=d.copy()
        # looks like iteritems won't stand side-effects
        for k in d.keys():
            if not isinstance(d[k],StringTypes):
                del d[k]

        element.attrib.update(d)

    def validate(self, schema):
        """
        Validate against rng schema
        """
        relaxng_doc = etree.parse(schema)
        relaxng = etree.RelaxNG(relaxng_doc)
        if not relaxng(self.root):
            error = relaxng.error_log.last_error
            message = "%s (line %s)" % (error.message, error.line)
            raise InvalidXML(message)
        return True

    def xpath(self, xpath, namespaces=None):
        if not namespaces:
            namespaces = self.namespaces
        return self.root.xpath(xpath, namespaces=namespaces)

    def set(self, key, value):
        return self.root.set(key, value)

    def remove_attribute(self, name, element=None):
        if not element:
            element = self.root
        element.remove_attribute(name)

    def add_element(self, *args, **kwds):
        """
        Wrapper around etree.SubElement(). Adds an element to
        specified parent node. Adds element to root node is parent is
        not specified.
        """
        return self.root.add_element(*args, **kwds)

    def remove_elements(self, name, element = None):
        """
        Removes all occurences of an element from the tree. Start at
        specified root_node if specified, otherwise start at tree's root.
        """
        if not element:
            element = self.root

        element.remove_elements(name)

    def add_instance(self, *args, **kwds):
        return self.root.add_instance(*args, **kwds)

    def get_instance(self, *args, **kwds):
        return self.root.get_instnace(*args, **kwds)

    def get_element_attributes(self, elem=None, depth=0):
        if elem == None:
            elem = self.root
        if not hasattr(elem, 'attrib'):
            # this is probably not an element node with attribute. could be just and an
            # attribute, return it
            return elem
        attrs = dict(elem.attrib)
        attrs['text'] = str(elem.text).strip()
        attrs['parent'] = elem.getparent()
        if isinstance(depth, int) and depth > 0:
            for child_elem in list(elem):
                key = str(child_elem.tag)
                if key not in attrs:
                    attrs[key] = [self.get_element_attributes(child_elem, depth-1)]
                else:
                    attrs[key].append(self.get_element_attributes(child_elem, depth-1))
        else:
            attrs['child_nodes'] = list(elem)
        return attrs

    def append(self, elem):
        return self.root.append(elem)

    def iterchildren(self):
        return self.root.iterchildren()

    def merge(self, in_xml):
        pass

    def __str__(self):
        return self.toxml()

    def toxml(self):
        return etree.tostring(self.root.element, encoding='UTF-8', pretty_print=True)

    # XXX smbaker, for record.load_from_string
    def todict(self, elem=None):
        if elem is None:
            elem = self.root
        d = {}
        d.update(elem.attrib)
        d['text'] = elem.text
        for child in elem.iterchildren():
            if child.tag not in d:
                d[child.tag] = []
            d[child.tag].append(self.todict(child))

        if len(d)==1 and ("text" in d):
            d = d["text"]

        return d

    def save(self, filename):
        f = open(filename, 'w')
        f.write(self.toxml())
        f.close()

# no RSpec in scope
#if __name__ == '__main__':
#    rspec = RSpec('/tmp/resources.rspec')
#    print rspec

<|endoftext|>


<commit_before>from __future__ import division
import math
import torch
import torch.optim as optim

sample_lr =[0.0001, 9e-05, 8.1e-05, 7.290000000000001e-05, 6.561000000000002e-05, 5.904900000000002e-05, 5.314410000000002e-05, 4.782969000000002e-05, 4.304672100000002e-05, 3.874204890000002e-05, 3.4867844010000016e-05, 3.138105960900002e-05, 2.8242953648100018e-05, 2.5418658283290016e-05, 2.2876792454961016e-05, 2.0589113209464913e-05, 1.8530201888518422e-05, 1.667718169966658e-05, 1.5009463529699922e-05, 1.350851717672993e-05, 1.2157665459056937e-05]

class SharedRMSprop(optim.RMSprop):
    """Implements RMSprop algorithm with shared states.
    """

    def __init__(self,
                 params,
                 lr=1e-2,
                 alpha=0.99,
                 eps=1e-8,
                 weight_decay=0,
                 momentum=0,
                 centered=False):
        super(SharedRMSprop, self).__init__(params, lr, alpha, eps,
                                            weight_decay, momentum, centered)

        for group in self.param_groups:
            for p in group['params']:
                state = self.state[p]
                state['step'] = torch.zeros(1)
                state['grad_avg'] = p.data.new().resize_as_(p.data).zero_()
                state['square_avg'] = p.data.new().resize_as_(p.data).zero_()
                state['momentum_buffer'] = p.data.new().resize_as_(
                    p.data).zero_()

    def share_memory(self):
        for group in self.param_groups:
            for p in group['params']:
                state = self.state[p]
                state['square_avg'].share_memory_()
                state['step'].share_memory_()
                state['grad_avg'].share_memory_()

    def step(self, closure=None):
        """Performs a single optimization step.
        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        """
        loss = None
        if closure is not None:
            loss = closure()

        for group in self.param_groups:
            for p in group['params']:
                if p.grad is None:
                    continue
                grad = p.grad.data
                state = self.state[p]

                square_avg = state['square_avg']
                alpha = group['alpha']

                state['step'] += 1

                if group['weight_decay'] != 0:
                    grad = grad.add(group['weight_decay'], p.data)

                square_avg.mul_(alpha).addcmul_(1 - alpha, grad, grad)

                if group['centered']:
                    grad_avg = state['grad_avg']
                    grad_avg.mul_(alpha).add_(1 - alpha, grad)
                    avg = square_avg.addcmul(
                        -1, grad_avg, grad_avg).sqrt().add_(group['eps'])
                else:
                    avg = square_avg.sqrt().add_(group['eps'])

                if group['momentum'] > 0:
                    buf = state['momentum_buffer']
                    buf.mul_(group['momentum']).addcdiv_(grad, avg)
                    p.data.add_(-group['lr'], buf)
                else:
                    p.data.addcdiv_(-group['lr'], grad, avg)

        return loss

class SharedAdam(optim.Adam):
    """Implements Adam algorithm with shared states.
    """

    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-3,
                 weight_decay=0):
        super(SharedAdam, self).__init__(params, lr, betas, eps, weight_decay)

        for group in self.param_groups:
            for p in group['params']:
                state = self.state[p]
                state['step'] = torch.zeros(1)
                state['exp_avg'] = p.data.new().resize_as_(p.data).zero_()
                state['exp_avg_sq'] = p.data.new().resize_as_(p.data).zero_()

    def share_memory(self):
        for group in self.param_groups:
            for p in group['params']:
                state = self.state[p]
                state['step'].share_memory_()
                state['exp_avg'].share_memory_()
                state['exp_avg_sq'].share_memory_()

    def step(self, closure=None):
        """Performs a single optimization step.
        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        """
        loss = None
        if closure is not None:
            loss = closure()

        for group in self.param_groups:
            for p in group['params']:
                if p.grad is None:
                    continue
                grad = p.grad.data
                state = self.state[p]

                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
                beta1, beta2 = group['betas']

                state['step'] += 1

                if group['weight_decay'] != 0:
                    grad = grad.add(group['weight_decay'], p.data)

                # Decay the first and second moment running average coefficient
                exp_avg.mul_(beta1).add_(1 - beta1, grad)
                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)

                denom = exp_avg_sq.sqrt().add_(group['eps'])

                bias_correction1 = 1 - beta1 ** state['step'][0]
                bias_correction2 = 1 - beta2 ** state['step'][0]
                step_size = group['lr'] * \
                    math.sqrt(bias_correction2) / bias_correction1

                p.data.addcdiv_(-step_size, exp_avg, denom)

        return loss

sample_lr =[0.0001, 9e-05, 8.1e-05, 7.290000000000001e-05, 6.561000000000002e-05, 5.904900000000002e-05,
5.314410000000002e-05, 4.782969000000002e-05, 4.304672100000002e-05, 3.874204890000002e-05,
3.4867844010000016e-05, 3.138105960900002e-05, 2.8242953648100018e-05, 2.5418658283290016e-05,
2.2876792454961016e-05, 2.0589113209464913e-05, 1.8530201888518422e-05, 1.667718169966658e-05,
1.5009463529699922e-05, 1.350851717672993e-05, 1.2157665459056937e-05]

class SharedLrSchedAdam(optim.Adam):
    """Implements Adam algorithm with shared states.
    """

    def __init__(self,
                 params,
                 lr=1e-3,
                 betas=(0.9, 0.999),
                 eps=1e-3,
                 weight_decay=0):
        super(SharedLrSchedAdam, self).__init__(params, lr, betas, eps, weight_decay)

        for group in self.param_groups:
            for p in group['params']:
                state = self.state[p]
                state['step'] = torch.zeros(1)
                state['exp_avg'] = p.data.new().resize_as_(p.data).zero_()
                state['exp_avg_sq'] = p.data.new().resize_as_(p.data).zero_()

    def share_memory(self):
        for group in self.param_groups:
            for p in group['params']:
                state = self.state[p]
                state['step'].share_memory_()
                state['exp_avg'].share_memory_()
                state['exp_avg_sq'].share_memory_()

    def step(self, closure=None):
        """Performs a single optimization step.
        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        """
        loss = None
        if closure is not None:
            loss = closure()

        for group in self.param_groups:
            for p in group['params']:
                if p.grad is None:
                    continue
                grad = p.grad.data
                state = self.state[p]

                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
                beta1, beta2 = group['betas']

                state['step'] += 1

                if group['weight_decay'] != 0:
                    grad = grad.add(group['weight_decay'], p.data)

                # Decay the first and second moment running average coefficient
                exp_avg.mul_(beta1).add_(1 - beta1, grad)
                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)

                denom = exp_avg_sq.sqrt().add_(group['eps'])

                bias_correction1 = 1 - beta1**state['step'][0]
                bias_correction2 = 1 - beta2**state['step'][0]
                step_size = group['lr'] * \
                    math.sqrt(bias_correction2) / bias_correction1
                p.data.addcdiv_(-step_size, exp_avg, denom)

        lr = sample_lr[int(state['step'][0] // 40000000)]
        group['lr'] = lr

        return loss

<commit_msg>simplified lrschedadam<commit_after>from __future__ import division
import math
import torch
import torch.optim as optim

class SharedRMSprop(optim.RMSprop):
    """Implements RMSprop algorithm with shared states.
    """

    def __init__(self,
                 params,
                 lr=1e-2,
                 alpha=0.99,
                 eps=1e-8,
                 weight_decay=0,
                 momentum=0,
                 centered=False):
        super(SharedRMSprop, self).__init__(params, lr, alpha, eps,
                                            weight_decay, momentum, centered)

        for group in self.param_groups:
            for p in group['params']:
                state = self.state[p]
                state['step'] = torch.zeros(1)
                state['grad_avg'] = p.data.new().resize_as_(p.data).zero_()
                state['square_avg'] = p.data.new().resize_as_(p.data).zero_()
                state['momentum_buffer'] = p.data.new().resize_as_(
                    p.data).zero_()

    def share_memory(self):
        for group in self.param_groups:
            for p in group['params']:
                state = self.state[p]
                state['square_avg'].share_memory_()
                state['step'].share_memory_()
                state['grad_avg'].share_memory_()

    def step(self, closure=None):
        """Performs a single optimization step.
        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        """
        loss = None
        if closure is not None:
            loss = closure()

        for group in self.param_groups:
            for p in group['params']:
                if p.grad is None:
                    continue
                grad = p.grad.data
                state = self.state[p]

                square_avg = state['square_avg']
                alpha = group['alpha']

                state['step'] += 1

                if group['weight_decay'] != 0:
                    grad = grad.add(group['weight_decay'], p.data)

                square_avg.mul_(alpha).addcmul_(1 - alpha, grad, grad)

                if group['centered']:
                    grad_avg = state['grad_avg']
                    grad_avg.mul_(alpha).add_(1 - alpha, grad)
                    avg = square_avg.addcmul(
                        -1, grad_avg, grad_avg).sqrt().add_(group['eps'])
                else:
                    avg = square_avg.sqrt().add_(group['eps'])

                if group['momentum'] > 0:
                    buf = state['momentum_buffer']
                    buf.mul_(group['momentum']).addcdiv_(grad, avg)
                    p.data.add_(-group['lr'], buf)
                else:
                    p.data.addcdiv_(-group['lr'], grad, avg)

        return loss

class SharedAdam(optim.Adam):
    """Implements Adam algorithm with shared states.
    """

    def __init__(self,
                 params,
                 lr=1e-3,
                 betas=(0.9, 0.999),
                 eps=1e-3,
                 weight_decay=0):
        super(SharedAdam, self).__init__(params, lr, betas, eps, weight_decay)

        for group in self.param_groups:
            for p in group['params']:
                state = self.state[p]
                state['step'] = torch.zeros(1)
                state['exp_avg'] = p.data.new().resize_as_(p.data).zero_()
                state['exp_avg_sq'] = p.data.new().resize_as_(p.data).zero_()

    def share_memory(self):
        for group in self.param_groups:
            for p in group['params']:
                state = self.state[p]
                state['step'].share_memory_()
                state['exp_avg'].share_memory_()
                state['exp_avg_sq'].share_memory_()

    def step(self, closure=None):
        """Performs a single optimization step.
        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        """
        loss = None
        if closure is not None:
            loss = closure()

        for group in self.param_groups:
            for p in group['params']:
                if p.grad is None:
                    continue
                grad = p.grad.data
                state = self.state[p]

                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
                beta1, beta2 = group['betas']

                state['step'] += 1

                if group['weight_decay'] != 0:
                    grad = grad.add(group['weight_decay'], p.data)

                # Decay the first and second moment running average coefficient
                exp_avg.mul_(beta1).add_(1 - beta1, grad)
                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)

                denom = exp_avg_sq.sqrt().add_(group['eps'])

                bias_correction1 = 1 - beta1**state['step'][0]
                bias_correction2 = 1 - beta2**state['step'][0]
                step_size = group['lr'] * \
                    math.sqrt(bias_correction2) / bias_correction1

                p.data.addcdiv_(-step_size, exp_avg, denom)

        return loss

sample_lr = [
    0.0001, 0.00009, 0.00008, 0.00007, 0.00006, 0.00005, 0.00004, 0.00003,
    0.00002, 0.00001, 0.000009, 0.000008, 0.000007, 0.000006, 0.000005,
    0.000004, 0.000003, 0.000002, 0.000001
]

class SharedLrSchedAdam(optim.Adam):
    """Implements Adam algorithm with shared states.
    """

    def __init__(self,
                 params,
                 lr=1e-3,
                 betas=(0.9, 0.999),
                 eps=1e-3,
                 weight_decay=0):
        super(SharedLrSchedAdam, self).__init__(params, lr, betas, eps,
                                                weight_decay)

        for group in self.param_groups:
            for p in group['params']:
                state = self.state[p]
                state['step'] = torch.zeros(1)
                state['exp_avg'] = p.data.new().resize_as_(p.data).zero_()
                state['exp_avg_sq'] = p.data.new().resize_as_(p.data).zero_()

    def share_memory(self):
        for group in self.param_groups:
            for p in group['params']:
                state = self.state[p]
                state['step'].share_memory_()
                state['exp_avg'].share_memory_()
                state['exp_avg_sq'].share_memory_()

    def step(self, closure=None):
        """Performs a single optimization step.
        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        """
        loss = None
        if closure is not None:
            loss = closure()

        for group in self.param_groups:
            for p in group['params']:
                if p.grad is None:
                    continue
                grad = p.grad.data
                state = self.state[p]

                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
                beta1, beta2 = group['betas']

                state['step'] += 1

                if group['weight_decay'] != 0:
                    grad = grad.add(group['weight_decay'], p.data)

                # Decay the first and second moment running average coefficient
                exp_avg.mul_(beta1).add_(1 - beta1, grad)
                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)

                denom = exp_avg_sq.sqrt().add_(group['eps'])

                bias_correction1 = 1 - beta1**state['step'][0]
                bias_correction2 = 1 - beta2**state['step'][0]
                step_size = group['lr'] * \
                    math.sqrt(bias_correction2) / bias_correction1
                p.data.addcdiv_(-step_size, exp_avg, denom)

        lr = sample_lr[int(state['step'][0] // 40000000)]
        group['lr'] = lr

        return loss
<|endoftext|>


<commit_before>import shodan
import configparser
import json
from base import dict_add_source_prefix
from base import dict_clean_empty
import sys

class ShodanObject:

    def __init__(self):
        """Return a ShodanObject initialised with API key"""
        config = configparser.ConfigParser()
        config.read("config.ini")
        self.SHODAN_API_KEY = (config['SectionOne']['SHODAN_API_KEY'])
        self.api = api = shodan.Shodan(self.SHODAN_API_KEY)

    @staticmethod
    def to_es_convert(self, input_dict):
        """Return dict ready to be sent to Logstash."""
        try:
            # set ip and ip_int
            ip_int = input_dict['ip']
            del input_dict['ip']
            input_dict['ip'] = input_dict['ip_str']
            del input_dict['ip_str']
            input_dict['ip_int'] = ip_int
        except KeyError:
            try:
                input_dict['ip'] = input_dict['ip_str']
                del input_dict['ip_str']
            except KeyError:
                print(input_dict)
                print('Missing required \'ip\' field in the element above. Exiting now...')
                sys.exit(1)

        # if present, convert ssl.cert.serial to string
        try:
            input_dict['ssl']['cert']['serial'] = str(input_dict['ssl']['cert']['serial'])
        except KeyError:
            pass
        # if present, convert ssl.dhparams.generator to string
        try:
            input_dict['ssl']['dhparams']['generator'] = str(input_dict['ssl']['dhparams']['generator'])
        except (KeyError, TypeError):
            pass

        try:
            # rename_shodan.modules to protocols (used as prefix per banner for combining multiple banners into 1 IP)
            input_dict['protocols'] = input_dict['_shodan']['module']
            # the rest of the data in _shodan is irrelevant
            del input_dict['_shodan']
        except KeyError:
            pass

        # asn to int
        try:
            input_dict['asn'] = int((input_dict['asn'])[2:])
        except KeyError:
            pass
        try:
            # rename location.country_name to location.country
            input_dict['location']['country'] = input_dict['location']['country_name']
            del input_dict['location']['country_name']
            # rename latitude and longitude for geoip
            input_dict['location']['geo'] = {}
            input_dict['location']['geo']['lat'] = input_dict['location']['latitude']
            input_dict['location']['geo']['lon'] = input_dict['location']['longitude']
            del input_dict['location']['latitude']
            del input_dict['location']['longitude']
        except KeyError:
            pass
        # Convert the 'nodes' and it's nested fields to 1 string, otherwise Elasticsearch will make new fields
        # for EVERY node ID.
        try:
            input_dict['elastic'] = str(
                input_dict['elastic'])
        except KeyError:
            pass

        # prefix non-nested fields with 'shodan'
        input_dict = dict_add_source_prefix(input_dict, 'shodan', str(input_dict['protocols']))
        return input_dict

    @staticmethod
    def to_file_shodan(self, queries, path_output_file, should_convert):
        """Makes a Shodan API call with each given query and writes results to output file
        :param self: ShodanObject
        :param queries: Collection of strings which present Shodan queries
        :param path_output_file: String which points to existing output file
        :param should_convert: Boolean if results should be converted
        """
        nr_total_results = 0
        failed_queries = set()
        for query in queries:
            results = []
            try:
                for banner in self.api.search_cursor(query):
                    banner = dict_clean_empty(banner)
                    if should_convert:
                        self.to_es_convert(self, banner)
                    results.append(json.dumps(banner) + '\n')
                    print('\r' + str(len(results)) + ' results fetched...', end='')
            except shodan.APIError as e:
                print('Error: ', e)
                failed_queries.add(failed_queries)
            with open(path_output_file, "a") as output_file:
                for banner in results:
                    output_file.write(banner)
            print('\r' + str(len(results)) + ' results written from query(' + query + ')')
            nr_total_results += len(results)
        # Print failed queries if present
        if not failed_queries == set():
            print('Failed queries: ', failed_queries)
        print(str(nr_total_results) + ' total results written in ' + path_output_file)

    @staticmethod
    def get_input_choice(self):
        """Returns input_choice represented as integer"""
        items = {'1': 'console_input', '2': 'cidr_file_input', '3': 'csv_file_input', '4': 'query_file_input'}
        input_choice = '0'
        while input_choice not in items:
            input_choice = input("Console input[1], CIDR file input[2], csv file input[3] or query file input[4]?")
        return int(input_choice)

    # Returns a non empty set of query strings
    @staticmethod
    def get_user_input_console_queries():
        queries = set()
        done = False
        print('Enter Shodan queries, one at a time. Enter \'4\' when done.')
        while not done:
                query = ''
                while query is '':
                    query = input("Query:")
                if query is '4':
                    if queries != set():
                        done = True
                else:
                    queries.add(query)
        return queries

<commit_msg>Limiting shodan fields in convert process<commit_after>import shodan
import configparser
import json
from base import dict_add_source_prefix
from base import dict_clean_empty
import sys

class ShodanObject:

    def __init__(self):
        """Return a ShodanObject initialised with API key"""
        config = configparser.ConfigParser()
        config.read("config.ini")
        self.SHODAN_API_KEY = (config['SectionOne']['SHODAN_API_KEY'])
        self.api = api = shodan.Shodan(self.SHODAN_API_KEY)

    @staticmethod
    def to_es_convert(self, input_dict):
        """Return dict ready to be sent to Logstash."""
        try:
            # set ip and ip_int
            ip_int = input_dict['ip']
            del input_dict['ip']
            input_dict['ip'] = input_dict['ip_str']
            del input_dict['ip_str']
            input_dict['ip_int'] = ip_int
        except KeyError:
            try:
                input_dict['ip'] = input_dict['ip_str']
                del input_dict['ip_str']
            except KeyError:
                print(input_dict)
                print('Missing required \'ip\' field in the element above. Exiting now...')
                sys.exit(1)

        # if present, convert ssl.cert.serial to string
        try:
            input_dict['ssl']['cert']['serial'] = str(input_dict['ssl']['cert']['serial'])
        except KeyError:
            pass
        # if present, convert ssl.dhparams.generator to string
        try:
            input_dict['ssl']['dhparams']['generator'] = str(input_dict['ssl']['dhparams']['generator'])
        except (KeyError, TypeError):
            pass

        try:
            # rename_shodan.modules to protocols (used as prefix per banner for combining multiple banners into 1 IP)
            input_dict['protocols'] = input_dict['_shodan']['module']
            # the rest of the data in _shodan is irrelevant
            del input_dict['_shodan']
        except KeyError:
            pass
        # asn to int
        try:
            input_dict['asn'] = int((input_dict['asn'])[2:])
        except KeyError:
            pass
        try:
            # rename location.country_name to location.country
            input_dict['location']['country'] = input_dict['location']['country_name']
            del input_dict['location']['country_name']
            # rename latitude and longitude for geoip
            input_dict['location']['geo'] = {}
            input_dict['location']['geo']['lat'] = input_dict['location']['latitude']
            input_dict['location']['geo']['lon'] = input_dict['location']['longitude']
            del input_dict['location']['latitude']
            del input_dict['location']['longitude']
        except KeyError:
            pass

        # Limit the number of fields
        input_dict = self.limit_nr_of_elements(input_dict)

        # prefix non-nested fields with 'shodan'
        input_dict = dict_add_source_prefix(input_dict, 'shodan', str(input_dict['protocols']))
        return input_dict

    @staticmethod
    def limit_nr_of_elements(self, input_dict):
        """Converts some of the JSON elements containing (too) many nested elements to 1 string element.
        This prevents Elasticsearch from making too many fields, so it is still manageable in Kibana.
        """
        try:
            input_dict['http']['components'] = str(
                input_dict['http']['components'])
        except KeyError:
            pass
        try:
            input_dict['elastic'] = str(
                input_dict['elastic'])
        except KeyError:
            pass

    @staticmethod
    def to_file_shodan(self, queries, path_output_file, should_convert):
        """Makes a Shodan API call with each given query and writes results to output file
        :param self: ShodanObject
        :param queries: Collection of strings which present Shodan queries
        :param path_output_file: String which points to existing output file
        :param should_convert: Boolean if results should be converted
        """
        nr_total_results = 0
        failed_queries = set()
        for query in queries:
            results = []
            try:
                for banner in self.api.search_cursor(query):
                    banner = dict_clean_empty(banner)
                    if should_convert:
                        self.to_es_convert(self, banner)
                    results.append(json.dumps(banner) + '\n')
                    print('\r' + str(len(results)) + ' results fetched...', end='')
            except shodan.APIError as e:
                print('Error: ', e)
                failed_queries.add(failed_queries)
            with open(path_output_file, "a") as output_file:
                for banner in results:
                    output_file.write(banner)
            print('\r' + str(len(results)) + ' results written from query(' + query + ')')
            nr_total_results += len(results)
        # Print failed queries if present
        if not failed_queries == set():
            print('Failed queries: ', failed_queries)
        print(str(nr_total_results) + ' total results written in ' + path_output_file)

    @staticmethod
    def get_input_choice(self):
        """Returns input_choice represented as integer"""
        items = {'1': 'console_input', '2': 'cidr_file_input', '3': 'csv_file_input', '4': 'query_file_input'}
        input_choice = '0'
        while input_choice not in items:
            input_choice = input("Console input[1], CIDR file input[2], csv file input[3] or query file input[4]?")
        return int(input_choice)

    # Returns a non empty set of query strings
    @staticmethod
    def get_user_input_console_queries():
        queries = set()
        done = False
        print('Enter Shodan queries, one at a time. Enter \'4\' when done.')
        while not done:
                query = ''
                while query is '':
                    query = input("Query:")
                if query is '4':
                    if queries != set():
                        done = True
                else:
                    queries.add(query)
        return queries

<|endoftext|>


<commit_before># coding=utf-8
#
# ROSREPO
# Manage ROS workspaces with multiple Gitlab repositories
#
# Author: Timo Röhling
#
# Copyright 2016 Fraunhofer FKIE
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
__author__ = "Timo Röhling <timo.roehling@fkie.fraunhofer.de>"
__copyright__ = "Copyright (c) 2016 Fraunhofer FKIE"
__license__ = "Apache-2.0"
__version__ = "3.0.3"
<commit_msg>3.0.4<commit_after># coding=utf-8
#
# ROSREPO
# Manage ROS workspaces with multiple Gitlab repositories
#
# Author: Timo Röhling
#
# Copyright 2016 Fraunhofer FKIE
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
__author__ = "Timo Röhling <timo.roehling@fkie.fraunhofer.de>"
__copyright__ = "Copyright (c) 2016 Fraunhofer FKIE"
__license__ = "Apache-2.0"
__version__ = "3.0.4"
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# shredcompare -- find common cliques in shred lists
#
import getopt, sys, binascii

class SHIF:
    "SHIF-A file metadata container."
    def __init__(self, name):
        self.name = name
        self.fp = open(name)
        self.comments = []
        id = self.fp.readline()
        if not id.startswith("#SHIF-A "):
            sys.stderr.write("shredcompare: %s is not a SHIF-A file."%fp.name)
            sys.exit(1)
        while True:
            line = self.fp.readline()
            if line == '%%\n':
                break
            (tag, value) = line.split(":")
            value = value.strip()
            if tag == "Normalization":
                self.normalization = value.split(", ")
                self.normalization.sort()
            elif tag == "Shred-Size":
                self.shredsize = int(value)
            elif tag == "Hash-Method":
                self.hash = value
            elif tag == "Generator-Program":
                self.generator = value
            elif tag == "Comment":
                self.comments.append(value)
    def __check_match(self, other, attr):
        if getattr(self, attr) != getattr(other, attr):
            sys.stderr.write("shredcompare: %s of %s and %s doesn't match." \
                             % (attr, self.name, other.name))
            sys.exit(1)
    def compatible(self, other):
        self.__check_match(other, 'normalization')
        self.__check_match(other, 'shredsize')
        self.__check_match(other, 'hash')

class Shred:
    "Represent a range of lines."
    def __init__(self, file, start, end):
        self.file = file
        self.start = start
        self.end = end
    def dump(self):
        rfp = open(self.file)
        for i in range(self.start):
            rfp.readline()
        text = ""
        for i in range(self.start, self.end):
            text += rfp.readline()
        rfp.close()
        return text
    def __range(self):
        return range(self.start, self.end)
    def intersects(self, other):
        return self.file == other.file and \
           self.start in other.__range() or other.start in self.__range()
    def merge(self, other):
        self.start = min(self.start, other.start)
        self.end   = max(self.end,   other.end)
    # Make these sortable
    def __cmp__(self, other):
        return cmp((self.file, self.start, self.end),
                   (other.file, other.start, other.end))
    def __hash_(self):
        return hash((self.file, self.start, self.end))
    # External representation
    def __str__(self):
        # 0-origin line numbers internally, 1-origin externally.
        return "%s:%d-%d" % (self.file, self.start, self.end)
    __repr__ = __str__

def merge_hashes(fp, dict):
    "Read and merge hashes corresponding to one file."
    file = fp.readline().rstrip()
    while True:
        line = fp.readline()
        if not line:
            return False
        if not line.strip():
            return True
        (start, end, hash) = line.split()
        start = int(start)
        end = int(end)
        hash = binascii.unhexlify(hash)
        if hash not in dict:
            dict[hash] = []
        dict[hash].append((file, start, end))

if __name__ == '__main__':
    try:
        (optlist, args) = getopt.getopt(sys.argv[1:], 'd')
    except getopt.GetoptError:
        sys.stderr.write("usage: shredcompare [-h] [-d] file\n")
        sys.exit(2)
    local_duplicates = True
    for (opt, val) in optlist:
        if opt == '-d':
            local_duplicates = False
        elif opt == '-h':
	    sys.stderr.write("usage: shredcompare [-d] file\n");
	    sys.stderr.write(" -d      = discard local duplicates.\n");
	    sys.stderr.write(" -h      = help (display this message).\n");
	    sys.exit(0);

    shiflist = []
    # Read input metadata
    for file in args:
        shiflist.append(SHIF(file))
    # Metadata sanity check
    for i in range(len(args)-1):
        shiflist[i].compatible(shiflist[i+1])
    # Read in all hashes
    hashdict = {}
    for shif in shiflist:
        while merge_hashes(shif.fp, hashdict):
            continue
    # Nuke all unique hashes
    for (key, matches) in hashdict.items():
        if len(matches) == 1:
            del hashdict[key]
    # Maybe nuke all match sets that don't cross a tree boundary
    if not local_duplicates:
        pass			# FIXME
    # Turn the remaining matches into match objects, because we're
    # going to want to do logic on them.  We deferred it this long
    # to lower memory usage.
    matches = []
    for clique in hashdict.values():
        matches.append(map(lambda x: Shred(x[0], x[1], x[2]), clique))
    # Merge overlapping ranges if possible.
    # This is O(n**2) in the number of matching chunks.
    # Should performance ever become a problem, break this up into
    # separate passes over the chunks in each file. (Would correspond
    # to eliminating a bunch of off-diagonal entries.)
    # FIXME: code currently only does two-way merge.
    retry = True
    while retry:
        retry = False
        #print "Matches at start:", matches
        for i in range(len(matches)):
            for j in range(len(matches)):
                if i >= j:
                    continue
                # Neither shred must have been deleted
                if not matches[j] or not matches[i]:
                    continue
                #print "Checking:", matches[j], matches[i]
                # Toss them out if they don't intersect
                if not (matches[j][0].intersects(matches[i][0]) and \
                	matches[j][1].intersects(matches[i][1])):
                    #print "alpha and beta don't intersect"
                    continue
                # Merge lower chunk into upper; delete lower
                #print "Found a merge:", matches[j], matches[i]
                matches[j][0].merge(matches[i][0])
                matches[j][1].merge(matches[i][1])
                matches[i] = None
                retry = True
    matches = filter(lambda x: x, matches)
    matches.sort(lambda x, y: cmp(x[0], y[0]))	# by source chunk
    # OK, dump all matches.
    print "#SHIF-B 1.0"
    print "Merge-Program: shredcompare.py 1.0"
    print "Hash-Method: MD5"
    print "Shred-Size: %d" % shiflist[0].shredsize
    print "Normalization:", ",".join(shiflist[0].normalization)
    print "%%"
    # FIXME: do something appropriate with comments
    for match in matches:
        for range in match:
            print `range`
        print "-"

<commit_msg>Take it all out.<commit_after>#!/usr/bin/env python
#
# shredcompare -- find common cliques in shred lists
#
import getopt, sys, binascii

class SHIF:
    "SHIF-A file metadata container."
    def __init__(self, name):
        self.name = name
        self.fp = open(name)
        self.comments = []
        id = self.fp.readline()
        if not id.startswith("#SHIF-A "):
            sys.stderr.write("shredcompare: %s is not a SHIF-A file."%self.fp.name)
            sys.exit(1)
        while True:
            line = self.fp.readline()
            if line == '%%\n':
                break
            (tag, value) = line.split(":")
            value = value.strip()
            if tag == "Normalization":
                self.normalization = value.split(", ")
                self.normalization.sort()
            elif tag == "Shred-Size":
                self.shredsize = int(value)
            elif tag == "Hash-Method":
                self.hash = value
            elif tag == "Generator-Program":
                self.generator = value
            elif tag == "Comment":
                self.comments.append(value)
    def __check_match(self, other, attr):
        if getattr(self, attr) != getattr(other, attr):
            sys.stderr.write("shredcompare: %s of %s and %s doesn't match." \
                             % (attr, self.name, other.name))
            sys.exit(1)
    def compatible(self, other):
        self.__check_match(other, 'normalization')
        self.__check_match(other, 'shredsize')
        self.__check_match(other, 'hash')

class Shred:
    "Represent a range of lines."
    def __init__(self, file, start, end):
        self.file = file
        self.start = start
        self.end = end
    def dump(self):
        rfp = open(self.file)
        for i in range(self.start):
            rfp.readline()
        text = ""
        for i in range(self.start, self.end):
            text += rfp.readline()
        rfp.close()
        return text
    def __range(self):
        return range(self.start, self.end)
    def intersects(self, other):
        return self.file == other.file and \
           self.start in other.__range() or other.start in self.__range()
    def merge(self, other):
        self.start = min(self.start, other.start)
        self.end   = max(self.end,   other.end)
    # Make these sortable
    def __cmp__(self, other):
        return cmp((self.file, self.start, self.end),
                   (other.file, other.start, other.end))
    def __hash_(self):
        return hash((self.file, self.start, self.end))
    # External representation
    def __str__(self):
        # 0-origin line numbers internally, 1-origin externally.
        return "%s:%d-%d" % (self.file, self.start, self.end)
    __repr__ = __str__

def merge_hashes(fp, dict):
    "Read and merge hashes corresponding to one file."
    file = fp.readline().rstrip()
    while True:
        line = fp.readline()
        if not line:
            return False
        if not line.strip():
            return True
        (start, end, hash) = line.split()
        start = int(start)
        end = int(end)
        hash = binascii.unhexlify(hash)
        if hash not in dict:
            dict[hash] = []
        dict[hash].append((file, start, end))

if __name__ == '__main__':
    try:
        (optlist, args) = getopt.getopt(sys.argv[1:], 'd')
    except getopt.GetoptError:
        sys.stderr.write("usage: shredcompare [-h] [-d] file\n")
        sys.exit(2)
    local_duplicates = True
    for (opt, val) in optlist:
        if opt == '-d':
            local_duplicates = False
        elif opt == '-h':
	    sys.stderr.write("usage: shredcompare [-d] file\n");
	    sys.stderr.write(" -d      = discard local duplicates.\n");
	    sys.stderr.write(" -h      = help (display this message).\n");
	    sys.exit(0);

    shiflist = []
    # Read input metadata
    for file in args:
        shiflist.append(SHIF(file))
    # Metadata sanity check
    for i in range(len(args)-1):
        shiflist[i].compatible(shiflist[i+1])
    # Read in all hashes
    hashdict = {}
    for shif in shiflist:
        while merge_hashes(shif.fp, hashdict):
            continue
    # Nuke all unique hashes
    for (key, matches) in hashdict.items():
        if len(matches) == 1:
            del hashdict[key]
    # Maybe nuke all match sets that don't cross a tree boundary
    if not local_duplicates:
        pass			# FIXME
    # Turn the remaining matches into match objects, because we're
    # going to want to do logic on them.  We deferred it this long
    # to lower memory usage.
    matches = []
    for clique in hashdict.values():
        matches.append(map(lambda x: Shred(x[0], x[1], x[2]), clique))
    # Merge overlapping ranges if possible.
    # This is O(n**2) in the number of matching chunks.
    # Should performance ever become a problem, break this up into
    # separate passes over the chunks in each file. (Would correspond
    # to eliminating a bunch of off-diagonal entries.)
    # FIXME: code currently only does two-way merge.
    retry = True
    while retry:
        retry = False
        #print "Matches at start:", matches
        for i in range(len(matches)):
            for j in range(len(matches)):
                if i >= j:
                    continue
                # Neither shred must have been deleted
                if not matches[j] or not matches[i]:
                    continue
                #print "Checking:", matches[j], matches[i]
                # Toss them out if they don't intersect
                if not (matches[j][0].intersects(matches[i][0]) and \
                	matches[j][1].intersects(matches[i][1])):
                    #print "alpha and beta don't intersect"
                    continue
                # Merge lower chunk into upper; delete lower
                #print "Found a merge:", matches[j], matches[i]
                matches[j][0].merge(matches[i][0])
                matches[j][1].merge(matches[i][1])
                matches[i] = None
                retry = True
    matches = filter(lambda x: x, matches)
    matches.sort(lambda x, y: cmp(x[0], y[0]))	# by source chunk
    # OK, dump all matches.
    print "#SHIF-B 1.0"
    print "Merge-Program: shredcompare.py 1.0"
    print "Hash-Method: MD5"
    print "Shred-Size: %d" % shiflist[0].shredsize
    print "Normalization:", ",".join(shiflist[0].normalization)
    print "%%"
    # FIXME: do something appropriate with comments
    for match in matches:
        for range in match:
            print `range`
        print "-"

<|endoftext|>


<commit_before>from pymongo import MongoClient

#function hashes
qualified_for = {}
valid_entity_for = {}
oncreate_for = {}
form_text = {}
#mongo coll to persist the roles data
grants=None

class NotQualified(Exception):
    pass
class InvalidEntity(Exception):
    pass

def default_text(role):
    return {
        "title": role,
        "invalid": "invalid entity for role "+role,
        "grant_title": "Grant "+role,
        "grant_button": "GRANT",
        "revoke_title": "Revoke "+ role,
        "revoke_button": "REVOKE",
        "empty": "there is currently nothing with the role "+role,
        "granted_by": "granted by"
    }

def new(role, qualifier_func, validator_func = lambda e: e!="",
        custom_text={},
        oncreate=lambda e: "default"):
    """
    :Parameters:
    - `qualifier_func`: takes in a netid and returns wether or not a user is
    qualified to add to a specific role.
    - `validator func`: takes in an entity (like netid or link) and returns
    if its valid for a role. for example it could take a link like 'htp://fuz'
    and say its not a valid link
    - `custom_text`: custom text to show on the form. see default_text source for options
    - `oncreate`: callback for extra logic when granting a role. eg remove a users links on ban
    """
    text=default_text(role)
    text.update(custom_text)
    form_text[role] = text
    qualified_for[role] = qualifier_func
    valid_entity_for[role] = validator_func
    oncreate_for[role] = oncreate

def grant(role, grantor, grantee):
    if exists(role) and valid_entity_for[role](grantee):
        #guard against double insertions
        if not check(role, grantee):
            grants.insert({"role": role, "entity": grantee, "granted_by": grantor})
    else:
        raise InvalidEntity()

def check(role, entity):
    if grants.find_one({"role": role, "entity": entity}):
        return True
    return False

def list_all(role):
    return list(grants.find({"role": role}))

def revoke(role, entity):
    grants.remove({"role": role, "entity": entity})

def template_data(role, invalid=False):
    data = {
        "role": role,
        "grants": list_all(role)
    }
    if invalid:
        data["msg"] = form_text[role]["invalid"]
    data.update(form_text[role])
    return data

def exists(role):
    return role in qualified_for

def init(app, mongo_client=None):
    if not mongo_client:
        mongo_client = MongoClient(app.config["DB_HOST"], app.config["DB_PORT"])
        #this forces pymongo to connect instead of sitting on its hands and blocking
        #the server. idk why it behaves like this but if you remove the next like the
        #server does not respond. pymongo3.6 aug 2018
        mongo_client.admin.command("ismaster")
    global grants
    grants = mongo_client.shrunk_roles.grants
<commit_msg>and has_one_of and roles.get<commit_after>from pymongo import MongoClient

#function hashes
qualified_for = {}
valid_entity_for = {}
oncreate_for = {}
form_text = {}
#mongo coll to persist the roles data
grants=None

class NotQualified(Exception):
    pass
class InvalidEntity(Exception):
    pass

def default_text(role):
    return {
        "title": role,
        "invalid": "invalid entity for role "+role,
        "grant_title": "Grant "+role,
        "grant_button": "GRANT",
        "revoke_title": "Revoke "+ role,
        "revoke_button": "REVOKE",
        "empty": "there is currently nothing with the role "+role,
        "granted_by": "granted by"
    }

def new(role, qualifier_func, validator_func = lambda e: e!="",
        custom_text={},
        oncreate=lambda e: "default"):
    """
    :Parameters:
    - `qualifier_func`: takes in a netid and returns wether or not a user is
    qualified to add to a specific role.
    - `validator func`: takes in an entity (like netid or link) and returns
    if its valid for a role. for example it could take a link like 'htp://fuz'
    and say its not a valid link
    - `custom_text`: custom text to show on the form. see default_text source for options
    - `oncreate`: callback for extra logic when granting a role. eg remove a users links on ban
    """
    text=default_text(role)
    text.update(custom_text)
    form_text[role] = text
    qualified_for[role] = qualifier_func
    valid_entity_for[role] = validator_func
    oncreate_for[role] = oncreate

def grant(role, grantor, grantee):
    if exists(role) and valid_entity_for[role](grantee):
        #guard against double insertions
        if not check(role, grantee):
            grants.insert({"role": role, "entity": grantee, "granted_by": grantor})
    else:
        raise InvalidEntity()

def check(role, entity):
    if grants.find_one({"role": role, "entity": entity}):
        return True
    return False

def has_one_of(roles, entity):
    for role in roles:
        if grants.find_one({"role": role, "entity": entity}):
            return True
    return False

def get(entity):
    """gives list of strings for all roles an entity has"""
    grants = grants.find({"entity": entity})
    return [grant["role"] for grant in grants]

def list_all(role):
    return list(grants.find({"role": role}))

def revoke(role, entity):
    grants.remove({"role": role, "entity": entity})

def template_data(role, invalid=False):
    data = {
        "role": role,
        "grants": list_all(role)
    }
    if invalid:
        data["msg"] = form_text[role]["invalid"]
    data.update(form_text[role])
    return data

def exists(role):
    return role in qualified_for

def init(app, mongo_client=None):
    if not mongo_client:
        mongo_client = MongoClient(app.config["DB_HOST"], app.config["DB_PORT"])
        #this forces pymongo to connect instead of sitting on its hands and blocking
        #the server. idk why it behaves like this but if you remove the next like the
        #server does not respond. pymongo3.6 aug 2018
        mongo_client.admin.command("ismaster")
    global grants
    grants = mongo_client.shrunk_roles.grants
<|endoftext|>


<commit_before>from abc import ABCMeta, abstractmethod
from collections import OrderedDict, UserList
from io import StringIO
from typing import Any, Callable, Collection, FrozenSet, Generator, Generic, \
                   Iterator, Mapping, NewType, Optional, Sequence, TextIO, \
                   TypeVar, Union

from _private.serializable import Serializable

__all__ = ['BaseChart', 'BaseCharts', 'BaseSimfile']

class _ListWithRepr(UserList):
    """
    Subclass of UserLis that overrides __repr__.
    """
    def __repr__(self):
        return '%s(%s)' % (self.__class__.__name__, super().__repr__())

class BaseChart(Serializable, metaclass=ABCMeta):
    """
    One chart from a simfile.
    """
    @property
    @abstractmethod
    def stepstype(self) -> str:
        pass

    @property
    @abstractmethod
    def description(self) -> str:
        pass

    @property
    @abstractmethod
    def difficulty(self) -> str:
        pass

    @property
    @abstractmethod
    def meter(self) -> int:
        pass

    @property
    @abstractmethod
    def radarvalues(self) -> str:
        pass

    @property
    @abstractmethod
    def notes(self) -> str:
        pass

    def __repr__(self) -> str:
        cls = self.__class__.__name__
        return f'<{cls}: {self.stepstype} {self.difficulty} {self.meter}>'

class BaseCharts(_ListWithRepr, Serializable, metaclass=ABCMeta):
    """
    A filterable list of BaseChart objects.
    """
    @property
    @abstractmethod
    def supported_fields(self) -> FrozenSet[str]:
        pass

    def _match(self, chart: BaseChart, fields: Mapping[str, Any]):
        for field, value in fields.items():
            if getattr(chart, field) != value:
                return False
        return True

    def filter(self, **fields: Any):
        """
        Filter charts according to the provided field values.

        All provided fields must be present in the implementation's
        `supported_fields()`, otherwise KeyError will be raised.

        In order for a chart to be yielded, all fields must exactly match
        the provided value.
        """
        unsupported_fields = set(fields.keys()) - self.supported_fields
        if unsupported_fields:
            raise KeyError('unsupported fields:', unsupported_fields)
        for chart in self:
            if self._match(chart, fields):
                yield chart

    def get(self, **fields: Any):
        """
        Get a chart that matches the given criteria.

        Arguments are identical to those of `filter`. Raises ``LookupError``
        if any number of charts other than one are matched.
        """
        generator = self.filter(**fields)

        try:
            match = next(generator)
        except StopIteration:
            raise LookupError('no charts match these parameters')

        try:
            next(generator)
        except StopIteration:
            return match

        raise LookupError('multiple charts match these parameters')

    @Serializable.enable_string_output
    def serialize(self, file: TextIO):
        for chart in self:
            chart.serialize(file)
            file.write('\n')

class BaseSimfile(OrderedDict, Serializable, metaclass=ABCMeta):
    """
    The BaseSimfile class encapsulates simfile parameters and charts.

    BaseSimfile objects can be created from filenames or file-like objects. They
    can also be created from strings containing simfile data using the
    `from_string` class method.

    Parameters are accessed through a dict-like interface. Identifiers are
    case-sensitive, but coerced to uppercase when importing. BaseCharts are stored
    in a `BaseCharts` object under the `charts` attribute.
    """
    def __init__(self, *,
                 file: Optional[Union[TextIO, Iterator[str]]] = None,
                 string: Optional[str] = None):
        if file is None and string is None:
            raise ValueError("must provide either a file or a string")
        if file is not None and string is not None:
            raise ValueError("must provide either a file or a string, not both")
        self.file = file
        self.string = string
        self._parse()

    @abstractmethod
    def _parse(self):
        pass

    @Serializable.enable_string_output
    def serialize(self, file: TextIO):
        for (key, value) in self.items():
            file.write(f'#{key}:{value};\n')
        file.write('\n')
        self.charts.serialize(file)

    @property
    @abstractmethod
    def charts(self) -> BaseCharts:
        pass

    def __repr__(self):
        rtn = '<' + self.__class__.__name__
        if self.get('TITLE'):
            rtn += ': ' + self['TITLE']
            if self.get('SUBTITLE'):
                rtn += ' ' + self['SUBTITLE']
        return rtn + '>'

    def __eq__(self, other):
        """
        Test for equality with another BaseSimfile.

        Two simfiles are equal if they have the same type, parameters, and
        charts.
        """
        return (type(self) is type(other) and
                super(OrderedDict, self) == super(OrderedDict, other) and
                self.charts == other.charts)<commit_msg>Fix import error<commit_after>from abc import ABCMeta, abstractmethod
from collections import OrderedDict, UserList
from io import StringIO
from typing import Any, Callable, Collection, FrozenSet, Generator, Generic, \
                   Iterator, Mapping, NewType, Optional, Sequence, TextIO, \
                   TypeVar, Union

from ._private.serializable import Serializable

__all__ = ['BaseChart', 'BaseCharts', 'BaseSimfile']

class _ListWithRepr(UserList):
    """
    Subclass of UserList that overrides __repr__.
    """
    def __repr__(self):
        return '%s(%s)' % (self.__class__.__name__, super().__repr__())

class BaseChart(Serializable, metaclass=ABCMeta):
    """
    One chart from a simfile.
    """
    @property
    @abstractmethod
    def stepstype(self) -> str:
        pass

    @property
    @abstractmethod
    def description(self) -> str:
        pass

    @property
    @abstractmethod
    def difficulty(self) -> str:
        pass

    @property
    @abstractmethod
    def meter(self) -> int:
        pass

    @property
    @abstractmethod
    def radarvalues(self) -> str:
        pass

    @property
    @abstractmethod
    def notes(self) -> str:
        pass

    def __repr__(self) -> str:
        cls = self.__class__.__name__
        return f'<{cls}: {self.stepstype} {self.difficulty} {self.meter}>'

class BaseCharts(_ListWithRepr, Serializable, metaclass=ABCMeta):
    """
    A filterable list of BaseChart objects.
    """
    def __init__(self, data=None):
        super().__init__(data)

    @property
    @abstractmethod
    def supported_fields(self) -> FrozenSet[str]:
        pass

    def _match(self, chart: BaseChart, fields: Mapping[str, Any]):
        for field, value in fields.items():
            if getattr(chart, field) != value:
                return False
        return True

    def filter(self, **fields: Any):
        """
        Filter charts according to the provided field values.

        All provided fields must be present in the implementation's
        `supported_fields()`, otherwise KeyError will be raised.

        In order for a chart to be yielded, all fields must exactly match
        the provided value.
        """
        unsupported_fields = set(fields.keys()) - self.supported_fields
        if unsupported_fields:
            raise KeyError('unsupported fields:', unsupported_fields)
        for chart in self:
            if self._match(chart, fields):
                yield chart

    def get(self, **fields: Any):
        """
        Get a chart that matches the given criteria.

        Arguments are identical to those of `filter`. Raises ``LookupError``
        if any number of charts other than one are matched.
        """
        generator = self.filter(**fields)

        try:
            match = next(generator)
        except StopIteration:
            raise LookupError('no charts match these parameters')

        try:
            next(generator)
        except StopIteration:
            return match

        raise LookupError('multiple charts match these parameters')

    @Serializable.enable_string_output
    def serialize(self, file: TextIO):
        for chart in self:
            chart.serialize(file)
            file.write('\n')

class BaseSimfile(OrderedDict, Serializable, metaclass=ABCMeta):
    """
    The BaseSimfile class encapsulates simfile parameters and charts.

    BaseSimfile objects can be created from filenames or file-like objects. They
    can also be created from strings containing simfile data using the
    `from_string` class method.

    Parameters are accessed through a dict-like interface. Identifiers are
    case-sensitive, but coerced to uppercase when importing. BaseCharts are stored
    in a `BaseCharts` object under the `charts` attribute.
    """
    def __init__(self, *,
                 file: Optional[Union[TextIO, Iterator[str]]] = None,
                 string: Optional[str] = None):
        if file is None and string is None:
            raise ValueError("must provide either a file or a string")
        if file is not None and string is not None:
            raise ValueError("must provide either a file or a string, not both")
        self.file = file
        self.string = string
        self._parse()

    @abstractmethod
    def _parse(self):
        pass

    @Serializable.enable_string_output
    def serialize(self, file: TextIO):
        for (key, value) in self.items():
            file.write(f'#{key}:{value};\n')
        file.write('\n')
        self.charts.serialize(file)

    @property
    @abstractmethod
    def charts(self) -> BaseCharts:
        pass

    def __repr__(self):
        rtn = '<' + self.__class__.__name__
        if self.get('TITLE'):
            rtn += ': ' + self['TITLE']
            if self.get('SUBTITLE'):
                rtn += ' ' + self['SUBTITLE']
        return rtn + '>'

    def __eq__(self, other):
        """
        Test for equality with another BaseSimfile.

        Two simfiles are equal if they have the same type, parameters, and
        charts.
        """
        return (type(self) is type(other) and
                OrderedDict.__eq__(self, other) and
                self.charts == other.charts)

    def __ne__(self, other):
        return not self.__eq__(other)<|endoftext|>


<commit_before>#!/usr/bin/env python
import decimal
import os
import unittest

from simfile import *

class TestSimfile(unittest.TestCase):

    def get_simfile(self, filename, cache={}):
        if filename not in cache:
            cache[filename] = Simfile(os.path.join('testdata', filename))
        return cache[filename]

    def test_empty(self):
        sm = self.get_simfile('empty.sm')
        self.assertTrue(sm)
        self.assertFalse(sm.params)
        self.assertRaises(KeyError, sm.get, 'TITLE')
        # Nowhere else to put these at the moment
        self.assertRaises(TypeError, Simfile, filename='.', string='.')

    def test_get(self):
        sm = self.get_simfile('Tribal Style.sm')
        self.assertEqual(sm.get('TITLE'), Param(('TITLE', 'Tribal Style')))
        self.assertRaises(MultiInstanceError, sm.get, 'NOTES')

    def test_get_string(self):
        sm = self.get_simfile('Tribal Style.sm')
        self.assertEqual(sm.get_string('TITLE'), 'Tribal Style')
        self.assertRaises(MultiInstanceError, sm.get_string, 'NOTES')

    def test_comments(self):
        sm = self.get_simfile('comments.sm')
        self.assertEqual(sm.get('TITLE'), Param(('TITLE', 'Comments')))
        self.assertEqual(sm.get('SUBTITLE'), Param(('SUBTITLE', 'Split into two lines')))
        self.assertEqual(sm.get('ARTIST'), Param(('ARTIST', 'Grant Garcia')))
        self.assertEqual(sm.get_string('TITLE'), 'Comments')

    def test_duplicates(self):
        sm = self.get_simfile('duplicates.sm')
        self.assertRaises(MultiInstanceError, sm.get, 'TITLE')
        self.assertRaises(MultiInstanceError, sm.get, 'SUBTITLE')
        self.assertRaises(MultiInstanceError, sm.get, 'subtitle')

    def test_multivalue(self):
        sm = self.get_simfile('multivalue.sm')
        self.assertEqual(sm.get('TITLE'), Param(('TITLE', 'One value')))
        self.assertEqual(sm.get('SUBTITLE'), Param(('SUBTITLE', 'First value', 'second value')))
        self.assertEqual(sm.get('ARTIST'), Param(('ARTIST', 'One', 'two', 'three')))
        self.assertEqual(sm.get_string('SUBTITLE'), 'First value:second value')
        self.assertEqual(sm.get_string('ARTIST'), 'One:two:three')

    def test_no_semicolon(self):
        sm = self.get_simfile('no semicolon.sm')
        self.assertEqual(sm.get('TITLE'), Param(('TITLE', 'No semicolon')))
        self.assertEqual(sm.get('SUBTITLE'), Param(('SUBTITLE', 'EOF')))

    def test_types(self):
        sm = self.get_simfile('Tribal Style.sm')
        self.assertIsInstance(sm.get('TITLE'), Param)
        bpms_param = sm.get('BPMS')
        self.assertIsInstance(bpms_param, Param)
        self.assertIsInstance(bpms_param[1], Timing)
        stops_param = sm.get('STOPS')
        self.assertIsInstance(stops_param[1], Timing)
        chart = sm.get_chart(index=0)
        self.assertIsInstance(chart, Chart)
        self.assertIsInstance(chart.notes, Notes)

    def test_bpms(self):
        sm = self.get_simfile('Robotix.sm')
        bpms = sm.get('BPMS')[1]
        self.assertEqual(bpms[0][0], 0)
        self.assertEqual(bpms[0][1], 150)
        self.assertEqual(bpms[1][0], 144)
        self.assertEqual(bpms[1][1], decimal.Decimal('170.001'))

    def test_stops(self):
        sm = self.get_simfile('Robotix.sm')
        stops = sm.get('STOPS')[1]
        self.assertEqual(stops[0][0], 313)
        self.assertEqual(stops[0][1], decimal.Decimal('0.400'))
        self.assertEqual(stops[1][0], 344)
        self.assertEqual(stops[1][1], decimal.Decimal('0.400'))

    def test_get_chart(self):
        sm = self.get_simfile('Tribal Style.sm')
        chart_sx = sm.get_chart(stepstype='dance-single', difficulty='Challenge')
        self.assertEqual(chart_sx.meter, 10)
        chart_sh = sm.get_chart(stepstype='dance-single', difficulty='Hard')
        self.assertEqual(chart_sh.meter, 9)
        chart_dx = sm.get_chart(stepstype='dance-double', difficulty='Challenge')
        self.assertEqual(chart_dx.meter, 11)
        chart_dh = sm.get_chart(stepstype='dance-double', difficulty='Hard')
        self.assertEqual(chart_dh.meter, 9)
        sm.get_chart(difficulty='Beginner')
        sm.get_chart(meter=11)
        sm.get_chart(index=0)
        sm.get_chart(index=8)
        sm.get_chart(description='J.Casarino')
        self.assertRaises(MultiInstanceError, sm.get_chart, stepstype='dance-single')
        self.assertRaises(MultiInstanceError, sm.get_chart, difficulty='Challenge')
        self.assertRaises(MultiInstanceError, sm.get_chart, description='M.Emirzian')
        self.assertRaises(IndexError, sm.get_chart, index=9)
        self.assertRaises(IndexError, sm.get_chart, meter=9, index=2)
        self.assertRaises(KeyError, sm.get_chart, meter=100)

    def test_param_str(self):
        sm = self.get_simfile('Tribal Style.sm')
        self.assertEqual(str(sm.get('TITLE')), '#TITLE:Tribal Style;')
        self.assertEqual(str(sm.get('Artist')), '#ARTIST:KaW;')
        bpms_param = sm.get('BPMS')
        self.assertEqual(str(bpms_param), '#BPMS:0.000=140.000;')
        self.assertEqual(str(bpms_param[1]), '0.000=140.000')
        stops_param = sm.get('STOPS')
        self.assertEqual(str(stops_param), '#STOPS:;')
        self.assertEqual(str(stops_param[1]), '')

    def test_chart_str(self):
        sm = self.get_simfile('Tribal Style.sm')
        chart_sn = sm.get_chart(meter=1)
        expected_str = (
            "#NOTES:\n"
            "     dance-single:\n"
            "     K. Ward:\n"
            "     Beginner:\n"
            "     1:\n"
            "     0.104,0.115,0.045,0.000,0.000:\n"
            "0000\n0000\n0000\n0000\n,\n")
        self.assertEqual(str(chart_sn)[:len(expected_str)], expected_str)

    def test_simfile_str(self):
        # Comprehensive test that ensures str(simfile) returns a perfect
        # representation of the original simfile. This also serves as a "test"
        # of Simfile.save(), which essentially writes str(self) to a file.
        sm1 = self.get_simfile('Tribal Style.sm')
        sm2 = Simfile(string=str(sm1))
        self.assertEqual(sm1, sm2)

    def test_set_chart(self):
        sm1 = self.get_simfile('Tribal Style.sm')
        self.assertRaises(ValueError, sm.set_chart, notes='0000')
        # TODO

if __name__ == '__main__':
    unittest.main()<commit_msg>Fixed tests<commit_after>#!/usr/bin/env python
import decimal
import os
import unittest

from simfile import *

class TestSimfile(unittest.TestCase):

    def get_simfile(self, filename, cache={}):
        if filename not in cache:
            cache[filename] = Simfile(os.path.join('testdata', filename))
        return cache[filename]

    def test_empty(self):
        sm = self.get_simfile('empty.sm')
        self.assertTrue(sm)
        self.assertFalse(sm.params)
        self.assertRaises(KeyError, sm.get, 'TITLE')
        # Nowhere else to put these at the moment
        self.assertRaises(TypeError, Simfile, filename='.', string='.')

    def test_get(self):
        sm = self.get_simfile('Tribal Style.sm')
        self.assertEqual(sm.get('TITLE'), Param(('TITLE', 'Tribal Style')))
        self.assertRaises(MultiInstanceError, sm.get, 'NOTES')

    def test_get_string(self):
        sm = self.get_simfile('Tribal Style.sm')
        self.assertEqual(sm.get_string('TITLE'), 'Tribal Style')
        self.assertRaises(MultiInstanceError, sm.get_string, 'NOTES')

    def test_comments(self):
        sm = self.get_simfile('comments.sm')
        self.assertEqual(sm.get('TITLE'), Param(('TITLE', 'Comments')))
        self.assertEqual(sm.get('SUBTITLE'), Param(('SUBTITLE', 'Split into two lines')))
        self.assertEqual(sm.get('ARTIST'), Param(('ARTIST', 'Grant Garcia')))
        self.assertEqual(sm.get_string('TITLE'), 'Comments')

    def test_duplicates(self):
        sm = self.get_simfile('duplicates.sm')
        self.assertRaises(MultiInstanceError, sm.get, 'TITLE')
        self.assertRaises(MultiInstanceError, sm.get, 'SUBTITLE')
        self.assertRaises(MultiInstanceError, sm.get, 'subtitle')

    def test_multivalue(self):
        sm = self.get_simfile('multivalue.sm')
        self.assertEqual(sm.get('TITLE'), Param(('TITLE', 'One value')))
        self.assertEqual(sm.get('SUBTITLE'), Param(('SUBTITLE', 'First value', 'second value')))
        self.assertEqual(sm.get('ARTIST'), Param(('ARTIST', 'One', 'two', 'three')))
        self.assertEqual(sm.get_string('SUBTITLE'), 'First value:second value')
        self.assertEqual(sm.get_string('ARTIST'), 'One:two:three')

    def test_no_semicolon(self):
        sm = self.get_simfile('no semicolon.sm')
        self.assertEqual(sm.get('TITLE'), Param(('TITLE', 'No semicolon')))
        self.assertEqual(sm.get('SUBTITLE'), Param(('SUBTITLE', 'EOF')))

    def test_types(self):
        sm = self.get_simfile('Tribal Style.sm')
        self.assertIsInstance(sm.get('TITLE'), Param)
        bpms_param = sm.get('BPMS')
        self.assertIsInstance(bpms_param, Param)
        self.assertIsInstance(bpms_param[1], Timing)
        stops_param = sm.get('STOPS')
        self.assertIsInstance(stops_param[1], Timing)
        chart = sm.get_chart(index=0)
        self.assertIsInstance(chart, Chart)
        self.assertIsInstance(chart.notes, Notes)

    def test_bpms(self):
        sm = self.get_simfile('Robotix.sm')
        bpms = sm.get('BPMS')[1]
        self.assertEqual(bpms[0][0], 0)
        self.assertEqual(bpms[0][1], 150)
        self.assertEqual(bpms[1][0], 144)
        self.assertEqual(bpms[1][1], decimal.Decimal('170.001'))

    def test_stops(self):
        sm = self.get_simfile('Robotix.sm')
        stops = sm.get('STOPS')[1]
        self.assertEqual(stops[0][0], 313)
        self.assertEqual(stops[0][1], decimal.Decimal('0.400'))
        self.assertEqual(stops[1][0], 344)
        self.assertEqual(stops[1][1], decimal.Decimal('0.400'))

    def test_get_chart(self):
        sm = self.get_simfile('Tribal Style.sm')
        chart_sx = sm.get_chart(stepstype='dance-single', difficulty='Challenge')
        self.assertEqual(chart_sx.meter, 10)
        chart_sh = sm.get_chart(stepstype='dance-single', difficulty='Hard')
        self.assertEqual(chart_sh.meter, 9)
        chart_dx = sm.get_chart(stepstype='dance-double', difficulty='Challenge')
        self.assertEqual(chart_dx.meter, 11)
        chart_dh = sm.get_chart(stepstype='dance-double', difficulty='Hard')
        self.assertEqual(chart_dh.meter, 9)
        sm.get_chart(difficulty='Beginner')
        sm.get_chart(meter=11)
        sm.get_chart(index=0)
        sm.get_chart(index=8)
        sm.get_chart(description='J.Casarino')
        self.assertRaises(MultiInstanceError, sm.get_chart, stepstype='dance-single')
        self.assertRaises(MultiInstanceError, sm.get_chart, difficulty='Challenge')
        self.assertRaises(MultiInstanceError, sm.get_chart, description='M.Emirzian')
        self.assertRaises(IndexError, sm.get_chart, index=9)
        self.assertRaises(IndexError, sm.get_chart, meter=9, index=2)
        self.assertRaises(KeyError, sm.get_chart, meter=100)

    def test_param_str(self):
        sm = self.get_simfile('Tribal Style.sm')
        self.assertEqual(str(sm.get('TITLE')), '#TITLE:Tribal Style;')
        self.assertEqual(str(sm.get('Artist')), '#ARTIST:KaW;')
        bpms_param = sm.get('BPMS')
        self.assertEqual(str(bpms_param), '#BPMS:0.000=140.000;')
        self.assertEqual(str(bpms_param[1]), '0.000=140.000')
        stops_param = sm.get('STOPS')
        self.assertEqual(str(stops_param), '#STOPS:;')
        self.assertEqual(str(stops_param[1]), '')

    def test_chart_str(self):
        sm = self.get_simfile('Tribal Style.sm')
        chart_sn = sm.get_chart(meter=1)
        expected_str = (
            "#NOTES:\n"
            "     dance-single:\n"
            "     K. Ward:\n"
            "     Beginner:\n"
            "     1:\n"
            "     0.104,0.115,0.045,0.000,0.000:\n"
            "0000\n0000\n0000\n0000\n,\n")
        self.assertEqual(str(chart_sn)[:len(expected_str)], expected_str)

    def test_simfile_str(self):
        # Comprehensive test that ensures str(simfile) returns a perfect
        # representation of the original simfile. This also serves as a "test"
        # of Simfile.save(), which essentially writes str(self) to a file.
        sm1 = self.get_simfile('Tribal Style.sm')
        sm2 = Simfile(string=str(sm1))
        self.assertEqual(sm1, sm2)

if __name__ == '__main__':
    unittest.main()<|endoftext|>


<commit_before>#from cnv import *

#__all__ = ['CNV', 'fCNV']

class CNVError(Exception):
    """Base class for exceptions in this module."""
    def __init__(self, tag, msg):
        self.tag = tag
        self.msg = msg
<commit_msg>Default message for CNVError.<commit_after>#from cnv import *

#__all__ = ['CNV', 'fCNV']

class CNVError(Exception):
    """Base class for exceptions in this module."""
    def __init__(self, tag, msg=None):
        self.tag = tag

        if msg is not None:
            self.msg = msg
        elif tag == 'noparsingrule':
            self.msg = "There are no rules able to parse the input."
<|endoftext|>