<commit_before># -*- coding: ISO-8859-15 -*-
# =============================================================================
# Copyright (c) 2010 Tom Kralidis
#
# Authors : Tom Kralidis <tomkralidis@hotmail.com>
#
# Contact email: tomkralidis@hotmail.com
# =============================================================================

""" FGDC metadata parser """

from owslib.etree import etree
from owslib import util

class Metadata(object):
    """ Process metadata """
    def __init__(self, md):
        if hasattr(md, 'getroot'):  # standalone document
            self.xml = etree.tostring(md.getroot())
        else:  # part of a larger document
            self.xml = etree.tostring(md)

        self.idinfo = Idinfo(md)
        self.eainfo = Eainfo(md)
        self.metainfo = Metainfo(md)

        if self.idinfo.datasetid:
            self.identifier = self.idinfo.datasetid

class Idinfo(object):
    """ Process idinfo """
    def __init__(self, md):
        val = md.find('idinfo/datasetid')
        self.datasetid = util.testXMLValue(val)

        val = md.find('idinfo/citation')
        self.citation = Citation(val)

        val = md.find('idinfo/descript')
        if val is not None:
            self.descript = Descript(val)

        val = md.find('idinfo/timeperd')
        self.timeperd = Timeperd(val)

        val = md.find('idinfo/status')
        if val is not None:
            self.status = Status(val)

        val = md.find('idinfo/spdom')
        if val is not None:
            self.spdom = Spdom(val)

        val = md.find('idinfo/keywords')
        if val is not None:
            self.keywords = Keywords(val)

        val = md.find('idinfo/accconst')
        self.accconst = util.testXMLValue(val)

        val = md.find('idinfo/useconst')
        self.useconst = util.testXMLValue(val)

        val = md.find('idinfo/ptcontac')
        if val is not None:
            self.ptcontac = Ptcontac(val)

        val = md.find('idinfo/datacred')
        self.datacred = util.testXMLValue(val)

        val = md.find('idinfo/crossref')
        self.crossref = Citation(val)

class Citation(object):
    """ Process citation """
    def __init__(self, md):
        if md is not None:
            self.citeinfo = {}

            val = md.find('citeinfo/origin')
            self.citeinfo['origin'] = util.testXMLValue(val)

            val = md.find('citeinfo/pubdate')
            self.citeinfo['pubdate'] = util.testXMLValue(val)

            val = md.find('citeinfo/title')
            self.citeinfo['title'] = util.testXMLValue(val)

            val = md.find('citeinfo/geoform')
            self.citeinfo['geoform'] = util.testXMLValue(val)

            val = md.find('citeinfo/pubinfo/pubplace')
            self.citeinfo['pubplace'] = util.testXMLValue(val)

            val = md.find('citeinfo/pubinfo/publish')
            self.citeinfo['publish'] = util.testXMLValue(val)

            self.citeinfo['onlink'] = []
            for link in md.findall('citeinfo/onlink'):
                self.citeinfo['onlink'].append(util.testXMLValue(link))

class Descript(object):
    """ Process descript """
    def __init__(self, md):
        val = md.find('abstract')
        self.abstract = util.testXMLValue(val)

        val = md.find('purpose')
        self.purpose = util.testXMLValue(val)

        val = md.find('supplinf')
        self.supplinf = util.testXMLValue(val)

class Timeperd(object):
    """ Process timeperd """
    def __init__(self, md):
        if md is not None:
            val = md.find('current')
            self.current = util.testXMLValue(val)

            val = md.find('timeinfo')
            if val is not None:
                self.timeinfo = Timeinfo(val)

class Timeinfo(object):
    """ Process timeinfo """
    def __init__(self, md):
        val = md.find('sngdate')
        if val is not None:
            self.sngdate = Sngdate(val)

        val = md.find('rngdates')
        if val is not None:
            self.rngdates = Rngdates(val)

class Sngdate(object):
    """ Process sngdate """
    def __init__(self, md):
        val = md.find('caldate')
        self.caldate = util.testXMLValue(val)
        val = md.find('time')
        self.time = util.testXMLValue(val)

class Rngdates(object):
    """ Process rngdates """
    def __init__(self, md):
        val = md.find('begdate')
        self.begdate = util.testXMLValue(val)
        val = md.find('begtime')
        self.begtime = util.testXMLValue(val)
        val = md.find('enddate')
        self.enddate = util.testXMLValue(val)
        val = md.find('endtime')
        self.endtime = util.testXMLValue(val)

class Status(object):
    """ Process status """
    def __init__(self, md):
        val = md.find('progress')
        self.progress = util.testXMLValue(val)

        val = md.find('update')
        self.update = util.testXMLValue(val)

class Spdom(object):
    """ Process spdom """
    def __init__(self, md):
        val = md.find('bounding/westbc')
        self.westbc = util.testXMLValue(val)

        val = md.find('bounding/eastbc')
        self.eastbc = util.testXMLValue(val)

        val = md.find('bounding/northbc')
        self.northbc = util.testXMLValue(val)

        val = md.find('bounding/southbc')
        self.southbc = util.testXMLValue(val)

        if (self.southbc is not None and self.northbc is not None and
        self.eastbc is not None and self.westbc is not None):
            self.bbox = Bbox(self)

class Bbox(object):
    """ Generate bbox for spdom (convenience function) """
    def __init__(self, spdom):
        self.minx = spdom.westbc
        self.miny = spdom.southbc
        self.maxx = spdom.eastbc
        self.maxy = spdom.northbc

class Keywords(object):
    """ Process keywords """
    def __init__(self, md):
        self.theme = []
        self.place = []
        self.temporal = []

        for i in md.findall('theme'):
            theme = {}
            val = i.find('themekt')
            theme['themekt'] = util.testXMLValue(val)
            theme['themekey'] = []
            for j in i.findall('themekey'):
                themekey = util.testXMLValue(j)
                if themekey is not None:
                    theme['themekey'].append(themekey)
            self.theme.append(theme)

        for i in md.findall('place'):
            theme = {}
            place = {}
            val = i.find('placekt')
            theme['placekt'] = util.testXMLValue(val)
            theme['placekey'] = []
            for j in i.findall('placekey'):
                theme['placekey'].append(util.testXMLValue(j))
            self.place.append(place)

        for i in md.findall('temporal'):
            theme = {}
            temporal = {}
            val = i.find('tempkt')
            theme['tempkt'] = util.testXMLValue(val)
            theme['tempkey'] = []
            for j in i.findall('tempkey'):
                theme['tempkey'].append(util.testXMLValue(j))
            self.temporal.append(temporal)

class Ptcontac(object):
    """ Process ptcontac """
    def __init__(self, md):
        val = md.find('cntinfo/cntorgp/cntorg')
        self.cntorg = util.testXMLValue(val)

        val = md.find('cntinfo/cntorgp/cntper')
        self.cntper = util.testXMLValue(val)

        val = md.find('cntinfo/cntpos')
        self.cntpos = util.testXMLValue(val)

        val = md.find('cntinfo/cntaddr/addrtype')
        self.addrtype = util.testXMLValue(val)

        val = md.find('cntinfo/cntaddr/address')
        self.address = util.testXMLValue(val)

        val = md.find('cntinfo/cntaddr/city')
        self.city = util.testXMLValue(val)

        val = md.find('cntinfo/cntaddr/state')
        self.state = util.testXMLValue(val)

        val = md.find('cntinfo/cntaddr/postal')
        self.postal = util.testXMLValue(val)

        val = md.find('cntinfo/cntaddr/country')
        self.country = util.testXMLValue(val)

        val = md.find('cntinfo/cntvoice')
        self.voice = util.testXMLValue(val)

        val = md.find('cntinfo/cntemail')
        self.email = util.testXMLValue(val)

class Eainfo(object):
    """ Process eainfo """
    def __init__(self, md):
        val = md.find('eainfo/detailed/enttyp/enttypl')
        self.enttypl = util.testXMLValue(val)

        val = md.find('eainfo/detailed/enttyp/enttypd')
        self.enttypd = util.testXMLValue(val)

        val = md.find('eainfo/detailed/enttyp/enttypds')
        self.enttypds = util.testXMLValue(val)

        self.attr = []
        for i in md.findall('eainfo/detailed/attr'):
            attr = {}
            val = i.find('attrlabl')
            attr['attrlabl'] = util.testXMLValue(val)

            val = i.find('attrdef')
            attr['attrdef'] = util.testXMLValue(val)

            val = i.find('attrdefs')
            attr['attrdefs'] = util.testXMLValue(val)

            val = i.find('attrdomv/udom')
            attr['udom'] = util.testXMLValue(val)

            self.attr.append(attr)

class Metainfo(object):
    """ Process metainfo """
    def __init__(self, md):
        val = md.find('metainfo/metd')
        self.metd = util.testXMLValue(val)

        val = md.find('metainfo/metrd')
        self.metrd = util.testXMLValue(val)

        val = md.find('metainfo/metc')
        if val is not None:
            self.metc = Ptcontac(val)

        val = md.find('metainfo/metstdn')
        self.metstdn = util.testXMLValue(val)

        val = md.find('metainfo/metstdv')
        self.metstdv = util.testXMLValue(val)

        val = md.find('metainfo/metac')
        self.metac = util.testXMLValue(val)

        val = md.find('metainfo/metuc')
        self.metuc = util.testXMLValue(val)
<commit_msg>add support for distinfo<commit_after># -*- coding: ISO-8859-15 -*-
# =============================================================================
# Copyright (c) 2010 Tom Kralidis
#
# Authors : Tom Kralidis <tomkralidis@hotmail.com>
#
# Contact email: tomkralidis@hotmail.com
# =============================================================================

""" FGDC metadata parser """

from owslib.etree import etree
from owslib import util

class Metadata(object):
    """ Process metadata """
    def __init__(self, md):
        if hasattr(md, 'getroot'):  # standalone document
            self.xml = etree.tostring(md.getroot())
        else:  # part of a larger document
            self.xml = etree.tostring(md)

        self.idinfo = Idinfo(md)
        self.eainfo = Eainfo(md)
        self.distinfo = Distinfo(md)
        self.metainfo = Metainfo(md)

        if self.idinfo.datasetid:
            self.identifier = self.idinfo.datasetid

class Idinfo(object):
    """ Process idinfo """
    def __init__(self, md):
        val = md.find('idinfo/datasetid')
        self.datasetid = util.testXMLValue(val)

        val = md.find('idinfo/citation')
        self.citation = Citation(val)

        val = md.find('idinfo/descript')
        if val is not None:
            self.descript = Descript(val)

        val = md.find('idinfo/timeperd')
        self.timeperd = Timeperd(val)

        val = md.find('idinfo/status')
        if val is not None:
            self.status = Status(val)

        val = md.find('idinfo/spdom')
        if val is not None:
            self.spdom = Spdom(val)

        val = md.find('idinfo/keywords')
        if val is not None:
            self.keywords = Keywords(val)

        val = md.find('idinfo/accconst')
        self.accconst = util.testXMLValue(val)

        val = md.find('idinfo/useconst')
        self.useconst = util.testXMLValue(val)

        val = md.find('idinfo/ptcontac')
        if val is not None:
            self.ptcontac = Ptcontac(val)

        val = md.find('idinfo/datacred')
        self.datacred = util.testXMLValue(val)

        val = md.find('idinfo/crossref')
        self.crossref = Citation(val)

class Citation(object):
    """ Process citation """
    def __init__(self, md):
        if md is not None:
            self.citeinfo = {}

            val = md.find('citeinfo/origin')
            self.citeinfo['origin'] = util.testXMLValue(val)

            val = md.find('citeinfo/pubdate')
            self.citeinfo['pubdate'] = util.testXMLValue(val)

            val = md.find('citeinfo/title')
            self.citeinfo['title'] = util.testXMLValue(val)

            val = md.find('citeinfo/geoform')
            self.citeinfo['geoform'] = util.testXMLValue(val)

            val = md.find('citeinfo/pubinfo/pubplace')
            self.citeinfo['pubplace'] = util.testXMLValue(val)

            val = md.find('citeinfo/pubinfo/publish')
            self.citeinfo['publish'] = util.testXMLValue(val)

            self.citeinfo['onlink'] = []
            for link in md.findall('citeinfo/onlink'):
                self.citeinfo['onlink'].append(util.testXMLValue(link))

class Descript(object):
    """ Process descript """
    def __init__(self, md):
        val = md.find('abstract')
        self.abstract = util.testXMLValue(val)

        val = md.find('purpose')
        self.purpose = util.testXMLValue(val)

        val = md.find('supplinf')
        self.supplinf = util.testXMLValue(val)

class Timeperd(object):
    """ Process timeperd """
    def __init__(self, md):
        if md is not None:
            val = md.find('current')
            self.current = util.testXMLValue(val)

            val = md.find('timeinfo')
            if val is not None:
                self.timeinfo = Timeinfo(val)

class Timeinfo(object):
    """ Process timeinfo """
    def __init__(self, md):
        val = md.find('sngdate')
        if val is not None:
            self.sngdate = Sngdate(val)

        val = md.find('rngdates')
        if val is not None:
            self.rngdates = Rngdates(val)

class Sngdate(object):
    """ Process sngdate """
    def __init__(self, md):
        val = md.find('caldate')
        self.caldate = util.testXMLValue(val)
        val = md.find('time')
        self.time = util.testXMLValue(val)

class Rngdates(object):
    """ Process rngdates """
    def __init__(self, md):
        val = md.find('begdate')
        self.begdate = util.testXMLValue(val)
        val = md.find('begtime')
        self.begtime = util.testXMLValue(val)
        val = md.find('enddate')
        self.enddate = util.testXMLValue(val)
        val = md.find('endtime')
        self.endtime = util.testXMLValue(val)

class Status(object):
    """ Process status """
    def __init__(self, md):
        val = md.find('progress')
        self.progress = util.testXMLValue(val)

        val = md.find('update')
        self.update = util.testXMLValue(val)

class Spdom(object):
    """ Process spdom """
    def __init__(self, md):
        val = md.find('bounding/westbc')
        self.westbc = util.testXMLValue(val)

        val = md.find('bounding/eastbc')
        self.eastbc = util.testXMLValue(val)

        val = md.find('bounding/northbc')
        self.northbc = util.testXMLValue(val)

        val = md.find('bounding/southbc')
        self.southbc = util.testXMLValue(val)

        if (self.southbc is not None and self.northbc is not None and
        self.eastbc is not None and self.westbc is not None):
            self.bbox = Bbox(self)

class Bbox(object):
    """ Generate bbox for spdom (convenience function) """
    def __init__(self, spdom):
        self.minx = spdom.westbc
        self.miny = spdom.southbc
        self.maxx = spdom.eastbc
        self.maxy = spdom.northbc

class Keywords(object):
    """ Process keywords """
    def __init__(self, md):
        self.theme = []
        self.place = []
        self.temporal = []

        for i in md.findall('theme'):
            theme = {}
            val = i.find('themekt')
            theme['themekt'] = util.testXMLValue(val)
            theme['themekey'] = []
            for j in i.findall('themekey'):
                themekey = util.testXMLValue(j)
                if themekey is not None:
                    theme['themekey'].append(themekey)
            self.theme.append(theme)

        for i in md.findall('place'):
            theme = {}
            place = {}
            val = i.find('placekt')
            theme['placekt'] = util.testXMLValue(val)
            theme['placekey'] = []
            for j in i.findall('placekey'):
                theme['placekey'].append(util.testXMLValue(j))
            self.place.append(place)

        for i in md.findall('temporal'):
            theme = {}
            temporal = {}
            val = i.find('tempkt')
            theme['tempkt'] = util.testXMLValue(val)
            theme['tempkey'] = []
            for j in i.findall('tempkey'):
                theme['tempkey'].append(util.testXMLValue(j))
            self.temporal.append(temporal)

class Ptcontac(object):
    """ Process ptcontac """
    def __init__(self, md):
        val = md.find('cntinfo/cntorgp/cntorg')
        self.cntorg = util.testXMLValue(val)

        val = md.find('cntinfo/cntorgp/cntper')
        self.cntper = util.testXMLValue(val)

        val = md.find('cntinfo/cntpos')
        self.cntpos = util.testXMLValue(val)

        val = md.find('cntinfo/cntaddr/addrtype')
        self.addrtype = util.testXMLValue(val)

        val = md.find('cntinfo/cntaddr/address')
        self.address = util.testXMLValue(val)

        val = md.find('cntinfo/cntaddr/city')
        self.city = util.testXMLValue(val)

        val = md.find('cntinfo/cntaddr/state')
        self.state = util.testXMLValue(val)

        val = md.find('cntinfo/cntaddr/postal')
        self.postal = util.testXMLValue(val)

        val = md.find('cntinfo/cntaddr/country')
        self.country = util.testXMLValue(val)

        val = md.find('cntinfo/cntvoice')
        self.voice = util.testXMLValue(val)

        val = md.find('cntinfo/cntemail')
        self.email = util.testXMLValue(val)

class Eainfo(object):
    """ Process eainfo """
    def __init__(self, md):
        val = md.find('eainfo/detailed/enttyp/enttypl')
        self.enttypl = util.testXMLValue(val)

        val = md.find('eainfo/detailed/enttyp/enttypd')
        self.enttypd = util.testXMLValue(val)

        val = md.find('eainfo/detailed/enttyp/enttypds')
        self.enttypds = util.testXMLValue(val)

        self.attr = []
        for i in md.findall('eainfo/detailed/attr'):
            attr = {}
            val = i.find('attrlabl')
            attr['attrlabl'] = util.testXMLValue(val)

            val = i.find('attrdef')
            attr['attrdef'] = util.testXMLValue(val)

            val = i.find('attrdefs')
            attr['attrdefs'] = util.testXMLValue(val)

            val = i.find('attrdomv/udom')
            attr['udom'] = util.testXMLValue(val)

            self.attr.append(attr)

class Distinfo(object):
    """ Process distinfo """
    def __init__(self, md):
        val = md.find('distinfo')
        if val is not None:
            val2 = val.find('stdorder')
            if val2 is not None:
                self.stdorder = {'digform': []}
                for link in val2.findall('digform'):
                    digform = {}
                    digform['name'] = util.testXMLValue(link.find('digtinfo/formname'))
                    digform['url'] = util.testXMLValue(link.find('digtopt/onlinopt/computer/networka/networkr/'))
                    self.stdorder['digform'].append(digform)

class Metainfo(object):
    """ Process metainfo """
    def __init__(self, md):
        val = md.find('metainfo/metd')
        self.metd = util.testXMLValue(val)

        val = md.find('metainfo/metrd')
        self.metrd = util.testXMLValue(val)

        val = md.find('metainfo/metc')
        if val is not None:
            self.metc = Ptcontac(val)

        val = md.find('metainfo/metstdn')
        self.metstdn = util.testXMLValue(val)

        val = md.find('metainfo/metstdv')
        self.metstdv = util.testXMLValue(val)

        val = md.find('metainfo/metac')
        self.metac = util.testXMLValue(val)

        val = md.find('metainfo/metuc')
        self.metuc = util.testXMLValue(val)
<|endoftext|>


<commit_before>"""Oz"""

from __future__ import absolute_import, division, print_function, with_statement, unicode_literals

import types
import collections
import tornado.web
import tornado.options
import tornado.util
import tornado.log
import inspect
import functools
import os

# On trigger execution, trigger listeners can return this to notify the
# request handler to cancel execution of the next functions in the trigger
# chain.
break_trigger = object()

# Mapping of action name -> callback
_actions = {}

# Mapping of uimodule name -> class
_uimodules = {}

# List of routes
_routes = []

# List of test classes
_tests = []

# Mapping of setting name -> value
settings = {}

def _add_to_dict(t, container, name, value):
    """
    Adds an item to a dictionary, or raises an exception if an item with the
    specified key already exists in the dictionary.
    """

    if name in container:
        raise Exception("%s '%s' already exists" % (t, name))
    else:
        container[name] = value

def action(fun):
    """Exposes an action"""
    _add_to_dict("Action", _actions, fun.__name__, fun)
    return fun

def uimodule(cls):
    """Exposes a UIModule"""
    _add_to_dict("UIModules", _uimodules, cls.__name__, cls)
    return cls

def route(new_route):
    """Exposes a route"""
    _routes.append(new_route)

def routes(*new_routes):
    """Exposes a list of routes"""
    _routes.extend(new_routes)

def option(name, **args):
    """Exposes an option"""
    tornado.options.define(name, **args)

def options(**kwargs):
    """Exposes several options"""
    for name, args in kwargs.items():
        option(name, **args)

def test(cls):
    """Exposes a unit test class, to be run on the `test` action."""
    _tests.append(cls)
    return cls

def plugin(namespace):
    """Loads an oz plugin"""
    __import__(namespace, globals(), locals(), [], 0)

class RequestHandler(tornado.web.RequestHandler):
    """
    Builds on top of tornado's RequestHandler to provide template helpers and triggers
    """

    def __init__(self, *args, **kwargs):
        self._template_helpers = {}
        self._triggers = collections.defaultdict(lambda: [])
        super(RequestHandler, self).__init__(*args, **kwargs)

    def template_helper(self, name, callback):
        """Adds a template helper"""
        _add_to_dict("Template helper", self._template_helpers, name, callback)

    def trigger_listener(self, name, callback):
        """Adds a trigger listener"""
        self._triggers[name].append(callback)

    def trigger(self, name, *args, **kwargs):
        """
        Triggers an event to run through middleware. This method will execute
        a chain of relevant trigger callbacks, until one of the callbacks
        returns the `break_trigger`.
        """

        # Relevant middleware is cached so we don't have to rediscover it
        # every time. Fetch the cached value if possible.

        listeners = self._triggers.get(name, [])

        # Execute each piece of middleware
        for listener in listeners:
            result = listener(*args, **kwargs)

            if result == break_trigger:
                return False

        return True

    def get_template_namespace(self):
        # Start with the default namespace
        namespace = super(RequestHandler, self).get_template_namespace()

        # Add the template helpers
        namespace.update(self._template_helpers)

        return namespace

    def initialize(self, **kwargs):
        self.trigger("initialize", **kwargs)

    def set_secure_cookie(self, name, value, **kwargs):
        if self.settings["use_secure_cookie"]:
            if "secure" not in kwargs:
                kwargs["secure"] = True
            if "httponly" not in kwargs:
                kwargs["httponly"] = True
        super(RequestHandler, self).set_secure_cookie(name, value, **kwargs)

    def prepare(self):
        if self.settings["use_hsts"]:
            self.set_header("Strict-Transport-Security", "max-age=2592000; includeSubDomains")
            # Redirect to HTTPS if necessary
            if self.request.headers.get("X-Forwarded-Proto", self.request.protocol) != "https":
                self.redirect("https://%s%s" % (self.request.host, self.request.uri), permanent=True)
                return

        self.trigger("prepare")

    def on_finish(self):
        self.trigger("on_finish")

    def on_connection_close(self):
        self.trigger("on_connection_close")

    def write_error(self, *args, **kwargs):
        # Call super's implementation of `write_error` unless we hit a
        # `break_trigger`.
        if self.trigger("write_error", *args, **kwargs):
            super(RequestHandler, self).write_error(*args, **kwargs)

def initialize(config=None):
    """Initializes oz"""

    # Load the config file
    if config == None:
        config = {}
        config_source = None

        try:
            with open(os.environ.get("OZ_CONFIG", "config.py")) as f:
                config_source = f.read()
        except Exception:
            tornado.log.gen_log.info("Could not read config.py", exc_info=True)

        if config_source != None:
            tornado.util.exec_in(config_source, config, config)

    # Load the plugins
    for p in config.get("plugins", ["oz.core"]):
        plugin(p)

    # Set the options
    for key, value in config.get("app_options", {}).items():
        setattr(tornado.options.options, key, value)

    # Generate the application settings
    global settings
    settings = tornado.options.options.as_dict()
    settings["ui_modules"] = _uimodules
    settings["project_name"] = config.get("project_name")
<commit_msg>remove initial redirect<commit_after>"""Oz"""

from __future__ import absolute_import, division, print_function, with_statement, unicode_literals

import types
import collections
import tornado.web
import tornado.options
import tornado.util
import tornado.log
import inspect
import functools
import os

# On trigger execution, trigger listeners can return this to notify the
# request handler to cancel execution of the next functions in the trigger
# chain.
break_trigger = object()

# Mapping of action name -> callback
_actions = {}

# Mapping of uimodule name -> class
_uimodules = {}

# List of routes
_routes = []

# List of test classes
_tests = []

# Mapping of setting name -> value
settings = {}

def _add_to_dict(t, container, name, value):
    """
    Adds an item to a dictionary, or raises an exception if an item with the
    specified key already exists in the dictionary.
    """

    if name in container:
        raise Exception("%s '%s' already exists" % (t, name))
    else:
        container[name] = value

def action(fun):
    """Exposes an action"""
    _add_to_dict("Action", _actions, fun.__name__, fun)
    return fun

def uimodule(cls):
    """Exposes a UIModule"""
    _add_to_dict("UIModules", _uimodules, cls.__name__, cls)
    return cls

def route(new_route):
    """Exposes a route"""
    _routes.append(new_route)

def routes(*new_routes):
    """Exposes a list of routes"""
    _routes.extend(new_routes)

def option(name, **args):
    """Exposes an option"""
    tornado.options.define(name, **args)

def options(**kwargs):
    """Exposes several options"""
    for name, args in kwargs.items():
        option(name, **args)

def test(cls):
    """Exposes a unit test class, to be run on the `test` action."""
    _tests.append(cls)
    return cls

def plugin(namespace):
    """Loads an oz plugin"""
    __import__(namespace, globals(), locals(), [], 0)

class RequestHandler(tornado.web.RequestHandler):
    """
    Builds on top of tornado's RequestHandler to provide template helpers and triggers
    """

    def __init__(self, *args, **kwargs):
        self._template_helpers = {}
        self._triggers = collections.defaultdict(lambda: [])
        super(RequestHandler, self).__init__(*args, **kwargs)

    def template_helper(self, name, callback):
        """Adds a template helper"""
        _add_to_dict("Template helper", self._template_helpers, name, callback)

    def trigger_listener(self, name, callback):
        """Adds a trigger listener"""
        self._triggers[name].append(callback)

    def trigger(self, name, *args, **kwargs):
        """
        Triggers an event to run through middleware. This method will execute
        a chain of relevant trigger callbacks, until one of the callbacks
        returns the `break_trigger`.
        """

        # Relevant middleware is cached so we don't have to rediscover it
        # every time. Fetch the cached value if possible.

        listeners = self._triggers.get(name, [])

        # Execute each piece of middleware
        for listener in listeners:
            result = listener(*args, **kwargs)

            if result == break_trigger:
                return False

        return True

    def get_template_namespace(self):
        # Start with the default namespace
        namespace = super(RequestHandler, self).get_template_namespace()

        # Add the template helpers
        namespace.update(self._template_helpers)

        return namespace

    def initialize(self, **kwargs):
        self.trigger("initialize", **kwargs)

    def set_secure_cookie(self, name, value, **kwargs):
        if self.settings["use_secure_cookie"]:
            if "secure" not in kwargs:
                kwargs["secure"] = True
            if "httponly" not in kwargs:
                kwargs["httponly"] = True
        super(RequestHandler, self).set_secure_cookie(name, value, **kwargs)

    def prepare(self):
        if self.settings["use_hsts"]:
            self.set_header("Strict-Transport-Security", "max-age=2592000; includeSubDomains")

        self.trigger("prepare")

    def on_finish(self):
        self.trigger("on_finish")

    def on_connection_close(self):
        self.trigger("on_connection_close")

    def write_error(self, *args, **kwargs):
        # Call super's implementation of `write_error` unless we hit a
        # `break_trigger`.
        if self.trigger("write_error", *args, **kwargs):
            super(RequestHandler, self).write_error(*args, **kwargs)

def initialize(config=None):
    """Initializes oz"""

    # Load the config file
    if config == None:
        config = {}
        config_source = None

        try:
            with open(os.environ.get("OZ_CONFIG", "config.py")) as f:
                config_source = f.read()
        except Exception:
            tornado.log.gen_log.info("Could not read config.py", exc_info=True)

        if config_source != None:
            tornado.util.exec_in(config_source, config, config)

    # Load the plugins
    for p in config.get("plugins", ["oz.core"]):
        plugin(p)

    # Set the options
    for key, value in config.get("app_options", {}).items():
        setattr(tornado.options.options, key, value)

    # Generate the application settings
    global settings
    settings = tornado.options.options.as_dict()
    settings["ui_modules"] = _uimodules
    settings["project_name"] = config.get("project_name")
<|endoftext|>


<commit_before># Copyright (C) 2002 Python Software Foundation
#
# A torture test of the email package.  This should not be run as part of the
# standard Python test suite since it requires several meg of email messages
# collected in the wild.  These source messages are not checked into the
# Python distro, but are available as part of the standalone email package at
# http://sf.net/projects/mimelib

import sys
import os
import unittest
from cStringIO import StringIO
from types import ListType

from email.test.test_email import TestEmailBase
from test.test_support import TestSkipped

import email
from email import __file__ as testfile
from email.Iterators import _structure

def openfile(filename):
    from os.path import join, dirname, abspath
    path = abspath(join(dirname(testfile), os.pardir, 'moredata', filename))
    return open(path)

# Prevent this test from running in the Python distro
try:
    openfile('crispin-torture.txt')
except IOError:
    raise TestSkipped

class TortureBase(TestEmailBase):
    def _msgobj(self, filename):
        fp = openfile(filename)
        try:
            msg = email.message_from_file(fp)
        finally:
            fp.close()
        return msg

class TestCrispinTorture(TortureBase):
    # Mark Crispin's torture test from the SquirrelMail project
    def test_mondo_message(self):
        eq = self.assertEqual
        msg = self._msgobj('crispin-torture.txt')
        payload = msg.get_payload()
        eq(type(payload), ListType)
        eq(len(payload), 12)
        eq(msg.preamble, None)
        eq(msg.epilogue, '\n\n')
        # Probably the best way to verify the message is parsed correctly is to
        # dump its structure and compare it against the known structure.
        fp = StringIO()
        _structure(msg, fp=fp)
        eq(fp.getvalue(), """\
multipart/mixed
    text/plain
    message/rfc822
        multipart/alternative
            text/plain
            multipart/mixed
                text/richtext
            application/andrew-inset
    message/rfc822
        audio/basic
    audio/basic
    image/pbm
    message/rfc822
        multipart/mixed
            multipart/mixed
                text/plain
                audio/x-sun
            multipart/mixed
                image/gif
                image/gif
                application/x-be2
                application/atomicmail
            audio/x-sun
    message/rfc822
        multipart/mixed
            text/plain
            image/pgm
            text/plain
    message/rfc822
        multipart/mixed
            text/plain
            image/pbm
    message/rfc822
        application/postscript
    image/gif
    message/rfc822
        multipart/mixed
            audio/basic
            audio/basic
    message/rfc822
        multipart/mixed
            application/postscript
            binary
            message/rfc822
                multipart/mixed
                    text/plain
                    multipart/parallel
                        image/gif
                        audio/basic
                    application/atomicmail
                    message/rfc822
                        audio/x-sun
""")

def _testclasses():
    mod = sys.modules[__name__]
    return [getattr(mod, name) for name in dir(mod) if name.startswith('Test')]

def suite():
    suite = unittest.TestSuite()
    for testclass in _testclasses():
        suite.addTest(unittest.makeSuite(testclass))
    return suite

def test_main():
    for testclass in _testclasses():
        test_support.run_unittest(testclass)

if __name__ == '__main__':
    unittest.main(defaultTest='suite')
<commit_msg>test_mondo_message(): "binary" is not a legal content type, so with the previous RFC 2045, $5.2 repair to get_content_type() this subpart's type will now be text/plain.<commit_after># Copyright (C) 2002 Python Software Foundation
#
# A torture test of the email package.  This should not be run as part of the
# standard Python test suite since it requires several meg of email messages
# collected in the wild.  These source messages are not checked into the
# Python distro, but are available as part of the standalone email package at
# http://sf.net/projects/mimelib

import sys
import os
import unittest
from cStringIO import StringIO
from types import ListType

from email.test.test_email import TestEmailBase
from test.test_support import TestSkipped

import email
from email import __file__ as testfile
from email.Iterators import _structure

def openfile(filename):
    from os.path import join, dirname, abspath
    path = abspath(join(dirname(testfile), os.pardir, 'moredata', filename))
    return open(path)

# Prevent this test from running in the Python distro
try:
    openfile('crispin-torture.txt')
except IOError:
    raise TestSkipped

class TortureBase(TestEmailBase):
    def _msgobj(self, filename):
        fp = openfile(filename)
        try:
            msg = email.message_from_file(fp)
        finally:
            fp.close()
        return msg

class TestCrispinTorture(TortureBase):
    # Mark Crispin's torture test from the SquirrelMail project
    def test_mondo_message(self):
        eq = self.assertEqual
        neq = self.ndiffAssertEqual
        msg = self._msgobj('crispin-torture.txt')
        payload = msg.get_payload()
        eq(type(payload), ListType)
        eq(len(payload), 12)
        eq(msg.preamble, None)
        eq(msg.epilogue, '\n\n')
        # Probably the best way to verify the message is parsed correctly is to
        # dump its structure and compare it against the known structure.
        fp = StringIO()
        _structure(msg, fp=fp)
        neq(fp.getvalue(), """\
multipart/mixed
    text/plain
    message/rfc822
        multipart/alternative
            text/plain
            multipart/mixed
                text/richtext
            application/andrew-inset
    message/rfc822
        audio/basic
    audio/basic
    image/pbm
    message/rfc822
        multipart/mixed
            multipart/mixed
                text/plain
                audio/x-sun
            multipart/mixed
                image/gif
                image/gif
                application/x-be2
                application/atomicmail
            audio/x-sun
    message/rfc822
        multipart/mixed
            text/plain
            image/pgm
            text/plain
    message/rfc822
        multipart/mixed
            text/plain
            image/pbm
    message/rfc822
        application/postscript
    image/gif
    message/rfc822
        multipart/mixed
            audio/basic
            audio/basic
    message/rfc822
        multipart/mixed
            application/postscript
            text/plain
            message/rfc822
                multipart/mixed
                    text/plain
                    multipart/parallel
                        image/gif
                        audio/basic
                    application/atomicmail
                    message/rfc822
                        audio/x-sun
""")

def _testclasses():
    mod = sys.modules[__name__]
    return [getattr(mod, name) for name in dir(mod) if name.startswith('Test')]

def suite():
    suite = unittest.TestSuite()
    for testclass in _testclasses():
        suite.addTest(unittest.makeSuite(testclass))
    return suite

def test_main():
    for testclass in _testclasses():
        test_support.run_unittest(testclass)

if __name__ == '__main__':
    unittest.main(defaultTest='suite')
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (C) 2017  Daniel Watkins <daniel@daniel-watkins.co.uk>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Run a series of checks against compiled job XML.
"""
import argparse
import os
import sys
from typing import Optional
from xml.etree import ElementTree

class Linter(object):
    """A super-class capturing the common linting pattern."""

    def __init__(self, tree: ElementTree) -> None:
        self._tree = tree

    def actual_check(self) -> (bool, Optional[str]):
        """This is where the actual check should happen."""
        raise NotImplementedError

    @property
    def description(self) -> str:
        """The output-friendly description of what this Linter does."""
        raise NotImplementedError

    def check(self) -> bool:
        """Wrap actual_check in nice output."""
        print(' ... {}:'.format(self.description), end='')
        result, text = self.actual_check()
        print(' OK' if result else ' FAILURE')
        if text:
            print('     {}'.format(text))
        return result

class EnsureTimestamps(Linter):

    description = 'checking for timestamps'
    _xpath = (
        './buildWrappers/hudson.plugins.timestamper.TimestamperBuildWrapper')

    def actual_check(self) -> (bool, Optional[str]):
        """Check that the TimestamperBuildWrapper element is present."""
        return self._tree.find(self._xpath) is not None, None

def lint_job_xml(tree: ElementTree) -> bool:
    """Run all the linters against an XML tree."""
    linters = [EnsureTimestamps]
    results = [linter(tree).check() for linter in linters]
    return all(results)

def lint_jobs_from_directory(compiled_job_directory: str) -> bool:
    """Load jobs from a directory and run linters against each one."""
    success = True
    for job_file in os.listdir(compiled_job_directory):
        print('Linting', job_file)
        job_path = os.path.join(compiled_job_directory, job_file)
        result = lint_job_xml(ElementTree.parse(job_path))
        success = success and result
    return success

def main() -> None:
    """Take a directory of Jenkins job XML and run some checks against it."""
    parser = argparse.ArgumentParser()
    parser.add_argument('compiled_job_directory')
    args = parser.parse_args()
    result = lint_jobs_from_directory(args.compiled_job_directory)
    if not result:
        sys.exit(1)
    sys.exit(0)

if __name__ == '__main__':
    main()
<commit_msg>Add CheckShebang linter<commit_after>#!/usr/bin/env python3
# Copyright (C) 2017  Daniel Watkins <daniel@daniel-watkins.co.uk>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Run a series of checks against compiled job XML.
"""
import argparse
import os
import sys
from typing import Optional
from xml.etree import ElementTree

class Linter(object):
    """A super-class capturing the common linting pattern."""

    def __init__(self, tree: ElementTree) -> None:
        self._tree = tree

    def actual_check(self) -> (bool, Optional[str]):
        """This is where the actual check should happen."""
        raise NotImplementedError

    @property
    def description(self) -> str:
        """The output-friendly description of what this Linter does."""
        raise NotImplementedError

    def check(self) -> bool:
        """Wrap actual_check in nice output."""
        print(' ... {}:'.format(self.description), end='')
        result, text = self.actual_check()
        print(' OK' if result else ' FAILURE')
        if text:
            print('     {}'.format(text))
        return result

class EnsureTimestamps(Linter):

    description = 'checking for timestamps'
    _xpath = (
        './buildWrappers/hudson.plugins.timestamper.TimestamperBuildWrapper')

    def actual_check(self) -> (bool, Optional[str]):
        """Check that the TimestamperBuildWrapper element is present."""
        return self._tree.find(self._xpath) is not None, None

class CheckShebang(Linter):

    description = 'checking shebang of shell builders'

    def actual_check(self) -> (bool, Optional[str]):
        shell_parts = self._tree.findall(
            './builders/hudson.tasks.Shell/command')
        for shell_part in shell_parts:
            script = shell_part.text
            first_line = script.splitlines()[0]
            if first_line.startswith('#!') and first_line != '#!/bin/sh -eux':
                return False, 'Shebang is {}'.format(first_line)
        return True, None

def lint_job_xml(tree: ElementTree) -> bool:
    """Run all the linters against an XML tree."""
    linters = [CheckShebang, EnsureTimestamps]
    results = [linter(tree).check() for linter in linters]
    return all(results)

def lint_jobs_from_directory(compiled_job_directory: str) -> bool:
    """Load jobs from a directory and run linters against each one."""
    success = True
    for job_file in os.listdir(compiled_job_directory):
        print('Linting', job_file)
        job_path = os.path.join(compiled_job_directory, job_file)
        result = lint_job_xml(ElementTree.parse(job_path))
        success = success and result
    return success

def main() -> None:
    """Take a directory of Jenkins job XML and run some checks against it."""
    parser = argparse.ArgumentParser()
    parser.add_argument('compiled_job_directory')
    args = parser.parse_args()
    result = lint_jobs_from_directory(args.compiled_job_directory)
    if not result:
        sys.exit(1)
    sys.exit(0)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env/python
##============================================================

import os
import numpy
import time
from sys import argv
from OSMethods import file_seek

##============================================================

## Seek out all fits image files in directory and make psfs
	## argv[1] is the directory to look at
	## argv[2] is option for PSFEx (maybe later)

def main(args):
	"""
	NAME
	  SourceDiagnostics.py

	PURPOSE
		Identifies sources in a FITS image, and calculates the image.

	COMMENTS
		Uses SExtractor and PSFEx software.

	USAGE
		python SourceDiagnostics.py [flags] directory/file

	FLAGS
	  -h            Print this message
	  -vb						Verbose

	INPUTS
		Requires the following configuration files:
		- default.conv
		- default.nnw
		- prepsfex.param
		- prepsfex.sex
		- default.psfex

		Input directory or filename

	OPTIONAL INPUTS

	OUTPUTS
	  stdout        Useful information
	  image.cat file	Diagnostics about each source
	  snap_image.fits	PSF model
	  image.psf				More sophisticated PSF model

	EXAMPLES
		py SourceDiagnostics.py CFHTLS_03_g_sci.fits
		py SourceDiagnostics.py CFHTLS_03/

	BUGS / ISSUES
		- Hard coded paths to configuration files

	HISTORY
	  2012-06 started Sandford (NYU)
	"""

	## -------------------------------------------------------------------
	## Options and help-string

	try:
		opts, args = getopt.getopt(argv, "hv",["help"])
	except getopt.GetoptError, err:
		## Print help information and exit:
		print str(err)
		print main.__doc__
		return

	vb=False

	for o,a in opts:
		print o,a
		if o in ("-h", "--help"):
			print main.__doc__
			return
		elif o in ("-v"):
			vb=True
		else:
			assert False, "Unhandled option"

	##--------------------------------------------------------------------

	t0=time.time()

	## If the input is a single file
	if os.path.isfile(args[0]):	fits_list = [args[0]]
	## If the input is a directory of files
	elif os.path.isdir(args[0]):	fits_list = file_seek(args[0], "CFHTLS_*sci.fits")

	## Run SExtractor and PSFEx on the image
	for fitsfile in fits_list:
		## Run SExtractor
		SEx(fitsfile, vb)
		## Run PSFEx on fresh .cat
		PSFEx(fitsfile[:-4]+"cat", "snap", vb)

	if vb:
		print "\nSourceDiagnostics.py: analysed",len(fits_list),"FITS images."
		print "\tTook",round(time.time()-t0,2),"seconds."

	return

##============================================================

## DEFUNCT
## Runs SExtractor or PSFEx depending on input file
	## argv[1] is a FILE
def choose(argv):

	if argv[1][-4:]=="fits": SEx(argv)
	elif argv[1][-3:]=="cat": PSFEx(argv)
	else: print "SourceDiagnostics.py: provide suitable input file."

	return

##============================================================

def SEx(imgname, vb):

	## Make absolute path
	imgname = os.path.abspath(imgname)

	## Check the input
	if imgname[-5:]!=".fits":
		print "SourceDiagnostics.py: SEx: invalid input image."
		return

	## PSFEx needs output in FITS_LDAC format, and different data in parameter file
	confile = " -c ~/Dropbox/Hogg_2012/Code/HumVI/decon/AstromaticConfig/prepsfex.sex "
	confile += " -FILTER_NAME ~/Dropbox/Hogg_2012/Code/HumVI/decon/AstromaticConfig/default.conv "
	params  = " -PARAMETERS_NAME ~/Dropbox/Hogg_2012/Code/HumVI/decon/AstromaticConfig/prepsfex.param "
	cattype = " -CATALOG_TYPE FITS_LDAC "
	catname = imgname[:-5]+".cat"
	catdecl = " -CATALOG_NAME "+catname

	## Put it all together
	commands = "sex "+imgname+confile+params+cattype+catdecl
	if vb: commands += " -VERBOSE_TYPE NORMAL "

	## Execute commands
	os.system(commands)

	if vb: print "SourceDiagnostics.py: SEx: saved source catalogue as "+catname

	return catname

##============================================================

## Modify arguments of PSFEx

def PSFEx(catfile, mode, vb):

	## Check the input
	if catfile[-4:]!=".cat":
		print "SourceDiagnostics.py: PSFEx: invalid input catalogue. Abort."
		return

	## Input catalogue
	outdir,catname = os.path.split(catfile)
	## Destination directory
	outdir = outdir+"/Diagnostics/"
	if os.path.isdir(outdir)==False: os.mkdir(outdir)

##------------------------------------------------------------
	imgname = None
	config = " -c ~/Dropbox/Hogg_2012/Code/HumVI/decon/AstromaticConfig/default.psfex "

	## If we want to return only a PSF snapshot
	if mode=="snap":

		## Only one check image: snap_n1d0*.fits
		imgname = outdir+"snap_n1d0_"+catname[:-4]+".fits"
		imgdest = " -CHECKIMAGE_TYPE SNAPSHOTS  -CHECKIMAGE_NAME "+outdir+"snap_n1d0"
		params  = " -PSFVAR_NSNAP 1  -PSFVAR_DEGREES 0  "
		## No check plots (png)
		pltdest = " -CHECKPLOT_TYPE NONE "
		## No XML file
		xmldest = " -WRITE_XML N "

	elif mode=="none":
		## No check image
		imgdest = " -CHECKIMAGE_TYPE NONE "
		## No check plots (png)
		pltdest = " -CHECKPLOT_TYPE NONE "
		## No XML file (xml)
		xmldest = " -WRITE_XML N "
		## Default 1-degree fit
		params  = " -PSFVAR_DEGREES 1   BASIS_TYPE PIXEL_AUTO "

	## Compute everything -- takes longer
			## Need to re-check this stuff
	elif mode=="all":
		## Destination for check images (fits)
		imgdest = outdir+"chi.fits,"+outdir+"proto.fits,"+outdir+"samp.fits,"+\
							outdir+"resi.fits,"+outdir+"snap_n1d0.fits,"+\
							outdir+"moffat.fits,"+outdir+"submoffat.fits,"+outdir+"subsym.fits"
		imgdest = " -CHECKIMAGE_NAME "+imgdest
		## Destination for check plots (png)
		pltdest = outdir+"fwhm,"+outdir+"ellipticity,"+outdir+"counts,"+\
							outdir+"countfrac,"+outdir+"chi2,"+outdir+"resi"
		pltdest = " -CHECKPLOT_NAME "+pltdest
		## Destination for XML file (xml)
		xmldest = " -XML_NAME "+outdir+catname+".xml "
		params  = " "

	else:
		print "SourceDiagnostics.py: invalid PSFEx option specified. Abort."
		return

##------------------------------------------------------------

	## Stick it all together
	commands = "psfex "+catfile+imgdest+pltdest+xmldest+params+config
	if vb: commands += " -VERBOSE_TYPE NORMAL "

	## Execute commands
	os.system(commands)

	if vb:
		if mode is "snap":
			print "SourceDiagnostics.py: PSF model image saved to",imgname

	return imgname

##============================================================

## DEFUNCT
## Generate snapshots of varying quality and coverage
	## Called from python line (use OSScript.file_seek first)
def snapall(catfile):
	for nsnap in [1,9]:
		for deg in [0,1,2,3]:
			PSFEx(["dummy",catfile,"snap",str(nsnap),str(deg)])
	return

##============================================================

if __name__=="__main__":
	main(argv[1:])
<commit_msg>Minor changes to SD<commit_after>#!/usr/bin/env/python
##============================================================

import os
import numpy
import time
from sys import argv
from OSMethods import file_seek

##============================================================

## Seek out all fits image files in directory and make psfs
	## argv[1] is the directory to look at
	## argv[2] is option for PSFEx (maybe later)

def main(args):
	"""
	NAME
	  SourceDiagnostics.py

	PURPOSE
		Identifies sources in a FITS image, and calculates the image.

	COMMENTS
		Uses SExtractor and PSFEx software.

	USAGE
		python SourceDiagnostics.py [flags] directory/file

	FLAGS
	  -h            Print this message
	  -vb						Verbose

	INPUTS
		Requires the following configuration files:
		- default.conv
		- default.nnw
		- prepsfex.param
		- prepsfex.sex
		- default.psfex

		Input directory or filename

	OPTIONAL INPUTS

	OUTPUTS
	  stdout        Useful information
	  image.cat file	Diagnostics about each source
	  snap_image.fits	PSF model
	  image.psf				More sophisticated PSF model

	EXAMPLES
		py SourceDiagnostics.py CFHTLS_03_g_sci.fits
		py SourceDiagnostics.py CFHTLS_03/

	BUGS / ISSUES
		- Hard coded paths to configuration files

	HISTORY
	  2012-06 started Sandford (NYU)
	"""

	## -------------------------------------------------------------------
	## Options and help-string

	try:
		opts, args = getopt.getopt(argv, "hv",["help"])
	except getopt.GetoptError, err:
		## Print help information and exit:
		print str(err)
		print main.__doc__
		return

	vb=False

	for o,a in opts:
		print o,a
		if o in ("-h", "--help"):
			print main.__doc__
			return
		elif o in ("-v"):
			vb=True
		else:
			assert False, "Unhandled option"

	##--------------------------------------------------------------------

	t0=time.time()

	## If the input is a single file
	if os.path.isfile(args[0]):	fits_list = [args[0]]
	## If the input is a directory of files
	elif os.path.isdir(args[0]):	fits_list = file_seek(args[0], "CFHTLS_*sci.fits")

	## Run SExtractor and PSFEx on the image
	for fitsfile in fits_list:
		## Run SExtractor
		SEx(fitsfile, vb)
		## Run PSFEx on fresh .cat
		PSFEx(fitsfile[:-4]+"cat", "snap", vb)

	if vb:
		print "\nSourceDiagnostics.py: analysed",len(fits_list),"FITS images."
		print "\tTook",round(time.time()-t0,2),"seconds."

	return

##============================================================

## DEFUNCT
## Runs SExtractor or PSFEx depending on input file
	## argv[1] is a FILE
def choose(argv):

	if argv[1][-4:]=="fits": SEx(argv)
	elif argv[1][-3:]=="cat": PSFEx(argv)
	else: print "SourceDiagnostics.py: provide suitable input file."

	return

##============================================================

def SEx(imgname, vb):

	## Make absolute path
	imgname = os.path.abspath(imgname)

	## Check the input
	if imgname[-5:]!=".fits":
		print "SourceDiagnostics.py: SEx: invalid input image."
		return

	## PSFEx needs output in FITS_LDAC format, and different data in parameter file
	confile = " -c ~/Dropbox/Hogg_2012/HumVI/decon/AstromaticConfig/prepsfex.sex "
	confile += " -FILTER_NAME ~/Dropbox/Hogg_2012/HumVI/decon/AstromaticConfig/default.conv "
	params  = " -PARAMETERS_NAME ~/Dropbox/Hogg_2012/HumVI/decon/AstromaticConfig/prepsfex.param "
	cattype = " -CATALOG_TYPE FITS_LDAC "
	catname = imgname[:-5]+".cat"
	catdecl = " -CATALOG_NAME "+catname

	## Put it all together
	commands = "sex "+imgname+confile+params+cattype+catdecl
	if vb: commands += " -VERBOSE_TYPE NORMAL "

	## Execute commands
	os.system(commands)

	if vb: print "SourceDiagnostics.py: SEx: saved source catalogue as "+catname

	return catname

##============================================================

## Modify arguments of PSFEx

def PSFEx(catfile, mode, vb):

	## Check the input
	if catfile[-4:]!=".cat":
		print "SourceDiagnostics.py: PSFEx: invalid input catalogue. Abort."
		return

	## Input catalogue
	outdir,catname = os.path.split(catfile)
	## Destination directory
	outdir = outdir+"/Diagnostics/"
	if os.path.isdir(outdir)==False: os.mkdir(outdir)

##------------------------------------------------------------
	imgname = None
	config = " -c ~/Dropbox/Hogg_2012/HumVI/decon/AstromaticConfig/default.psfex "

	## If we want to return only a PSF snapshot
	if mode=="snap":

		## Only one check image: snap_n1d0*.fits
		imgname = outdir+"snap_n1d0_"+catname[:-4]+".fits"
		imgdest = " -CHECKIMAGE_TYPE SNAPSHOTS  -CHECKIMAGE_NAME "+outdir+"snap_n1d0"
		params  = " -PSFVAR_NSNAP 1  -PSFVAR_DEGREES 0  "
		## No check plots (png)
		pltdest = " -CHECKPLOT_TYPE NONE "
		## No XML file
		xmldest = " -WRITE_XML N "

	elif mode=="none":
		## No check image
		imgdest = " -CHECKIMAGE_TYPE NONE "
		## No check plots (png)
		pltdest = " -CHECKPLOT_TYPE NONE "
		## No XML file (xml)
		xmldest = " -WRITE_XML N "
		## Default 1-degree fit
		params  = " -PSFVAR_DEGREES 1   BASIS_TYPE PIXEL_AUTO "

	## Compute everything -- takes longer
			## Need to re-check this stuff
	elif mode=="all":
		## Destination for check images (fits)
		imgdest = outdir+"chi.fits,"+outdir+"proto.fits,"+outdir+"samp.fits,"+\
							outdir+"resi.fits,"+outdir+"snap_n1d0.fits,"+\
							outdir+"moffat.fits,"+outdir+"submoffat.fits,"+outdir+"subsym.fits"
		imgdest = " -CHECKIMAGE_NAME "+imgdest
		## Destination for check plots (png)
		pltdest = outdir+"fwhm,"+outdir+"ellipticity,"+outdir+"counts,"+\
							outdir+"countfrac,"+outdir+"chi2,"+outdir+"resi"
		pltdest = " -CHECKPLOT_NAME "+pltdest
		## Destination for XML file (xml)
		xmldest = " -XML_NAME "+outdir+catname+".xml "
		params  = " "

	else:
		print "SourceDiagnostics.py: invalid PSFEx option specified. Abort."
		return

##------------------------------------------------------------

	## Stick it all together
	commands = "psfex "+catfile+imgdest+pltdest+xmldest+params+config
	if vb: commands += " -VERBOSE_TYPE NORMAL "

	## Execute commands
	os.system(commands)

	if vb:
		if mode is "snap":
			print "SourceDiagnostics.py: PSF model image saved to",imgname

	return imgname

##============================================================

## DEFUNCT
## Generate snapshots of varying quality and coverage
	## Called from python line (use OSScript.file_seek first)
def snapall(catfile):
	for nsnap in [1,9]:
		for deg in [0,1,2,3]:
			PSFEx(["dummy",catfile,"snap",str(nsnap),str(deg)])
	return

##============================================================

if __name__=="__main__":
	main(argv[1:])
<|endoftext|>


<commit_before>from __future__ import (
    print_function, division, absolute_import, unicode_literals)

from ufo2ft.filters import BaseFilter, logger
from booleanOperations import union, BooleanOperationsError

class RemoveOverlapsFilter(BaseFilter):

    def filter(self, glyph, glyphSet=None):
        if not len(glyph):
            return False

        contours = list(glyph)
        glyph.clearContours()
        try:
            union(contours, glyph.getPointPen())
        except BooleanOperationsError:
            logger.error("Failed to remove overlaps for %s",
                         glyph.name)
            raise
        return True
<commit_msg>[removeOverlaps] use module's logger instead of global one<commit_after>from __future__ import (
    print_function, division, absolute_import, unicode_literals)

from ufo2ft.filters import BaseFilter
from booleanOperations import union, BooleanOperationsError

import logging

logger = logging.getLogger(__name__)

class RemoveOverlapsFilter(BaseFilter):

    def filter(self, glyph, glyphSet=None):
        if not len(glyph):
            return False

        contours = list(glyph)
        glyph.clearContours()
        try:
            union(contours, glyph.getPointPen())
        except BooleanOperationsError:
            logger.error("Failed to remove overlaps for %s",
                         glyph.name)
            raise
        return True
<|endoftext|>


<commit_before>from .model import DeepLabResNetModel
from .image_reader import ImageReader
from .utils import decode_labels, prepare_label<commit_msg>Update __init__.py<commit_after>from .model import DeepLabResNetModel
from .image_reader import ImageReader
from .utils import decode_labels, inv_preprocess, prepare_label
<|endoftext|>


<commit_before>#! /usr/bin/env python
# :noTabs=true:
# (c) Copyright (c) 2017  Gijs Kant
# (c) This file is distributed under the MIT License,
# (c) see the file LICENSE.
"""
tools.py

Brief: Utility classes for tools used in the experiments.

Author: Gijs Kant <gijskant@protonmail.com>

"""
import os
import shutil
import subprocess
import sys
import tempfile
import time
import traceback
from humanfriendly import AutomaticSpinner, Spinner, Timer

"""
Runs a command in a shell and returns the stdout.
"""
def run_command(label, command, logfile = None):
    print >> sys.stderr, '-', command
    with AutomaticSpinner(label) as spinner:
        start =  time.time()
        task = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        (stdoutdata, stderrdata) = task.communicate()
        result = task.wait()
        if not result == 0:
            raise Exception('Command failed.')
        if not logfile is None:
            f = open(logfile, 'a')
            f.write(command)
            f.write('\n')
            if not stderrdata is None:
                f.write(stderrdata)
            f.close()
        end = time.time()
    print >> sys.stderr, '  ({:.2f} seconds)'.format((end - start))
    return

"""
Runs a command in a shell and returns the stdout.
"""
def run_simple_command(command):
    task = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
    data = task.stdout.read()
    result = task.wait()
    return data

"""
Runs a command in a shell and returns the stdout.
"""
def run_boolean_command(command):
    task = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
    result = task.wait()
    return result

"""
Gets the filename without extension.
"""
def get_model_name(filename):
    return os.path.splitext(os.path.basename(filename))[0]

def create_tempfile(filename, suffix):
    modelname = get_model_name(filename)
    (f, filename) = tempfile.mkstemp(suffix, modelname + '.')
    return filename

def get_path_from_config(config, tool):
    c = config.get('tools')
    if c is None:
        return None
    t = c.get(tool)
    if t is None:
        return None
    return t.get('path')

class Tool:
    def prepare(self, data):
        pass

    def run(self, experiments, index):
        pass

    def list(self, experiments):
        pass

    def print_list(self, experiments):
        pass

class Mcrl2(Tool):

    path = ""

    def __init__(self, config):
        path = get_path_from_config(config, 'mcrl2')
        if path is None:
            self.path = self.find_path()
        else:
            self.path = os.path.abspath(path)
        self.path += '/'
        print >> sys.stderr, 'mcrl2 path:', self.path
        test_program = self.path + 'mcrl22lps'
        if not os.path.isfile(test_program):
            raise Exception('mcrl22lps not found.')

    def find_path(self):
        path = run_simple_command('which mcrl22lps')
        mcrl2_dir = os.path.dirname(os.path.abspath(path))
        print >> sys.stderr, 'Found mcrl2 binary in path: ', path,
        return mcrl2_dir

    def check_pbes(self, pbes_filename):
        command = self.path + 'pbesinfo {pbes_filename}'.format(pbes_filename = pbes_filename)
        result = run_boolean_command(command)
        return result == 0

    def generate_pbes(self, data):
        pbes_filename = data['pbes_filename']
        lps_filename = data['lps_filename']
        input_mcf = data['input_mcf']
        preparation_options = data['preparation_options']
        lps2pbes_options = preparation_options['lps2pbes']

        # lps2pbes
        command = self.path + 'lps2pbes -v {lps2pbes_options} -f {input_mcf} {lps_filename} {pbes_filename}'.format(
            lps2pbes_options = lps2pbes_options,
            input_mcf = input_mcf,
            lps_filename = lps_filename,
            pbes_filename = pbes_filename
        )
        # redirect log messages
        logfile = '{pbes_filename}.lps2pbes.log'.format(pbes_filename = pbes_filename)
        try:
            print >> sys.stderr, ''
            start = time.time()

            run_command('Generating ' + pbes_filename, command, logfile)

            end = time.time()
            print >> sys.stderr, 'Generating {} took {:.2f} seconds.'.format(pbes_filename, (end - start))
        except Exception as e:
            print >> sys.stderr, 'Error:', e
            os.remove(pbes_filename)
            sys.exit(1)

    def check_lps(self, lps_filename):
        command = self.path + 'lpsinfo {lps_filename}'.format(lps_filename = lps_filename)
        result = run_boolean_command(command)
        return result == 0

    def generate_lps(self, data):
        lps_filename = data['lps_filename']
        input_mcrl2 = data['input_mcrl2']
        preparation_options = data['preparation_options']
        lin_options = preparation_options['linearisation']
        parunfold_steps = preparation_options['lpsparunfold']

        # redirect log messages
        logfile = '{lps_filename}.log'.format(lps_filename = lps_filename)

        lps_in = ""
        lps_out = ""

        try:
            print >> sys.stderr, ''
            start = time.time()

            # linearisation
            lps_out = create_tempfile(lps_filename, '.lps')
            command = self.path + 'mcrl22lps -v {lin_options} {input_mcrl2} {lps_out}'.format(
                lin_options = lin_options,
                input_mcrl2 = input_mcrl2,
                lps_out = lps_out
            )
            run_command('Linearising ' + input_mcrl2, command, logfile)

            # lpssuminst
            lps_in = lps_out
            lps_out = create_tempfile(lps_filename, '.lps')
            command = self.path + 'lpssuminst {lps_in} {lps_out}'.format(lps_in = lps_in, lps_out = lps_out)
            run_command('lpssuminst ' + input_mcrl2, command, logfile)
            os.remove(lps_in)

            # parunfold
            for step in parunfold_steps:
                lps_in = lps_out
                lps_out = create_tempfile(lps_filename, '.lps')
                command = self.path + 'lpsparunfold -v {options} {lps_in} {lps_out}'.format(
                    options = step,
                    lps_in = lps_in,
                    lps_out = lps_out
                )
                data = run_command('lpsparunfold ' + input_mcrl2, command, logfile)
                os.remove(lps_in)

            # rewrite
            lps_in = lps_out
            lps_out = create_tempfile(lps_filename, '.lps')
            command = self.path + 'lpsrewr -v {lps_in} {lps_out}'.format(lps_in = lps_in, lps_out = lps_out)
            run_command('lpsrewr ' + input_mcrl2, command, logfile)
            os.remove(lps_in)

            # constelm
            lps_in = lps_out
            lps_out = create_tempfile(lps_filename, '.lps')
            command = self.path + 'lpsconstelm -v -c {lps_in} {lps_out}'.format(lps_in = lps_in, lps_out = lps_out)
            run_command('lpsconstelm ' + input_mcrl2, command, logfile)
            os.remove(lps_in)

            # move output to lps_filename
            shutil.move(lps_out, lps_filename)

            end = time.time()
            print >> sys.stderr, 'Generating {} took {:.2f} seconds.'.format(lps_filename, (end - start))
        except Exception as e:
            print >> sys.stderr, 'Error:', e
            traceback.print_exc(file=sys.stderr)
            if os.path.isfile(lps_in):
                os.remove(lps_in)
            if os.path.isfile(lps_out):
                os.remove(lps_out)
            sys.exit(1)

    def prepare(self, data):
        experiment_type = data['type']
        assert experiment_type in ['lps', 'pbes']

        # check if an invalid LPS exists
        lps_filename = data['lps_filename']
        if os.path.isfile(lps_filename):
            if not self.check_lps(lps_filename):
                print >> sys.stderr, 'Found invalid LPS:', lps_filename
                os.remove(lps_filename)

        # generate lps if not exists
        if not os.path.isfile(lps_filename):
            self.generate_lps(data)
        else:
            print >> sys.stderr, 'Existing LPS found:', lps_filename
        if not os.path.isfile(lps_filename):
            raise Exception('Error creating LPS: ' + lps_filename)

        if not experiment_type == 'pbes':
            return

        # check if an invalid PBES exists
        pbes_filename = data['pbes_filename']
        if os.path.isfile(pbes_filename):
            if not self.check_pbes(pbes_filename):
                print >> sys.stderr, 'Found invalid PBES:', pbes_filename
                os.remove(pbes_filename)

        # generate pbes if not exists
        if not os.path.isfile(pbes_filename):
            self.generate_pbes(data)
        else:
            print >> sys.stderr, 'Existing PBES found:', pbes_filename
        if not os.path.isfile(pbes_filename):
            raise Exception('Error creating PBES: ' + pbes_filename)

class Ltsmin(Tool):
    def __init__(self, config):
        path = get_path_from_config(config, 'ltsmin')
        if path is None:
            self.path = self.find_path()
        else:
            self.path = os.path.abspath(path)
        self.path += '/'
        print >> sys.stderr, 'ltsmin path:', self.path
        test_program = self.path + 'pbes2lts-sym'
        if not os.path.isfile(test_program):
            raise Exception('pbes2lts-sym not found.')

    def find_path(self):
        path = run_simple_command('which pbes2lts-sym')
        ltsmin_dir = os.path.dirname(os.path.abspath(path))
        print >> sys.stderr, 'Found ltsmin binary in path:', path,
        return ltsmin_dir

    def list(self, experiments):
        runs = []
        for data in experiments:
            experiment_type = data['type']
            assert experiment_type in ['lps', 'pbes']
            if experiment_type is 'lps':
                input_filename = data['lps_filename']
            else:
                input_filename = data['pbes_filename']
            run_options = data.get('run_options')
            if run_options is None:
                print >> sys.stderr, 'No run options found for experiment', experiment_type, input_filename
            else:
                n_cores = run_options['n_cores']
                for n in n_cores:
                    runs.append({
                        'type': experiment_type,
                        'input': input_filename,
                        'cores': n,
                        'data': data
                    })
        return runs

    def print_list(self, experiments):
        runs = self.list(experiments)
        for run in runs:
            print run['type'], run['input'], run['cores']
        print >> sys.stderr, 'Total:', len(runs)

    def lps_instantiate(self, input_lps, n_cores):
        raise Exception('not implemented yet.')

    def pbes_instantiate(self, input_pbes, output_spg, n_cores):
        # pbes2lts-sym
        pbes2lts_options = '--mcrl2-rewriter=jitty -rgs --vset=lddmc --order=par-prev'
        lace_options = '--lace-workers={}'.format(n_cores)
        command = self.path + 'pbes2lts-sym {pbes2lts_options} {lace_options} {input_pbes} --pg-write={output_spg}'.format(
            pbes2lts_options = pbes2lts_options,
            input_pbes = input_pbes,
            output_spg = output_spg,
            lace_options = lace_options
        )
        # redirect log messages
        logfile = '{input_pbes}.pbes2spg.{n_cores}.log'.format(input_pbes = input_pbes, n_cores = n_cores)
        try:
            print >> sys.stderr, ''
            start = time.time()

            run_command('Instantiating ' + input_pbes, command, logfile)

            end = time.time()
            print >> sys.stderr, 'Instantiating {} took {:.2f} seconds.'.format(input_pbes, (end - start))
        except Exception as e:
            print >> sys.stderr, 'Error:', e
            if os.path.isfile(output_spg):
                os.remove(output_spg)
            sys.exit(1)

    def pbes_solve(self, input_spg, n_cores):
        # spgsolver
        spgsolver_options = ''
        lace_options = '--lace-workers={}'.format(n_cores)
        command = self.path + 'spgsolver {spgsolver_options} {lace_options} {input_spg}'.format(
            spgsolver_options = spgsolver_options,
            input_spg = input_spg,
            lace_options = lace_options
        )
        # redirect log messages
        logfile = '{input_spg}.spgsolver.{n_cores}.log'.format(input_spg = input_spg, n_cores = n_cores)
        try:
            print >> sys.stderr, ''
            start = time.time()

            run_command('Solving ' + input_spg, command, logfile)

            end = time.time()
            print >> sys.stderr, 'Solving {} took {:.2f} seconds.'.format(input_spg, (end - start))
        except Exception as e:
            print >> sys.stderr, 'Error:', e
            sys.exit(1)

    def run(self, experiments, index):
        runs = self.list(experiments)
        if index > len(runs):
            print >> sys.stderr, 'Choose a number between 1 and', len(runs), '( was:', index, ')'
            raise Exception('Index out of bounds.')
        run = runs[index - 1]
        type = run['type']
        input_filename = run['input']
        cores = run['cores']
        assert type in ['lps', 'pbes']
        if type is 'lps':
            self.lps_instantiate(input_filename, cores)
        else:
            spg_filename = input_filename + '.spg'
            self.pbes_instantiate(input_filename, spg_filename, cores)
            self.pbes_solve(spg_filename, cores)

class ToolRegistry:

    tools = {}

    def __init__(self, config):
        self.tools = {
            'mcrl2': Mcrl2(config),
            'ltsmin': Ltsmin(config)
        }

<commit_msg>Update log dirs in experiments script.<commit_after>#! /usr/bin/env python
# :noTabs=true:
# (c) Copyright (c) 2017  Gijs Kant
# (c) This file is distributed under the MIT License,
# (c) see the file LICENSE.
"""
tools.py

Brief: Utility classes for tools used in the experiments.

Author: Gijs Kant <gijskant@protonmail.com>

"""
import os
import shutil
import subprocess
import sys
import tempfile
import time
from datetime import datetime
import traceback
from humanfriendly import AutomaticSpinner, Spinner, Timer

"""
Runs a command in a shell and returns the stdout.
"""
def run_command(label, command, logfile = None):
    print >> sys.stderr, '-', command
    with AutomaticSpinner(label) as spinner:
        start =  time.time()
        task = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        (stdoutdata, stderrdata) = task.communicate()
        result = task.wait()
        if not result == 0:
            raise Exception('Command failed.')
        if not logfile is None:
            f = open(logfile, 'a')
            f.write(command)
            f.write('\n')
            if not stderrdata is None:
                f.write(stderrdata)
            f.close()
        end = time.time()
    print >> sys.stderr, '  ({:.2f} seconds)'.format((end - start))
    return

"""
Runs a command in a shell and returns the stdout.
"""
def run_simple_command(command):
    task = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
    data = task.stdout.read()
    result = task.wait()
    return data

"""
Runs a command in a shell and returns the stdout.
"""
def run_boolean_command(command):
    task = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
    result = task.wait()
    return result

"""
Gets the filename without extension.
"""
def get_model_name(filename):
    return os.path.splitext(os.path.basename(filename))[0]

def create_tempfile(filename, suffix):
    modelname = get_model_name(filename)
    (f, filename) = tempfile.mkstemp(suffix, modelname + '.')
    return filename

def get_path_from_config(config, tool):
    c = config.get('tools')
    if c is None:
        return None
    t = c.get(tool)
    if t is None:
        return None
    return t.get('path')

class Tool:
    def prepare(self, data):
        pass

    def run(self, experiments, index):
        pass

    def list(self, experiments):
        pass

    def print_list(self, experiments):
        pass

class Mcrl2(Tool):

    path = ""

    def __init__(self, config):
        path = get_path_from_config(config, 'mcrl2')
        if path is None:
            self.path = self.find_path()
        else:
            self.path = os.path.abspath(path)
        self.path += '/'
        print >> sys.stderr, 'mcrl2 path:', self.path
        test_program = self.path + 'mcrl22lps'
        if not os.path.isfile(test_program):
            raise Exception('mcrl22lps not found.')

    def find_path(self):
        path = run_simple_command('which mcrl22lps')
        mcrl2_dir = os.path.dirname(os.path.abspath(path))
        print >> sys.stderr, 'Found mcrl2 binary in path: ', path,
        return mcrl2_dir

    def check_pbes(self, pbes_filename):
        command = self.path + 'pbesinfo {pbes_filename}'.format(pbes_filename = pbes_filename)
        result = run_boolean_command(command)
        return result == 0

    def generate_pbes(self, data):
        pbes_filename = data['pbes_filename']
        lps_filename = data['lps_filename']
        input_mcf = data['input_mcf']
        preparation_options = data['preparation_options']
        lps2pbes_options = preparation_options['lps2pbes']

        # lps2pbes
        command = self.path + 'lps2pbes -v {lps2pbes_options} -f {input_mcf} {lps_filename} {pbes_filename}'.format(
            lps2pbes_options = lps2pbes_options,
            input_mcf = input_mcf,
            lps_filename = lps_filename,
            pbes_filename = pbes_filename
        )
        # redirect log messages
        logfile = '{pbes_filename}.lps2pbes.log'.format(pbes_filename = pbes_filename)
        try:
            print >> sys.stderr, ''
            start = time.time()

            run_command('Generating ' + pbes_filename, command, logfile)

            end = time.time()
            print >> sys.stderr, 'Generating {} took {:.2f} seconds.'.format(pbes_filename, (end - start))
        except Exception as e:
            print >> sys.stderr, 'Error:', e
            os.remove(pbes_filename)
            sys.exit(1)

    def check_lps(self, lps_filename):
        command = self.path + 'lpsinfo {lps_filename}'.format(lps_filename = lps_filename)
        result = run_boolean_command(command)
        return result == 0

    def generate_lps(self, data):
        lps_filename = data['lps_filename']
        input_mcrl2 = data['input_mcrl2']
        preparation_options = data['preparation_options']
        lin_options = preparation_options['linearisation']
        parunfold_steps = preparation_options['lpsparunfold']

        # redirect log messages
        logfile = '{lps_filename}.log'.format(lps_filename = lps_filename)

        lps_in = ""
        lps_out = ""

        try:
            print >> sys.stderr, ''
            start = time.time()

            # linearisation
            lps_out = create_tempfile(lps_filename, '.lps')
            command = self.path + 'mcrl22lps -v {lin_options} {input_mcrl2} {lps_out}'.format(
                lin_options = lin_options,
                input_mcrl2 = input_mcrl2,
                lps_out = lps_out
            )
            run_command('Linearising ' + input_mcrl2, command, logfile)

            # lpssuminst
            lps_in = lps_out
            lps_out = create_tempfile(lps_filename, '.lps')
            command = self.path + 'lpssuminst {lps_in} {lps_out}'.format(lps_in = lps_in, lps_out = lps_out)
            run_command('lpssuminst ' + input_mcrl2, command, logfile)
            os.remove(lps_in)

            # parunfold
            for step in parunfold_steps:
                lps_in = lps_out
                lps_out = create_tempfile(lps_filename, '.lps')
                command = self.path + 'lpsparunfold -v {options} {lps_in} {lps_out}'.format(
                    options = step,
                    lps_in = lps_in,
                    lps_out = lps_out
                )
                data = run_command('lpsparunfold ' + input_mcrl2, command, logfile)
                os.remove(lps_in)

            # rewrite
            lps_in = lps_out
            lps_out = create_tempfile(lps_filename, '.lps')
            command = self.path + 'lpsrewr -v {lps_in} {lps_out}'.format(lps_in = lps_in, lps_out = lps_out)
            run_command('lpsrewr ' + input_mcrl2, command, logfile)
            os.remove(lps_in)

            # constelm
            lps_in = lps_out
            lps_out = create_tempfile(lps_filename, '.lps')
            command = self.path + 'lpsconstelm -v -c {lps_in} {lps_out}'.format(lps_in = lps_in, lps_out = lps_out)
            run_command('lpsconstelm ' + input_mcrl2, command, logfile)
            os.remove(lps_in)

            # move output to lps_filename
            shutil.move(lps_out, lps_filename)

            end = time.time()
            print >> sys.stderr, 'Generating {} took {:.2f} seconds.'.format(lps_filename, (end - start))
        except Exception as e:
            print >> sys.stderr, 'Error:', e
            traceback.print_exc(file=sys.stderr)
            if os.path.isfile(lps_in):
                os.remove(lps_in)
            if os.path.isfile(lps_out):
                os.remove(lps_out)
            sys.exit(1)

    def prepare(self, data):
        experiment_type = data['type']
        assert experiment_type in ['lps', 'pbes']

        # check if an invalid LPS exists
        lps_filename = data['lps_filename']
        if os.path.isfile(lps_filename):
            if not self.check_lps(lps_filename):
                print >> sys.stderr, 'Found invalid LPS:', lps_filename
                os.remove(lps_filename)

        # generate lps if not exists
        if not os.path.isfile(lps_filename):
            self.generate_lps(data)
        else:
            print >> sys.stderr, 'Existing LPS found:', lps_filename
        if not os.path.isfile(lps_filename):
            raise Exception('Error creating LPS: ' + lps_filename)

        if not experiment_type == 'pbes':
            return

        # check if an invalid PBES exists
        pbes_filename = data['pbes_filename']
        if os.path.isfile(pbes_filename):
            if not self.check_pbes(pbes_filename):
                print >> sys.stderr, 'Found invalid PBES:', pbes_filename
                os.remove(pbes_filename)

        # generate pbes if not exists
        if not os.path.isfile(pbes_filename):
            self.generate_pbes(data)
        else:
            print >> sys.stderr, 'Existing PBES found:', pbes_filename
        if not os.path.isfile(pbes_filename):
            raise Exception('Error creating PBES: ' + pbes_filename)

class Ltsmin(Tool):
    def __init__(self, config):
        path = get_path_from_config(config, 'ltsmin')
        if path is None:
            self.path = self.find_path()
        else:
            self.path = os.path.abspath(path)
        self.path += '/'
        print >> sys.stderr, 'ltsmin path:', self.path
        test_program = self.path + 'pbes2lts-sym'
        if not os.path.isfile(test_program):
            raise Exception('pbes2lts-sym not found.')

    def find_path(self):
        path = run_simple_command('which pbes2lts-sym')
        ltsmin_dir = os.path.dirname(os.path.abspath(path))
        print >> sys.stderr, 'Found ltsmin binary in path:', path,
        return ltsmin_dir

    def list(self, experiments):
        runs = []
        for data in experiments:
            experiment_type = data['type']
            assert experiment_type in ['lps', 'pbes']
            if experiment_type is 'lps':
                input_filename = data['lps_filename']
            else:
                input_filename = data['pbes_filename']
            run_options = data.get('run_options')
            if run_options is None:
                print >> sys.stderr, 'No run options found for experiment', experiment_type, input_filename
            else:
                n_cores = run_options['n_cores']
                for n in n_cores:
                    runs.append({
                        'type': experiment_type,
                        'input': input_filename,
                        'cores': n,
                        'data': data
                    })
        return runs

    def print_list(self, experiments):
        runs = self.list(experiments)
        for run in runs:
            print run['type'], run['input'], run['cores']
        print >> sys.stderr, 'Total:', len(runs)

    def report(self, action, result, output_dir):
        print >> sys.stderr, result
        result_file = '{}/{}.result'.format(output_dir, action)
        with open(result_file, 'w') as f:
            f.write(result)

    def lps_instantiate(self, input_lps, n_cores, output_dir):
        raise Exception('not implemented yet.')

    def pbes_instantiate(self, input_pbes, output_spg, n_cores, output_dir):
        # pbes2lts-sym
        pbes2lts_options = '--mcrl2-rewriter=jitty -rgs --vset=lddmc --order=par-prev'
        lace_options = '--lace-workers={}'.format(n_cores)
        command = self.path + 'pbes2lts-sym {pbes2lts_options} {lace_options} {input_pbes} --pg-write={output_spg}'.format(
            pbes2lts_options = pbes2lts_options,
            input_pbes = input_pbes,
            output_spg = output_spg,
            lace_options = lace_options
        )
        # redirect log messages
        logfile = '{}/pbes2spg.log'.format(output_dir)
        try:
            print >> sys.stderr, ''
            start = time.time()

            run_command('Instantiating ' + input_pbes, command, logfile)

            end = time.time()
            self.report('pbes2spg', 'Instantiating {} took {:.2f} seconds.'.format(input_pbes, (end - start)), output_dir)
        except Exception as e:
            print >> sys.stderr, 'Error:', e
            if os.path.isfile(output_spg):
                os.remove(output_spg)
            sys.exit(1)

    def pbes_solve(self, input_spg, n_cores, output_dir):
        # spgsolver
        spgsolver_options = ''
        lace_options = '--lace-workers={}'.format(n_cores)
        command = self.path + 'spgsolver {spgsolver_options} {lace_options} {input_spg}'.format(
            spgsolver_options = spgsolver_options,
            input_spg = input_spg,
            lace_options = lace_options
        )
        # redirect log messages
        logfile = '{}/spgsolver.log'.format(output_dir)
        try:
            print >> sys.stderr, ''
            start = time.time()

            run_command('Solving ' + input_spg, command, logfile)

            end = time.time()
            self.report('spgsolver', 'Solving {} took {:.2f} seconds.'.format(input_spg, (end - start)), output_dir)

        except Exception as e:
            print >> sys.stderr, 'Error:', e
            sys.exit(1)

    def prepare_output_dir(self, input_path, cores, timestamp):
        parent = os.path.dirname(input_path)
        input_filename = os.path.basename(input_path)
        output_dir = '{}/runs/{}/{}/{}'.format(parent, input_filename, cores, timestamp)
        os.makedirs(output_dir)
        return (output_dir, input_filename)

    def run(self, experiments, index):
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S%f')
        runs = self.list(experiments)
        if index > len(runs):
            print >> sys.stderr, 'Choose a number between 1 and', len(runs), '( was:', index, ')'
            raise Exception('Index out of bounds.')
        run = runs[index - 1]
        type = run['type']
        input_filename = run['input']
        cores = run['cores']
        assert type in ['lps', 'pbes']
        if type is 'lps':
            (output_dir, filename) = self.prepare_output_dir(input_filename, cores, timestamp)
            self.lps_instantiate(input_filename, cores, output_dir)
        else:
            (output_dir, filename) = self.prepare_output_dir(input_filename, cores, timestamp)
            spg_filename = '{}/{}.spg'.format(output_dir, filename)
            # FIXME: derive spg filename without directory prefix
            self.pbes_instantiate(input_filename, spg_filename, cores, output_dir)
            self.pbes_solve(spg_filename, cores, output_dir)

class ToolRegistry:

    tools = {}

    def __init__(self, config):
        self.tools = {
            'mcrl2': Mcrl2(config),
            'ltsmin': Ltsmin(config)
        }

<|endoftext|>


<commit_before>from scraper import search_CL
from scraper import read_search_results
from scraper import parse_source
from scraper import extract_listings
from scraper import fetch_json_results
from scraper import read_json_results
from scraper import add_location
from types import GeneratorType
import bs4

def test_search_CL():
    test_body, test_encoding = search_CL(minAsk=500, maxAsk=1000, bedrooms=2)
    assert "craigslist" in test_body
    assert test_encoding == 'utf-8'

def test_json_search():
    json = fetch_json_results()
    assert isinstance(json, list)
    assert isinstance(json[0], list)
    assert isinstance(json[0][0], dict)
    assert json[-1][u'baseurl'] == u'//seattle.craigslist.org'

def test_read_json():
    json = read_json_results()
    assert isinstance(json, list)
    assert isinstance(json[0], list)
    assert isinstance(json[0][0], dict)
    assert json[-1][u'baseurl'] == u'//seattle.craigslist.org'

def test_read_search_result():
    test_body, test_encoding = read_search_results()
    assert "craigslist" in test_body
    assert test_encoding == 'utf-8'

def test_parse_source():
    test_body, test_encoding = read_search_results()
    test_parse = parse_source(test_body, test_encoding)
    assert isinstance(test_parse, bs4.BeautifulSoup)

def test_extract_listings():
    test_body, test_encoding = read_search_results()
    test_parse = parse_source(test_body, test_encoding)
    test_data = extract_listings(test_parse)
    assert isinstance(test_data, GeneratorType)
    for dict_ in test_data:
        assert isinstance(dict_, dict)

def test_add_location_1():
    listing = {'pid': u'1'}
    search = {
        u'1': {'Latitude': 45.0, 'Longitude': 90.0},
        u'2': {'Latitude': 30.0, 'Longitude': 90.0}
    }
    assert add_location(listing, search) == True

def test_add_location_2():
    listing = {'pid': u'3'}
    search = {
        u'1': {'Latitude': 45.0, 'Longitude': 90.0},
        u'2': {'Latitude': 30.0, 'Longitude': 90.0}
    }
    assert add_location(listing, search) == False<commit_msg>Revise test_add_location_1 and _2 to use is for comparision<commit_after>from scraper import search_CL
from scraper import read_search_results
from scraper import parse_source
from scraper import extract_listings
from scraper import fetch_json_results
from scraper import read_json_results
from scraper import add_location
from types import GeneratorType
import bs4

def test_search_CL():
    test_body, test_encoding = search_CL(minAsk=500, maxAsk=1000, bedrooms=2)
    assert "craigslist" in test_body
    assert test_encoding == 'utf-8'

def test_json_search():
    json = fetch_json_results()
    assert isinstance(json, list)
    assert isinstance(json[0], list)
    assert isinstance(json[0][0], dict)
    assert json[-1][u'baseurl'] == u'//seattle.craigslist.org'

def test_read_json():
    json = read_json_results()
    assert isinstance(json, list)
    assert isinstance(json[0], list)
    assert isinstance(json[0][0], dict)
    assert json[-1][u'baseurl'] == u'//seattle.craigslist.org'

def test_read_search_result():
    test_body, test_encoding = read_search_results()
    assert "craigslist" in test_body
    assert test_encoding == 'utf-8'

def test_parse_source():
    test_body, test_encoding = read_search_results()
    test_parse = parse_source(test_body, test_encoding)
    assert isinstance(test_parse, bs4.BeautifulSoup)

def test_extract_listings():
    test_body, test_encoding = read_search_results()
    test_parse = parse_source(test_body, test_encoding)
    test_data = extract_listings(test_parse)
    assert isinstance(test_data, GeneratorType)
    for dict_ in test_data:
        assert isinstance(dict_, dict)

def test_add_location_1():
    listing = {'pid': u'1'}
    search = {
        u'1': {'Latitude': 45.0, 'Longitude': 90.0},
        u'2': {'Latitude': 30.0, 'Longitude': 90.0}
    }
    assert add_location(listing, search) is True

def test_add_location_2():
    listing = {'pid': u'3'}
    search = {
        u'1': {'Latitude': 45.0, 'Longitude': 90.0},
        u'2': {'Latitude': 30.0, 'Longitude': 90.0}
    }
    assert add_location(listing, search) is False

def test_add_address():
    pass<|endoftext|>


<commit_before>import mock
import json

from worker import PublicTitlesWorker, queue, queue_key

test_message = ('titles_queue',
            '''{'proprietors': [
                    {'first_name': 'Gustavo', 'last_name': 'Fring'}, {'first_name': '', 'last_name': ''}],
            'title_number': 'TN1234567',
            'property': {'address':
                                    {'house_number': '1', 'town': 'Albuquerque', 'postcode': '98765', 'road': 'Somewhere'},
            'class_of_title': 'qualified',
            'tenure': 'freehold'},
            'payment': {'titles': ['1234'],
            'price_paid': '987654321'}}''')

expected_public_data = {'title_number': 'TN1234567', 'house_number': '1',  'town': 'Albuquerque', 'postcode': '98765', 'road': 'Somewhere', 'price_paid': '987654321'}

def test_build_public_data():
    worker = PublicTitlesWorker(queue, queue_key)

    public_data = worker.build_public_data(test_message)

    assert public_data == expected_public_data

@mock.patch("requests.put")
def test_worker_should_put_data_to_destinations(mock_put):
    worker = PublicTitlesWorker(queue, queue_key)

    public_data = worker.build_public_data(test_message)
    headers = {"Content-Type": "application/json"}
    base_destination_url = 'http://public-titles-api/title'
    full_destination_url = '%s/%s' % (base_destination_url,  public_data['title_number'])

    worker.send(base_destination_url, public_data)

    mock_put.assert_called_with(full_destination_url, data=json.dumps(public_data), headers=headers)
<commit_msg>Fixed tests<commit_after>import mock
import json

from worker import PublicTitlesWorker, queue, queue_key

test_message = ('titles_queue',
            '''{'proprietors': [
                    {'first_name': 'Gustavo', 'last_name': 'Fring'}, {'first_name': '', 'last_name': ''}],
            'title_number': 'TN1234567',
            'property': {'address':
                                    {'house_number': '1', 'town': 'Albuquerque', 'postcode': '98765', 'road': 'Somewhere'},
            'class_of_title': 'qualified',
            'tenure': 'freehold'},
            'payment': {'titles': ['1234'],
            'price_paid': '987654321'}}''')

expected_public_data = {'title_number': 'TN1234567', 'house_number': '1',  'town': 'Albuquerque', 'postcode': '98765', 'road': 'Somewhere', 'price_paid': '987654321'}

base_titles_api_url = 'http://public-titles-api'

def test_build_public_data():
    worker = PublicTitlesWorker(queue, queue_key, [])

    public_data = worker.build_public_data(test_message)

    assert public_data == expected_public_data

@mock.patch("requests.put")
def test_worker_should_put_data_to_destinations(mock_put):
    worker = PublicTitlesWorker(queue, queue_key, [base_titles_api_url])

    public_data = worker.build_public_data(test_message)
    headers = {"Content-Type": "application/json"}

    titles_api_url =  '%s/title' %  base_titles_api_url
    full_title_url = '%s/%s' % (titles_api_url,  public_data['title_number'])

    worker.send(titles_api_url, public_data)

    mock_put.assert_called_with(full_title_url, data=json.dumps(public_data), headers=headers)
<|endoftext|>