<commit_before>from os import listdir
from os.path import abspath
from os.path import dirname
from os.path import isfile
from os.path import realpath
import cherrypy
import json
import os.path
import re

from PressUI.cherrypy.PressConfig import PressConfig
from PressUI.cherrypy.react import compile_react
from PressUI.cherrypy.static import press_get_static_file_by_dig
from PressUI.cherrypy.static import press_static_file
from PressUI.utils.browser_cache import add_cache_control_header
import PressUI.cherrypy.PressProduction as PressProduction

class PressApp():
    __JQUERY_VERSION = '2.1.1'
    __REACT_VERSION = '0.12.0'

    def __init__(self):
        with open('PressUI/reactive/index.html', 'r') as f:
            self.__index_html = f.read()
        if PressProduction.is_production():
            to_remove = 'development'
        else:
            to_remove = 'production'
        self.__index_html = re.sub(
            '<!-- {to_remove} -->.*<!-- end_{to_remove} -->'.format(
                to_remove = to_remove,
            ),
            '',
            self.__index_html,
            flags = re.DOTALL,
        )

    def _js_sources(self):
        return []

    def _json(self, output):
        cherrypy.response.headers['content-type'] = 'application/json'
        add_cache_control_header()
        return json.dumps(output).encode('utf-8')

    @cherrypy.tools.allow(methods = ['GET'])
    @cherrypy.expose
    def fb_login_info_json(self):
        ret = self._json({
            'app_id': PressConfig.get('fb_app_id'),
            'hostname': cherrypy.request.base,
        })
        add_cache_control_header(years = 1)
        return ret

    @cherrypy.tools.allow(methods = ['GET'])
    @cherrypy.expose
    def favicon_png(self):
        cherrypy.response.headers['content-type'] = 'image/png'
        add_cache_control_header(years = 1)
        path = os.path.join(dirname(dirname(__file__)), 'FirefoxOS/icon.png')
        with open(path, 'rb') as f:
            return f.read()

    @cherrypy.tools.allow(methods = ['GET'])
    @cherrypy.expose
    def channel_html(self):
        add_cache_control_header(years = 1)
        return '<script src="//connect.facebook.net/en_US/all.js"></script>'

    # Same as `all_js` but for development.  Diffrerent path to avoid cache
    # problems.
    @cherrypy.tools.allow(methods = ['GET'])
    @cherrypy.expose
    def all_jsx_js(self, dig):
        return press_get_static_file_by_dig(dig, 'text/jsx')

    @cherrypy.tools.allow(methods = ['GET'])
    @cherrypy.expose
    def all_js(self, dig):
        return press_get_static_file_by_dig(dig, 'application/javascript')

    @cherrypy.tools.allow(methods = ['GET'])
    @cherrypy.expose
    def all_css(self, dig):
        return press_get_static_file_by_dig(dig, 'text/css')

    @cherrypy.tools.allow(methods = ['GET'])
    @cherrypy.expose
    def default(self, *args, **kwargs):
        add_cache_control_header(hours = 1)

        # Will be added twice but why not.
        js_sources = ['PressUI/reactive/jsx_header.js']
        for path in listdir('PressUI/reactive'):
            path = realpath('PressUI/reactive/' + path)
            if isfile(path) and path.endswith('.js'):
                js_sources.append(path)
        js_sources.extend(self._js_sources())

        compile_fun = None
        if PressProduction.is_production():
            compile_fun = lambda x: compile_react(self.__REACT_VERSION, x)
        js_dig = press_static_file(list(map(abspath, js_sources)), compile_fun)
        css_dig = press_static_file([abspath('style/all.css')])

        return self.__index_html.format(
            js_dig = js_dig,
            css_dig = css_dig,
            jquery_version = self.__JQUERY_VERSION,
            react_version = self.__REACT_VERSION,
        ).encode('utf-8')
<commit_msg>Update jQuery<commit_after>from os import listdir
from os.path import abspath
from os.path import dirname
from os.path import isfile
from os.path import realpath
import cherrypy
import json
import os.path
import re

from PressUI.cherrypy.PressConfig import PressConfig
from PressUI.cherrypy.react import compile_react
from PressUI.cherrypy.static import press_get_static_file_by_dig
from PressUI.cherrypy.static import press_static_file
from PressUI.utils.browser_cache import add_cache_control_header
import PressUI.cherrypy.PressProduction as PressProduction

class PressApp():
    __JQUERY_VERSION = '2.1.3'
    __REACT_VERSION = '0.12.0'

    def __init__(self):
        with open('PressUI/reactive/index.html', 'r') as f:
            self.__index_html = f.read()
        if PressProduction.is_production():
            to_remove = 'development'
        else:
            to_remove = 'production'
        self.__index_html = re.sub(
            '<!-- {to_remove} -->.*<!-- end_{to_remove} -->'.format(
                to_remove = to_remove,
            ),
            '',
            self.__index_html,
            flags = re.DOTALL,
        )

    def _js_sources(self):
        return []

    def _json(self, output):
        cherrypy.response.headers['content-type'] = 'application/json'
        add_cache_control_header()
        return json.dumps(output).encode('utf-8')

    @cherrypy.tools.allow(methods = ['GET'])
    @cherrypy.expose
    def fb_login_info_json(self):
        ret = self._json({
            'app_id': PressConfig.get('fb_app_id'),
            'hostname': cherrypy.request.base,
        })
        add_cache_control_header(years = 1)
        return ret

    @cherrypy.tools.allow(methods = ['GET'])
    @cherrypy.expose
    def favicon_png(self):
        cherrypy.response.headers['content-type'] = 'image/png'
        add_cache_control_header(years = 1)
        path = os.path.join(dirname(dirname(__file__)), 'FirefoxOS/icon.png')
        with open(path, 'rb') as f:
            return f.read()

    @cherrypy.tools.allow(methods = ['GET'])
    @cherrypy.expose
    def channel_html(self):
        add_cache_control_header(years = 1)
        return '<script src="//connect.facebook.net/en_US/all.js"></script>'

    # Same as `all_js` but for development.  Diffrerent path to avoid cache
    # problems.
    @cherrypy.tools.allow(methods = ['GET'])
    @cherrypy.expose
    def all_jsx_js(self, dig):
        return press_get_static_file_by_dig(dig, 'text/jsx')

    @cherrypy.tools.allow(methods = ['GET'])
    @cherrypy.expose
    def all_js(self, dig):
        return press_get_static_file_by_dig(dig, 'application/javascript')

    @cherrypy.tools.allow(methods = ['GET'])
    @cherrypy.expose
    def all_css(self, dig):
        return press_get_static_file_by_dig(dig, 'text/css')

    @cherrypy.tools.allow(methods = ['GET'])
    @cherrypy.expose
    def default(self, *args, **kwargs):
        add_cache_control_header(hours = 1)

        # Will be added twice but why not.
        js_sources = ['PressUI/reactive/jsx_header.js']
        for path in listdir('PressUI/reactive'):
            path = realpath('PressUI/reactive/' + path)
            if isfile(path) and path.endswith('.js'):
                js_sources.append(path)
        js_sources.extend(self._js_sources())

        compile_fun = None
        if PressProduction.is_production():
            compile_fun = lambda x: compile_react(self.__REACT_VERSION, x)
        js_dig = press_static_file(list(map(abspath, js_sources)), compile_fun)
        css_dig = press_static_file([abspath('style/all.css')])

        return self.__index_html.format(
            js_dig = js_dig,
            css_dig = css_dig,
            jquery_version = self.__JQUERY_VERSION,
            react_version = self.__REACT_VERSION,
        ).encode('utf-8')
<|endoftext|>


<commit_before>"""Error classes for CherryPy."""

from cgi import escape as _escape
from sys import exc_info as _exc_info
from traceback import format_exception as _format_exception
from urlparse import urljoin as _urljoin
from cherrypy.lib import httputil as _httputil

class CherryPyException(Exception):
    pass

class MaxSizeExceeded(CherryPyException):
    """Exception raised when the request body is longer than allowed."""
    pass

class TimeoutError(CherryPyException):
    """Exception raised when Response.timed_out is detected."""
    pass

class InternalRedirect(CherryPyException):
    """Exception raised to switch to the handler for a different URL.

    Any request.params must be supplied in a query string.
    """

    def __init__(self, path):
        import cherrypy
        request = cherrypy.request

        self.query_string = ""
        if "?" in path:
            # Separate any params included in the path
            path, self.query_string = path.split("?", 1)

        # Note that urljoin will "do the right thing" whether url is:
        #  1. a URL relative to root (e.g. "/dummy")
        #  2. a URL relative to the current path
        # Note that any query string will be discarded.
        path = _urljoin(request.path_info, path)

        # Set a 'path' member attribute so that code which traps this
        # error can have access to it.
        self.path = path

        CherryPyException.__init__(self, path, self.query_string)

class HTTPRedirect(CherryPyException):
    """Exception raised when the request should be redirected.

    The new URL must be passed as the first argument to the Exception,
    e.g., HTTPRedirect(newUrl). Multiple URLs are allowed. If a URL is
    absolute, it will be used as-is. If it is relative, it is assumed
    to be relative to the current cherrypy.request.path_info.
    """

    def __init__(self, urls, status=None):
        import cherrypy
        request = cherrypy.request

        if isinstance(urls, basestring):
            urls = [urls]

        abs_urls = []
        for url in urls:
            # Note that urljoin will "do the right thing" whether url is:
            #  1. a complete URL with host (e.g. "http://www.example.com/test")
            #  2. a URL relative to root (e.g. "/dummy")
            #  3. a URL relative to the current path
            # Note that any query string in cherrypy.request is discarded.
            url = _urljoin(cherrypy.url(), url)
            abs_urls.append(url)
        self.urls = abs_urls

        # RFC 2616 indicates a 301 response code fits our goal; however,
        # browser support for 301 is quite messy. Do 302/303 instead. See
        # http://www.alanflavell.org.uk/www/post-redirect.html
        if status is None:
            if request.protocol >= (1, 1):
                status = 303
            else:
                status = 302
        else:
            status = int(status)
            if status < 300 or status > 399:
                raise ValueError("status must be between 300 and 399.")

        self.status = status
        CherryPyException.__init__(self, abs_urls, status)

    def set_response(self):
        """Modify cherrypy.response status, headers, and body to represent self.

        CherryPy uses this internally, but you can also use it to create an
        HTTPRedirect object and set its output without *raising* the exception.
        """
        import cherrypy
        response = cherrypy.response
        response.status = status = self.status

        if status in (300, 301, 302, 303, 307):
            response.headers['Content-Type'] = "text/html"
            # "The ... URI SHOULD be given by the Location field
            # in the response."
            response.headers['Location'] = self.urls[0]

            # "Unless the request method was HEAD, the entity of the response
            # SHOULD contain a short hypertext note with a hyperlink to the
            # new URI(s)."
            msg = {300: "This resource can be found at %s.",
                   301: "This resource has permanently moved to %s.",
                   302: "This resource resides temporarily at %s.",
                   303: "This resource can be found at %s.",
                   307: "This resource has moved temporarily to %s.",
                   }[status]
            response.body = "
\n".join([msg % (u, u) for u in self.urls])
            # Previous code may have set C-L, so we have to reset it
            # (allow finalize to set it).
            response.headers.pop('Content-Length', None)
        elif status == 304:
            # Not Modified.
            # "The response MUST include the following header fields:
            # Date, unless its omission is required by section 14.18.1"
            # The "Date" header should have been set in Response.__init__

            # "...the response SHOULD NOT include other entity-headers."
            for key in ('Allow', 'Content-Encoding', 'Content-Language',
                        'Content-Length', 'Content-Location', 'Content-MD5',
                        'Content-Range', 'Content-Type', 'Expires',
                        'Last-Modified'):
                if key in response.headers:
                    del response.headers[key]

            # "The 304 response MUST NOT contain a message-body."
            response.body = None
            # Previous code may have set C-L, so we have to reset it.
            response.headers.pop('Content-Length', None)
        elif status == 305:
            # Use Proxy.
            # self.urls[0] should be the URI of the proxy.
            response.headers['Location'] = self.urls[0]
            response.body = None
            # Previous code may have set C-L, so we have to reset it.
            response.headers.pop('Content-Length', None)
        else:
            raise ValueError("The %s status code is unknown." % status)

    def __call__(self):
        """Use this exception as a request.handler (raise self)."""
        raise self

def clean_headers(status):
    """Remove any headers which should not apply to an error response."""
    import cherrypy

    response = cherrypy.response

    # Remove headers which applied to the original content,
    # but do not apply to the error page.
    respheaders = response.headers
    for key in ["Accept-Ranges", "Age", "ETag", "Location", "Retry-After",
                "Vary", "Content-Encoding", "Content-Length", "Expires",
                "Content-Location", "Content-MD5", "Last-Modified"]:
        if respheaders.has_key(key):
            del respheaders[key]

    if status != 416:
        # A server sending a response with status code 416 (Requested
        # range not satisfiable) SHOULD include a Content-Range field
        # with a byte-range-resp-spec of "*". The instance-length
        # specifies the current length of the selected resource.
        # A response with status code 206 (Partial Content) MUST NOT
        # include a Content-Range field with a byte-range- resp-spec of "*".
        if respheaders.has_key("Content-Range"):
            del respheaders["Content-Range"]

class HTTPError(CherryPyException):
    """ Exception used to return an HTTP error code (4xx-5xx) to the client.
        This exception will automatically set the response status and body.

        A custom message (a long description to display in the browser)
        can be provided in place of the default.
    """

    def __init__(self, status=500, message=None):
        self.status = status
        try:
            self.code, self.reason, defaultmsg = _httputil.valid_status(status)
        except ValueError, x:
            raise cherrypy.__class__(500, x.args[0])

        if self.code < 400 or self.code > 599:
            raise ValueError("status must be between 400 and 599.")

        # See http://www.python.org/dev/peps/pep-0352/
        # self.message = message
        self._message = message or defaultmsg
        CherryPyException.__init__(self, status, message)

    def set_response(self):
        """Modify cherrypy.response status, headers, and body to represent self.

        CherryPy uses this internally, but you can also use it to create an
        HTTPError object and set its output without *raising* the exception.
        """
        import cherrypy

        response = cherrypy.response

        clean_headers(self.code)

        # In all cases, finalize will be called after this method,
        # so don't bother cleaning up response values here.
        response.status = self.status
        tb = None
        if cherrypy.request.show_tracebacks:
            tb = format_exc()
        response.headers['Content-Type'] = "text/html"

        content = self.get_error_page(self.status, traceback=tb,
                                      message=self._message)
        response.body = content
        response.headers['Content-Length'] = len(content)

        _be_ie_unfriendly(self.code)

    def get_error_page(self, *args, **kwargs):
        return get_error_page(*args, **kwargs)

    def __call__(self):
        """Use this exception as a request.handler (raise self)."""
        raise self

class NotFound(HTTPError):
    """Exception raised when a URL could not be mapped to any handler (404)."""

    def __init__(self, path=None):
        if path is None:
            import cherrypy
            path = cherrypy.request.script_name + cherrypy.request.path_info
        self.args = (path,)
        HTTPError.__init__(self, 404, "The path %r was not found." % path)

_HTTPErrorTemplate = '''<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8"></meta>
    <title>%(status)s</title>

%(status)s

%(message)s

%(traceback)s

    Powered by CherryPy %(version)s

'''

def get_error_page(status, **kwargs):
    """Return an HTML page, containing a pretty error response.

    status should be an int or a str.
    kwargs will be interpolated into the page template.
    """
    import cherrypy

    try:
        code, reason, message = _httputil.valid_status(status)
    except ValueError, x:
        raise cherrypy.HTTPError(500, x.args[0])

    # We can't use setdefault here, because some
    # callers send None for kwarg values.
    if kwargs.get('status') is None:
        kwargs['status'] = "%s %s" % (code, reason)
    if kwargs.get('message') is None:
        kwargs['message'] = message
    if kwargs.get('traceback') is None:
        kwargs['traceback'] = ''
    if kwargs.get('version') is None:
        kwargs['version'] = cherrypy.__version__

    for k, v in kwargs.iteritems():
        if v is None:
            kwargs[k] = ""
        else:
            kwargs[k] = _escape(kwargs[k])

    # Use a custom template or callable for the error page?
    pages = cherrypy.request.error_page
    error_page = pages.get(code) or pages.get('default')
    if error_page:
        try:
            if callable(error_page):
                return error_page(**kwargs)
            else:
                return file(error_page, 'rb').read() % kwargs
        except:
            e = _format_exception(*_exc_info())[-1]
            m = kwargs['message']
            if m:
                m += "
"
            m += "In addition, the custom error page failed:\n
%s" % e
            kwargs['message'] = m

    return _HTTPErrorTemplate % kwargs

_ie_friendly_error_sizes = {
    400: 512, 403: 256, 404: 512, 405: 256,
    406: 512, 408: 512, 409: 512, 410: 256,
    500: 512, 501: 512, 505: 512,
    }

def _be_ie_unfriendly(status):
    import cherrypy
    response = cherrypy.response

    # For some statuses, Internet Explorer 5+ shows "friendly error
    # messages" instead of our response.body if the body is smaller
    # than a given size. Fix this by returning a body over that size
    # (by adding whitespace).
    # See http://support.microsoft.com/kb/q218155/
    s = _ie_friendly_error_sizes.get(status, 0)
    if s:
        s += 1
        # Since we are issuing an HTTP error status, we assume that
        # the entity is short, and we should just collapse it.
        content = response.collapse_body()
        l = len(content)
        if l and l < s:
            # IN ADDITION: the response must be written to IE
            # in one chunk or it will still get replaced! Bah.
            content = content + (" " * (s - l))
        response.body = content
        response.headers[u'Content-Length'] = str(len(content))

def format_exc(exc=None):
    """Return exc (or sys.exc_info if None), formatted."""
    if exc is None:
        exc = _exc_info()
    if exc == (None, None, None):
        return ""
    import traceback
    return "".join(traceback.format_exception(*exc))

def bare_error(extrabody=None):
    """Produce status, headers, body for a critical error.

    Returns a triple without calling any other questionable functions,
    so it should be as error-free as possible. Call it from an HTTP server
    if you get errors outside of the request.

    If extrabody is None, a friendly but rather unhelpful error message
    is set in the body. If extrabody is a string, it will be appended
    as-is to the body.
    """

    # The whole point of this function is to be a last line-of-defense
    # in handling errors. That is, it must not raise any errors itself;
    # it cannot be allowed to fail. Therefore, don't add to it!
    # In particular, don't call any other CP functions.

    body = "Unrecoverable error in the server."
    if extrabody is not None:
        if not isinstance(extrabody, str):
            extrabody = extrabody.encode('utf-8')
        body += "\n" + extrabody

    return ("500 Internal Server Error",
            [('Content-Type', 'text/plain'),
             ('Content-Length', str(len(body)))],
            [body])

<commit_msg>trunk - prefer 'in' over has_key as it's more python3 compatible<commit_after>"""Error classes for CherryPy."""

from cgi import escape as _escape
from sys import exc_info as _exc_info
from traceback import format_exception as _format_exception
from urlparse import urljoin as _urljoin
from cherrypy.lib import httputil as _httputil

class CherryPyException(Exception):
    pass

class MaxSizeExceeded(CherryPyException):
    """Exception raised when the request body is longer than allowed."""
    pass

class TimeoutError(CherryPyException):
    """Exception raised when Response.timed_out is detected."""
    pass

class InternalRedirect(CherryPyException):
    """Exception raised to switch to the handler for a different URL.

    Any request.params must be supplied in a query string.
    """

    def __init__(self, path):
        import cherrypy
        request = cherrypy.request

        self.query_string = ""
        if "?" in path:
            # Separate any params included in the path
            path, self.query_string = path.split("?", 1)

        # Note that urljoin will "do the right thing" whether url is:
        #  1. a URL relative to root (e.g. "/dummy")
        #  2. a URL relative to the current path
        # Note that any query string will be discarded.
        path = _urljoin(request.path_info, path)

        # Set a 'path' member attribute so that code which traps this
        # error can have access to it.
        self.path = path

        CherryPyException.__init__(self, path, self.query_string)

class HTTPRedirect(CherryPyException):
    """Exception raised when the request should be redirected.

    The new URL must be passed as the first argument to the Exception,
    e.g., HTTPRedirect(newUrl). Multiple URLs are allowed. If a URL is
    absolute, it will be used as-is. If it is relative, it is assumed
    to be relative to the current cherrypy.request.path_info.
    """

    def __init__(self, urls, status=None):
        import cherrypy
        request = cherrypy.request

        if isinstance(urls, basestring):
            urls = [urls]

        abs_urls = []
        for url in urls:
            # Note that urljoin will "do the right thing" whether url is:
            #  1. a complete URL with host (e.g. "http://www.example.com/test")
            #  2. a URL relative to root (e.g. "/dummy")
            #  3. a URL relative to the current path
            # Note that any query string in cherrypy.request is discarded.
            url = _urljoin(cherrypy.url(), url)
            abs_urls.append(url)
        self.urls = abs_urls

        # RFC 2616 indicates a 301 response code fits our goal; however,
        # browser support for 301 is quite messy. Do 302/303 instead. See
        # http://www.alanflavell.org.uk/www/post-redirect.html
        if status is None:
            if request.protocol >= (1, 1):
                status = 303
            else:
                status = 302
        else:
            status = int(status)
            if status < 300 or status > 399:
                raise ValueError("status must be between 300 and 399.")

        self.status = status
        CherryPyException.__init__(self, abs_urls, status)

    def set_response(self):
        """Modify cherrypy.response status, headers, and body to represent self.

        CherryPy uses this internally, but you can also use it to create an
        HTTPRedirect object and set its output without *raising* the exception.
        """
        import cherrypy
        response = cherrypy.response
        response.status = status = self.status

        if status in (300, 301, 302, 303, 307):
            response.headers['Content-Type'] = "text/html"
            # "The ... URI SHOULD be given by the Location field
            # in the response."
            response.headers['Location'] = self.urls[0]

            # "Unless the request method was HEAD, the entity of the response
            # SHOULD contain a short hypertext note with a hyperlink to the
            # new URI(s)."
            msg = {300: "This resource can be found at %s.",
                   301: "This resource has permanently moved to %s.",
                   302: "This resource resides temporarily at %s.",
                   303: "This resource can be found at %s.",
                   307: "This resource has moved temporarily to %s.",
                   }[status]
            response.body = "
\n".join([msg % (u, u) for u in self.urls])
            # Previous code may have set C-L, so we have to reset it
            # (allow finalize to set it).
            response.headers.pop('Content-Length', None)
        elif status == 304:
            # Not Modified.
            # "The response MUST include the following header fields:
            # Date, unless its omission is required by section 14.18.1"
            # The "Date" header should have been set in Response.__init__

            # "...the response SHOULD NOT include other entity-headers."
            for key in ('Allow', 'Content-Encoding', 'Content-Language',
                        'Content-Length', 'Content-Location', 'Content-MD5',
                        'Content-Range', 'Content-Type', 'Expires',
                        'Last-Modified'):
                if key in response.headers:
                    del response.headers[key]

            # "The 304 response MUST NOT contain a message-body."
            response.body = None
            # Previous code may have set C-L, so we have to reset it.
            response.headers.pop('Content-Length', None)
        elif status == 305:
            # Use Proxy.
            # self.urls[0] should be the URI of the proxy.
            response.headers['Location'] = self.urls[0]
            response.body = None
            # Previous code may have set C-L, so we have to reset it.
            response.headers.pop('Content-Length', None)
        else:
            raise ValueError("The %s status code is unknown." % status)

    def __call__(self):
        """Use this exception as a request.handler (raise self)."""
        raise self

def clean_headers(status):
    """Remove any headers which should not apply to an error response."""
    import cherrypy

    response = cherrypy.response

    # Remove headers which applied to the original content,
    # but do not apply to the error page.
    respheaders = response.headers
    for key in ["Accept-Ranges", "Age", "ETag", "Location", "Retry-After",
                "Vary", "Content-Encoding", "Content-Length", "Expires",
                "Content-Location", "Content-MD5", "Last-Modified"]:
        if key in respheaders:
            del respheaders[key]

    if status != 416:
        # A server sending a response with status code 416 (Requested
        # range not satisfiable) SHOULD include a Content-Range field
        # with a byte-range-resp-spec of "*". The instance-length
        # specifies the current length of the selected resource.
        # A response with status code 206 (Partial Content) MUST NOT
        # include a Content-Range field with a byte-range- resp-spec of "*".
        if "Content-Range" in respheaders:
            del respheaders["Content-Range"]

class HTTPError(CherryPyException):
    """ Exception used to return an HTTP error code (4xx-5xx) to the client.
        This exception will automatically set the response status and body.

        A custom message (a long description to display in the browser)
        can be provided in place of the default.
    """

    def __init__(self, status=500, message=None):
        self.status = status
        try:
            self.code, self.reason, defaultmsg = _httputil.valid_status(status)
        except ValueError, x:
            raise cherrypy.__class__(500, x.args[0])

        if self.code < 400 or self.code > 599:
            raise ValueError("status must be between 400 and 599.")

        # See http://www.python.org/dev/peps/pep-0352/
        # self.message = message
        self._message = message or defaultmsg
        CherryPyException.__init__(self, status, message)

    def set_response(self):
        """Modify cherrypy.response status, headers, and body to represent self.

        CherryPy uses this internally, but you can also use it to create an
        HTTPError object and set its output without *raising* the exception.
        """
        import cherrypy

        response = cherrypy.response

        clean_headers(self.code)

        # In all cases, finalize will be called after this method,
        # so don't bother cleaning up response values here.
        response.status = self.status
        tb = None
        if cherrypy.request.show_tracebacks:
            tb = format_exc()
        response.headers['Content-Type'] = "text/html"

        content = self.get_error_page(self.status, traceback=tb,
                                      message=self._message)
        response.body = content
        response.headers['Content-Length'] = len(content)

        _be_ie_unfriendly(self.code)

    def get_error_page(self, *args, **kwargs):
        return get_error_page(*args, **kwargs)

    def __call__(self):
        """Use this exception as a request.handler (raise self)."""
        raise self

class NotFound(HTTPError):
    """Exception raised when a URL could not be mapped to any handler (404)."""

    def __init__(self, path=None):
        if path is None:
            import cherrypy
            path = cherrypy.request.script_name + cherrypy.request.path_info
        self.args = (path,)
        HTTPError.__init__(self, 404, "The path %r was not found." % path)

_HTTPErrorTemplate = '''<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8"></meta>
    <title>%(status)s</title>

%(status)s

%(message)s

%(traceback)s

    Powered by CherryPy %(version)s

'''

def get_error_page(status, **kwargs):
    """Return an HTML page, containing a pretty error response.

    status should be an int or a str.
    kwargs will be interpolated into the page template.
    """
    import cherrypy

    try:
        code, reason, message = _httputil.valid_status(status)
    except ValueError, x:
        raise cherrypy.HTTPError(500, x.args[0])

    # We can't use setdefault here, because some
    # callers send None for kwarg values.
    if kwargs.get('status') is None:
        kwargs['status'] = "%s %s" % (code, reason)
    if kwargs.get('message') is None:
        kwargs['message'] = message
    if kwargs.get('traceback') is None:
        kwargs['traceback'] = ''
    if kwargs.get('version') is None:
        kwargs['version'] = cherrypy.__version__

    for k, v in kwargs.iteritems():
        if v is None:
            kwargs[k] = ""
        else:
            kwargs[k] = _escape(kwargs[k])

    # Use a custom template or callable for the error page?
    pages = cherrypy.request.error_page
    error_page = pages.get(code) or pages.get('default')
    if error_page:
        try:
            if callable(error_page):
                return error_page(**kwargs)
            else:
                return file(error_page, 'rb').read() % kwargs
        except:
            e = _format_exception(*_exc_info())[-1]
            m = kwargs['message']
            if m:
                m += "
"
            m += "In addition, the custom error page failed:\n
%s" % e
            kwargs['message'] = m

    return _HTTPErrorTemplate % kwargs

_ie_friendly_error_sizes = {
    400: 512, 403: 256, 404: 512, 405: 256,
    406: 512, 408: 512, 409: 512, 410: 256,
    500: 512, 501: 512, 505: 512,
    }

def _be_ie_unfriendly(status):
    import cherrypy
    response = cherrypy.response

    # For some statuses, Internet Explorer 5+ shows "friendly error
    # messages" instead of our response.body if the body is smaller
    # than a given size. Fix this by returning a body over that size
    # (by adding whitespace).
    # See http://support.microsoft.com/kb/q218155/
    s = _ie_friendly_error_sizes.get(status, 0)
    if s:
        s += 1
        # Since we are issuing an HTTP error status, we assume that
        # the entity is short, and we should just collapse it.
        content = response.collapse_body()
        l = len(content)
        if l and l < s:
            # IN ADDITION: the response must be written to IE
            # in one chunk or it will still get replaced! Bah.
            content = content + (" " * (s - l))
        response.body = content
        response.headers[u'Content-Length'] = str(len(content))

def format_exc(exc=None):
    """Return exc (or sys.exc_info if None), formatted."""
    if exc is None:
        exc = _exc_info()
    if exc == (None, None, None):
        return ""
    import traceback
    return "".join(traceback.format_exception(*exc))

def bare_error(extrabody=None):
    """Produce status, headers, body for a critical error.

    Returns a triple without calling any other questionable functions,
    so it should be as error-free as possible. Call it from an HTTP server
    if you get errors outside of the request.

    If extrabody is None, a friendly but rather unhelpful error message
    is set in the body. If extrabody is a string, it will be appended
    as-is to the body.
    """

    # The whole point of this function is to be a last line-of-defense
    # in handling errors. That is, it must not raise any errors itself;
    # it cannot be allowed to fail. Therefore, don't add to it!
    # In particular, don't call any other CP functions.

    body = "Unrecoverable error in the server."
    if extrabody is not None:
        if not isinstance(extrabody, str):
            extrabody = extrabody.encode('utf-8')
        body += "\n" + extrabody

    return ("500 Internal Server Error",
            [('Content-Type', 'text/plain'),
             ('Content-Length', str(len(body)))],
            [body])

<|endoftext|>


<commit_before>try:
    set
except NameError:
    from sets import Set as set
import cherrypy

class Hook(object):
    """A point at which CherryPy will call registered callbacks."""

    def __init__(self, name, failsafe = False):
        self.name = name
        self.failsafe = failsafe
        self.callbacks = []

class HookDispatcher(object):

    _input_hooks = [Hook('on_start_resource', failsafe=True),
                    Hook('before_request_body'),
                    Hook('before_main')]
    _output_hooks = [Hook('before_finalize'),
                     Hook('on_end_resource', failsafe=True),
                     Hook('on_end_request', failsafe=True),
                     Hook('before_error_response'),
                     Hook('after_error_response'),
                     ]

    def __init__(self):
        self.hooks = dict([(k.name, k) for k
                           in self._input_hooks + self._output_hooks])

    def run(self, hook_name):
        """Execute all registered hooks for the given name."""
        hook = self.hooks[hook_name]
        for c in hook.callbacks:
            if cherrypy.config.enabled(c.name):
                kwargs = dict([(k, v) for k, v in conf if k.startswith(c.name + ".")])
                # The on_start_resource, on_end_resource, and on_end_request methods
                # are guaranteed to run even if other methods of the same name fail.
                # We will still log the failure, but proceed on to the next method.
                # The only way to stop all processing from one of these methods is
                # to raise SystemExit and stop the whole server. So, trap your own
                # errors in these methods!
                if hook.failsafe:
                    try:
                        c(**kwargs)
                    except (KeyboardInterrupt, SystemExit):
                        raise
                    except:
                        cherrypy.log(traceback=True)
                else:
                    c(**kwargs)

    def register(self, hook_name, callback):
        self.hooks[hook_name].callbacks.append(callback)

    def register_builtin(self, name):
        from cherrypy.lib import cptools
        if name == "base_url":
            self.register('before_request_body', cptools.base_url)
        elif name == "virtual_host":
            self.register('before_request_body', cptools.virtual_host)
        elif name == "static.file":
            from cherrypy.lib import static
            self.register(static.File(filename))

dispatcher = HookDispatcher()

<commit_msg>Oops. _cphooks was transitory.<commit_after><|endoftext|>


<commit_before># Module containing generally handy functions used for plotting

global lnz, nz
lnz, nz = r'$\ln[n(z)]$', r'$n(z)$'

global s_tru, w_tru, a_tru, c_tru, d_tru, l_tru
s_tru, w_tru, a_tru, c_tru, d_tru, l_tru = '--', 0.5, 1., 'k', [(0,(1,0.0001))], 'True '
global s_int,w_int,a_int,c_int,d_int,l_int
s_int, w_int, a_int, c_int, d_int, l_int = '--', 0.5, 0.5, 'k', [(0,(1,0.0001))], 'Interim '
global s_stk,w_stk,a_stk,c_stk,d_stk,l_stk
s_stk, w_stk, a_stk, c_stk, d_stk, l_stk = '--', 1.5, 1., 'k',[(0,(3,2))], 'Stacked '#[(0,(2,1))]
global s_map, w_map, a_map, c_map, d_map, l_map
s_map, w_map, a_map, c_map, d_map, l_map = '--', 1., 1., 'k', [(0,(3,2))], 'MMAP '#[(0,(1,1,3,1))]
global s_exp,w_exp,a_exp,c_exp,d_exp,l_exp
s_exp, w_exp, a_exp, c_exp, d_exp, l_exp = '--', 1., 1., 'k', [(0,(1,1))], 'MExp '#[(0,(3,3,1,3))]
global s_mle,w_mle,a_mle,c_mle,d_mle,l_mle
s_mle, w_mle, a_mle, c_mle, d_mle, l_mle = '--', 2., 1., 'k', [(0,(1,1))], 'MMLE '#[(0,(3,2))]
# global s_smp,w_smp,a_smp,c_smp,d,smp,l_smp
# s_smp,w_smp,a_smp,c_smp,d_smp,l_smp = '--',1.,1.,'k',[(0,(1,0.0001))],'Sampled '
# global s_bfe,w_bfe,a_bfe,c_bfe,d_bfe,l_bfe
# s_bfe,w_bfe,a_bfe,c_bfe,d_bfe,l_bfe = '--',2.,1.,'k',[(0,(1,0.0001))],'Mean of\n Samples '

def plot_step(sub_plot, bin_ends, plot, s='--', c='k', a=1, w=1, d=[(0,(1,0.0001))], l=None, r=False):
    """
    Plots a step function

    Parameters
    ----------
    sub_plot: matplotlib.pyplot subplot object
        subplot into which step function is drawn
    bin_ends: list or ndarray
        list or array of endpoints of bins
    plot: list or ndarray
        list or array of values within each bin
    s: string, optional
        matplotlib.pyplot linestyle
    c: string, optional
        matplotlib.pyplot color
    a: int or float, [0., 1.], optional
        matplotlib.pyplot alpha (transparency)
    w: int or float, optional
        matplotlib.pyplot linewidth
    d: list of tuple, optional
        matplotlib.pyplot dash style, of form [(start_point, (points_on, points_off, ...))]
    l: string, optional
        label for function
    r: boolean, optional
        True for rasterized, False for vectorized
    """

    plot_h(sub_plot, bin_ends, plot, s, c, a, w, d, l, r)
    plot_v(sub_plot, bin_ends, plot, s, c, a, w, d, r)

def plot_h(sub_plot, bin_ends,plot, s='--', c='k', a=1, w=1, d=[(0,(1,0.0001))], l=None, r=False):
    """
    Helper function to plot horizontal lines of a step function

    Parameters
    ----------
    sub_plot: matplotlib.pyplot subplot object
        subplot into which step function is drawn
    bin_ends: list or ndarray
        list or array of endpoints of bins
    plot: list or ndarray
        list or array of values within each bin
    s: string, optional
        matplotlib.pyplot linestyle
    c: string, optional
        matplotlib.pyplot color
    a: int or float, [0., 1.], optional
        matplotlib.pyplot alpha (transparency)
    w: int or float, optional
        matplotlib.pyplot linewidth
    d: list of tuple, optional
        matplotlib.pyplot dash style, of form [(start_point, (points_on, points_off, ...))]
    l: string, optional
        label for function
    r: boolean, optional
        True for rasterized, False for vectorized
    """

    sub_plot.hlines(plot,
                   bin_ends[:-1],
                   bin_ends[1:],
                   linewidth=w,
                   linestyle=s,
                   dashes=d,
                   color=c,
                   alpha=a,
                   label=l,
                   rasterized=r)
def plot_v(sub_plot, bin_ends, plot, s='--', c='k', a=1, w=1, d=[(0,(1,0.0001))], r=False):
    """
    Helper function to plot vertical lines of a step function

    Parameters
    ----------
    sub_plot: matplotlib.pyplot subplot object
        subplot into which step function is drawn
    bin_ends: list or ndarray
        list or array of endpoints of bins
    plot: list or ndarray
        list or array of values within each bin
    s: string, optional
        matplotlib.pyplot linestyle
    c: string, optional
        matplotlib.pyplot color
    a: int or float, [0., 1.], optional
        matplotlib.pyplot alpha (transparency)
    w: int or float, optional
        matplotlib.pyplot linewidth
    d: list of tuple, optional
        matplotlib.pyplot dash style, of form [(start_point, (points_on, points_off, ...))]
    r: boolean, optional
        True for rasterized, False for vectorized
    """

    sub_plot.vlines(bin_ends[1:-1],
                   plot[:-1],
                   plot[1:],
                   linewidth=w,
                   linestyle=s,
                   dashes=d,
                   color=c,
                   alpha=a,
                   rasterized=r)
<commit_msg>fixed dashes issue<commit_after># Module containing generally handy functions used for plotting

lnz, nz = r'$\ln[n(z)]$', r'$n(z)$'

s_tru, w_tru, a_tru, c_tru, d_tru, l_tru = '--', 0.5, 1., 'k', [(0, (1, 1))], 'True '
s_int, w_int, a_int, c_int, d_int, l_int = '--', 0.5, 0.5, 'k', [(0, (1, 1))], 'Interim '
s_stk, w_stk, a_stk, c_stk, d_stk, l_stk = '--', 1.5, 1., 'k', [(0, (7.5, 7.5))], 'Stacked '#[(0,(2,1))]
s_map, w_map, a_map, c_map, d_map, l_map = '--', 1., 1., 'k', [(0, (7.5, 7.5))], 'MMAP '#[(0,(1,1,3,1))]
s_exp, w_exp, a_exp, c_exp, d_exp, l_exp = '--', 1., 1., 'k', [(0, (2.5, 2.5))], 'MExp '#[(0,(3,3,1,3))]
s_mle, w_mle, a_mle, c_mle, d_mle, l_mle = '--', 2., 1., 'k', [(0, (2.5, 2.5))], 'MMLE '#[(0,(3,2))]
# global s_smp,w_smp,a_smp,c_smp,d,smp,l_smp
# s_smp,w_smp,a_smp,c_smp,d_smp,l_smp = '--',1.,1.,'k',[(0,(1,0.0001))],'Sampled '
# global s_bfe,w_bfe,a_bfe,c_bfe,d_bfe,l_bfe
# s_bfe,w_bfe,a_bfe,c_bfe,d_bfe,l_bfe = '--',2.,1.,'k',[(0,(1,0.0001))],'Mean of\n Samples '

def plot_step(sub_plot, bin_ends, plot, s='--', c='k', a=1, w=1, d=[(0,(1,0.0001))], l=None, r=False):
    """
    Plots a step function

    Parameters
    ----------
    sub_plot: matplotlib.pyplot subplot object
        subplot into which step function is drawn
    bin_ends: list or ndarray
        list or array of endpoints of bins
    plot: list or ndarray
        list or array of values within each bin
    s: string, optional
        matplotlib.pyplot linestyle
    c: string, optional
        matplotlib.pyplot color
    a: int or float, [0., 1.], optional
        matplotlib.pyplot alpha (transparency)
    w: int or float, optional
        matplotlib.pyplot linewidth
    d: list of tuple, optional
        matplotlib.pyplot dash style, of form [(start_point, (points_on, points_off, ...))]
    l: string, optional
        label for function
    r: boolean, optional
        True for rasterized, False for vectorized
    """

    plot_h(sub_plot, bin_ends, plot, s, c, a, w, d, l, r)
    plot_v(sub_plot, bin_ends, plot, s, c, a, w, d, r)

def plot_h(sub_plot, bin_ends,plot, s='--', c='k', a=1, w=1, d=[(0,(1,0.0001))], l=None, r=False):
    """
    Helper function to plot horizontal lines of a step function

    Parameters
    ----------
    sub_plot: matplotlib.pyplot subplot object
        subplot into which step function is drawn
    bin_ends: list or ndarray
        list or array of endpoints of bins
    plot: list or ndarray
        list or array of values within each bin
    s: string, optional
        matplotlib.pyplot linestyle
    c: string, optional
        matplotlib.pyplot color
    a: int or float, [0., 1.], optional
        matplotlib.pyplot alpha (transparency)
    w: int or float, optional
        matplotlib.pyplot linewidth
    d: list of tuple, optional
        matplotlib.pyplot dash style, of form [(start_point, (points_on, points_off, ...))]
    l: string, optional
        label for function
    r: boolean, optional
        True for rasterized, False for vectorized
    """

    sub_plot.hlines(plot,
                   bin_ends[:-1],
                   bin_ends[1:],
                   linewidth=w,
                   linestyle=s,
                   dashes=d,
                   color=c,
                   alpha=a,
                   label=l,
                   rasterized=r)
def plot_v(sub_plot, bin_ends, plot, s='--', c='k', a=1, w=1, d=[(0,(1,0.0001))], r=False):
    """
    Helper function to plot vertical lines of a step function

    Parameters
    ----------
    sub_plot: matplotlib.pyplot subplot object
        subplot into which step function is drawn
    bin_ends: list or ndarray
        list or array of endpoints of bins
    plot: list or ndarray
        list or array of values within each bin
    s: string, optional
        matplotlib.pyplot linestyle
    c: string, optional
        matplotlib.pyplot color
    a: int or float, [0., 1.], optional
        matplotlib.pyplot alpha (transparency)
    w: int or float, optional
        matplotlib.pyplot linewidth
    d: list of tuple, optional
        matplotlib.pyplot dash style, of form [(start_point, (points_on, points_off, ...))]
    r: boolean, optional
        True for rasterized, False for vectorized
    """

    sub_plot.vlines(bin_ends[1:-1],
                   plot[:-1],
                   plot[1:],
                   linewidth=w,
                   linestyle=s,
                   dashes=d,
                   color=c,
                   alpha=a,
                   rasterized=r)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import rospy
from flexbe_core import EventState, Logger

from flexbe_core.proxy import ProxyActionClient

from hector_perception_msgs.msg import DetectObjectAction, DetectObjectGoal
from flexbe_core.proxy import ProxyTransformListener

'''
Created on 15.06.2015

@author: Philipp Schillinger
'''

class GetPipePose(EventState):
	'''
	># centerpose	PoseStamped()		Pose of centered pipe.
	#> pipepose	PoseStamped()		Pose of desired pipe.
	'''
	TOP_LEFT = 1
	TOP_RIGHT = 2
	CENTER = 3
	DOWN_LEFT = 4
	DOWN_RIGHT = 5

	def __init__(self, choice):
		'''
		Constructor
		'''
		super(GetPipePose, self).__init__(outcomes=['succeeded'], input_keys = ['centerpose'], output_keys = ['pipepose'])
		self._choice = choice

		self._tf = ProxyTransformListener().listener()
		self._succeeded = False

	def execute(self, userdata):
		'''
		Execute this state
		'''
		if self._succeeded:
			return 'succeeded'

	def on_enter(self, userdata):
		tempPose = PoseStamped()
		tempPose = self._tf.transformPose('base_link', centerpose)

		if self._choice == TOP_LEFT:
			tempPose.position.y = tempPose.position.y + 0.105
			tempPose.position.z = tempPose.position.z + 0.105
		if self._choice == TOP_RIGHT:
			tempPose.position.y = tempPose.position.y - 0.105
			tempPose.position.z = tempPose.position.z + 0.105
		if self._choice == DOWN_RIGHT:
			tempPose.position.y = tempPose.position.y + 0.105
			tempPose.position.z = tempPose.position.z - 0.105
		if self._choice == DOWN_LEFT:
			tempPose.position.y = tempPose.position.y - 0.105
			tempPose.position.z = tempPose.position.z - 0.105

		userdata.pipepose = tempPose

		self._succeeded = True

<commit_msg>[hector_flexbe_states] Fixed pipe translation state<commit_after>#!/usr/bin/env python

import rospy
from flexbe_core import EventState, Logger

from flexbe_core.proxy import ProxyActionClient

from hector_perception_msgs.msg import DetectObjectAction, DetectObjectGoal
from flexbe_core.proxy import ProxyTransformListener
from geometry_msgs.msg import PoseStamped

'''
Created on 15.06.2015

@author: Philipp Schillinger
'''

class GetPipePose(EventState):
	'''
	># centerpose	PoseStamped()		Pose of centered pipe.
	#> pipepose	PoseStamped()		Pose of desired pipe.
	'''
	TOP_LEFT = 1
	TOP_RIGHT = 2
	CENTER = 3
	DOWN_LEFT = 4
	DOWN_RIGHT = 5

	def __init__(self, choice):
		'''
		Constructor
		'''
		super(GetPipePose, self).__init__(outcomes=['succeeded'], input_keys = ['centerpose'], output_keys = ['pipepose'])
		self._choice = choice

		self._tf = ProxyTransformListener().listener()
		self._succeeded = False

	def execute(self, userdata):
		'''
		Execute this state
		'''
		if self._succeeded:
			return 'succeeded'

	def on_enter(self, userdata):
		tempPose = PoseStamped()
		tempPose = self._tf.transformPose('base_link', userdata.centerpose)

		if self._choice == GetPipePose.TOP_LEFT:
			tempPose.pose.position.y = tempPose.pose.position.y + 0.105
			tempPose.pose.position.z = tempPose.pose.position.z + 0.105
		if self._choice == GetPipePose.TOP_RIGHT:
			tempPose.pose.position.y = tempPose.pose.position.y - 0.105
			tempPose.pose.position.z = tempPose.pose.position.z + 0.105
		if self._choice == GetPipePose.DOWN_RIGHT:
			tempPose.pose.position.y = tempPose.pose.position.y + 0.105
			tempPose.pose.position.z = tempPose.pose.position.z - 0.105
		if self._choice == GetPipePose.DOWN_LEFT:
			tempPose.pose.position.y = tempPose.pose.position.y - 0.105
			tempPose.pose.position.z = tempPose.pose.position.z - 0.105

		userdata.pipepose = tempPose

		self._succeeded = True

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from nose.tools import assert_equal
from nose.tools import assert_true
import os.path as osp

import numpy as np

import jsk_apc2016_common

_this_dir = osp.dirname(osp.realpath(osp.abspath(__file__)))

def test_get_object_data():
    obj_data = jsk_apc2016_common.get_object_data()
    obj_names = map(lambda d: d['name'], obj_data)
    assert_true(isinstance(obj_data, list))
    assert_equal(39, len(obj_data))
    assert_equal(sorted(obj_names), obj_names)
    for d in obj_data:
        assert_true(isinstance(d, dict))
        assert_true('name' in d)
        assert_true('weight' in d)
        assert_true('graspability' in d)
        assert_true('gripper2015' in d['graspability'])
        assert_true('gripper2016' in d['graspability'])
        assert_true('stock' in d)

def test_get_object_data_2015():
    obj_data = jsk_apc2016_common.get_object_data_2015()
    assert_true(isinstance(obj_data, list))
    for d in obj_data:
        assert_true(isinstance(d, dict))
        assert_true('name' in d)

def test__load_stow_json():
    json_file = osp.join(_this_dir, 'data', 'stow_layout_1.json')
    bin_contents, work_order = jsk_apc2016_common._load_stow_json(json_file)
    assert_true(isinstance(bin_contents, dict))
    for bin_, contents in bin_contents.items():
        assert_true(isinstance(bin_, basestring))
        assert_true(isinstance(contents, list))
    assert_true(isinstance(work_order, list))

def test_visualize_stow_contents():
    json_file = osp.join(_this_dir, 'data', 'stow_layout_1.json')
    _, work_order = jsk_apc2016_common._load_stow_json(json_file)
    img = jsk_apc2016_common.visualize_stow_contents(work_order)
    assert_equal(img.shape, (960, 1280, 3))
    assert_equal(img.dtype, np.uint8)

def test_visualize_stow_json():
    json_file = osp.join(_this_dir, 'data', 'stow_layout_1.json')
    img = jsk_apc2016_common.visualize_stow_json(json_file)
    assert_equal(img.shape, (3985, 2067, 3))
    assert_equal(img.dtype, np.uint8)

def test_visualize_pick_json():
    json_file = osp.join(_this_dir, 'data', 'pick_layout_1.json')
    img = jsk_apc2016_common.visualize_pick_json(json_file)
    assert_equal(img.shape, (2435, 2067, 3))
    assert_equal(img.dtype, np.uint8)
<commit_msg>add get_object_data graspability test checking range in [1, 4]<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from nose.tools import assert_equal
from nose.tools import assert_true
import os.path as osp

import numpy as np

import jsk_apc2016_common

_this_dir = osp.dirname(osp.realpath(osp.abspath(__file__)))

def test_get_object_data():
    obj_data = jsk_apc2016_common.get_object_data()
    obj_names = map(lambda d: d['name'], obj_data)
    assert_true(isinstance(obj_data, list))
    assert_equal(39, len(obj_data))
    assert_equal(sorted(obj_names), obj_names)
    for d in obj_data:
        assert_true(isinstance(d, dict))
        assert_true('name' in d)
        assert_true('weight' in d)
        assert_true('graspability' in d)
        graspability = d['graspability']
        assert_true('gripper2015' in graspability)
        assert_true('gripper2016' in graspability)
        assert_true(graspability['gripper2015'] in range(1, 5))
        assert_true(graspability['gripper2016'] in range(1, 5))
        assert_true('stock' in d)

def test_get_object_data_2015():
    obj_data = jsk_apc2016_common.get_object_data_2015()
    assert_true(isinstance(obj_data, list))
    for d in obj_data:
        assert_true(isinstance(d, dict))
        assert_true('name' in d)

def test__load_stow_json():
    json_file = osp.join(_this_dir, 'data', 'stow_layout_1.json')
    bin_contents, work_order = jsk_apc2016_common._load_stow_json(json_file)
    assert_true(isinstance(bin_contents, dict))
    for bin_, contents in bin_contents.items():
        assert_true(isinstance(bin_, basestring))
        assert_true(isinstance(contents, list))
    assert_true(isinstance(work_order, list))

def test_visualize_stow_contents():
    json_file = osp.join(_this_dir, 'data', 'stow_layout_1.json')
    _, work_order = jsk_apc2016_common._load_stow_json(json_file)
    img = jsk_apc2016_common.visualize_stow_contents(work_order)
    assert_equal(img.shape, (960, 1280, 3))
    assert_equal(img.dtype, np.uint8)

def test_visualize_stow_json():
    json_file = osp.join(_this_dir, 'data', 'stow_layout_1.json')
    img = jsk_apc2016_common.visualize_stow_json(json_file)
    assert_equal(img.shape, (3985, 2067, 3))
    assert_equal(img.dtype, np.uint8)

def test_visualize_pick_json():
    json_file = osp.join(_this_dir, 'data', 'pick_layout_1.json')
    img = jsk_apc2016_common.visualize_pick_json(json_file)
    assert_equal(img.shape, (2435, 2067, 3))
    assert_equal(img.dtype, np.uint8)
<|endoftext|>


<commit_before>import os
import re
import joblib
from feature_extraction.regex.regex_lib import RegexLib

regex_lib = RegexLib()

def get_ask_lease_modification():
    return meta_regex("ask modification of lease", [
        re.compile(
            '\[[123]\].+produit une demande en modifications du bail', re.IGNORECASE)
    ])

def get_not_three_weeks_late():
    return meta_regex("is not 3 week late", [
        re.compile(
            '\[\d+\].+locataire(s|) (n\'est|ne sont) pas en retard (\w+(\s|\'|,\s)){1,8}trois semaines', re.IGNORECASE)
    ])

def meta_regex(name, regexes):
    returnlist = set()
    precfiles = os.listdir('precedents/text_bk')
    for file in precfiles:
        with open('precedents/text_bk/' + file, 'r', encoding="iso-8859-1") as f:
            s = f.read()
            for reg in regexes:
                if reg.search(s):
                    returnlist.add(file)
    print("Percent of precedents with {} : {}".format(name,
                                                len(returnlist) * 100.0 / len(precfiles)))
    return returnlist

def fill_dict(precedentDict, fileSet, fact):
    for precedent in precedentDict.keys():
        if precedent in fileSet:
            precedentDict[precedent][fact] = 1
        else:
            precedentDict[precedent][fact] = 0

def get_matching_precedent_file_names(fact):
    return meta_regex(fact, regex_lib.get_regexes(fact))

lease = get_matching_precedent_file_names("lease")
asker_landlord = get_matching_precedent_file_names("asker_is_landlord")
asker_tenant = get_matching_precedent_file_names("asker_is_tenant")
ask_lease_modification = get_ask_lease_modification()
ask_retake_rental = get_matching_precedent_file_names("landlord_demand_retake_apartment")
ask_lower_rent = get_matching_precedent_file_names("tenant_demand_rent_decrease")
ask_damages = get_matching_precedent_file_names("landlord_demand_damage")
ask_fixation_of_rent = get_matching_precedent_file_names("landlord_fix_rent")
asking_rent_payment = get_matching_precedent_file_names("landlord_money_cover_rent")
ask_access_rental = get_matching_precedent_file_names("landlord_demand_access_rental")

ask_resiliation = get_matching_precedent_file_names("demand_resiliation")
tenant_left = get_matching_precedent_file_names("tenant_left_without_paying")
late_payment = get_matching_precedent_file_names("tenant_continuous_late_payment")
menacing = get_matching_precedent_file_names("violent")
three_weeks_late = get_matching_precedent_file_names("three_weeks_late")
proof_of_late = get_matching_precedent_file_names("proof_of_late")
increased_rent = get_matching_precedent_file_names("rent_increased")
not_three_weeks_late = get_not_three_weeks_late()
lack_of_proof = get_matching_precedent_file_names("tenant_lacks_proof")
serious_prejudice = get_matching_precedent_file_names("landlord_serious_prejudice")
money_owed = get_matching_precedent_file_names("tenant_owes_rent")
no_rent_owed = get_matching_precedent_file_names("no_rent_owed")
proof_of_revenu = get_matching_precedent_file_names("proof_of_revenu")
bothers_others = get_matching_precedent_file_names("bothers_others")
non_respect_of_judgement = get_matching_precedent_file_names("disrespect_previous_judgement")
tenant_died = get_matching_precedent_file_names("tenant_dead")
tenant_damaged_rental = get_matching_precedent_file_names("tenant_damaged_rental")
tenant_negligence = get_matching_precedent_file_names("tenant_negligence")
tenant_is_bothered = get_matching_precedent_file_names("tenant_is_bothered")
is_not_habitable = get_matching_precedent_file_names("apartment_impropre")

bad_mutual_agreement = get_matching_precedent_file_names("tenant_landlord_agreement")
bad_paid_before_audience = get_matching_precedent_file_names("tenant_rent_paid_before_hearing")
bad_absent = get_matching_precedent_file_names("absent")
bad_incorrect_facts = get_matching_precedent_file_names("incorrect_facts")

meta = money_owed.union(
    tenant_left).union(
    is_not_habitable).union(
    proof_of_revenu).union(
    late_payment).union(
    tenant_negligence).union(
    menacing).union(
    three_weeks_late).union(
    tenant_is_bothered).union(
    increased_rent).union(
    proof_of_late).union(
    not_three_weeks_late).union(
    lack_of_proof).union(
    serious_prejudice).union(
    tenant_damaged_rental).union(
    tenant_died).union(
    no_rent_owed).union(
    non_respect_of_judgement).union(
    bothers_others)

rem = ask_resiliation - bad_mutual_agreement - \
    bad_paid_before_audience - bad_absent - bad_incorrect_facts
inter = rem - meta.intersection(ask_resiliation)

all_precedents = set(os.listdir('precedents/text_bk'))
bad_precedents = bad_mutual_agreement.union(
    bad_paid_before_audience).union(bad_absent).union(bad_incorrect_facts)
good_precedents = all_precedents - bad_precedents
precedent_vector = dict((el, {}) for el in good_precedents)

fill_dict(precedent_vector, lease, 'lease')
fill_dict(precedent_vector, asker_landlord, 'asker_landlord')
fill_dict(precedent_vector, asker_tenant, 'asker_tenant')
fill_dict(precedent_vector, ask_lease_modification, 'ask_lease_modification')
fill_dict(precedent_vector, ask_retake_rental, 'ask_retake_rental')
fill_dict(precedent_vector, ask_lower_rent, 'ask_lower_rent')
fill_dict(precedent_vector, ask_damages, 'ask_damages')
fill_dict(precedent_vector, ask_fixation_of_rent, 'ask_fixation_of_rent')
fill_dict(precedent_vector, asking_rent_payment, 'asking_rent_payment')
fill_dict(precedent_vector, ask_access_rental, 'ask_access_rental')
fill_dict(precedent_vector, ask_resiliation, 'ask_resiliation')
fill_dict(precedent_vector, tenant_left, 'tenant_left')
fill_dict(precedent_vector, late_payment, 'late_payment')
fill_dict(precedent_vector, menacing, 'menacing')
fill_dict(precedent_vector, three_weeks_late, 'three_weeks_late')
fill_dict(precedent_vector, proof_of_late, 'proof_of_late')
fill_dict(precedent_vector, increased_rent, 'increased_rent')
fill_dict(precedent_vector, not_three_weeks_late, 'not_three_weeks_late')
fill_dict(precedent_vector, lack_of_proof, 'lack_of_proof')
fill_dict(precedent_vector, serious_prejudice, 'serious_prejudice')
fill_dict(precedent_vector, money_owed, 'money_owed')
fill_dict(precedent_vector, no_rent_owed, 'no_rent_owed')
fill_dict(precedent_vector, proof_of_revenu, 'proof_of_revenu')
fill_dict(precedent_vector, bothers_others, 'bothers_others')
fill_dict(precedent_vector, non_respect_of_judgement,'non_respect_of_judgement')
fill_dict(precedent_vector, tenant_died, 'tenant_died')
fill_dict(precedent_vector, tenant_damaged_rental, 'tenant_damaged_rental')
fill_dict(precedent_vector, tenant_negligence, 'tenant_negligence')
fill_dict(precedent_vector, tenant_is_bothered, 'tenant_is_bothered')
fill_dict(precedent_vector, is_not_habitable, 'is_not_habitable')

joblib.dump(precedent_vector, 'prec.bin')

print("diff total:")
print(len(inter))

print("diff %:")
print(len(inter) * 100.0 / len(rem))

with open('diff.txt', 'w') as file:
    for item in inter:
        file.write("%s\n" % item)
<commit_msg>[#183] moved all regexes to regex_lib<commit_after>import os
import joblib
from feature_extraction.regex.regex_lib import RegexLib

regex_lib = RegexLib()

def meta_regex(name, regexes):
    returnlist = set()
    precfiles = os.listdir('precedents/text_bk')
    for file in precfiles:
        with open('precedents/text_bk/' + file, 'r', encoding="iso-8859-1") as f:
            s = f.read()
            for reg in regexes:
                if reg.search(s):
                    returnlist.add(file)
    print("Percent of precedents with {} : {}".format(name,
                                                len(returnlist) * 100.0 / len(precfiles)))
    return returnlist

def fill_dict(precedentDict, fileSet, fact):
    for precedent in precedentDict.keys():
        if precedent in fileSet:
            precedentDict[precedent][fact] = 1
        else:
            precedentDict[precedent][fact] = 0

def get_matching_precedent_file_names(fact):
    return meta_regex(fact, regex_lib.get_regexes(fact))

lease = get_matching_precedent_file_names("lease")
asker_landlord = get_matching_precedent_file_names("asker_is_landlord")
asker_tenant = get_matching_precedent_file_names("asker_is_tenant")
ask_lease_modification = get_matching_precedent_file_names("landlord_demand_lease_modification")
ask_retake_rental = get_matching_precedent_file_names("landlord_demand_retake_apartment")
ask_lower_rent = get_matching_precedent_file_names("tenant_demand_rent_decrease")
ask_damages = get_matching_precedent_file_names("landlord_demand_damage")
ask_fixation_of_rent = get_matching_precedent_file_names("landlord_fix_rent")
asking_rent_payment = get_matching_precedent_file_names("landlord_money_cover_rent")
ask_access_rental = get_matching_precedent_file_names("landlord_demand_access_rental")

ask_resiliation = get_matching_precedent_file_names("demand_resiliation")
tenant_left = get_matching_precedent_file_names("tenant_left_without_paying")
late_payment = get_matching_precedent_file_names("tenant_continuous_late_payment")
menacing = get_matching_precedent_file_names("violent")
three_weeks_late = get_matching_precedent_file_names("three_weeks_late")
proof_of_late = get_matching_precedent_file_names("proof_of_late")
increased_rent = get_matching_precedent_file_names("rent_increased")
not_three_weeks_late = get_matching_precedent_file_names("tenant_rent_late_less_than_3_weeks")
lack_of_proof = get_matching_precedent_file_names("tenant_lacks_proof")
serious_prejudice = get_matching_precedent_file_names("landlord_serious_prejudice")
money_owed = get_matching_precedent_file_names("tenant_owes_rent")
no_rent_owed = get_matching_precedent_file_names("no_rent_owed")
proof_of_revenu = get_matching_precedent_file_names("proof_of_revenu")
bothers_others = get_matching_precedent_file_names("bothers_others")
non_respect_of_judgement = get_matching_precedent_file_names("disrespect_previous_judgement")
tenant_died = get_matching_precedent_file_names("tenant_dead")
tenant_damaged_rental = get_matching_precedent_file_names("tenant_damaged_rental")
tenant_negligence = get_matching_precedent_file_names("tenant_negligence")
tenant_is_bothered = get_matching_precedent_file_names("tenant_is_bothered")
is_not_habitable = get_matching_precedent_file_names("apartment_impropre")

bad_mutual_agreement = get_matching_precedent_file_names("tenant_landlord_agreement")
bad_paid_before_audience = get_matching_precedent_file_names("tenant_rent_paid_before_hearing")
bad_absent = get_matching_precedent_file_names("absent")
bad_incorrect_facts = get_matching_precedent_file_names("incorrect_facts")

meta = money_owed.union(
    tenant_left).union(
    is_not_habitable).union(
    proof_of_revenu).union(
    late_payment).union(
    tenant_negligence).union(
    menacing).union(
    three_weeks_late).union(
    tenant_is_bothered).union(
    increased_rent).union(
    proof_of_late).union(
    not_three_weeks_late).union(
    lack_of_proof).union(
    serious_prejudice).union(
    tenant_damaged_rental).union(
    tenant_died).union(
    no_rent_owed).union(
    non_respect_of_judgement).union(
    bothers_others)

rem = ask_resiliation - bad_mutual_agreement - \
    bad_paid_before_audience - bad_absent - bad_incorrect_facts
inter = rem - meta.intersection(ask_resiliation)

all_precedents = set(os.listdir('precedents/text_bk'))
bad_precedents = bad_mutual_agreement.union(
    bad_paid_before_audience).union(bad_absent).union(bad_incorrect_facts)
good_precedents = all_precedents - bad_precedents
precedent_vector = dict((el, {}) for el in good_precedents)

fill_dict(precedent_vector, lease, 'lease')
fill_dict(precedent_vector, asker_landlord, 'asker_landlord')
fill_dict(precedent_vector, asker_tenant, 'asker_tenant')
fill_dict(precedent_vector, ask_lease_modification, 'ask_lease_modification')
fill_dict(precedent_vector, ask_retake_rental, 'ask_retake_rental')
fill_dict(precedent_vector, ask_lower_rent, 'ask_lower_rent')
fill_dict(precedent_vector, ask_damages, 'ask_damages')
fill_dict(precedent_vector, ask_fixation_of_rent, 'ask_fixation_of_rent')
fill_dict(precedent_vector, asking_rent_payment, 'asking_rent_payment')
fill_dict(precedent_vector, ask_access_rental, 'ask_access_rental')
fill_dict(precedent_vector, ask_resiliation, 'ask_resiliation')
fill_dict(precedent_vector, tenant_left, 'tenant_left')
fill_dict(precedent_vector, late_payment, 'late_payment')
fill_dict(precedent_vector, menacing, 'menacing')
fill_dict(precedent_vector, three_weeks_late, 'three_weeks_late')
fill_dict(precedent_vector, proof_of_late, 'proof_of_late')
fill_dict(precedent_vector, increased_rent, 'increased_rent')
fill_dict(precedent_vector, not_three_weeks_late, 'not_three_weeks_late')
fill_dict(precedent_vector, lack_of_proof, 'lack_of_proof')
fill_dict(precedent_vector, serious_prejudice, 'serious_prejudice')
fill_dict(precedent_vector, money_owed, 'money_owed')
fill_dict(precedent_vector, no_rent_owed, 'no_rent_owed')
fill_dict(precedent_vector, proof_of_revenu, 'proof_of_revenu')
fill_dict(precedent_vector, bothers_others, 'bothers_others')
fill_dict(precedent_vector, non_respect_of_judgement,'non_respect_of_judgement')
fill_dict(precedent_vector, tenant_died, 'tenant_died')
fill_dict(precedent_vector, tenant_damaged_rental, 'tenant_damaged_rental')
fill_dict(precedent_vector, tenant_negligence, 'tenant_negligence')
fill_dict(precedent_vector, tenant_is_bothered, 'tenant_is_bothered')
fill_dict(precedent_vector, is_not_habitable, 'is_not_habitable')

joblib.dump(precedent_vector, 'prec.bin')

print("diff total:")
print(len(inter))

print("diff %:")
print(len(inter) * 100.0 / len(rem))

with open('diff.txt', 'w') as file:
    for item in inter:
        file.write("%s\n" % item)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import absolute_import

import logging
import requests
import json

from ast import literal_eval as make_tuple

from StringIO import StringIO

from celery import shared_task

from django.conf import settings

from apps.modeling.tr55.utils import (aoi_resolution,
                                      precipitation,
                                      apply_modifications_to_census,
                                      )

from tr55.model import simulate_day
from gwlfe import gwlfe, parser

logger = logging.getLogger(__name__)

KG_PER_POUND = 0.453592
CM_PER_INCH = 2.54

def format_quality(model_output):
    measures = ['Total Suspended Solids',
                'Total Nitrogen',
                'Total Phosphorus']
    codes = ['tss', 'tn', 'tp']

    quality = {}
    for key in model_output:
        def map_and_convert_units(input):
            measure, code = input
            return {
                'measure': measure,
                'load': model_output[key][code] * KG_PER_POUND,
                'runoff': model_output[key]['runoff']  # Already CM
            }

        quality[key] = map(map_and_convert_units, zip(measures, codes))

    return quality

def format_runoff(model_output):
    """
    Convert model output values that are in inches into centimeters.
    """
    inch_keys = ['runoff', 'inf', 'et']
    for key in model_output:
        for item in inch_keys:
            model_output[key][item] = model_output[key][item] * CM_PER_INCH

        distribution = model_output[key]['distribution']
        for k in distribution:
            for item in inch_keys:
                model_output[key]['distribution'][k][item] = \
                    model_output[key]['distribution'][k][item] * CM_PER_INCH

    return model_output

def format_for_srat(huc12_id, model_output):
    formatted = {'huc12': huc12_id,
                 # Tile Drain may be calculated by future versions of
                 # Mapshed. The SRAT API requires a placeholder
                 'tpload_tiledrain': 0,
                 'tnload_tiledrain': 0,
                 'tssload_tiledrain': 0,
                 }

    for load in model_output['Loads']:
        source_key = settings.SRAT_KEYS.get(load['Source'], None)

        if source_key is None:
            continue

        formatted['tpload_' + source_key] = load['TotalP']
        formatted['tnload_' + source_key] = load['TotalN']

        if source_key not in ['farman',
                              'subsurface',
                              'septics',
                              'pointsource']:
            formatted['tssload_' + source_key] = load['Sediment']

    return formatted

@shared_task(throws=Exception)
def nlcd_soil(result):
    if 'error' in result:
        raise Exception('[nlcd_soil] {}'.format(result['error']))

    dist = {}
    total_count = 0

    for key, count in result.iteritems():
        # Extract (3, 4) from "List(3,4)"
        (n, s) = make_tuple(key[4:])
        # Map [NODATA, ad, bd] to c, [cd] to d
        s2 = 3 if s in [settings.NODATA, 5, 6] else 4 if s == 7 else s
        # Only count those values for which we have mappings
        if n in settings.NLCD_MAPPING and s2 in settings.SOIL_MAPPING:
            total_count += count
            label = '{soil}:{nlcd}'.format(soil=settings.SOIL_MAPPING[s2][0],
                                           nlcd=settings.NLCD_MAPPING[n][0])
            dist[label] = {'cell_count': (
                count + (dist[label]['cell_count'] if label in dist else 0)
            )}

    return [{
        'cell_count': total_count,
        'distribution': dist,
    }]

@shared_task
def run_tr55(censuses, aoi, model_input, cached_aoi_census=None):
    """
    A Celery wrapper around our TR55 implementation.
    censuses is either output from previous tasks in the job
    chain or are provided directly (in the case where the AoI
    census and modification censuses are cached).
    If cached_aoi_census is provided, censuses will only contain
    the modification_censuses, which were generated in the
    previous task. If cached_aoi_census isn't provided, the AoI
    census will be the first census in censuses, and everything
    else is a modification census.
    """

    # Get precipitation and cell resolution
    precip = precipitation(model_input)

    width = aoi_resolution(aoi)
    resolution = width * width

    if precip is None:
        raise Exception('No precipitation value defined')

    # Modification/BMP fragments and their censuses
    # The original modifications are not POSTed. We only
    # send the altered modifications/modification pieces.
    modification_pieces = model_input.get('modification_pieces')
    modification_censuses = (censuses[1:] if cached_aoi_census is None
                             else censuses[0:])

    # Calculate total areas for each type modification
    area_sums = {}
    for piece in modification_pieces:
        kinds = piece['value']
        area = piece['area']

        if 'bmp' in kinds:
            kind = kinds['bmp']
        else:
            kind = kinds['reclass']

        if kind in area_sums:
            area_sums[kind] += area
        else:
            area_sums[kind] = area

    # The area of interest census
    aoi_census = cached_aoi_census if cached_aoi_census else censuses[0]

    if modification_pieces and not modification_censuses:
        raise Exception('Missing censuses for modifications')
    elif modification_censuses and not modification_pieces:
        modification_censuses = []

    modifications = apply_modifications_to_census(modification_pieces,
                                                  modification_censuses)
    aoi_census['modifications'] = modifications
    aoi_census['BMPs'] = area_sums

    # Run the model under both current conditions and Pre-Columbian
    # conditions.
    try:
        model_output = simulate_day(aoi_census, precip,
                                    cell_res=resolution)

        precolumbian_output = simulate_day(aoi_census,
                                           precip,
                                           cell_res=resolution,
                                           precolumbian=True)

        model_output['pc_unmodified'] = precolumbian_output['unmodified']
        model_output['pc_modified'] = precolumbian_output['modified']
        runoff = format_runoff(model_output)
        quality = format_quality(model_output)

    except KeyError as e:
        runoff = {}
        quality = []
        logger.error('Bad input data to TR55: %s' % e)

    # Modifications were added to aoi_census for TR-55, but we do
    # not want to persist it since we have it stored seperately
    # and it may cause problems when sharing the aoi_census
    # for other model runs and scenarios.
    aoi_census.pop('modifications', None)

    # Return all results
    return {
        'inputmod_hash': model_input['inputmod_hash'],
        'modification_hash': model_input['modification_hash'],
        'aoi_census': aoi_census,
        'modification_censuses': modification_censuses,
        'runoff': runoff,
        'quality': quality
    }

@shared_task
def run_gwlfe(model_input, inputmod_hash, watershed_id=None):
    """
    Given a model_input resulting from a MapShed run, converts that dictionary
    to an intermediate GMS file representation, which is then parsed by GWLF-E
    to create the final data model z. We run GWLF-E on this final data model
    and return the results.

    This intermediate GMS file representation needs to be created because most
    of GWLF-E logic is written to handle GMS files, and to support dictionaries
    directly we would have to replicate all that logic. Thus, it is easier to
    simply create a GMS file and have it read that.
    """
    output = to_gms_file(model_input)

    reader = parser.GmsReader(output)
    z = reader.read()

    result = gwlfe.run(z)
    result['inputmod_hash'] = inputmod_hash
    result['watershed_id'] = watershed_id

    return result

@shared_task
def run_srat(watersheds):
    try:
        data = [format_for_srat(id, w) for id, w in watersheds.iteritems()]
    except Exception as e:
        raise Exception('Formatting sub-basin GWLF-E results failed: %s' % e)

    headers = {'x-api-key': settings.SRAT_CATCHMENT_API['api_key']}

    try:
        r = requests.post(settings.SRAT_CATCHMENT_API['url'],
                          headers=headers,
                          data=json.dumps(data))
    except Exception as e:
        raise Exception('Request to SRAT Catchment API failed: %s' % e)

    if (r.status_code != 200):
        raise Exception('SRAT Catchment API request failed: %s %s' %
                        (r.status_code, r.text))

    try:
        result = r.json()
    except ValueError:
        raise Exception('SRAT Catchment API did not return JSON')

    if (r.status_code != 200):
        logger.error('SRAT Catchment API request failed: %s' % result)

    return result

@shared_task
def subbasin_results_to_dict(subbasin_results):
    def popped_key_result(result):
        watershed_id = result.pop('watershed_id')
        return (watershed_id, result)

    popped_key_results = [popped_key_result(r) for r in subbasin_results]
    return dict(popped_key_results)

def to_gms_file(mapshed_data):
    """
    Given a dictionary of MapShed data, uses GWLF-E to convert it to a GMS file
    """
    mapshed_areas = [round(a, 1) for a in mapshed_data['Area']]
    mapshed_data['Area'] = mapshed_areas

    pre_z = parser.DataModel(mapshed_data)
    output = StringIO()
    writer = parser.GmsWriter(output)
    writer.write(pre_z)

    output.seek(0)

    return output
<commit_msg>Format SRAT Response For App<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import absolute_import

import logging
import requests
import json

from ast import literal_eval as make_tuple

from StringIO import StringIO

from celery import shared_task

from django.conf import settings

from apps.modeling.tr55.utils import (aoi_resolution,
                                      precipitation,
                                      apply_modifications_to_census,
                                      )

from tr55.model import simulate_day
from gwlfe import gwlfe, parser

logger = logging.getLogger(__name__)

KG_PER_POUND = 0.453592
CM_PER_INCH = 2.54

def format_quality(model_output):
    measures = ['Total Suspended Solids',
                'Total Nitrogen',
                'Total Phosphorus']
    codes = ['tss', 'tn', 'tp']

    quality = {}
    for key in model_output:
        def map_and_convert_units(input):
            measure, code = input
            return {
                'measure': measure,
                'load': model_output[key][code] * KG_PER_POUND,
                'runoff': model_output[key]['runoff']  # Already CM
            }

        quality[key] = map(map_and_convert_units, zip(measures, codes))

    return quality

def format_runoff(model_output):
    """
    Convert model output values that are in inches into centimeters.
    """
    inch_keys = ['runoff', 'inf', 'et']
    for key in model_output:
        for item in inch_keys:
            model_output[key][item] = model_output[key][item] * CM_PER_INCH

        distribution = model_output[key]['distribution']
        for k in distribution:
            for item in inch_keys:
                model_output[key]['distribution'][k][item] = \
                    model_output[key]['distribution'][k][item] * CM_PER_INCH

    return model_output

def format_for_srat(huc12_id, model_output):
    formatted = {'huc12': huc12_id,
                 # Tile Drain may be calculated by future versions of
                 # Mapshed. The SRAT API requires a placeholder
                 'tpload_tiledrain': 0,
                 'tnload_tiledrain': 0,
                 'tssload_tiledrain': 0,
                 }

    for load in model_output['Loads']:
        source_key = settings.SRAT_KEYS.get(load['Source'], None)

        if source_key is None:
            continue

        formatted['tpload_' + source_key] = load['TotalP']
        formatted['tnload_' + source_key] = load['TotalN']

        if source_key not in ['farman',
                              'subsurface',
                              'septics',
                              'pointsource']:
            formatted['tssload_' + source_key] = load['Sediment']

    return formatted

def format_subbasin(huc12_gwlfe_results, srat_catchment_results):
    def empty_source(s):
        return {'Source': s, 'TotalN': 0, 'TotalP': 0, 'Sediment': 0}

    def add_huc12_source(source, srat_huc12):
        source_name = source['Source']
        source_key = settings.SRAT_KEYS[source_name]
        total_n = srat_huc12.get('tnload_' + source_key, 0)
        total_p = srat_huc12.get('tpload_' + source_key, 0)
        sediment = srat_huc12.get('tssload_' + source_key, 0)

        source['TotalN'] += total_n
        source['TotalP'] += total_p
        source['Sediment'] += sediment

        return {
            'Source': source_name,
            'TotalN': total_n,
            'TotalP': total_p,
            'Sediment': sediment
        }

    def format_catchment(srat_catchment):
        return {
            'TotalLoadingRates': {
                'TotalN': srat_catchment['tnloadrate_total'],
                'TotalP': srat_catchment['tploadrate_total'],
                'Sediment': srat_catchment['tssloadrate_total'],
            },
            'LoadingRateConcentrations': {
                'TotalN': srat_catchment['tnloadrate_conc'],
                'TotalP': srat_catchment['tploadate_conc'],
                'Sediment': srat_catchment['tssloadrate_conc'],
            },
        }

    def add_huc12(srat_huc12, aggregate):
        area = huc12_gwlfe_results[srat_huc12['huc12']]['AreaTotal']
        aggregate['AreaTotal'] += area
        return {
            'Loads': [add_huc12_source(s, srat_huc12)
                      for s in aggregate['Loads']],
            'AreaTotal': area,
            'Catchments': {comid: format_catchment(result)
                           for comid, result
                           in srat_huc12['catchments'].iteritems()}
        }

    aggregate = {
        'Loads': [empty_source(source_name)
                  for source_name in settings.SRAT_KEYS.keys()],
        'AreaTotal': 0,
        # All gwlf-e results should have the same inputmod hash,
        # so grab any of them
        'inputmod_hash': huc12_gwlfe_results.itervalues()
                                            .next()['inputmod_hash'],
    }

    aggregate['HUC-12s'] = {huc12_id: add_huc12(result, aggregate)
                            for huc12_id, result
                            in srat_catchment_results['huc12s'].iteritems()}

    return aggregate

@shared_task(throws=Exception)
def nlcd_soil(result):
    if 'error' in result:
        raise Exception('[nlcd_soil] {}'.format(result['error']))

    dist = {}
    total_count = 0

    for key, count in result.iteritems():
        # Extract (3, 4) from "List(3,4)"
        (n, s) = make_tuple(key[4:])
        # Map [NODATA, ad, bd] to c, [cd] to d
        s2 = 3 if s in [settings.NODATA, 5, 6] else 4 if s == 7 else s
        # Only count those values for which we have mappings
        if n in settings.NLCD_MAPPING and s2 in settings.SOIL_MAPPING:
            total_count += count
            label = '{soil}:{nlcd}'.format(soil=settings.SOIL_MAPPING[s2][0],
                                           nlcd=settings.NLCD_MAPPING[n][0])
            dist[label] = {'cell_count': (
                count + (dist[label]['cell_count'] if label in dist else 0)
            )}

    return [{
        'cell_count': total_count,
        'distribution': dist,
    }]

@shared_task
def run_tr55(censuses, aoi, model_input, cached_aoi_census=None):
    """
    A Celery wrapper around our TR55 implementation.
    censuses is either output from previous tasks in the job
    chain or are provided directly (in the case where the AoI
    census and modification censuses are cached).
    If cached_aoi_census is provided, censuses will only contain
    the modification_censuses, which were generated in the
    previous task. If cached_aoi_census isn't provided, the AoI
    census will be the first census in censuses, and everything
    else is a modification census.
    """

    # Get precipitation and cell resolution
    precip = precipitation(model_input)

    width = aoi_resolution(aoi)
    resolution = width * width

    if precip is None:
        raise Exception('No precipitation value defined')

    # Modification/BMP fragments and their censuses
    # The original modifications are not POSTed. We only
    # send the altered modifications/modification pieces.
    modification_pieces = model_input.get('modification_pieces')
    modification_censuses = (censuses[1:] if cached_aoi_census is None
                             else censuses[0:])

    # Calculate total areas for each type modification
    area_sums = {}
    for piece in modification_pieces:
        kinds = piece['value']
        area = piece['area']

        if 'bmp' in kinds:
            kind = kinds['bmp']
        else:
            kind = kinds['reclass']

        if kind in area_sums:
            area_sums[kind] += area
        else:
            area_sums[kind] = area

    # The area of interest census
    aoi_census = cached_aoi_census if cached_aoi_census else censuses[0]

    if modification_pieces and not modification_censuses:
        raise Exception('Missing censuses for modifications')
    elif modification_censuses and not modification_pieces:
        modification_censuses = []

    modifications = apply_modifications_to_census(modification_pieces,
                                                  modification_censuses)
    aoi_census['modifications'] = modifications
    aoi_census['BMPs'] = area_sums

    # Run the model under both current conditions and Pre-Columbian
    # conditions.
    try:
        model_output = simulate_day(aoi_census, precip,
                                    cell_res=resolution)

        precolumbian_output = simulate_day(aoi_census,
                                           precip,
                                           cell_res=resolution,
                                           precolumbian=True)

        model_output['pc_unmodified'] = precolumbian_output['unmodified']
        model_output['pc_modified'] = precolumbian_output['modified']
        runoff = format_runoff(model_output)
        quality = format_quality(model_output)

    except KeyError as e:
        runoff = {}
        quality = []
        logger.error('Bad input data to TR55: %s' % e)

    # Modifications were added to aoi_census for TR-55, but we do
    # not want to persist it since we have it stored seperately
    # and it may cause problems when sharing the aoi_census
    # for other model runs and scenarios.
    aoi_census.pop('modifications', None)

    # Return all results
    return {
        'inputmod_hash': model_input['inputmod_hash'],
        'modification_hash': model_input['modification_hash'],
        'aoi_census': aoi_census,
        'modification_censuses': modification_censuses,
        'runoff': runoff,
        'quality': quality
    }

@shared_task
def run_gwlfe(model_input, inputmod_hash, watershed_id=None):
    """
    Given a model_input resulting from a MapShed run, converts that dictionary
    to an intermediate GMS file representation, which is then parsed by GWLF-E
    to create the final data model z. We run GWLF-E on this final data model
    and return the results.

    This intermediate GMS file representation needs to be created because most
    of GWLF-E logic is written to handle GMS files, and to support dictionaries
    directly we would have to replicate all that logic. Thus, it is easier to
    simply create a GMS file and have it read that.
    """
    output = to_gms_file(model_input)

    reader = parser.GmsReader(output)
    z = reader.read()

    result = gwlfe.run(z)
    result['inputmod_hash'] = inputmod_hash
    result['watershed_id'] = watershed_id

    return result

@shared_task
def run_srat(watersheds):
    try:
        data = [format_for_srat(id, w) for id, w in watersheds.iteritems()]
    except Exception as e:
        raise Exception('Formatting sub-basin GWLF-E results failed: %s' % e)

    headers = {'x-api-key': settings.SRAT_CATCHMENT_API['api_key']}

    try:
        r = requests.post(settings.SRAT_CATCHMENT_API['url'],
                          headers=headers,
                          data=json.dumps(data))
    except Exception as e:
        raise Exception('Request to SRAT Catchment API failed: %s' % e)

    if (r.status_code != 200):
        raise Exception('SRAT Catchment API request failed: %s %s' %
                        (r.status_code, r.text))

    try:
        srat_catchment_result = r.json()
    except ValueError:
        raise Exception('SRAT Catchment API did not return JSON')

    try:
        result = format_subbasin(watersheds, srat_catchment_result)
    except KeyError as e:
        raise Exception('SRAT Catchment API returned malformed result: %s' % e)

    return result

@shared_task
def subbasin_results_to_dict(subbasin_results):
    def popped_key_result(result):
        watershed_id = result.pop('watershed_id')
        return (watershed_id, result)

    popped_key_results = [popped_key_result(r) for r in subbasin_results]
    return dict(popped_key_results)

def to_gms_file(mapshed_data):
    """
    Given a dictionary of MapShed data, uses GWLF-E to convert it to a GMS file
    """
    mapshed_areas = [round(a, 1) for a in mapshed_data['Area']]
    mapshed_data['Area'] = mapshed_areas

    pre_z = parser.DataModel(mapshed_data)
    output = StringIO()
    writer = parser.GmsWriter(output)
    writer.write(pre_z)

    output.seek(0)

    return output
<|endoftext|>


<commit_before>import modelspec

PWARelational = modelspec.ModelGeneric

class KPath(modelspec.PartitionedDiscreteAffineModel):
    """Generelization of Relational Modeling"""
    def __init__(self, m, p, pnexts, p_future):
        """
        The k-length path modeled as:
            p -> pnext -> ... p2 -> pk ==> x' = m(x)
        where p, pnext, p_future enforces the order: [p, pnext, p2..., pk]
        [p2,...,pk] \in p_future.
        Currently, p_future is used only for modeling and not for BMC
        queries.
        """
        # Model's ID is its first partition's ID
        # This is acceptable because even a k-relational submodel is
        # identified by which cell it begins in and nothing else.
        #self.ID = p
        self.p = p
        self.pnexts = pnexts
        self.p_future = p_future
        self.m = m
        # make p = p1, i.e., check sat against p[0] and ID is p[0]'s ID
        super(KPath, self).__init__(p, m)
        return

    def __repr__(self):
        s = '({},{},{},{})'.format(self.p, self.pnexts, self.p_future, self.m)
        return s

    def __str__(self):
        s = 'SubModel ->(\n{},\n{},\n{},\n{})'.format(self.p, self.pnexts, self.p_future, self.m)
        return s

class Relation(modelspec.PartitionedDiscreteAffineModel):
    def __init__(self, p1, p2, m):
        """
        The edge between p1 (source parition) and p2 (target parition)
        is modeled as:
            p1 -> p2 ==> x' = m(x)
        """
        self.p1 = p1
        self.p2 = p2
        self.m = m
        # make p = p1, i.e., check sat against p1 and ID is p1's ID
        super(Relation, self).__init__(p1, m)
        return

    def __repr__(self):
        s = '({},{},{})'.format(self.p1, self.p2, self.m)
        return s

    def __str__(self):
        s = 'SubModel ->(\n{},\n{},\n{})'.format(self.p1, self.p2, self.m)
        return s

Partition = modelspec.Partition

DiscreteAffineMap = modelspec.DiscreteAffineMap
<commit_msg>ignore<commit_after>import modelspec

PWARelational = modelspec.ModelGeneric

class KPath(modelspec.PartitionedDiscreteAffineModel):
    """Generelization of Relational Modeling"""
    def __init__(self, m, p, pnexts, p_future):
        """
        The k-length path modeled as:
            p -> pnext -> ... p2 -> pk ==> x' = m(x)
        where p, pnext, p_future enforces the order: [p, pnext, p2..., pk]
        [p2,...,pk] \in p_future.
        Currently, p_future is used only for modeling and not for BMC
        queries.
        """
        # Model's ID is its first partition's ID
        # This is acceptable because even a k-relational submodel is
        # identified by which cell it begins in and nothing else.
        #self.ID = p
        self.p = p
        self.pnexts = pnexts

        # convinience function for now
        self.pnext = self.pnexts[0] if len(pnexts) == 1 else None

        self.p_future = p_future
        self.m = m
        # make p = p1, i.e., check sat against p[0] and ID is p[0]'s ID
        super(KPath, self).__init__(p, m)
        return

    def __repr__(self):
        s = '({},{},{},{})'.format(self.p, self.pnexts, self.p_future, self.m)
        return s

    def __str__(self):
        s = 'SubModel ->(\n{},\n{},\n{},\n{})'.format(self.p, self.pnexts, self.p_future, self.m)
        return s

class Relation(modelspec.PartitionedDiscreteAffineModel):
    def __init__(self, p1, p2, m):
        """
        The edge between p1 (source parition) and p2 (target parition)
        is modeled as:
            p1 -> p2 ==> x' = m(x)
        """
        self.p1 = p1
        self.p2 = p2
        self.m = m
        # make p = p1, i.e., check sat against p1 and ID is p1's ID
        super(Relation, self).__init__(p1, m)
        return

    def __repr__(self):
        s = '({},{},{})'.format(self.p1, self.p2, self.m)
        return s

    def __str__(self):
        s = 'SubModel ->(\n{},\n{},\n{})'.format(self.p1, self.p2, self.m)
        return s

Partition = modelspec.Partition

DiscreteAffineMap = modelspec.DiscreteAffineMap
<|endoftext|>