<commit_before>#
# Jasy - Web Tooling Framework
# Copyright 2010-2012 Zynga Inc.
#

import sys, os, jasy, logging, base64, json, requests
from urllib.parse import urlparse
from collections import namedtuple

from jasy.core.Logging import debug, info, error, header, indent, outdent
from jasy.env.State import session
from jasy.core.Util import getKey
from jasy.core.Cache import Cache
from jasy.core.Types import CaseInsensitiveDict

Result = namedtuple('Result', ['headers', 'content'])

# Disable logging HTTP request being created
logging.getLogger("requests").setLevel(logging.WARNING)

import cherrypy
from cherrypy.lib.static import serve_file as serveFile
from cherrypy.lib.static import serve_file as serveFile

__all__ = ["serve"]

#
# UTILITIES
#

def enableCrossDomain():
    # See also: https://developer.mozilla.org/En/HTTP_Access_Control

    # Allow requests from all locations
    cherrypy.response.headers["Access-Control-Allow-Origin"] = "*"

    # Allow all methods supported by urlfetch
    cherrypy.response.headers["Access-Control-Allow-Methods"] = "GET, POST, HEAD, PUT, DELETE"

    # Allow cache-control and our custom headers
    cherrypy.response.headers["Access-Control-Allow-Headers"] = "Cache-Control, X-Proxy-Authorization, X-Requested-With"

    # Cache allowence for cross domain for 7 days
    cherrypy.response.headers["Access-Control-Max-Age"] = "604800"

def findIndex(path):
    all = ["index.html", "index.php"]
    for candidate in all:
        rel = os.path.join(path, candidate)
        if os.path.exists(rel):
            return candidate

    return None

#
# ROUTERS
#

class Proxy(object):

    def __init__(self, id, config):
        self.id = id
        self.config = config
        self.host = getKey(config, "host")
        self.auth = getKey(config, "auth")

        self.enableDebug = getKey(config, "debug", False)
        self.enableMirror = getKey(config, "mirror", False)
        self.enableOffline = getKey(config, "offline", False)

        if self.enableMirror:
            self.mirror = Cache(os.getcwd(), "jasymirror-%s" % self.id, hashkeys=True)

        info('Proxy "%s" => "%s" [debug:%s|mirror:%s|offline:%s]', self.id, self.host, self.enableDebug, self.enableMirror, self.enableOffline)

    # These headers will be blocked between header copies
    __blockHeaders = CaseInsensitiveDict.fromkeys([
        "content-encoding",
        "content-length",
        "connection",
        "keep-alive",
        "proxy-authenticate",
        "proxy-authorization",
        "transfer-encoding",
        "remote-addr",
        "host"
    ])

    @cherrypy.expose
    def default(self, *args, **query):
        """
        This method returns the content of existing files on the file system.
        Query string might be used for cache busting and are otherwise ignored.
        """

        url = self.config["host"] + "/".join(args)
        result = None

        # Try using offline mirror if feasible
        if self.enableMirror and cherrypy.request.method == "GET":
            mirrorId = "%s[%s]" % (url, json.dumps(query, separators=(',',':'), sort_keys=True))
            result = self.mirror.read(mirrorId)
            if result is not None and self.enableDebug:
                info("Mirrored: %s" % url)

        # Check if we're in forced offline mode
        if self.enableOffline and result is None:
            info("Offline: %s" % url)
            raise cherrypy.NotFound(url)

        # Load URL from remote server
        if result is None:

            # Prepare headers
            headers = CaseInsensitiveDict()
            for name in cherrypy.request.headers:
                if not name in self.__blockHeaders:
                    headers[name] = cherrypy.request.headers[name]

            # Load URL from remote host
            try:
                if self.enableDebug:
                    info("Requesting: %s", url)

                # Apply headers for basic HTTP authentification
                if "X-Proxy-Authorization" in headers:
                    headers["Authorization"] = headers["X-Proxy-Authorization"]
                    del headers["X-Proxy-Authorization"]

                # Add headers for different authentification approaches
                if self.auth:

                    # Basic Auth
                    if self.auth["method"] == "basic":
                        headers["Authorization"] = b"Basic " + base64.b64encode(("%s:%s" % (self.auth["user"], self.auth["password"])).encode("ascii"))

                # We disable verifícation of SSL certificates to be more tolerant on test servers
                result = requests.request(cherrypy.request.method, url, params=query, headers=headers, verify=False)

            except Exception as err:
                if self.enableDebug:
                    info("Request failed: %s", err)

                raise cherrypy.HTTPError(403)

            # Storing result into mirror
            if self.enableMirror and cherrypy.request.method == "GET":

                # Wrap result into mirrorable entry
                resultCopy = Result(result.headers, result.content)
                self.mirror.store(mirrorId, resultCopy)

        # Copy response headers to our reponse
        for name in result.headers:
            if not name.lower() in self.__blockHeaders:
                cherrypy.response.headers[name] = result.headers[name]

        # Append special header to all responses
        cherrypy.response.headers["X-Jasy-Version"] = jasy.__version__

        # Enable cross domain access to this server
        enableCrossDomain()

        return result.content

class Static(object):

    def __init__(self, id, config):
        self.id = id
        self.config = config
        self.root = getKey(config, "root", ".")
        self.enableDebug = getKey(config, "debug", False)

        info('Static "%s" => "%s" [debug:%s]', self.id, self.root, self.enableDebug)

    @cherrypy.expose
    def default(self, *args, **query):
        """
        This method returns the content of existing files on the file system.
        Query string might be used for cache busting and are otherwise ignored.
        """

        # Append special header to all responses
        cherrypy.response.headers["X-Jasy-Version"] = jasy.__version__

        # Enable cross domain access to this server
        enableCrossDomain()

        # When it's a file name in the local folder... load it
        if args:
            path = os.path.join(*args)
        else:
            path = "index.html"

        path = os.path.join(self.root, path)

        # Check for existance first
        if os.path.isfile(path):
            if self.enableDebug:
                info("Serving file: %s", path)

            return serveFile(os.path.abspath(path))

        # Otherwise return a classic 404
        else:
            if self.enableDebug:
                warn("File at location %s not found at %s!", path, os.path.abspath(path))

            raise cherrypy.NotFound(path)

#
# ADDITIONAL MIME TYPES
#

additionalContentTypes = {
    "js": "application/javascript",
    "jsonp": "application/javascript",
    "json": "application/json",
    "oga": "audio/ogg",
    "ogg": "audio/ogg",
    "m4a": "audio/mp4",
    "f4a": "audio/mp4",
    "f4b": "audio/mp4",
    "ogv": "video/ogg",
    "mp4": "video/mp4",
    "m4v": "video/mp4",
    "f4v": "video/mp4",
    "f4p": "video/mp4",
    "webm": "video/webm",
    "flv": "video/x-flv",
    "svg": "image/svg+xml",
    "svgz": "image/svg+xml",
    "eot": "application/vnd.ms-fontobject",
    "ttf": "application/x-font-ttf",
    "ttc": "application/x-font-ttf",
    "otf": "font/opentype",
    "woff": "application/x-font-woff",
    "ico": "image/x-icon",
    "webp": "image/webp",
    "appcache": "text/cache-manifest",
    "manifest": "text/cache-manifest",
    "htc": "text/x-component",
    "rss": "application/xml",
    "atom": "application/xml",
    "xml": "application/xml",
    "rdf": "application/xml",
    "crx": "application/x-chrome-extension",
    "oex": "application/x-opera-extension",
    "xpi": "application/x-xpinstall",
    "safariextz": "application/octet-stream",
    "webapp": "application/x-web-app-manifest+json",
    "vcf": "text/x-vcard",
    "swf": "application/x-shockwave-flash",
    "vtt": "text/vtt"
}

#
# START
#

def serve(routes=None, customContentTypes=None, port=8080, host="127.0.0.1"):

    header("HTTP Server")

    # We need to pause the session to make room for other jasy executions
    session.pause()

    # Shared configuration (global/app)
    config = {
        "global" : {
            "environment" : "production",
            "log.screen" : False,
            "server.socket_port": port,
            "server.socket_host": host,
            "engine.autoreload_on" : False
        },

        "/" : {
            "log.screen" : False
        }
    }

    contentTypes = {}
    contentTypes.update(additionalContentTypes)
    if customContentTypes:
            contentTypes.update(customContentTypes)

    config["/"]["tools.staticfile.content_types"] = contentTypes

    # Update global config
    cherrypy.config.update(config)

    # Somehow this screen disabling does not work
    # This hack to disable all access/error logging works
    def empty(*param, **args): pass
    def inspect(*param, **args):
        if args["severity"] > 20:
            error("Critical error occoured:")
            error(param[0])

    cherrypy.log.access = empty
    cherrypy.log.error = inspect
    cherrypy.log.screen = False

    # Initialize routing
    info("Initialize routing...")
    indent()
    root = Static("/", {})
    if routes:
        for key in routes:
            entry = routes[key]
            if "host" in entry:
                node = Proxy(key, entry)
            else:
                node = Static(key, entry)

            setattr(root, key, node)
    outdent()

    # Finally start the server
    app = cherrypy.tree.mount(root, "", config)
    cherrypy.process.plugins.PIDFile(cherrypy.engine, "jasylock-http-%s" % port).subscribe()

    cherrypy.engine.start()
    info("Started HTTP server at port %s... [PID=%s]", port, os.getpid())
    indent()
    cherrypy.engine.block()

    outdent()
    info("Stopped HTTP server at port %s.", port)

    # Resume session to continue work on next task (if given)
    session.resume()

<commit_msg>Implemented support for contentTypes for serve_file usage<commit_after>#
# Jasy - Web Tooling Framework
# Copyright 2010-2012 Zynga Inc.
#

import sys, os, jasy, logging, base64, json, requests
from urllib.parse import urlparse
from collections import namedtuple

from jasy.core.Logging import debug, info, error, header, indent, outdent
from jasy.env.State import session
from jasy.core.Util import getKey
from jasy.core.Cache import Cache
from jasy.core.Types import CaseInsensitiveDict

Result = namedtuple('Result', ['headers', 'content'])

# Disable logging HTTP request being created
logging.getLogger("requests").setLevel(logging.WARNING)

import cherrypy
from cherrypy.lib.static import serve_file as serveFile
from cherrypy.lib.static import serve_file as serveFile

__all__ = ["serve"]

#
# UTILITIES
#

def enableCrossDomain():
    # See also: https://developer.mozilla.org/En/HTTP_Access_Control

    # Allow requests from all locations
    cherrypy.response.headers["Access-Control-Allow-Origin"] = "*"

    # Allow all methods supported by urlfetch
    cherrypy.response.headers["Access-Control-Allow-Methods"] = "GET, POST, HEAD, PUT, DELETE"

    # Allow cache-control and our custom headers
    cherrypy.response.headers["Access-Control-Allow-Headers"] = "Cache-Control, X-Proxy-Authorization, X-Requested-With"

    # Cache allowence for cross domain for 7 days
    cherrypy.response.headers["Access-Control-Max-Age"] = "604800"

def findIndex(path):
    all = ["index.html", "index.php"]
    for candidate in all:
        rel = os.path.join(path, candidate)
        if os.path.exists(rel):
            return candidate

    return None

#
# ROUTERS
#

class Proxy(object):

    def __init__(self, id, config):
        self.id = id
        self.config = config
        self.host = getKey(config, "host")
        self.auth = getKey(config, "auth")

        self.enableDebug = getKey(config, "debug", False)
        self.enableMirror = getKey(config, "mirror", False)
        self.enableOffline = getKey(config, "offline", False)

        if self.enableMirror:
            self.mirror = Cache(os.getcwd(), "jasymirror-%s" % self.id, hashkeys=True)

        info('Proxy "%s" => "%s" [debug:%s|mirror:%s|offline:%s]', self.id, self.host, self.enableDebug, self.enableMirror, self.enableOffline)

    # These headers will be blocked between header copies
    __blockHeaders = CaseInsensitiveDict.fromkeys([
        "content-encoding",
        "content-length",
        "connection",
        "keep-alive",
        "proxy-authenticate",
        "proxy-authorization",
        "transfer-encoding",
        "remote-addr",
        "host"
    ])

    @cherrypy.expose
    def default(self, *args, **query):
        """
        This method returns the content of existing files on the file system.
        Query string might be used for cache busting and are otherwise ignored.
        """

        url = self.config["host"] + "/".join(args)
        result = None

        # Try using offline mirror if feasible
        if self.enableMirror and cherrypy.request.method == "GET":
            mirrorId = "%s[%s]" % (url, json.dumps(query, separators=(',',':'), sort_keys=True))
            result = self.mirror.read(mirrorId)
            if result is not None and self.enableDebug:
                info("Mirrored: %s" % url)

        # Check if we're in forced offline mode
        if self.enableOffline and result is None:
            info("Offline: %s" % url)
            raise cherrypy.NotFound(url)

        # Load URL from remote server
        if result is None:

            # Prepare headers
            headers = CaseInsensitiveDict()
            for name in cherrypy.request.headers:
                if not name in self.__blockHeaders:
                    headers[name] = cherrypy.request.headers[name]

            # Load URL from remote host
            try:
                if self.enableDebug:
                    info("Requesting: %s", url)

                # Apply headers for basic HTTP authentification
                if "X-Proxy-Authorization" in headers:
                    headers["Authorization"] = headers["X-Proxy-Authorization"]
                    del headers["X-Proxy-Authorization"]

                # Add headers for different authentification approaches
                if self.auth:

                    # Basic Auth
                    if self.auth["method"] == "basic":
                        headers["Authorization"] = b"Basic " + base64.b64encode(("%s:%s" % (self.auth["user"], self.auth["password"])).encode("ascii"))

                # We disable verifícation of SSL certificates to be more tolerant on test servers
                result = requests.request(cherrypy.request.method, url, params=query, headers=headers, verify=False)

            except Exception as err:
                if self.enableDebug:
                    info("Request failed: %s", err)

                raise cherrypy.HTTPError(403)

            # Storing result into mirror
            if self.enableMirror and cherrypy.request.method == "GET":

                # Wrap result into mirrorable entry
                resultCopy = Result(result.headers, result.content)
                self.mirror.store(mirrorId, resultCopy)

        # Copy response headers to our reponse
        for name in result.headers:
            if not name.lower() in self.__blockHeaders:
                cherrypy.response.headers[name] = result.headers[name]

        # Append special header to all responses
        cherrypy.response.headers["X-Jasy-Version"] = jasy.__version__

        # Enable cross domain access to this server
        enableCrossDomain()

        return result.content

class Static(object):

    def __init__(self, id, config, contentTypes=None):
        self.id = id
        self.config = config
        self.contentTypes = contentTypes
        self.root = getKey(config, "root", ".")
        self.enableDebug = getKey(config, "debug", False)

        info('Static "%s" => "%s" [debug:%s]', self.id, self.root, self.enableDebug)

    @cherrypy.expose
    def default(self, *args, **query):
        """
        This method returns the content of existing files on the file system.
        Query string might be used for cache busting and are otherwise ignored.
        """

        # Append special header to all responses
        cherrypy.response.headers["X-Jasy-Version"] = jasy.__version__

        # Enable cross domain access to this server
        enableCrossDomain()

        # When it's a file name in the local folder... load it
        if args:
            path = os.path.join(*args)
        else:
            path = "index.html"

        path = os.path.join(self.root, path)

        # Check for existance first
        if os.path.isfile(path):
            if self.enableDebug:
                info("Serving file: %s", path)

            # Default content type to autodetection by Python mimetype API
            contenttype = None

            # Support overriding by extensions
            extension = os.path.splitext(path)[1]
            if extension:
                extension = extension.lower()[1:]
                if extension in self.contentTypes:
                    contentType = self.contentTypes[extension]

            return serveFile(os.path.abspath(path), content_type=contentType)

        # Otherwise return a classic 404
        else:
            if self.enableDebug:
                warn("File at location %s not found at %s!", path, os.path.abspath(path))

            raise cherrypy.NotFound(path)

#
# ADDITIONAL MIME TYPES
#

additionalContentTypes = {
    "js": "application/javascript",
    "jsonp": "application/javascript",
    "json": "application/json",
    "oga": "audio/ogg",
    "ogg": "audio/ogg",
    "m4a": "audio/mp4",
    "f4a": "audio/mp4",
    "f4b": "audio/mp4",
    "ogv": "video/ogg",
    "mp4": "video/mp4",
    "m4v": "video/mp4",
    "f4v": "video/mp4",
    "f4p": "video/mp4",
    "webm": "video/webm",
    "flv": "video/x-flv",
    "svg": "image/svg+xml",
    "svgz": "image/svg+xml",
    "eot": "application/vnd.ms-fontobject",
    "ttf": "application/x-font-ttf",
    "ttc": "application/x-font-ttf",
    "otf": "font/opentype",
    "woff": "application/x-font-woff",
    "ico": "image/x-icon",
    "webp": "image/webp",
    "appcache": "text/cache-manifest",
    "manifest": "text/cache-manifest",
    "htc": "text/x-component",
    "rss": "application/xml",
    "atom": "application/xml",
    "xml": "application/xml",
    "rdf": "application/xml",
    "crx": "application/x-chrome-extension",
    "oex": "application/x-opera-extension",
    "xpi": "application/x-xpinstall",
    "safariextz": "application/octet-stream",
    "webapp": "application/x-web-app-manifest+json",
    "vcf": "text/x-vcard",
    "swf": "application/x-shockwave-flash",
    "vtt": "text/vtt"
}

#
# START
#

def serve(routes=None, customContentTypes=None, port=8080, host="127.0.0.1"):

    header("HTTP Server")

    # We need to pause the session to make room for other jasy executions
    session.pause()

    # Shared configuration (global/app)
    config = {
        "global" : {
            "environment" : "production",
            "log.screen" : False,
            "server.socket_port": port,
            "server.socket_host": host,
            "engine.autoreload_on" : False
        },

        "/" : {
            "log.screen" : False
        }
    }

    # Build dict of content types to override native mimetype detection
    contentTypes = {}
    contentTypes.update(additionalContentTypes)
    if customContentTypes:
            contentTypes.update(customContentTypes)

    # Update global config
    cherrypy.config.update(config)

    # Somehow this screen disabling does not work
    # This hack to disable all access/error logging works
    def empty(*param, **args): pass
    def inspect(*param, **args):
        if args["severity"] > 20:
            error("Critical error occoured:")
            error(param[0])

    cherrypy.log.access = empty
    cherrypy.log.error = inspect
    cherrypy.log.screen = False

    # Initialize routing
    info("Initialize routing...")
    indent()
    root = Static("/", {}, contentTypes=contentTypes)
    if routes:
        for key in routes:
            entry = routes[key]
            if "host" in entry:
                node = Proxy(key, entry)
            else:
                node = Static(key, entry, contentTypes=contentTypes)

            setattr(root, key, node)
    outdent()

    # Finally start the server
    app = cherrypy.tree.mount(root, "", config)
    cherrypy.process.plugins.PIDFile(cherrypy.engine, "jasylock-http-%s" % port).subscribe()

    cherrypy.engine.start()
    info("Started HTTP server at port %s... [PID=%s]", port, os.getpid())
    indent()
    cherrypy.engine.block()

    outdent()
    info("Stopped HTTP server at port %s.", port)

    # Resume session to continue work on next task (if given)
    session.resume()

<|endoftext|>


<commit_before>import sublime
import sublime_plugin

from Context.base import Base

try:
  from Expression import expression
  from Statement import statement
except ImportError:
  sublime.error_message("Dependency import failed; please read readme for " +
   "Context plugin for installation instructions; to disable this " +
   "message remove this plugin")

class InArguments(Base):
  def on_query_context(self, *args):
    callback = lambda view, sel: statement.is_arguments(view, sel.b)
    return self._check_sel('in_arguments', callback, *args)<commit_msg>context in arguments moved to separate plugin<commit_after><|endoftext|>


<commit_before>#
# Jasy - Web Tooling Framework
# Copyright 2013-2014 Sebastian Werner
#

import itertools, re
import jasy.style.parse.Node as Node

RE_ENGINE_PROPERTY = re.compile(r"^(?:\-(apple|chrome|moz|ms|o|webkit)\-)?([a-z\-]+)$")

MATH_OPERATORS = ("plus", "minus", "mul", "div", "mod")
COMPARE_OPERATORS = ("eq", "ne", "gt", "lt", "ge", "le")
LOGIC_OPERATORS = ("not", "and", "or")

ALL_OPERATORS = MATH_OPERATORS + COMPARE_OPERATORS + LOGIC_OPERATORS

def extractVendor(name):
    match = RE_ENGINE_PROPERTY.match(name)
    if match:
        return match.group(1)

def extractName(name):
    match = RE_ENGINE_PROPERTY.match(name)
    if match:
        return match.group(2)
    else:
        return name

def executeCommand(node, profile):
    command = node.name

    params = []
    for param in node.params:
        # Variable not yet processed (possible e.g. during permutation apply)
        if param.type == "variable":
            return node
        elif param.type == "unary_minus":
            value = -param[0].value
        else:
            value = param.value

        params.append(value)

    # print("Looking for command: %s(%s)" % (command, ", ".join([str(param) for param in params])))
    result, restype = profile.executeCommand(command, params)

    if restype == "px":
        repl = Node.Node(type="number")
        repl.value = result
        repl.unit = restype

    elif restype == "url":
        repl = Node.Node(type="function")
        repl.name = "url"
        listChild = Node.Node(type="list")
        repl.append(listChild, "params")
        valueChild = Node.Node(type="identifier")
        valueChild.value = result
        listChild.append(valueChild)

    elif restype == "number":
        repl = Node.Node(type="number")
        repl.value = result

    else:
        repl = Node.Node(type="identifier")
        repl.value = result

    return repl

def assembleDot(node, result=None):
    """
    Joins a dot node (cascaded supported, too) into a single string like "foo.bar.Baz"
    """

    if result == None:
        result = []

    for child in node:
        if child.type == "identifier":
            result.append(child.value)
        elif child.type == "dot":
            assembleDot(child, result)
        else:
            return None

    return ".".join(result)

def castNativeToNode(value):
    if value is True:
        node = Node.Node(type="true")
    elif value is False:
        node = Node.Node(type="false")
    elif isinstance(value, str):
        node = Node.Node(type="string")
        node.value = value
    elif isinstance(value, (float, int)):
        node = Node.Node(type="number")
        node.value = value
    elif value is None:
        node = Node.Node(type="null")
    else:
        raise ResolverError("Could not transform field %s=%s to style value" % (name, value))

    return node

# list of media types which should be prepended in "and" connections
cssMediaTypes = set(["all", "aural", "braille", "handheld", "print", "projection", "screen", "tty", "tv", "embossed"])

# Prepend all media queries which contain not, only or any valid media type
RE_PREPEND_QUERY = re.compile(r'\b(not|only|all|aural|braille|handheld|print|projection|screen|tty|tv|embossed)\b')

def combineSelectorList(selector, stop, root=None):
    if not selector:
        return None

    if root is None:
        root = [""]

    combinedSelectors = []

    for currentRoot in root:
        for item in itertools.product(*reversed(selector)):
            combined = ""
            for part in item:
                if combined:
                    if "&" in part:
                        combined = part.replace("&", currentRoot+combined)
                    else:
                        combined = "%s %s" % (combined, part)
                else:
                    if "&" in part:
                        if currentRoot:
                            combined = part.replace("&", currentRoot)

                        # Tolerate open/unsolvable "&" parent reference when we stop too early
                        elif not stop:
                            raise Exception("Can't merge selector %s - parent missing!" % part)
                    else:
                        combined = part

            combinedSelectors.append(combined)

    return combinedSelectors

def combineMediaQueryList(media):
    if not media:
        return None

    if len(media) == 1:
        return media[0]

    combinedMedia = []
    for item in itertools.product(*reversed(media)):
        parts = []
        for entry in item:
            # Prepend media types and not/only operators
            if RE_PREPEND_QUERY.search(entry):
                parts.insert(0, entry)
            else:
                parts.append(entry)

        combinedMedia.append(" and ".join(parts))

    return combinedMedia

def combineSupportList(supports):
    if not supports:
        return None

    # Supports rules are not a list of different @support rule sets,
    # but just a plain list of "and" connected expressions.

    if len(supports) > 1:
        return " and ".join(supports)
    else:
        return supports[0]

def combineSelector(node, stop=None):
    """
    Figures out the fully qualified selector, media query
    and @supports value of the given Node.
    """

    root = None

    # Fast path and fix for identical start/stop
    if node is stop:
        return ["&"], None, None

    # list of list of selectors
    selector = []

    # list of list of media queries
    media = []

    # list of @support directives (no list in list)
    supports = []

    # Selector and media lists are in reversed order...
    current = node
    while current and current is not stop and current.type is not "root":
        if current.type == "mixin" and current.selector:
            selector.append(current.selector) # extend for this mixin
        elif hasattr(current, "name") and current.name:
            if current.type == "selector":
                selector.append(current.name)
            elif current.type == "media":
                media.append(current.name)
            elif current.type == "supports":
                supports.append(current.name)

        current = getattr(current, "parent", None)
        if current is not None and current.type == "root":
            root = current

    #if not selector and not media and not supports:
    #    raise Exception("Node %s at line %s is not a selector/mixin/@media/@supports and is no child of any selector/mixin/@media/@supports." % (node.type, node.line))

    rootSelectors = None
    rootMedia = None
    rootSupports = None

    if root:
        rootSelectors, rootMedia, rootSupports = combineSelector(root.parent)

    # So we need process collected selector data etc. in reversed
    # order, too, to get the normal order back.
    combinedSelectors = combineSelectorList(selector, stop, rootSelectors)
    combinedMedia = combineMediaQueryList(media)
    combinedSupports = combineSupportList(supports)

    return combinedSelectors, combinedMedia, combinedSupports

<commit_msg>Minor fix<commit_after>#
# Jasy - Web Tooling Framework
# Copyright 2013-2014 Sebastian Werner
#

import itertools, re
import jasy.style.parse.Node as Node

RE_ENGINE_PROPERTY = re.compile(r"^(?:\-(apple|chrome|moz|ms|o|webkit)\-)?([a-z\-]+)$")

MATH_OPERATORS = ("plus", "minus", "mul", "div", "mod")
COMPARE_OPERATORS = ("eq", "ne", "gt", "lt", "ge", "le")
LOGIC_OPERATORS = ("not", "and", "or")

ALL_OPERATORS = MATH_OPERATORS + COMPARE_OPERATORS + LOGIC_OPERATORS

def extractVendor(name):
    match = RE_ENGINE_PROPERTY.match(name)
    if match:
        return match.group(1)

def extractName(name):
    match = RE_ENGINE_PROPERTY.match(name)
    if match:
        return match.group(2)
    else:
        return name

def executeCommand(node, profile):
    command = node.name

    params = []
    for param in node.params:
        # Variable not yet processed (possible e.g. during permutation apply)
        if param.type == "variable":
            return node
        elif param.type == "unary_minus":
            value = -param[0].value
        else:
            value = param.value

        params.append(value)

    # print("Looking for command: %s(%s)" % (command, ", ".join([str(param) for param in params])))
    result, restype = profile.executeCommand(command, params)

    if restype == "px":
        repl = Node.Node(type="number")
        repl.value = result
        repl.unit = restype

    elif restype == "url":
        repl = Node.Node(type="function")
        repl.name = "url"
        listChild = Node.Node(type="list")
        repl.append(listChild, "params")
        valueChild = Node.Node(type="identifier")
        valueChild.value = result
        listChild.append(valueChild)

    elif restype == "number":
        repl = Node.Node(type="number")
        repl.value = result

    else:
        repl = Node.Node(type="identifier")
        repl.value = result

    return repl

def assembleDot(node, result=None):
    """
    Joins a dot node (cascaded supported, too) into a single string like "foo.bar.Baz"
    """

    if result == None:
        result = []

    for child in node:
        if child.type == "identifier":
            result.append(child.value)
        elif child.type == "dot":
            assembleDot(child, result)
        else:
            return None

    return ".".join(result)

def castNativeToNode(value):
    if value is True:
        node = Node.Node(type="true")
    elif value is False:
        node = Node.Node(type="false")
    elif isinstance(value, str):
        node = Node.Node(type="string")
        node.value = value
    elif isinstance(value, (float, int)):
        node = Node.Node(type="number")
        node.value = value
    elif value is None:
        node = Node.Node(type="null")
    else:
        raise ResolverError("Could not transform field %s=%s to style value" % (name, value))

    return node

# list of media types which should be prepended in "and" connections
cssMediaTypes = set(["all", "aural", "braille", "handheld", "print", "projection", "screen", "tty", "tv", "embossed"])

# Prepend all media queries which contain not, only or any valid media type
RE_PREPEND_QUERY = re.compile(r'\b(not|only|all|aural|braille|handheld|print|projection|screen|tty|tv|embossed)\b')

def combineSelectorList(selector, stop, root=None):
    if not selector:
        return None

    if root is None:
        root = [""]

    combinedSelectors = []

    for currentRoot in root:
        for item in itertools.product(*reversed(selector)):
            combined = ""
            for part in item:
                if combined:
                    if "&" in part:
                        combined = part.replace("&", currentRoot+combined)
                    else:
                        combined = "%s %s" % (combined, part)
                else:
                    if "&" in part:
                        if currentRoot:
                            combined = part.replace("&", currentRoot)
                        elif not stop:
                            # Tolerate open/unsolvable "&" parent reference only when we stop early
                            raise Exception("Can't merge selector %s - parent missing!" % part)
                        else:
                            combined = part
                    else:
                        combined = part

            combinedSelectors.append(combined)

    return combinedSelectors

def combineMediaQueryList(media):
    if not media:
        return None

    if len(media) == 1:
        return media[0]

    combinedMedia = []
    for item in itertools.product(*reversed(media)):
        parts = []
        for entry in item:
            # Prepend media types and not/only operators
            if RE_PREPEND_QUERY.search(entry):
                parts.insert(0, entry)
            else:
                parts.append(entry)

        combinedMedia.append(" and ".join(parts))

    return combinedMedia

def combineSupportList(supports):
    if not supports:
        return None

    # Supports rules are not a list of different @support rule sets,
    # but just a plain list of "and" connected expressions.

    if len(supports) > 1:
        return " and ".join(supports)
    else:
        return supports[0]

def combineSelector(node, stop=None):
    """
    Figures out the fully qualified selector, media query
    and @supports value of the given Node.
    """

    root = None

    # Fast path and fix for identical start/stop
    if node is stop:
        return ["&"], None, None

    # list of list of selectors
    selector = []

    # list of list of media queries
    media = []

    # list of @support directives (no list in list)
    supports = []

    # Selector and media lists are in reversed order...
    current = node
    while current and current is not stop and current.type is not "root":
        if current.type == "mixin" and current.selector:
            selector.append(current.selector) # extend for this mixin
        elif hasattr(current, "name") and current.name:
            if current.type == "selector":
                selector.append(current.name)
            elif current.type == "media":
                media.append(current.name)
            elif current.type == "supports":
                supports.append(current.name)

        current = getattr(current, "parent", None)
        if current is not None and current.type == "root":
            root = current

    #if not selector and not media and not supports:
    #    raise Exception("Node %s at line %s is not a selector/mixin/@media/@supports and is no child of any selector/mixin/@media/@supports." % (node.type, node.line))

    rootSelectors = None
    rootMedia = None
    rootSupports = None

    if root:
        rootSelectors, rootMedia, rootSupports = combineSelector(root.parent)

    # So we need process collected selector data etc. in reversed
    # order, too, to get the normal order back.
    combinedSelectors = combineSelectorList(selector, stop, rootSelectors)
    combinedMedia = combineMediaQueryList(media)
    combinedSupports = combineSupportList(supports)

    return combinedSelectors, combinedMedia, combinedSupports

<|endoftext|>


<commit_before>
import re

import matplotlib
matplotlib.use('Agg')

from iris.unit import Unit

import matplotlib.pyplot as plt
import iris.quickplot as qplt
import cartopy.crs as ccrs
import iris
import iris.plot as iplt

import matplotlib.ticker as mticker
from cartopy.mpl.gridliner import LATITUDE_FORMATTER, LONGITUDE_FORMATTER

import logging
log = logging.getLogger(__name__)

class GribMapping(object):
    def __init__(self):
        self._grib_id_map = {}

    def add(self, grib_id, standard_name, long_name, units):
        self._grib_id_map[grib_id] = (standard_name, long_name, units)

    def _metadata_callback(self, cube, field, filename):
        # Use IRIS values when present
        if cube.standard_name:
            return

        match = re.match(r'UNKNOWN LOCAL PARAM (\d+)\.(\d+)', cube.long_name)
        if match:
            parameter, table_version = (int(x) for x in match.groups())
            log.debug('Cube "{0}" --> {1}.{2}'.format(cube.long_name, parameter, table_version))
            if parameter in self._grib_id_map:
                standard_name, long_name, unit = self._grib_id_map[parameter]
                if standard_name:
                    cube.long_name = standard_name
                if long_name:
                    cube.long_name = long_name
                if unit:
                    cube.units = Unit(unit)
            else:
                log.error('Grib Parameter {0} not in grib mapping'.format(parameter))

    def load_cubes(self, filename):
        cubes = iris.load(filename, callback=self._metadata_callback)

        return cubes

grib_mapping = GribMapping()

grib_mapping.add(1, 'atmosphere_horizontal_streamfunction', None, 'm2 s-1') # Stream function, strf, m2 s-1
grib_mapping.add(34, 'sea_surface_temperature', None, 'K') # Sea surface temperature, sst, K
grib_mapping.add(59, 'atmosphere_specific_convective_available_potential_energy', None, 'J kg-1') # Convective available potential energy, cape, J kg-1
grib_mapping.add(60, 'ertel_potential_vorticity', None, 'K m2 kg-1 s-1')
grib_mapping.add(130, 'air_temperature', None, 'K') # Temperature, t, K
grib_mapping.add(131, 'eastward_wind', None, 'm s-1') # U component of wind, u, m s-1
grib_mapping.add(132, 'northward_wind', None, 'm s-1') # V component of wind, v, m s-1
grib_mapping.add(133, 'specific_humidity', None, 'kg kg-1') # Specific humidity, q, kj kj-1
grib_mapping.add(135, 'lagrangian_tendency_of_air_pressure', None, 'Pa s-1') # Vertical velocity, w, Pa s-1
grib_mapping.add(138, 'atmosphere_relative_vorticity', None, 's-1') # Vorticity (relative), vo, s-1
grib_mapping.add(142, 'lwe_thickness_of_large_scale_precipitation_amount', None, 'm') # Large-scale precipitation, lsp, m
grib_mapping.add(143, 'lwe_thickness_of_convective_precipitation_amount', None, 'm') # Convective precipitation, cp, m
grib_mapping.add(155, 'divergence_of_wind', None, 's-1') # Divergence, d, s-1
grib_mapping.add(156, 'geopotential_height', None, None) # Geopotential Height, gh, gpm
grib_mapping.add(157, 'relative_humidity', None, '%') # Relative humidity, r, %
grib_mapping.add(159, 'atmospheric_boundary_layer_thickness', None, 'm') # Boundary layer height, blh, m
grib_mapping.add(186, 'cloud_area_fraction', None, '1') # Low cloud cover, lcc, (0-1)
grib_mapping.add(151, 'air_pressure_at_sea_level', None, 'Pa') # Top net radiation, tnr, W m-2
grib_mapping.add(165, None, '10 METER U WIND COMPONENT', 'm s-1') # 10 meter U wind component, 10u, m s-1
grib_mapping.add(166, None, '10 METER V WIND COMPONENT', 'm s-1') # 10 meter V wind component, 10v, m s-1
grib_mapping.add(167, None, '2 METER TEMPERATURE', 'K') # 2 meter temperature, 2t, K
grib_mapping.add(168, None, '2 METER DEWPOINT TEMPERATURE', 'K') # 2 meter dwepoint temperature, 2d, K
grib_mapping.add(187, None, 'MEDIUM CLOUD COVER', '1') # Medium cloud cover, lcc, (0-1)
grib_mapping.add(188, None, 'HIGH CLOUD COVER', '1') # High cloud cover, hcc, (0-1)
grib_mapping.add(206, None, 'TOTAL COLUMN OZONE', 'kg m-2') # Total column ozone, tco3, kg m-2
grib_mapping.add(228, None, 'HUMIDITY TENDENCY BY LARGE SCALE CONDENSATION', 'kg kg-1') # Humidity tendency by large-scale condensation, htlc, kg kg-1

def plot_and_save(cube, out_filename, pressure_level=None, format=None):
    fig = plot_cube(cube, pressure_level)

    # Reasonable values for balancing plot and colourbar
    fig.set_size_inches(10, 7)
    fig.savefig(out_filename, format=format)
    log.info('Saving figure %s' % out_filename)

def plot_cube(cube, pressure_level=None):

    # Detect whether the cube is 2D or 3D
    coord_names = [d.name() for d in cube.dim_coords]
    if len(coord_names) == 3:
        assert 'pressure' in coord_names
        is_3d = True
    else:
        assert len(coord_names) == 2
        is_3d = False

    # Select slice
    if is_3d:
        p = cube.coord('pressure')
        if pressure_level:
            cslice = cube.subset(p[p==pressure_level])[0]
            level = pressure_level
        else:
            cslice = cube[0]
            level = p.points[0]
    else:
        cslice = cube
        level = 'surface'

    fig = plt.figure()

    log.info('Preparing plot for %s at level %s' % (cslice.name(), level))
    ax = plt.axes(projection=ccrs.PlateCarree(central_longitude=180))
    ax.coastlines()
    fig.add_axes(ax)

    gl = ax.gridlines(draw_labels=True)
    gl.xlabels_top = False
    gl.ylabels_right = False
    gl.xformatter = LONGITUDE_FORMATTER
    gl.yformatter = LATITUDE_FORMATTER
    gl.xlocator = mticker.FixedLocator([100, 140, 180, -140])

    contours = iplt.contourf(cslice, axes=ax)
    cb = plt.colorbar(contours, orientation='horizontal')
    cb.ax.set_xlabel(cslice.units)
    cb.ax.set_aspect(0.03)

    ax.set_xlabel('longitude')
    ax.set_ylabel('latitude')
    ax.set_title('%s (level: %s)' % (cslice.name(), level))

    return fig

CAST_STANDARD_NAMES = [
    'relative_humidity',
    'air_temperature',
    'atmosphere_relative_vorticity',
    'geopotential_height',
    'eastward_wind',
    'northward_wind',
    'x_wind',
    'y_wind',
]
CAST_LONG_NAMES = [
  '10 METER U WIND COMPONENT',
  '10 METER V WIND COMPONENT',
  '2 METER TEMPERATURE',
]

def plot_cast_file(filename, outdir='.'):
    cubes = grib_mapping.load_cubes(filename)

    for i, cube in enumerate(cubes):
        if ((cube.standard_name not in CAST_STANDARD_NAMES) and
            (cube.long_name not in CAST_LONG_NAMES)):
            log.info('Skipping cube {0} ({1})'.format(cube.name(), cube.long_name))
            continue

        plot_name = '{0}_{1}.png'.format(cube.name().lower().replace(' ', '_'),
                                         i)

        #!TODO: Configure pressure level
        fig = plot_and_save(cube, plot_name)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    import sys
    cubes_file, = sys.argv[1:]

    plot_cast_file(cubes_file)
<commit_msg>First pass at command line interface<commit_after>
import re

import matplotlib
matplotlib.use('Agg')

from iris.unit import Unit

import matplotlib.pyplot as plt
import iris.quickplot as qplt
import cartopy.crs as ccrs
import iris
import iris.plot as iplt

import matplotlib.ticker as mticker
from cartopy.mpl.gridliner import LATITUDE_FORMATTER, LONGITUDE_FORMATTER

import logging
log = logging.getLogger(__name__)

import os, sys, getopt

class GribMapping(object):
    def __init__(self):
        self._grib_id_map = {}

    def add(self, grib_id, standard_name, long_name, units):
        self._grib_id_map[grib_id] = (standard_name, long_name, units)

    def _metadata_callback(self, cube, field, filename):
        # Use IRIS values when present
        if cube.standard_name:
            return

        match = re.match(r'UNKNOWN LOCAL PARAM (\d+)\.(\d+)', cube.long_name)
        if match:
            parameter, table_version = (int(x) for x in match.groups())
            log.debug('Cube "{0}" --> {1}.{2}'.format(cube.long_name, parameter, table_version))
            if parameter in self._grib_id_map:
                standard_name, long_name, unit = self._grib_id_map[parameter]
                if standard_name:
                    cube.long_name = standard_name
                if long_name:
                    cube.long_name = long_name
                if unit:
                    cube.units = Unit(unit)
            else:
                log.error('Grib Parameter {0} not in grib mapping'.format(parameter))

    def load_cubes(self, filename):
        cubes = iris.load(filename, callback=self._metadata_callback)

        return cubes

grib_mapping = GribMapping()

grib_mapping.add(1, 'atmosphere_horizontal_streamfunction', None, 'm2 s-1') # Stream function, strf, m2 s-1
grib_mapping.add(34, 'sea_surface_temperature', None, 'K') # Sea surface temperature, sst, K
grib_mapping.add(59, 'atmosphere_specific_convective_available_potential_energy', None, 'J kg-1') # Convective available potential energy, cape, J kg-1
grib_mapping.add(60, 'ertel_potential_vorticity', None, 'K m2 kg-1 s-1')
grib_mapping.add(130, 'air_temperature', None, 'K') # Temperature, t, K
grib_mapping.add(131, 'eastward_wind', None, 'm s-1') # U component of wind, u, m s-1
grib_mapping.add(132, 'northward_wind', None, 'm s-1') # V component of wind, v, m s-1
grib_mapping.add(133, 'specific_humidity', None, 'kg kg-1') # Specific humidity, q, kj kj-1
grib_mapping.add(135, 'lagrangian_tendency_of_air_pressure', None, 'Pa s-1') # Vertical velocity, w, Pa s-1
grib_mapping.add(138, 'atmosphere_relative_vorticity', None, 's-1') # Vorticity (relative), vo, s-1
grib_mapping.add(142, 'lwe_thickness_of_large_scale_precipitation_amount', None, 'm') # Large-scale precipitation, lsp, m
grib_mapping.add(143, 'lwe_thickness_of_convective_precipitation_amount', None, 'm') # Convective precipitation, cp, m
grib_mapping.add(155, 'divergence_of_wind', None, 's-1') # Divergence, d, s-1
grib_mapping.add(156, 'geopotential_height', None, None) # Geopotential Height, gh, gpm
grib_mapping.add(157, 'relative_humidity', None, '%') # Relative humidity, r, %
grib_mapping.add(159, 'atmospheric_boundary_layer_thickness', None, 'm') # Boundary layer height, blh, m
grib_mapping.add(186, 'cloud_area_fraction', None, '1') # Low cloud cover, lcc, (0-1)
grib_mapping.add(151, 'air_pressure_at_sea_level', None, 'Pa') # Top net radiation, tnr, W m-2
grib_mapping.add(165, None, '10 METER U WIND COMPONENT', 'm s-1') # 10 meter U wind component, 10u, m s-1
grib_mapping.add(166, None, '10 METER V WIND COMPONENT', 'm s-1') # 10 meter V wind component, 10v, m s-1
grib_mapping.add(167, None, '2 METER TEMPERATURE', 'K') # 2 meter temperature, 2t, K
grib_mapping.add(168, None, '2 METER DEWPOINT TEMPERATURE', 'K') # 2 meter dwepoint temperature, 2d, K
grib_mapping.add(187, None, 'MEDIUM CLOUD COVER', '1') # Medium cloud cover, lcc, (0-1)
grib_mapping.add(188, None, 'HIGH CLOUD COVER', '1') # High cloud cover, hcc, (0-1)
grib_mapping.add(206, None, 'TOTAL COLUMN OZONE', 'kg m-2') # Total column ozone, tco3, kg m-2
grib_mapping.add(228, None, 'HUMIDITY TENDENCY BY LARGE SCALE CONDENSATION', 'kg kg-1') # Humidity tendency by large-scale condensation, htlc, kg kg-1

def plot_and_save(cube, out_filename, pressure_level=None, format=None):
    fig = plot_cube(cube, pressure_level)

    # Reasonable values for balancing plot and colourbar
    fig.set_size_inches(10, 7)
    fig.savefig(out_filename, format=format)
    log.info('Saving figure %s' % out_filename)

def plot_cube(cube, pressure_level=None):

    # Detect whether the cube is 2D or 3D
    coord_names = [d.name() for d in cube.dim_coords]
    if len(coord_names) == 3:
        assert 'pressure' in coord_names
        is_3d = True
    else:
        assert len(coord_names) == 2
        is_3d = False

    # Select slice
    if is_3d:
        p = cube.coord('pressure')
        if pressure_level:
            cslice = cube.subset(p[p==pressure_level])[0]
            level = pressure_level
        else:
            cslice = cube[0]
            level = p.points[0]
    else:
        cslice = cube
        level = 'surface'

    fig = plt.figure()

    log.info('Preparing plot for %s at level %s' % (cslice.name(), level))
    ax = plt.axes(projection=ccrs.PlateCarree(central_longitude=180))
    ax.coastlines()
    fig.add_axes(ax)

    gl = ax.gridlines(draw_labels=True)
    gl.xlabels_top = False
    gl.ylabels_right = False
    gl.xformatter = LONGITUDE_FORMATTER
    gl.yformatter = LATITUDE_FORMATTER
    gl.xlocator = mticker.FixedLocator([100, 140, 180, -140])

    contours = iplt.contourf(cslice, axes=ax)
    cb = plt.colorbar(contours, orientation='horizontal')
    cb.ax.set_xlabel(cslice.units)
    cb.ax.set_aspect(0.03)

    ax.set_xlabel('longitude')
    ax.set_ylabel('latitude')
    ax.set_title('%s (level: %s)' % (cslice.name(), level))

    return fig

CAST_STANDARD_NAMES = [
    'relative_humidity',
    'air_temperature',
    'atmosphere_relative_vorticity',
    'geopotential_height',
    'eastward_wind',
    'northward_wind',
    'x_wind',
    'y_wind',
]
CAST_LONG_NAMES = [
  '10 METER U WIND COMPONENT',
  '10 METER V WIND COMPONENT',
  '2 METER TEMPERATURE',
]

def plot_cast_file(filename, outdir='.'):
    cubes = grib_mapping.load_cubes(filename)

    for i, cube in enumerate(cubes):
        if ((cube.standard_name not in CAST_STANDARD_NAMES) and
            (cube.long_name not in CAST_LONG_NAMES)):
            log.info('Skipping cube {0} ({1})'.format(cube.name(), cube.long_name))
            continue

        plot_name = '{0}_{1}.png'.format(cube.name().lower().replace(' ', '_'),
                                         i)

        #!TODO: Configure pressure level
        fig = plot_and_save(cube, os.path.join(outdir,plot_name))

def usage():
    print '''usage: python2.7 cast_plot.py [-h | --help] [-d | --debug] [-o | --outdir OUTDIR ] file
    positional arguments:
        file        A GRIB file to be processed

    Optional arguments:
        -h, --help              show this help message and exit
        -d, --debug             show debug logs
        -o, --outdir OUTDIR     Output directory. Must be writable by the current user.
    '''

if __name__ == '__main__':
    try:
        opts, args = getopt.getopt(sys.argv[1:],'hdo:',['help','debug','outdir='])
    except getopt.GetoptError as err:
        print str(err);
        usage()
        sys.exit(2)
    #defaults
    outdir = '.'
    level=logging.WARNING

    for o, a in opts:
        if o in ("-h", "--help"):
            usage()
            sys.exit()
        elif o in ("-d", "--debug"):
            level=logging.DEBUG
        elif o in ("-o", "--outdir"):
            outdir = a
        else:
            assert False, "unhandled option"

    logging.basicConfig(level=level)
    #os.access is technically vulnerable to TOCTOU errors, but...
    if(not os.access(a, os.W_OK)):
        log.error("directory '" + a +"' is not writable")
        sys.exit(2)
    cubes_file, = args[0:]

    plot_cast_file(cubes_file, outdir)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse
import os
import random
import sys
import time

import google.protobuf as pb

import mesos
import mesos_pb2

#################################### Schedulers implement the integration tests

class Scheduler(mesos.Scheduler):
    def __init__(self, trials=10):
        self.token    = "%016x" % random.getrandbits(64)
        self.trials   = trials
        self.tasks    = []
        self.statuses = {}
    def __repr__(self):
        return "%s(%r)" % (self.__class__, self.__dict__)
    def registered(self, driver, framework_id, master):
        self.framework_id = framework_id
        print >>sys.stderr, "Registered as framework %s" % framework_id.value
    def statusUpdate(self, driver, update):
        task, code = update.task_id.value, update.state
        self.statuses[task] = code
        name = mesos_pb2.TaskState.Name(code)
        info = "%s: %s" % (task, name)
        if update.HasField("message"):
            info += "\n  " + update.message
        print >>sys.stderr, info
    def next_task_id(self):
        return "medea-test-%s-task%02d" % (self.token, len(self.tasks))
    terminal = set([ mesos_pb2.TASK_FINISHED,
                     mesos_pb2.TASK_FAILED,
                     mesos_pb2.TASK_KILLED,
                     mesos_pb2.TASK_LOST ])
    failed   = set([ mesos_pb2.TASK_FAILED,
                     mesos_pb2.TASK_KILLED,
                     mesos_pb2.TASK_LOST ])

class ExecutorScheduler(Scheduler):                # TODO: Make this class work
    def __init__(self, command, uris=[], container=None, trials=10):
        Scheduler.__init__(self, trials)
        self.command   = command
        self.uris      = uris
        self.container = container
        self.messages  = []
    def statusUpdate(self, driver, update):
        super(ExecutorScheduler, self).statusUpdate(driver, update)
        if update.state == mesos_pb2.TASK_RUNNING:
            pass                  # TODO: Send a message if we get TASK_RUNNING
        task_terminated = update.state in Scheduler.terminal
        enough_tasks    = len(self.tasks) >= self.trials
        if task_terminated and enough_tasks:
            driver.stop()
    def frameworkMessage(self, driver, executor_id, slave_id, msg):
        self.messages += [msg]
        driver.killTask(update.task_id)
    def resourceOffers(self, driver, offers):
        for offer in offers:
            if len(self.tasks) >= self.trials: break
            tid  = self.next_task_id()
            sid  = offer.slave_id
            task = task_with_executor(tid, sid)
            self.tasks += [task]
            driver.launchTasks(offer.id, [task])

class SleepScheduler(Scheduler):
    def __init__(self, sleep=20, uris=[], container=None, trials=10):
        Scheduler.__init__(self, trials)
        self.sleep     = sleep
        self.uris      = uris
        self.container = container
    def statusUpdate(self, driver, update):
        super(SleepScheduler, self).statusUpdate(driver, update)
        task_terminated = update.state in Scheduler.terminal
        enough_tasks    = len(self.tasks) >= self.trials
        if task_terminated and enough_tasks:
            print >>sys.stderr, "Tried enough times"
            err = None
            for task in self.tasks:
                try:
                    driver.killTask(task.task_id)
                except Exception as e:
                    err = e
            driver.stop()
            if err: raise err
    def resourceOffers(self, driver, offers):
        delay = int(float(self.sleep) / self.trials)
        for offer in offers:
            if len(self.tasks) >= self.trials: break
          # time.sleep(self.sleep + 0.5)
            time.sleep(delay)                    # Space out the requests a bit
            tid  = self.next_task_id()
            sid  = offer.slave_id
            cmd  = "date -u +%T ; sleep " + str(self.sleep) + " ; date -u +%T"
            task = task_with_command(tid, sid, cmd, self.uris, self.container)
            self.tasks += [task]
            print >>sys.stderr, present_task(task)
            driver.launchTasks(offer.id, [task])

class PGScheduler(Scheduler):
    def __init__(self, container="docker:///zaiste/postgresql", trials=10):
        Scheduler.__init__(self, trials)
        self.container = container
    def statusUpdate(self, driver, update):
        super(PGScheduler, self).statusUpdate(driver, update)
        if update.state == mesos_pb2.TASK_RUNNING:
            time.sleep(2)
            driver.killTask(update.task_id)       # Shutdown Postgres container
        task_terminated = update.state in Scheduler.terminal
        enough_tasks    = len(self.tasks) >= self.trials
        if task_terminated and enough_tasks:
            driver.stop()
    def resourceOffers(self, driver, offers):
        for offer in offers:
            if len(self.tasks) >= self.trials: break
            time.sleep(2)
            tid  = self.next_task_id()
            sid  = offer.slave_id
            task = task_with_daemon(tid, sid, self.container)
            self.tasks += [task]
            print >>sys.stderr, present_task(task)
            driver.launchTasks(offer.id, [task])

################################################################ Task factories

def task_with_executor(tid, sid, *args):
    executor = mesos_pb2.ExecutorInfo()
    executor.executor_id.value = tid
    executor.name = tid
    executor.source = "medea-test"
    executor.command.MergeFrom(command(*args))
    task = task_base(tid, sid)
    task.executor.MergeFrom(executor)
    return task

def task_with_command(tid, sid, *args):
    task = task_base(tid, sid)
    task.command.MergeFrom(command(*args))
    return task

def task_with_daemon(tid, sid, image):
    task = task_base(tid, sid)
    task.command.MergeFrom(command(image=image))
    return task

def task_base(tid, sid, cpu=0.5, ram=256):
    task = mesos_pb2.TaskInfo()
    task.task_id.value = tid
    task.slave_id.value = sid.value
    task.name = tid
    cpus = task.resources.add()
    cpus.name = "cpus"
    cpus.type = mesos_pb2.Value.SCALAR
    cpus.scalar.value = cpu
    mem = task.resources.add()
    mem.name = "mem"
    mem.type = mesos_pb2.Value.SCALAR
    mem.scalar.value = ram
    return task

def command(shell="", uris=[], image=None):
    command = mesos_pb2.CommandInfo()
    command.value = shell                                          # TODO: URIs
    if image:                                       # Rely on the default image
        container = mesos_pb2.CommandInfo.ContainerInfo()
        container.image = image
        command.container.MergeFrom(container)
    return command

def present_task(task):
    label = task.task_id.value
    if task.HasField("executor"):
        token, body = "executor", task.executor
    else:
        token, body = "command", task.command
    lines = pb.text_format.MessageToString(body).strip().split("\n")
    return "task_id: %s\n%s {\n  %s\n}" % (label, token, "\n  ".join(lines))

########################################################################## Main

def cli():
    schedulers = { "sleep" : SleepScheduler,
                   "pg"    : PGScheduler }
    p = argparse.ArgumentParser(prog="medea-test.py")
    p.add_argument("--master", default="localhost:5050",
                   help="Mesos master URL")
    p.add_argument("--test", choices=schedulers.keys(), default="sleep",
                   help="Test suite to use")
    p.add_argument("--test.container",
                   help="Image URL to use (for any test)")
    p.add_argument("--test.sleep", type=int,
                   help="Seconds to sleep (for sleep test)")
    p.add_argument("--test.trials", type=int,
                   help="Number of tasks to run (for any test)")
    p.add_argument("--test.command",
                   help="Command to use (for executor test)")
    p.add_argument("--test.uris", nargs="*", default=[],
                   help="Pass any number of times to pass URIs (for any test)")
    parsed = p.parse_args()

    pairs = [ (k.split("test.")[1:], v) for k, v in vars(parsed).items() ]
    constructor_args = dict( (k[0], v) for k, v in pairs if len(k) == 1 and v )
    scheduler_class = schedulers[parsed.test]
    args = ", ".join( "%s=%r" % (k, v) for k, v in constructor_args.items() )
    print >>sys.stderr, "Test class: %s(%s)" % (scheduler_class.__name__, args)
    scheduler = scheduler_class(**constructor_args)

    framework = mesos_pb2.FrameworkInfo()
    framework.name = "medea-test"
    framework.user = ""
    driver = mesos.MesosSchedulerDriver(scheduler, framework, parsed.master)
    code = driver.run()
    print >>sys.stderr, mesos_pb2.Status.Name(code)
    driver.stop()
    os._exit(0 if code == mesos_pb2.DRIVER_STOPPED else 1)

if __name__ == "__main__": cli()

<commit_msg>Pass URIs.<commit_after>#!/usr/bin/env python
import argparse
import os
import random
import sys
import time

import google.protobuf as pb

import mesos
import mesos_pb2

#################################### Schedulers implement the integration tests

class Scheduler(mesos.Scheduler):
    def __init__(self, trials=10):
        self.token    = "%016x" % random.getrandbits(64)
        self.trials   = trials
        self.tasks    = []
        self.statuses = {}
    def __repr__(self):
        return "%s(%r)" % (self.__class__, self.__dict__)
    def registered(self, driver, framework_id, master):
        self.framework_id = framework_id
        print >>sys.stderr, "Registered as framework %s" % framework_id.value
    def statusUpdate(self, driver, update):
        task, code = update.task_id.value, update.state
        self.statuses[task] = code
        name = mesos_pb2.TaskState.Name(code)
        info = "%s: %s" % (task, name)
        if update.HasField("message"):
            info += "\n  " + update.message
        print >>sys.stderr, info
    def next_task_id(self):
        return "medea-test-%s-task%02d" % (self.token, len(self.tasks))
    terminal = set([ mesos_pb2.TASK_FINISHED,
                     mesos_pb2.TASK_FAILED,
                     mesos_pb2.TASK_KILLED,
                     mesos_pb2.TASK_LOST ])
    failed   = set([ mesos_pb2.TASK_FAILED,
                     mesos_pb2.TASK_KILLED,
                     mesos_pb2.TASK_LOST ])

class ExecutorScheduler(Scheduler):                # TODO: Make this class work
    def __init__(self, command, uris=[], container=None, trials=10):
        Scheduler.__init__(self, trials)
        self.command   = command
        self.uris      = uris
        self.container = container
        self.messages  = []
    def statusUpdate(self, driver, update):
        super(ExecutorScheduler, self).statusUpdate(driver, update)
        if update.state == mesos_pb2.TASK_RUNNING:
            pass                  # TODO: Send a message if we get TASK_RUNNING
        task_terminated = update.state in Scheduler.terminal
        enough_tasks    = len(self.tasks) >= self.trials
        if task_terminated and enough_tasks:
            driver.stop()
    def frameworkMessage(self, driver, executor_id, slave_id, msg):
        self.messages += [msg]
        driver.killTask(update.task_id)
    def resourceOffers(self, driver, offers):
        for offer in offers:
            if len(self.tasks) >= self.trials: break
            tid  = self.next_task_id()
            sid  = offer.slave_id
            task = task_with_executor(tid, sid)
            self.tasks += [task]
            driver.launchTasks(offer.id, [task])

class SleepScheduler(Scheduler):
    def __init__(self, sleep=20, uris=[], container=None, trials=10):
        Scheduler.__init__(self, trials)
        self.sleep     = sleep
        self.uris      = uris
        self.container = container
    def statusUpdate(self, driver, update):
        super(SleepScheduler, self).statusUpdate(driver, update)
        task_terminated = update.state in Scheduler.terminal
        enough_tasks    = len(self.tasks) >= self.trials
        if task_terminated and enough_tasks:
            print >>sys.stderr, "Tried enough times"
            err = None
            for task in self.tasks:
                try:
                    driver.killTask(task.task_id)
                except Exception as e:
                    err = e
            driver.stop()
            if err: raise err
    def resourceOffers(self, driver, offers):
        delay = int(float(self.sleep) / self.trials)
        for offer in offers:
            if len(self.tasks) >= self.trials: break
          # time.sleep(self.sleep + 0.5)
            time.sleep(delay)                    # Space out the requests a bit
            tid  = self.next_task_id()
            sid  = offer.slave_id
            cmd  = "date -u +%T ; sleep " + str(self.sleep) + " ; date -u +%T"
            task = task_with_command(tid, sid, cmd, self.uris, self.container)
            self.tasks += [task]
            print >>sys.stderr, present_task(task)
            driver.launchTasks(offer.id, [task])

class PGScheduler(Scheduler):
    def __init__(self, container="docker:///zaiste/postgresql", trials=10):
        Scheduler.__init__(self, trials)
        self.container = container
    def statusUpdate(self, driver, update):
        super(PGScheduler, self).statusUpdate(driver, update)
        if update.state == mesos_pb2.TASK_RUNNING:
            time.sleep(2)
            driver.killTask(update.task_id)       # Shutdown Postgres container
        task_terminated = update.state in Scheduler.terminal
        enough_tasks    = len(self.tasks) >= self.trials
        if task_terminated and enough_tasks:
            driver.stop()
    def resourceOffers(self, driver, offers):
        for offer in offers:
            if len(self.tasks) >= self.trials: break
            time.sleep(2)
            tid  = self.next_task_id()
            sid  = offer.slave_id
            task = task_with_daemon(tid, sid, self.container)
            self.tasks += [task]
            print >>sys.stderr, present_task(task)
            driver.launchTasks(offer.id, [task])

################################################################ Task factories

def task_with_executor(tid, sid, *args):
    executor = mesos_pb2.ExecutorInfo()
    executor.executor_id.value = tid
    executor.name = tid
    executor.source = "medea-test"
    executor.command.MergeFrom(command(*args))
    task = task_base(tid, sid)
    task.executor.MergeFrom(executor)
    return task

def task_with_command(tid, sid, *args):
    task = task_base(tid, sid)
    task.command.MergeFrom(command(*args))
    return task

def task_with_daemon(tid, sid, image):
    task = task_base(tid, sid)
    task.command.MergeFrom(command(image=image))
    return task

def task_base(tid, sid, cpu=0.5, ram=256):
    task = mesos_pb2.TaskInfo()
    task.task_id.value = tid
    task.slave_id.value = sid.value
    task.name = tid
    cpus = task.resources.add()
    cpus.name = "cpus"
    cpus.type = mesos_pb2.Value.SCALAR
    cpus.scalar.value = cpu
    mem = task.resources.add()
    mem.name = "mem"
    mem.type = mesos_pb2.Value.SCALAR
    mem.scalar.value = ram
    return task

def command(shell="", uris=[], image=None):
    command = mesos_pb2.CommandInfo()
    command.value = shell
    for uri in uris:
        command.uris.add().value = uri
    if image:                      # Rely on the default image when none is set
        container = mesos_pb2.CommandInfo.ContainerInfo()
        container.image = image
        command.container.MergeFrom(container)
    return command

def present_task(task):
    label = task.task_id.value
    if task.HasField("executor"):
        token, body = "executor", task.executor
    else:
        token, body = "command", task.command
    lines = pb.text_format.MessageToString(body).strip().split("\n")
    return "task_id: %s\n%s {\n  %s\n}" % (label, token, "\n  ".join(lines))

########################################################################## Main

def cli():
    schedulers = { "sleep" : SleepScheduler,
                   "pg"    : PGScheduler }
    p = argparse.ArgumentParser(prog="medea-test.py")
    p.add_argument("--master", default="localhost:5050",
                   help="Mesos master URL")
    p.add_argument("--test", choices=schedulers.keys(), default="sleep",
                   help="Test suite to use")
    p.add_argument("--test.container",
                   help="Image URL to use (for any test)")
    p.add_argument("--test.sleep", type=int,
                   help="Seconds to sleep (for sleep test)")
    p.add_argument("--test.trials", type=int,
                   help="Number of tasks to run (for any test)")
    p.add_argument("--test.command",
                   help="Command to use (for executor test)")
    p.add_argument("--test.uris", nargs="*", default=[],
                   help="Pass any number of times to pass URIs (for any test)")
    parsed = p.parse_args()

    pairs = [ (k.split("test.")[1:], v) for k, v in vars(parsed).items() ]
    constructor_args = dict( (k[0], v) for k, v in pairs if len(k) == 1 and v )
    scheduler_class = schedulers[parsed.test]
    args = ", ".join( "%s=%r" % (k, v) for k, v in constructor_args.items() )
    print >>sys.stderr, "Test class: %s(%s)" % (scheduler_class.__name__, args)
    scheduler = scheduler_class(**constructor_args)

    framework = mesos_pb2.FrameworkInfo()
    framework.name = "medea-test"
    framework.user = ""
    driver = mesos.MesosSchedulerDriver(scheduler, framework, parsed.master)
    code = driver.run()
    print >>sys.stderr, mesos_pb2.Status.Name(code)
    driver.stop()
    os._exit(0 if code == mesos_pb2.DRIVER_STOPPED else 1)

if __name__ == "__main__": cli()

<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import collections
import contextlib
import logging
import os
import sys

import fuzzywuzzy.fuzz as fuzz

logger = logging.getLogger(__name__)

@contextlib.contextmanager
def open_history(filename):
    history = read_history(filename)
    yield history
    write_history(filename, history)

def read_history(filename):
    history = collections.Counter()
    if os.path.exists(filename):
        with open(filename) as fp:
            for line in fp:
                if line:
                    count, path = line.split()
                    history[path] = int(count)

    return history

def write_history(filename, history):
    data = sorted([(history[p], p) for p in history], reverse=True)
    with open(filename, 'w') as fp:
        for count, path in data:
            fp.write('%d %s\n' % (count, path))

def rank_paths(history, test, limit=10):
    def score(path):
        token_score = max(fuzz.ratio(token, test) for token in path.split('/'))
        total_score = fuzz.ratio(path, test)
        freq_score = history[path]
        return ((token_score, total_score, freq_score), path)

    results = sorted([score(path) for path in history], reverse=True)

    if logger.getEffectiveLevel() <= logging.DEBUG:
        for result in results:
            logger.debug(result)

    return [path for _, path in results[:limit]]

def main(argv=sys.argv[1:]):
    logging.basicConfig(format='%(asctime)-15s %(message)s')

    parser = argparse.ArgumentParser()
    parser.add_argument('-a', '--add', action='store_true')
    parser.add_argument('-l', '--list', action='store_true')
    parser.add_argument('-c', '--clear', action='store_true')
    parser.add_argument('-r', '--refresh', action='store_true')
    parser.add_argument('-m', '--match', action='store_true')
    parser.add_argument('-v', '--verbose', action='store_true')
    parser.add_argument('-f', '--file',
            default=os.path.realpath(os.path.expanduser('~/.cdhistory')))
    parser.add_argument('paths', nargs=argparse.REMAINDER)

    args = parser.parse_args(argv)

    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG if args.verbose else logging.INFO)

    if args.clear:
        if os.path.isfile(args.file):
            os.remove(args.file)

    if args.add:
        with open_history(args.file) as history:
            for path in args.paths:
                rpath = os.path.realpath(path)
                if os.path.exists(rpath):
                    history[rpath] += 1

    elif args.refresh:
        with open_history(args.file) as history:
            remove = [p for p in history if not os.path.exists(p)]
            for path in remove:
                del history[path]

    elif args.match:
        with open_history(args.file) as history:
            path = args.paths[0] if args.paths else os.getcwd()
            for result in rank_paths(history, path):
                print(result)

    elif args.list:
        history = read_history(args.file)
        paths = sorted(history.keys(), key=lambda p: history[p], reverse=True)
        for path in paths:
            print(path)

if __name__ == "__main__":
    main()
<commit_msg>cdhistory: added --max-results argument<commit_after>#!/usr/bin/env python

import argparse
import collections
import contextlib
import logging
import os
import sys

import fuzzywuzzy.fuzz as fuzz

logger = logging.getLogger(__name__)

@contextlib.contextmanager
def open_history(filename):
    history = read_history(filename)
    yield history
    write_history(filename, history)

def read_history(filename):
    history = collections.Counter()
    if os.path.exists(filename):
        with open(filename) as fp:
            for line in fp:
                if line:
                    count, path = line.split()
                    history[path] = int(count)

    return history

def write_history(filename, history):
    data = sorted([(history[p], p) for p in history], reverse=True)
    with open(filename, 'w') as fp:
        for count, path in data:
            fp.write('%d %s\n' % (count, path))

def rank_paths(history, test, limit=10):
    def score(path):
        token_score = max(fuzz.ratio(token, test) for token in path.split('/'))
        total_score = fuzz.ratio(path, test)
        freq_score = history[path]
        return ((token_score, total_score, freq_score), path)

    results = sorted([score(path) for path in history], reverse=True)

    if logger.getEffectiveLevel() <= logging.DEBUG:
        for result in results:
            logger.debug(result)

    return [path for _, path in results[:limit]]

def main(argv=sys.argv[1:]):
    logging.basicConfig(format='%(asctime)-15s %(message)s')

    parser = argparse.ArgumentParser()
    parser.add_argument('-a', '--add', action='store_true')
    parser.add_argument('-l', '--list', action='store_true')
    parser.add_argument('-c', '--clear', action='store_true')
    parser.add_argument('-r', '--refresh', action='store_true')
    parser.add_argument('-m', '--match', action='store_true')
    parser.add_argument('-n', '--max-results', type=int, default=10)
    parser.add_argument('-v', '--verbose', action='store_true')
    parser.add_argument('-f', '--file',
            default=os.path.realpath(os.path.expanduser('~/.cdhistory')))
    parser.add_argument('paths', nargs=argparse.REMAINDER)

    args = parser.parse_args(argv)

    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG if args.verbose else logging.INFO)

    if args.clear:
        if os.path.isfile(args.file):
            os.remove(args.file)

    if args.add:
        with open_history(args.file) as history:
            for path in args.paths:
                rpath = os.path.realpath(path)
                if os.path.exists(rpath):
                    history[rpath] += 1

    elif args.refresh:
        with open_history(args.file) as history:
            remove = [p for p in history if not os.path.exists(p)]
            for path in remove:
                del history[path]

    elif args.match:
        with open_history(args.file) as history:
            path = args.paths[0] if args.paths else os.getcwd()
            for result in rank_paths(history, path, limit=args.max_results):
                print(result)

    elif args.list:
        history = read_history(args.file)
        paths = sorted(history.keys(), key=lambda p: history[p], reverse=True)
        for path in paths:
            print(path)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import pickle, rsa, os

if __name__ == "__main__":
    master_tup = pickle.load(open("masterKey.pickle", "rb"))
    master_key = rsa.PrivateKey(*master_tup)

    if os.path.exists("to_sign"):
        for key in os.listdir("." + os.sep + "to_sign"):
            tup = pickle.load(open("to_sign" + os.sep + key, "rb"))
            signed = rsa.sign(str(tup), master_key, "SHA-256")
            print("Key " + str(key) + " signed")
            print("Signiture valid? " + str(rsa.verify(str(tup), signed, master_key)))
            if not os.path.exists("signed"):
                os.mkdir("signed")
            pickle.dump(signed, open("signed" + os.sep + key, "wb"), 0)

def exportProof(priv):
    tup = (priv.n, priv.e)
    proof = rsa.sign(str(tup), priv,"SHA-256")
    print("Post this proof somewhere associated with your")
    print("organization, to verify your identity.")
    print("Send that link to gappleto97+development@gmail.com")
    return (tup,proof)
<commit_msg>More style guide issues [skip ci]<commit_after>import pickle, os
from rsa import PrivateKey, sign, verify

if __name__ == "__main__":
    master_tup = pickle.load(open("masterKey.pickle", "rb"))
    master_key = PrivateKey(*master_tup)

    if os.path.exists("to_sign"):
        for key in os.listdir("." + os.sep + "to_sign"):
            tup = pickle.load(open("to_sign" + os.sep + key, "rb"))
            signed = sign(str(tup), master_key, "SHA-256")
            print("Key " + str(key) + " signed.")
            print("Sig valid? " + str(verify(str(tup), signed, master_key)))
            if not os.path.exists("signed"):
                os.mkdir("signed")
            pickle.dump(signed, open("signed" + os.sep + key, "wb"), 0)

def exportProof(priv):
    tup = (priv.n, priv.e)
    proof = rsa.sign(str(tup), priv, "SHA-256")
    print("Post this proof somewhere associated with your")
    print("organization, to verify your identity.")
    print("Send that link to gappleto97+development@gmail.com")
    return (tup, proof)
<|endoftext|>


<commit_before># Copyright 2014 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Routines to generate root and server certificates.

Certificate Naming Conventions:
  ca_cert:  crypto.X509 for the certificate authority (w/ both the pub &
                priv keys)
  cert:  a crypto.X509 certificate (w/ just the pub key)
  cert_str:  a certificate string (w/ just the pub cert)
  key:  a private crypto.PKey  (from ca or pem)
  ca_cert_str:  a certificae authority string (w/ both the pub & priv certs)
"""

import logging
import os
import socket
import time

openssl_import_error = None

SSL_METHOD = None
VERIFY_PEER = None
SysCallError = None
Error = None
ZeroReturnError = None

try:
  from OpenSSL import crypto, SSL

  SSL_METHOD = SSL.SSLv23_METHOD
  VERIFY_PEER = SSL.VERIFY_PEER
  SysCallError = SSL.SysCallError
  Error = SSL.Error
  ZeroReturnError = SSL.ZeroReturnError
except ImportError, e:
  openssl_import_error = e

def get_ssl_context(method=SSL_METHOD):
  # One of: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, or TLSv1_METHOD
  return SSL.Context(method)

class WrappedConnection(object):

  def __init__(self, obj):
    self._wrapped_obj = obj

  def __getattr__(self, attr):
    if attr in self.__dict__:
      return getattr(self, attr)
    return getattr(self._wrapped_obj, attr)

  def recv(self, buflen=1024, flags=0):
    try:
      return self._wrapped_obj.recv(buflen, flags)
    except SSL.SysCallError, e:
      if e.args[1] == 'Unexpected EOF':
        return ''
      raise
    except SSL.ZeroReturnError:
      return ''

def get_ssl_connection(context, connection):
  return WrappedConnection(SSL.Connection(context, connection))

def load_privatekey(key, filetype=crypto.FILETYPE_PEM):
  """Loads obj private key object from string."""
  return crypto.load_privatekey(filetype, key)

def load_cert(cert_str, filetype=crypto.FILETYPE_PEM):
  """Loads obj cert object from string."""
  return crypto.load_certificate(filetype, cert_str)

def _dump_privatekey(key, filetype=crypto.FILETYPE_PEM):
  """Dumps obj private key object to string."""
  return crypto.dump_privatekey(filetype, key)

def _dump_cert(cert, filetype=crypto.FILETYPE_PEM):
  """Dumps obj cert object to string."""
  return crypto.dump_certificate(filetype, cert)

def generate_dummy_ca_cert(subject='_WebPageReplayCert'):
  """Generates dummy certificate authority.

  Args:
    subject: a string representing the desired root cert issuer
  Returns:
    A tuple of the public key and the private key strings for the root
    certificate
  """
  if openssl_import_error:
    raise openssl_import_error

  key = crypto.PKey()
  key.generate_key(crypto.TYPE_RSA, 1024)

  ca_cert = crypto.X509()
  ca_cert.set_serial_number(int(time.time()*10000))
  ca_cert.set_version(2)
  ca_cert.get_subject().CN = subject
  ca_cert.get_subject().O = subject
  ca_cert.gmtime_adj_notBefore(-60 * 60 * 24 * 365 * 2)
  ca_cert.gmtime_adj_notAfter(60 * 60 * 24 * 365 * 2)
  ca_cert.set_issuer(ca_cert.get_subject())
  ca_cert.set_pubkey(key)
  ca_cert.add_extensions([
      crypto.X509Extension('basicConstraints', True, 'CA:TRUE'),
      crypto.X509Extension('nsCertType', True, 'sslCA'),
      crypto.X509Extension('extendedKeyUsage', True,
                           ('serverAuth,clientAuth,emailProtection,'
                            'timeStamping,msCodeInd,msCodeCom,msCTLSign,'
                            'msSGC,msEFS,nsSGC')),
      crypto.X509Extension('keyUsage', False, 'keyCertSign, cRLSign'),
      crypto.X509Extension('subjectKeyIdentifier', False, 'hash',
                           subject=ca_cert),
      ])
  ca_cert.sign(key, 'sha1')
  key_str = _dump_privatekey(key)
  ca_cert_str = _dump_cert(ca_cert)
  return ca_cert_str, key_str

def get_host_cert(host, port=443):
  """Contacts the host and returns its certificate."""
  host_certs = []
  def verify_cb(conn, cert, errnum, depth, ok):
    host_certs.append(cert)
    # The return code of 1 indicates that the certificate was ok.
    return 1

  context = SSL.Context(SSL.SSLv23_METHOD)
  context.set_verify(SSL.VERIFY_PEER, verify_cb)  # Demand a certificate
  s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  connection = SSL.Connection(context, s)
  try:
    connection.connect((host, port))
    connection.send('')
  except SSL.SysCallError:
    pass
  except socket.gaierror:
    logging.debug('Host name is not valid')
  finally:
    connection.shutdown()
    connection.close()
  if len(host_certs) > 0:
    return _dump_cert(host_certs[-1])
  return ''

def write_dummy_ca_cert(ca_cert_str, key_str, cert_path):
  """Writes four certificate files.

  For example, if cert_path is "mycert.pem":
      mycert.pem - CA plus private key
      mycert-cert.pem - CA in PEM format
      mycert-cert.cer - CA for Android
      mycert-cert.p12 - CA in PKCS12 format for Windows devices
  Args:
    cert_path: path string such as "mycert.pem"
    ca_cert_str: certificate string
    key_str: private key string
  """
  dirname = os.path.dirname(cert_path)
  if dirname and not os.path.exists(dirname):
    os.makedirs(dirname)

  root_path = os.path.splitext(cert_path)[0]
  ca_cert_path = root_path + '-cert.pem'
  android_cer_path = root_path + '-cert.cer'
  windows_p12_path = root_path + '-cert.p12'

  # Dump the CA plus private key
  with open(cert_path, 'w') as f:
    f.write(key_str)
    f.write(ca_cert_str)

  # Dump the certificate in PEM format
  with open(ca_cert_path, 'w') as f:
    f.write(ca_cert_str)

  # Create a .cer file with the same contents for Android
  with open(android_cer_path, 'w') as f:
    f.write(ca_cert_str)

  ca_cert = load_cert(ca_cert_str)
  key = load_privatekey(key_str)
  # Dump the certificate in PKCS12 format for Windows devices
  with open(windows_p12_path, 'w') as f:
    p12 = crypto.PKCS12()
    p12.set_certificate(ca_cert)
    p12.set_privatekey(key)
    f.write(p12.export())

def generate_cert(root_ca_cert_str, server_cert_str, server_host):
  """Generates a cert_str with the sni field in server_cert_str signed by the
  root_ca_cert_str.

  Args:
    root_ca_cert_str: PEM formatted string representing the root cert
    server_cert_str: PEM formatted string representing cert
    server_host: host name to use if there is no server_cert_str
  Returns:
    a PEM formatted certificate string
  """
  if openssl_import_error:
    raise openssl_import_error

  common_name = server_host
  if server_cert_str:
    cert = load_cert(server_cert_str)
    common_name = cert.get_subject().commonName

  ca_cert = load_cert(root_ca_cert_str)
  key = load_privatekey(root_ca_cert_str)

  req = crypto.X509Req()
  subj = req.get_subject()
  subj.CN = common_name
  req.set_pubkey(ca_cert.get_pubkey())
  req.sign(key, 'sha1')

  cert = crypto.X509()
  cert.gmtime_adj_notBefore(-60 * 60)
  cert.gmtime_adj_notAfter(60 * 60 * 24 * 30)
  cert.set_issuer(ca_cert.get_subject())
  cert.set_subject(req.get_subject())
  cert.set_serial_number(int(time.time()*10000))
  cert.set_pubkey(req.get_pubkey())
  cert.sign(key, 'sha1')

  return _dump_cert(cert)
<commit_msg>Avoid referencing crypto constant if import failed.<commit_after># Copyright 2014 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Routines to generate root and server certificates.

Certificate Naming Conventions:
  ca_cert:  crypto.X509 for the certificate authority (w/ both the pub &
                priv keys)
  cert:  a crypto.X509 certificate (w/ just the pub key)
  cert_str:  a certificate string (w/ just the pub cert)
  key:  a private crypto.PKey  (from ca or pem)
  ca_cert_str:  a certificae authority string (w/ both the pub & priv certs)
"""

import logging
import os
import socket
import time

openssl_import_error = None

Error = None
SSL_METHOD = None
SysCallError = None
VERIFY_PEER = None
ZeroReturnError = None
FILETYPE_PEM = None

try:
  from OpenSSL import crypto, SSL

  Error = SSL.Error
  SSL_METHOD = SSL.SSLv23_METHOD
  SysCallError = SSL.SysCallError
  VERIFY_PEER = SSL.VERIFY_PEER
  ZeroReturnError = SSL.ZeroReturnError
  FILETYPE_PEM = crypto.FILETYPE_PEM
except ImportError, e:
  openssl_import_error = e

def get_ssl_context(method=SSL_METHOD):
  # One of: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, or TLSv1_METHOD
  return SSL.Context(method)

class WrappedConnection(object):

  def __init__(self, obj):
    self._wrapped_obj = obj

  def __getattr__(self, attr):
    if attr in self.__dict__:
      return getattr(self, attr)
    return getattr(self._wrapped_obj, attr)

  def recv(self, buflen=1024, flags=0):
    try:
      return self._wrapped_obj.recv(buflen, flags)
    except SSL.SysCallError, e:
      if e.args[1] == 'Unexpected EOF':
        return ''
      raise
    except SSL.ZeroReturnError:
      return ''

def get_ssl_connection(context, connection):
  return WrappedConnection(SSL.Connection(context, connection))

def load_privatekey(key, filetype=FILETYPE_PEM):
  """Loads obj private key object from string."""
  return crypto.load_privatekey(filetype, key)

def load_cert(cert_str, filetype=FILETYPE_PEM):
  """Loads obj cert object from string."""
  return crypto.load_certificate(filetype, cert_str)

def _dump_privatekey(key, filetype=FILETYPE_PEM):
  """Dumps obj private key object to string."""
  return crypto.dump_privatekey(filetype, key)

def _dump_cert(cert, filetype=FILETYPE_PEM):
  """Dumps obj cert object to string."""
  return crypto.dump_certificate(filetype, cert)

def generate_dummy_ca_cert(subject='_WebPageReplayCert'):
  """Generates dummy certificate authority.

  Args:
    subject: a string representing the desired root cert issuer
  Returns:
    A tuple of the public key and the private key strings for the root
    certificate
  """
  if openssl_import_error:
    raise openssl_import_error

  key = crypto.PKey()
  key.generate_key(crypto.TYPE_RSA, 1024)

  ca_cert = crypto.X509()
  ca_cert.set_serial_number(int(time.time()*10000))
  ca_cert.set_version(2)
  ca_cert.get_subject().CN = subject
  ca_cert.get_subject().O = subject
  ca_cert.gmtime_adj_notBefore(-60 * 60 * 24 * 365 * 2)
  ca_cert.gmtime_adj_notAfter(60 * 60 * 24 * 365 * 2)
  ca_cert.set_issuer(ca_cert.get_subject())
  ca_cert.set_pubkey(key)
  ca_cert.add_extensions([
      crypto.X509Extension('basicConstraints', True, 'CA:TRUE'),
      crypto.X509Extension('nsCertType', True, 'sslCA'),
      crypto.X509Extension('extendedKeyUsage', True,
                           ('serverAuth,clientAuth,emailProtection,'
                            'timeStamping,msCodeInd,msCodeCom,msCTLSign,'
                            'msSGC,msEFS,nsSGC')),
      crypto.X509Extension('keyUsage', False, 'keyCertSign, cRLSign'),
      crypto.X509Extension('subjectKeyIdentifier', False, 'hash',
                           subject=ca_cert),
      ])
  ca_cert.sign(key, 'sha1')
  key_str = _dump_privatekey(key)
  ca_cert_str = _dump_cert(ca_cert)
  return ca_cert_str, key_str

def get_host_cert(host, port=443):
  """Contacts the host and returns its certificate."""
  host_certs = []
  def verify_cb(conn, cert, errnum, depth, ok):
    host_certs.append(cert)
    # The return code of 1 indicates that the certificate was ok.
    return 1

  context = SSL.Context(SSL.SSLv23_METHOD)
  context.set_verify(SSL.VERIFY_PEER, verify_cb)  # Demand a certificate
  s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  connection = SSL.Connection(context, s)
  try:
    connection.connect((host, port))
    connection.send('')
  except SSL.SysCallError:
    pass
  except socket.gaierror:
    logging.debug('Host name is not valid')
  finally:
    connection.shutdown()
    connection.close()
  if len(host_certs) > 0:
    return _dump_cert(host_certs[-1])
  return ''

def write_dummy_ca_cert(ca_cert_str, key_str, cert_path):
  """Writes four certificate files.

  For example, if cert_path is "mycert.pem":
      mycert.pem - CA plus private key
      mycert-cert.pem - CA in PEM format
      mycert-cert.cer - CA for Android
      mycert-cert.p12 - CA in PKCS12 format for Windows devices
  Args:
    cert_path: path string such as "mycert.pem"
    ca_cert_str: certificate string
    key_str: private key string
  """
  dirname = os.path.dirname(cert_path)
  if dirname and not os.path.exists(dirname):
    os.makedirs(dirname)

  root_path = os.path.splitext(cert_path)[0]
  ca_cert_path = root_path + '-cert.pem'
  android_cer_path = root_path + '-cert.cer'
  windows_p12_path = root_path + '-cert.p12'

  # Dump the CA plus private key
  with open(cert_path, 'w') as f:
    f.write(key_str)
    f.write(ca_cert_str)

  # Dump the certificate in PEM format
  with open(ca_cert_path, 'w') as f:
    f.write(ca_cert_str)

  # Create a .cer file with the same contents for Android
  with open(android_cer_path, 'w') as f:
    f.write(ca_cert_str)

  ca_cert = load_cert(ca_cert_str)
  key = load_privatekey(key_str)
  # Dump the certificate in PKCS12 format for Windows devices
  with open(windows_p12_path, 'w') as f:
    p12 = crypto.PKCS12()
    p12.set_certificate(ca_cert)
    p12.set_privatekey(key)
    f.write(p12.export())

def generate_cert(root_ca_cert_str, server_cert_str, server_host):
  """Generates a cert_str with the sni field in server_cert_str signed by the
  root_ca_cert_str.

  Args:
    root_ca_cert_str: PEM formatted string representing the root cert
    server_cert_str: PEM formatted string representing cert
    server_host: host name to use if there is no server_cert_str
  Returns:
    a PEM formatted certificate string
  """
  if openssl_import_error:
    raise openssl_import_error

  common_name = server_host
  if server_cert_str:
    cert = load_cert(server_cert_str)
    common_name = cert.get_subject().commonName

  ca_cert = load_cert(root_ca_cert_str)
  key = load_privatekey(root_ca_cert_str)

  req = crypto.X509Req()
  subj = req.get_subject()
  subj.CN = common_name
  req.set_pubkey(ca_cert.get_pubkey())
  req.sign(key, 'sha1')

  cert = crypto.X509()
  cert.gmtime_adj_notBefore(-60 * 60)
  cert.gmtime_adj_notAfter(60 * 60 * 24 * 30)
  cert.set_issuer(ca_cert.get_subject())
  cert.set_subject(req.get_subject())
  cert.set_serial_number(int(time.time()*10000))
  cert.set_pubkey(req.get_pubkey())
  cert.sign(key, 'sha1')

  return _dump_cert(cert)
<|endoftext|>