<commit_before># -*- coding: utf-8 -*-

__all__ = ['Command']

import os
import shutil
import json

from .lib import SwaggerParserFactory
from .lib import Printer
from .lib import SDKWriter, DocWriter, CourgetteWriter
from .lib import ModelsProcessor
from .lib import GitManager
from .lib import Utils
from .lib import SwaggerToSpecConverter
from .lib import SpecParser

CODEGEN_DIRECTORY = './codegen'
DOCS_DIRECTORY = './docgen'
SPECGEN_DIRECTORY = './specgen'

API_URL = 'web/docs/api/'

class Command(object):
    """ Command

    """
    @classmethod
    def generate_specs(cls, vsdurl, path, apiversion, output_path=None):
        """ Generate specs

        """
        path = Utils.remove_slash(path)
        url = cls._get_api_url(vsdurl)

        if url is None and path is None:
            Printer.raiseError("Please provide a vsd url or a path to swagger json file")

        # Read Swagger
        swagger_parser = SwaggerParserFactory.create(url=url, path=path, apiversion=apiversion)
        resources = swagger_parser.grab_all()

        # Convert Swagger models
        specs = SwaggerToSpecConverter.convert(resources=resources)

        if not output_path:
            output_path = SPECGEN_DIRECTORY

        if not os.path.exists(output_path):
            os.makedirs(output_path)

        for name, spec in specs.iteritems():
            file_path = '%s/%s.spec' % (output_path, name.lower())
            with open(file_path, 'wb') as file:
                json.dump(spec, file, indent=4)

    @classmethod
    def generate_sdk(cls, vsdurl, path, apiversion, revision, git_repository, output_path=None, push=False, force_removal=False, specs_path=None):
        """ Generate the Python SDK according to given parameters

            It will generate a new SDK from vanilla/vsdk sources or update the targeted repository.

            Args:
                vsdurl: the url to the vsd api
                apiversion: the version of the vsd api in a dotted notation (ex: 3.0)
                revision: the revision number
                git_repository: the GIT repository URL from where to start
                output_path: the path to the output directory
                push: boolean to ask for the push or not
                force: force removal of the existing codegen directory

        """
        path = Utils.remove_slash(path)
        url = cls._get_api_url(vsdurl)

        if url is None and path is None:
            Printer.raiseError("Please provide a vsd url or a path to swagger json file")

        # Read Swagger
        swagger_parser = SwaggerParserFactory.create(url=url, path=path, apiversion=apiversion)
        resources = swagger_parser.grab_all()

        # Convert Swagger models
        specs = SwaggerToSpecConverter.convert(resources=resources)

        if specs_path is not None:
            candidates = SpecParser.grab_all(specs_path)
            specs.update(candidates)

        # Process Swagger models
        processed_resources = ModelsProcessor.process(resources=specs)

        # Compute output directory according to the version
        if apiversion is None:
            apiversion = swagger_parser.apiversion

        if output_path:
            directory = '%s/%s' % (output_path, apiversion)
        else:
            directory = '%s/%s' % (CODEGEN_DIRECTORY, apiversion)

        if force_removal and os.path.exists(directory):
            shutil.rmtree(directory)

        # Grab existing sources
        git_manager = None

        if git_repository:
            git_manager = GitManager(url=git_repository, branch=apiversion, directory=directory)

        # Write Python sources
        sdk_writer = SDKWriter(directory=directory)
        sdk_writer.write(resources=processed_resources, apiversion=apiversion, revision=revision)

        if git_manager and push:
            nb_diffs = git_manager.commit(message="Update from API %s" % apiversion)
            Printer.log("Ready to push %s modification to branch %s of repistory %s" % (nb_diffs, apiversion, git_repository))
            git_manager.push()

    @classmethod
    def generate_doc(cls, vsdurl, path, apiversion, output_path=None):
        """ Generate the Python SDK according to given parameters

            It will generate a new SDK from vanilla/vsdk sources or update the targeted repository.

            Args:
                vsdurl: the url to the vsd api
                apiversion: the version of the vsd api in a dotted notation (ex: 3.0)
                output_path: the path to the output directory

        """
        path = Utils.remove_slash(path)
        url = cls._get_api_url(vsdurl)

        # Read Swagger
        swagger_parser = SwaggerParserFactory.create(url=url, apiversion=apiversion, path=path)
        resources = swagger_parser.grab_all()

        # Convert Swagger models
        specs = SwaggerToSpecConverter.convert(resources=resources)

        # Process Swagger models
        processed_resources = ModelsProcessor.process(resources=specs)

        # Compute output directory according to the version
        if apiversion is None:
            apiversion = swagger_parser.apiversion

        if output_path:
            directory = '%s/%s' % (output_path, apiversion)
        else:
            directory = '%s/%s' % (DOCS_DIRECTORY, apiversion)

        # Write Python sources
        doc_writer = DocWriter(directory=directory)
        doc_writer.write(resources=processed_resources, apiversion=apiversion)

    @classmethod
    def generate_courgette(cls, vsdurl, apiversion, output_path, path=None):
        """ Generate Courgette sources

            It will generate all the environments and tests for the Courgette Framework.

            Args:
                vsdurl: the url to the vsd api
                apiversion: the version of the vsd api in a dotted notation (ex: 3.0)
                output_path: the path to the output directory

        """
        directory = '%s' % output_path

        path = Utils.remove_slash(path)
        url = cls._get_api_url(vsdurl)

        # Read Swagger
        swagger_parser = SwaggerParserFactory.create(url=url, apiversion=apiversion, path=path)
        resources = swagger_parser.grab_all()

        # Processed Swagger models
        processed_resources = ModelsProcessor.process(resources=resources)

        writer = CourgetteWriter(directory=directory)
        writer.write(resources=processed_resources)

    @classmethod
    def _get_api_url(cls, vsdurl=None):
        """ Compute API Url according to vsdurl parameter

            Args:
                vsdurl: the vsd url given

        """
        url = Utils.remove_slash(vsdurl)

        if url:
            return '%s/%s' % (vsdurl, API_URL)

        return None
<commit_msg>Added command get_spec<commit_after># -*- coding: utf-8 -*-

__all__ = ['Command']

import os
import shutil
import json

from .lib import SwaggerParserFactory
from .lib import Printer
from .lib import SDKWriter, DocWriter, CourgetteWriter
from .lib import ModelsProcessor
from .lib import GitManager
from .lib import Utils
from .lib import SwaggerToSpecConverter
from .lib import SpecParser

CODEGEN_DIRECTORY = './codegen'
DOCS_DIRECTORY = './docgen'
SPECGEN_DIRECTORY = './specgen'

API_URL = 'web/docs/api/'

class Command(object):
    """ Command

    """

    @classmethod
    def get_spec(cls, vsdurl, apiversion, entity_name, path=None):
        """

        """
        path = Utils.remove_slash(path)
        url = cls._get_api_url(vsdurl)

        if url is None and path is None:
            Printer.raiseError("Please provide a vsd url or a path to swagger json file")

        # Read Swagger
        swagger_parser = SwaggerParserFactory.create(url=url, path=path, apiversion=apiversion)
        resources = swagger_parser.grab_all()

        # Convert Swagger models
        specs = SwaggerToSpecConverter.convert(resources=resources)

        if entity_name in specs:
            return specs[entity_name]

        return None

    @classmethod
    def generate_specs(cls, vsdurl, path, apiversion, output_path=None):
        """ Generate specs

        """
        path = Utils.remove_slash(path)
        url = cls._get_api_url(vsdurl)

        if url is None and path is None:
            Printer.raiseError("Please provide a vsd url or a path to swagger json file")

        # Read Swagger
        swagger_parser = SwaggerParserFactory.create(url=url, path=path, apiversion=apiversion)
        resources = swagger_parser.grab_all()

        # Convert Swagger models
        specs = SwaggerToSpecConverter.convert(resources=resources)

        if not output_path:
            output_path = SPECGEN_DIRECTORY

        if not os.path.exists(output_path):
            os.makedirs(output_path)

        for name, spec in specs.iteritems():
            file_path = '%s/%s.spec' % (output_path, name.lower())
            with open(file_path, 'wb') as file:
                json.dump(spec, file, indent=4)

    @classmethod
    def generate_sdk(cls, vsdurl, path, apiversion, revision, git_repository, output_path=None, push=False, force_removal=False, specs_path=None):
        """ Generate the Python SDK according to given parameters

            It will generate a new SDK from vanilla/vsdk sources or update the targeted repository.

            Args:
                vsdurl: the url to the vsd api
                apiversion: the version of the vsd api in a dotted notation (ex: 3.0)
                revision: the revision number
                git_repository: the GIT repository URL from where to start
                output_path: the path to the output directory
                push: boolean to ask for the push or not
                force: force removal of the existing codegen directory

        """
        path = Utils.remove_slash(path)
        url = cls._get_api_url(vsdurl)

        if url is None and path is None:
            Printer.raiseError("Please provide a vsd url or a path to swagger json file")

        # Read Swagger
        swagger_parser = SwaggerParserFactory.create(url=url, path=path, apiversion=apiversion)
        resources = swagger_parser.grab_all()

        # Convert Swagger models
        specs = SwaggerToSpecConverter.convert(resources=resources)

        if specs_path is not None:
            candidates = SpecParser.grab_all(specs_path)
            specs.update(candidates)

        # Process Swagger models
        processed_resources = ModelsProcessor.process(resources=specs)

        # Compute output directory according to the version
        if apiversion is None:
            apiversion = swagger_parser.apiversion

        if output_path:
            directory = '%s/%s' % (output_path, apiversion)
        else:
            directory = '%s/%s' % (CODEGEN_DIRECTORY, apiversion)

        if force_removal and os.path.exists(directory):
            shutil.rmtree(directory)

        # Grab existing sources
        git_manager = None

        if git_repository:
            git_manager = GitManager(url=git_repository, branch=apiversion, directory=directory)

        # Write Python sources
        sdk_writer = SDKWriter(directory=directory)
        sdk_writer.write(resources=processed_resources, apiversion=apiversion, revision=revision)

        if git_manager and push:
            nb_diffs = git_manager.commit(message="Update from API %s" % apiversion)
            Printer.log("Ready to push %s modification to branch %s of repistory %s" % (nb_diffs, apiversion, git_repository))
            git_manager.push()

    @classmethod
    def generate_doc(cls, vsdurl, path, apiversion, output_path=None):
        """ Generate the Python SDK according to given parameters

            It will generate a new SDK from vanilla/vsdk sources or update the targeted repository.

            Args:
                vsdurl: the url to the vsd api
                apiversion: the version of the vsd api in a dotted notation (ex: 3.0)
                output_path: the path to the output directory

        """
        path = Utils.remove_slash(path)
        url = cls._get_api_url(vsdurl)

        # Read Swagger
        swagger_parser = SwaggerParserFactory.create(url=url, apiversion=apiversion, path=path)
        resources = swagger_parser.grab_all()

        # Convert Swagger models
        specs = SwaggerToSpecConverter.convert(resources=resources)

        # Process Swagger models
        processed_resources = ModelsProcessor.process(resources=specs)

        # Compute output directory according to the version
        if apiversion is None:
            apiversion = swagger_parser.apiversion

        if output_path:
            directory = '%s/%s' % (output_path, apiversion)
        else:
            directory = '%s/%s' % (DOCS_DIRECTORY, apiversion)

        # Write Python sources
        doc_writer = DocWriter(directory=directory)
        doc_writer.write(resources=processed_resources, apiversion=apiversion)

    @classmethod
    def generate_courgette(cls, vsdurl, apiversion, output_path, path=None):
        """ Generate Courgette sources

            It will generate all the environments and tests for the Courgette Framework.

            Args:
                vsdurl: the url to the vsd api
                apiversion: the version of the vsd api in a dotted notation (ex: 3.0)
                output_path: the path to the output directory

        """
        directory = '%s' % output_path

        path = Utils.remove_slash(path)
        url = cls._get_api_url(vsdurl)

        # Read Swagger
        swagger_parser = SwaggerParserFactory.create(url=url, apiversion=apiversion, path=path)
        resources = swagger_parser.grab_all()

        # Processed Swagger models
        processed_resources = ModelsProcessor.process(resources=resources)

        writer = CourgetteWriter(directory=directory)
        writer.write(resources=processed_resources)

    @classmethod
    def _get_api_url(cls, vsdurl=None):
        """ Compute API Url according to vsdurl parameter

            Args:
                vsdurl: the vsd url given

        """
        url = Utils.remove_slash(vsdurl)

        if url:
            return '%s/%s' % (vsdurl, API_URL)

        return None
<|endoftext|>


<commit_before>#import win32traceutil

import traceback
import win32con
import sys
import os
from ctypes import *
import time
import new

import _mshtml

import win32con

from ctypes import *
from ctypes.wintypes import *
import comtypes
from comtypes import IUnknown
from comtypes.automation import IDispatch, VARIANT
from comtypes.client import wrap, GetModule
from comtypes.client.dynamic import Dispatch

#from win32com.client import *
#cast = gencache.GetModuleForProgID('htmlfile')

if not hasattr(sys, 'frozen'):
    GetModule('atl.dll')
    GetModule('shdocvw.dll')

kernel32 = windll.kernel32
user32 = windll.user32
atl = windll.atl                  # If this fails, you need atl.dll

import win32con
from ctypes import *
from comtypes import IUnknown
from comtypes.automation import VARIANT
#from comtypes.client import GetEvents, ShowEvents
import mshtmlevents
from comtypes.gen import SHDocVw
from comtypes.gen import MSHTML

kernel32 = windll.kernel32
user32 = windll.user32

WNDPROC = WINFUNCTYPE(c_long, c_int, c_uint, c_int, c_int)

class WNDCLASS(Structure):
    _fields_ = [('style', c_uint),
                ('lpfnWndProc', WNDPROC),
                ('cbClsExtra', c_int),
                ('cbWndExtra', c_int),
                ('hInstance', c_int),
                ('hIcon', c_int),
                ('hCursor', c_int),
                ('hbrBackground', c_int),
                ('lpszMenuName', c_char_p),
                ('lpszClassName', c_char_p)]

class RECT(Structure):
    _fields_ = [('left', c_long),
                ('top', c_long),
                ('right', c_long),
                ('bottom', c_long)]

class PAINTSTRUCT(Structure):
    _fields_ = [('hdc', c_int),
                ('fErase', c_int),
                ('rcPaint', RECT),
                ('fRestore', c_int),
                ('fIncUpdate', c_int),
                ('rgbReserved', c_char * 32)]

class POINT(Structure):
    _fields_ = [('x', c_long),
                ('y', c_long)]

class MSG(Structure):
    _fields_ = [('hwnd', c_int),
                ('message', c_uint),
                ('wParam', c_int),
                ('lParam', c_int),
                ('time', c_int),
                ('pt', POINT)]

def ErrorIfZero(handle):
    if handle == 0:
        raise WinError
    else:
        return handle

def _createDiv(doc):
    div = doc.createElement('div')
    print div
    style = div.style
    print style
    #style2 = style.QueryInterface(MSHTML.IHTMLStyle2)
    #style3 = style.QueryInterface(MSHTML.IHTMLStyle3)
    style.position = 'absolute'
    print "pos", style.position
    style.top = 0
    print "top", style.top
    sys.stdout.flush()
    print "left before", style.left
    style.left = 0
    print "left", style.left
    style.overflow = 'scroll'
    print "overflow", style.overflow
    style.overflowX = 'hidden'
    print "overflowX", style.overflowX
    style.width = 300
    print "width", style.width
    style.height = 100
    print "height", style.height
    style.scrollbarBaseColor = '#3366CC'
    style.borderBottom = '2px solid black'
    style.scrollbarHighlightColor = '#99CCFF'
    style.scrollbarArrowColor = 'white'
    div.innerHTML = 'Hello'
    return div

#b = Dispatch('InternetExplorer.Application')
#b.Navigate('about:
Hello
')
#b.Visible = 1
#doc1 = cast.IHTMLDocument2(b.Document)
#header = doc1.all.item('header')
#frm = doc1.all.item('frm')
#frm2 = doc1.frames('frm')
#doc2 = cast.IHTMLDocument2(frm2.document)
#div = doc2.createElement('div')
#cast.DispHTMLBody(doc2.body).appendChild(div)

#popup = cast.DispHTMLWindow2(doc1.parentWindow).createPopup()
#doc3 = cast.IHTMLDocument2(popup.document)
#body = cast.DispHTMLBody(doc3.body)
#div = _createDiv()

class EventSink(object):
    # some DWebBrowserEvents
    def OnVisible(self, this, *args):
        print "OnVisible", args

    def BeforeNavigate(self, this, *args):
        print "BeforeNavigate", args

    def NavigateComplete(self, this, *args):
        print "NavigateComplete", this, args
        return

    # some DWebBrowserEvents2
    def BeforeNavigate2(self, this, *args):
        print "BeforeNavigate2", args

    def NavigateComplete2(self, this, *args):
        print "NavigateComplete2", args

    def DocumentComplete(self, this, *args):
        print "DocumentComplete", args
        if self.workaround_ignore_first_doc_complete == False:
            # ignore first about:blank.  *sigh*...
            # TODO: work out how to parse *args byref VARIANT
            # in order to get at the URI.
            self.workaround_ignore_first_doc_complete = True
            return

        self._loaded()

    def NewWindow2(self, this, *args):
        print "NewWindow2", args
        return
        v = cast(args[1]._.c_void_p, POINTER(VARIANT))[0]
        v.value = True

    def NewWindow3(self, this, *args):
        print "NewWindow3", args
        return
        v = cast(args[1]._.c_void_p, POINTER(VARIANT))[0]
        v.value = True

def addWindowEventListener(self, event_name, cb):
    #print self, event_name, cb
    if cb not in self._callbacks:
        self.connect("browser-event", cb)
        self._callbacks.append(cb)
    return self.addWindowEventListener(event_name, True)

def addXMLHttpRequestEventListener(element, event_name, cb):
    if not hasattr(element, "_callbacks"):
        element._callbacks = []
    if cb not in element._callbacks:
        element.connect("browser-event", cb)
        element._callbacks.append(cb)
    return element.addEventListener(event_name)

def addEventListener(element, event_name, cb):
    if not hasattr(element, "_callbacks"):
        element._callbacks = []
    if cb not in element._callbacks:
        element.connect("browser-event", cb)
        element._callbacks.append(cb)
    return element.addEventListener(event_name, True)

fn_txt = """\
def event_fn(self, *args):
    print "event %s", self, args, dir(self)
    print "event callbacks", self._listeners
    callbacks = self._listeners.get('%s', [])
    for fn in callbacks:
        try:
            fn(self._sender, Dispatch(args[0]), True)
        except:
            sys.stderr.write( traceback.print_exc() )
            sys.stderr.flush()
"""

class EventCaller:
    def __init__(self, handler, name):
        self.handler = handler
        self.name = name
    def __call__(self, *args):
        callbacks = self.handler._listeners.get(self.name, [])
        print "event", self.name, callbacks
        for fn in callbacks:
            try:
                fn(self.handler._sender, Dispatch(args[0]), True)
            except:
                print traceback.print_stack()

class EventHandler(object):
    def __init__(self, sender):
        self._sender = sender
        self._listeners = {}
    def __getattr__(self, name):
        if name.startswith('__') and name.endswith('__'):
            raise AttributeError(name)
        print "EventHandler requested ", name
        if name.startswith('_') or name == 'addEventListener':
            return self.__dict__[name]
        idx = name.find('_on')
        if idx >= 0:
            if idx > 0:
                name = name[idx+1:]
            #return EventCaller(self, name)
            exec fn_txt % (name[2:], name[2:])
            print event_fn
            return new.instancemethod(event_fn, self)
        raise AttributeError(name)

    def addEventListener(self, name, fn):
        if not self._listeners.has_key(name):
            self._listeners[name] = []
        self._listeners[name].append(fn)

class Browser(EventSink):
    def __init__(self, application, appdir):
        EventSink.__init__(self)
        self.platform = 'mshtml'
        self.application = application
        self.appdir = appdir
        self.already_initialised = False
        self.workaround_ignore_first_doc_complete = False
        self.window_handler = None
        self.node_handlers = {}

        CreateWindowEx = windll.user32.CreateWindowExA
        CreateWindowEx.argtypes = [c_int, c_char_p, c_char_p, c_int, c_int, c_int, c_int, c_int, c_int, c_int, c_int, c_int]
        CreateWindowEx.restype = ErrorIfZero

        # Create an instance of IE via AtlAxWin.
        atl.AtlAxWinInit()
        hInstance = kernel32.GetModuleHandleA(None)

        self.hwnd = CreateWindowEx(0,
                              "AtlAxWin",
                              "about:blank",
                              win32con.WS_OVERLAPPEDWINDOW |
                              win32con.WS_VISIBLE |
                              win32con.WS_HSCROLL | win32con.WS_VSCROLL,
                              win32con.CW_USEDEFAULT,
                              win32con.CW_USEDEFAULT,
                              win32con.CW_USEDEFAULT,
                              win32con.CW_USEDEFAULT,
                              win32con.NULL,
                              win32con.NULL,
                              hInstance,
                              win32con.NULL)

        # Get the IWebBrowser2 interface for the IE control.
        self.pBrowserUnk = POINTER(IUnknown)()
        atl.AtlAxGetControl(self.hwnd, byref(self.pBrowserUnk))
        # the wrap call querys for the default interface
        self.pBrowser = wrap(self.pBrowserUnk)
        self.pBrowser.RegisterAsBrowser = True
        self.pBrowser.AddRef()

        self.conn = mshtmlevents.GetEvents(self.pBrowser, sink=self,
                        interface=SHDocVw.DWebBrowserEvents2)

    def _alert(self, txt):
        print "TODO: alert", txt

    def load_app(self):

        uri = self.application
        if uri.find(":") == -1:
            # assume file
            uri = 'file://'+os.path.abspath(uri)

        print "load_app", uri

        self.application = uri
        v = byref(VARIANT())
        self.pBrowser.Navigate(uri, v, v, v, v)

        # Show Window
        cw = c_int(self.hwnd)
        windll.user32.ShowWindow(cw, c_int(win32con.SW_SHOWNORMAL))
        windll.user32.UpdateWindow(cw)

    def getDomDocument(self):
        return Dispatch(self.pBrowser.Document)

    def getDomWindow(self):
        return self.getDomDocument().parentWindow

    def _addXMLHttpRequestEventListener(self, node, event_name, event_fn):

        return None
        listener = xpcom.server.WrapObject(ContentInvoker(node, event_fn),
                                            interfaces.nsIDOMEventListener)
        print event_name, listener
        node.addEventListener(event_name, listener, False)
        return listener

    def addEventListener(self, node, event_name, event_fn):

        rcvr = mshtmlevents.GetDispEventReceiver(MSHTML.HTMLElementEvents2, event_fn, "on%s" % event_name)
        rcvr.sender = node
        ifc = rcvr.QueryInterface(IDispatch)
        node.attachEvent("on%s" % event_name, ifc)
        return ifc

    def mash_attrib(self, attrib_name):
        return attrib_name

    def _addWindowEventListener(self, event_name, event_fn):

        print "_addWindowEventListener", event_name, event_fn
        #rcvr = mshtmlevents.GetDispEventReceiver(MSHTML.HTMLWindowEvents,
        #                   event_fn, "on%s" % event_name)
        #print rcvr
        #rcvr.sender = self.getDomWindow()
        #print rcvr.sender
        #ifc = rcvr.QueryInterface(IDispatch)
        #print ifc
        #v = VARIANT(ifc)
        #print v
        #setattr(self.getDomWindow(), "on%s" % event_name, v)
        #return ifc

        wnd = self.pBrowser.Document.parentWindow
        if self.window_handler is None:
            self.window_handler = EventHandler(self)
            self.window_conn = mshtmlevents.GetEvents(wnd,
                                        sink=self.window_handler,
                                    interface=MSHTML.HTMLWindowEvents2)
        self.window_handler.addEventListener(event_name, event_fn)
        return event_name # hmmm...

    def getXmlHttpRequest(self):
        xml_svc_cls = components.classes[ \
            "@mozilla.org/xmlextras/xmlhttprequest;1"]
        return xml_svc_cls.createInstance(interfaces.nsIXMLHttpRequest)

    def getUri(self):
        return self.application

    def _loaded(self):

        print "loaded"

        if self.already_initialised:
            return
        self.already_initialised = True

        from __pyjamas__ import pygwt_processMetas, set_main_frame
        #from __pyjamas__ import set_gtk_module
        set_main_frame(self)
        #set_gtk_module(gtk)

        (pth, app) = os.path.split(self.application)
        if self.appdir:
            pth = os.path.abspath(self.appdir)
        sys.path.append(pth)

def MainWin(one_event):

    # Pump Messages
    msg = MSG()
    pMsg = pointer(msg)
    NULL = c_int(win32con.NULL)

    while windll.user32.GetMessageA( pMsg, NULL, 0, 0) != 0:

        windll.user32.TranslateMessage(pMsg)
        windll.user32.DispatchMessageA(pMsg)

        #print msg.message, msg.wParam, msg.lParam
        #if msg.message == 161: # win32con.WM_DESTROY:
        #    windll.user32.PostQuitMessage(0)

        if one_event:
            break

    return msg.wParam

class ContentInvoker:

    def __init__(self, node, event_fn):
        self._node = node
        self._event_fn = event_fn

    def handleEvent(self, event):
        self._event_fn(self._node, event, False)

global wv
wv = None

def is_loaded():
    return wv.already_initialised

def run(one_event=False, block=True):
    try:
        MainWin(one_event) # TODO: ignore block arg for now
    except:
        sys.stderr.write( traceback.print_exc() )
        sys.stderr.flush()

def setup(application, appdir=None, width=800, height=600):

    global wv
    wv = Browser(application, appdir)

    wv.load_app()

    while 1:
        if is_loaded():
            return
        run(one_event=True)

<commit_msg>quit on unload signal. yaay!;<commit_after>#import win32traceutil

import traceback
import win32con
import sys
import os
from ctypes import *
import time
import new

import _mshtml

import win32con

from ctypes import *
from ctypes.wintypes import *
import comtypes
from comtypes import IUnknown
from comtypes.automation import IDispatch, VARIANT
from comtypes.client import wrap, GetModule
from comtypes.client.dynamic import Dispatch

#from win32com.client import *
#cast = gencache.GetModuleForProgID('htmlfile')

if not hasattr(sys, 'frozen'):
    GetModule('atl.dll')
    GetModule('shdocvw.dll')

kernel32 = windll.kernel32
user32 = windll.user32
atl = windll.atl                  # If this fails, you need atl.dll

import win32con
from ctypes import *
from comtypes import IUnknown
from comtypes.automation import VARIANT
#from comtypes.client import GetEvents, ShowEvents
import mshtmlevents
from comtypes.gen import SHDocVw
from comtypes.gen import MSHTML

kernel32 = windll.kernel32
user32 = windll.user32

WNDPROC = WINFUNCTYPE(c_long, c_int, c_uint, c_int, c_int)

class WNDCLASS(Structure):
    _fields_ = [('style', c_uint),
                ('lpfnWndProc', WNDPROC),
                ('cbClsExtra', c_int),
                ('cbWndExtra', c_int),
                ('hInstance', c_int),
                ('hIcon', c_int),
                ('hCursor', c_int),
                ('hbrBackground', c_int),
                ('lpszMenuName', c_char_p),
                ('lpszClassName', c_char_p)]

class RECT(Structure):
    _fields_ = [('left', c_long),
                ('top', c_long),
                ('right', c_long),
                ('bottom', c_long)]

class PAINTSTRUCT(Structure):
    _fields_ = [('hdc', c_int),
                ('fErase', c_int),
                ('rcPaint', RECT),
                ('fRestore', c_int),
                ('fIncUpdate', c_int),
                ('rgbReserved', c_char * 32)]

class POINT(Structure):
    _fields_ = [('x', c_long),
                ('y', c_long)]

class MSG(Structure):
    _fields_ = [('hwnd', c_int),
                ('message', c_uint),
                ('wParam', c_int),
                ('lParam', c_int),
                ('time', c_int),
                ('pt', POINT)]

def ErrorIfZero(handle):
    if handle == 0:
        raise WinError
    else:
        return handle

def _createDiv(doc):
    div = doc.createElement('div')
    print div
    style = div.style
    print style
    #style2 = style.QueryInterface(MSHTML.IHTMLStyle2)
    #style3 = style.QueryInterface(MSHTML.IHTMLStyle3)
    style.position = 'absolute'
    print "pos", style.position
    style.top = 0
    print "top", style.top
    sys.stdout.flush()
    print "left before", style.left
    style.left = 0
    print "left", style.left
    style.overflow = 'scroll'
    print "overflow", style.overflow
    style.overflowX = 'hidden'
    print "overflowX", style.overflowX
    style.width = 300
    print "width", style.width
    style.height = 100
    print "height", style.height
    style.scrollbarBaseColor = '#3366CC'
    style.borderBottom = '2px solid black'
    style.scrollbarHighlightColor = '#99CCFF'
    style.scrollbarArrowColor = 'white'
    div.innerHTML = 'Hello'
    return div

#b = Dispatch('InternetExplorer.Application')
#b.Navigate('about:
Hello
')
#b.Visible = 1
#doc1 = cast.IHTMLDocument2(b.Document)
#header = doc1.all.item('header')
#frm = doc1.all.item('frm')
#frm2 = doc1.frames('frm')
#doc2 = cast.IHTMLDocument2(frm2.document)
#div = doc2.createElement('div')
#cast.DispHTMLBody(doc2.body).appendChild(div)

#popup = cast.DispHTMLWindow2(doc1.parentWindow).createPopup()
#doc3 = cast.IHTMLDocument2(popup.document)
#body = cast.DispHTMLBody(doc3.body)
#div = _createDiv()

class EventSink(object):
    # some DWebBrowserEvents
    def OnVisible(self, this, *args):
        print "OnVisible", args

    def BeforeNavigate(self, this, *args):
        print "BeforeNavigate", args

    def NavigateComplete(self, this, *args):
        print "NavigateComplete", this, args
        return

    # some DWebBrowserEvents2
    def BeforeNavigate2(self, this, *args):
        print "BeforeNavigate2", args

    def NavigateComplete2(self, this, *args):
        print "NavigateComplete2", args

    def DocumentComplete(self, this, *args):
        print "DocumentComplete", args
        if self.workaround_ignore_first_doc_complete == False:
            # ignore first about:blank.  *sigh*...
            # TODO: work out how to parse *args byref VARIANT
            # in order to get at the URI.
            self.workaround_ignore_first_doc_complete = True
            return

        self._loaded()

    def NewWindow2(self, this, *args):
        print "NewWindow2", args
        return
        v = cast(args[1]._.c_void_p, POINTER(VARIANT))[0]
        v.value = True

    def NewWindow3(self, this, *args):
        print "NewWindow3", args
        return
        v = cast(args[1]._.c_void_p, POINTER(VARIANT))[0]
        v.value = True

def addWindowEventListener(self, event_name, cb):
    #print self, event_name, cb
    if cb not in self._callbacks:
        self.connect("browser-event", cb)
        self._callbacks.append(cb)
    return self.addWindowEventListener(event_name, True)

def addXMLHttpRequestEventListener(element, event_name, cb):
    if not hasattr(element, "_callbacks"):
        element._callbacks = []
    if cb not in element._callbacks:
        element.connect("browser-event", cb)
        element._callbacks.append(cb)
    return element.addEventListener(event_name)

def addEventListener(element, event_name, cb):
    if not hasattr(element, "_callbacks"):
        element._callbacks = []
    if cb not in element._callbacks:
        element.connect("browser-event", cb)
        element._callbacks.append(cb)
    return element.addEventListener(event_name, True)

fn_txt = """\
def event_fn(self, *args):
    print "event %s", self, args, dir(self)
    print "event callbacks", self._listeners
    callbacks = self._listeners.get('%s', [])
    for fn in callbacks:
        try:
            fn(self._sender, Dispatch(args[0]), True)
        except:
            sys.stderr.write( traceback.print_exc() )
            sys.stderr.flush()
"""

class EventCaller:
    def __init__(self, handler, name):
        self.handler = handler
        self.name = name
    def __call__(self, *args):
        callbacks = self.handler._listeners.get(self.name, [])
        print "event", self.name, callbacks
        for fn in callbacks:
            try:
                fn(self.handler._sender, Dispatch(args[0]), True)
            except:
                print traceback.print_stack()

class EventHandler(object):
    def __init__(self, sender):
        self._sender = sender
        self._listeners = {}
    def __getattr__(self, name):
        if name.startswith('__') and name.endswith('__'):
            raise AttributeError(name)
        print "EventHandler requested ", name
        if name.startswith('_') or name == 'addEventListener':
            return self.__dict__[name]
        idx = name.find('_on')
        if idx >= 0:
            if idx > 0:
                name = name[idx+1:]
            #return EventCaller(self, name)
            exec fn_txt % (name[2:], name[2:])
            print event_fn
            return new.instancemethod(event_fn, self)
        raise AttributeError(name)

    def addEventListener(self, name, fn):
        if not self._listeners.has_key(name):
            self._listeners[name] = []
        self._listeners[name].append(fn)

class Browser(EventSink):
    def __init__(self, application, appdir):
        EventSink.__init__(self)
        self.platform = 'mshtml'
        self.application = application
        self.appdir = appdir
        self.already_initialised = False
        self.workaround_ignore_first_doc_complete = False
        self.window_handler = None
        self.node_handlers = {}

        CreateWindowEx = windll.user32.CreateWindowExA
        CreateWindowEx.argtypes = [c_int, c_char_p, c_char_p, c_int, c_int, c_int, c_int, c_int, c_int, c_int, c_int, c_int]
        CreateWindowEx.restype = ErrorIfZero

        # Create an instance of IE via AtlAxWin.
        atl.AtlAxWinInit()
        hInstance = kernel32.GetModuleHandleA(None)

        self.hwnd = CreateWindowEx(0,
                              "AtlAxWin",
                              "about:blank",
                              win32con.WS_OVERLAPPEDWINDOW |
                              win32con.WS_VISIBLE |
                              win32con.WS_HSCROLL | win32con.WS_VSCROLL,
                              win32con.CW_USEDEFAULT,
                              win32con.CW_USEDEFAULT,
                              win32con.CW_USEDEFAULT,
                              win32con.CW_USEDEFAULT,
                              win32con.NULL,
                              win32con.NULL,
                              hInstance,
                              win32con.NULL)

        # Get the IWebBrowser2 interface for the IE control.
        self.pBrowserUnk = POINTER(IUnknown)()
        atl.AtlAxGetControl(self.hwnd, byref(self.pBrowserUnk))
        # the wrap call querys for the default interface
        self.pBrowser = wrap(self.pBrowserUnk)
        self.pBrowser.RegisterAsBrowser = True
        self.pBrowser.AddRef()

        self.conn = mshtmlevents.GetEvents(self.pBrowser, sink=self,
                        interface=SHDocVw.DWebBrowserEvents2)

    def _alert(self, txt):
        print "TODO: alert", txt

    def load_app(self):

        uri = self.application
        if uri.find(":") == -1:
            # assume file
            uri = 'file://'+os.path.abspath(uri)

        print "load_app", uri

        self.application = uri
        v = byref(VARIANT())
        self.pBrowser.Navigate(uri, v, v, v, v)

        # Show Window
        cw = c_int(self.hwnd)
        windll.user32.ShowWindow(cw, c_int(win32con.SW_SHOWNORMAL))
        windll.user32.UpdateWindow(cw)

    def getDomDocument(self):
        return Dispatch(self.pBrowser.Document)

    def getDomWindow(self):
        return self.getDomDocument().parentWindow

    def _addXMLHttpRequestEventListener(self, node, event_name, event_fn):

        return None
        listener = xpcom.server.WrapObject(ContentInvoker(node, event_fn),
                                            interfaces.nsIDOMEventListener)
        print event_name, listener
        node.addEventListener(event_name, listener, False)
        return listener

    def addEventListener(self, node, event_name, event_fn):

        rcvr = mshtmlevents.GetDispEventReceiver(MSHTML.HTMLElementEvents2, event_fn, "on%s" % event_name)
        rcvr.sender = node
        ifc = rcvr.QueryInterface(IDispatch)
        node.attachEvent("on%s" % event_name, ifc)
        return ifc

    def mash_attrib(self, attrib_name):
        return attrib_name

    def _addWindowEventListener(self, event_name, event_fn):

        print "_addWindowEventListener", event_name, event_fn
        #rcvr = mshtmlevents.GetDispEventReceiver(MSHTML.HTMLWindowEvents,
        #                   event_fn, "on%s" % event_name)
        #print rcvr
        #rcvr.sender = self.getDomWindow()
        #print rcvr.sender
        #ifc = rcvr.QueryInterface(IDispatch)
        #print ifc
        #v = VARIANT(ifc)
        #print v
        #setattr(self.getDomWindow(), "on%s" % event_name, v)
        #return ifc

        wnd = self.pBrowser.Document.parentWindow
        if self.window_handler is None:
            self.window_handler = EventHandler(self)
            self.window_conn = mshtmlevents.GetEvents(wnd,
                                        sink=self.window_handler,
                                    interface=MSHTML.HTMLWindowEvents2)
        self.window_handler.addEventListener(event_name, event_fn)
        return event_name # hmmm...

    def getXmlHttpRequest(self):
        xml_svc_cls = components.classes[ \
            "@mozilla.org/xmlextras/xmlhttprequest;1"]
        return xml_svc_cls.createInstance(interfaces.nsIXMLHttpRequest)

    def getUri(self):
        return self.application

    def _loaded(self):

        print "loaded"

        if self.already_initialised:
            return
        self.already_initialised = True

        from __pyjamas__ import pygwt_processMetas, set_main_frame
        #from __pyjamas__ import set_gtk_module
        set_main_frame(self)
        #set_gtk_module(gtk)

        (pth, app) = os.path.split(self.application)
        if self.appdir:
            pth = os.path.abspath(self.appdir)
        sys.path.append(pth)

        self._addWindowEventListener("unload", self.on_unload_callback)

    def on_unload_callback(self, *args):
        windll.user32.PostQuitMessage(0)

def MainWin(one_event):

    # Pump Messages
    msg = MSG()
    pMsg = pointer(msg)
    NULL = c_int(win32con.NULL)

    while 1:
        res = windll.user32.GetMessageA( pMsg, NULL, 0, 0)
        if res == -1:
            return 0
        if res == 0:
            break

        windll.user32.TranslateMessage(pMsg)
        windll.user32.DispatchMessageA(pMsg)

        #print msg.message, msg.wParam, msg.lParam
        #if msg.message == 161: # win32con.WM_DESTROY:
        #    windll.user32.PostQuitMessage(0)

        if one_event:
            break

    return msg.wParam

class ContentInvoker:

    def __init__(self, node, event_fn):
        self._node = node
        self._event_fn = event_fn

    def handleEvent(self, event):
        self._event_fn(self._node, event, False)

global wv
wv = None

def is_loaded():
    return wv.already_initialised

def run(one_event=False, block=True):
    try:
        MainWin(one_event) # TODO: ignore block arg for now
    except:
        sys.stderr.write( traceback.print_exc() )
        sys.stderr.flush()

def setup(application, appdir=None, width=800, height=600):

    global wv
    wv = Browser(application, appdir)

    wv.load_app()

    while 1:
        if is_loaded():
            return
        run(one_event=True)

<|endoftext|>


<commit_before>import base64

import requests
import yaml

from .exceptions import KubernetesError

class APIClient(object):

    def __init__(self, kubeconfig_path, cluster_name, user_name, url=None, namespace="default", version="v1"):
        self.kubeconfig_path = kubeconfig_path
        self.cluster_name = cluster_name
        self.user_name = user_name
        self.url = url
        self.namespace = namespace
        self.version = version

        self.session = self.build_session()

    def build_session(self):
        with open(self.kubeconfig_path, "rb") as fp:
            kubeconfig = yaml.load(fp.read())

        # prepare the CA bundle
        for cluster in kubeconfig["clusters"]:
            if cluster["name"] == self.cluster_name:
                break
        else:
            raise KubernetesError("cannot find cluster `{}` cluster in {}".format(self.cluster_name, self.kubeconfig_path))
        # @@@ temporary file? need to investigate when this is opened and read by urllib3
        ca_cert_file = "/tmp/k8s-ca-cert.pem"
        with open(ca_cert_file, "wb") as fp:
            fp.write(base64.b64decode(cluster["cluster"]["certificate-authority-data"]))

        # set URL if needed
        if self.url is None:
            self.url = cluster["cluster"]["server"]

        # prepare client certificate
        for user in kubeconfig["users"]:
            if user["name"] == self.user_name:
                break
        else:
            raise KubernetesError("cannot find user `{}` cluster in {}".format(self.user_name, self.kubeconfig_path))
        # @@@ temporary files? need to investigate when these are opened and read by urllib3
        client_key_file = "/tmp/k8s-client-key.pem"
        client_cert_file = "/tmp/k8s-client-cert.pem"
        with open(client_key_file, "wb") as fp:
            fp.write(base64.b64decode(user["user"]["client-key-data"]))
        with open(client_cert_file, "wb") as fp:
            fp.write(base64.b64decode(user["user"]["client-certificate-data"]))

        # prepare requests session object
        s = requests.Session()
        s.verify = ca_cert_file
        s.cert = (client_cert_file, client_key_file)
        return s

    def get_kwargs(self, **kwargs):
        kwargs["url"] = "{}/api/{}/namespaces/{}{}".format(
            self.url,
            self.version,
            self.namespace,
            kwargs.get("url", "")
        )
        return kwargs

    def request(self, *args, **kwargs):
        return self.session.request(*args, **self.get_kwargs(**kwargs))

    def get(self, *args, **kwargs):
        return self.session.get(*args, **self.get_kwargs(**kwargs))

    def options(self, *args, **kwargs):
        return self.session.options(*args, **self.get_kwargs(**kwargs))

    def head(self, *args, **kwargs):
        return self.session.head(*args, **self.get_kwargs(**kwargs))

    def post(self, *args, **kwargs):
        return self.session.post(*args, **self.get_kwargs(**kwargs))

    def put(self, *args, **kwargs):
        return self.session.put(*args, **self.get_kwargs(**kwargs))

    def patch(self, *args, **kwargs):
        return self.session.patch(*args, **self.get_kwargs(**kwargs))

    def delete(self, *args, **kwargs):
        return self.session.delete(*args, **self.get_kwargs(**kwargs))
<commit_msg>renamed APIClient<commit_after>import base64

import requests
import yaml

from .exceptions import KubernetesError

class HTTPClient(object):

    def __init__(self, kubeconfig_path, cluster_name, user_name, url=None, namespace="default", version="v1"):
        self.kubeconfig_path = kubeconfig_path
        self.cluster_name = cluster_name
        self.user_name = user_name
        self.url = url
        self.namespace = namespace
        self.version = version

        self.session = self.build_session()

    def build_session(self):
        with open(self.kubeconfig_path, "rb") as fp:
            kubeconfig = yaml.load(fp.read())

        # prepare the CA bundle
        for cluster in kubeconfig["clusters"]:
            if cluster["name"] == self.cluster_name:
                break
        else:
            raise KubernetesError("cannot find cluster `{}` cluster in {}".format(self.cluster_name, self.kubeconfig_path))
        # @@@ temporary file? need to investigate when this is opened and read by urllib3
        ca_cert_file = "/tmp/k8s-ca-cert.pem"
        with open(ca_cert_file, "wb") as fp:
            fp.write(base64.b64decode(cluster["cluster"]["certificate-authority-data"]))

        # set URL if needed
        if self.url is None:
            self.url = cluster["cluster"]["server"]

        # prepare client certificate
        for user in kubeconfig["users"]:
            if user["name"] == self.user_name:
                break
        else:
            raise KubernetesError("cannot find user `{}` cluster in {}".format(self.user_name, self.kubeconfig_path))
        # @@@ temporary files? need to investigate when these are opened and read by urllib3
        client_key_file = "/tmp/k8s-client-key.pem"
        client_cert_file = "/tmp/k8s-client-cert.pem"
        with open(client_key_file, "wb") as fp:
            fp.write(base64.b64decode(user["user"]["client-key-data"]))
        with open(client_cert_file, "wb") as fp:
            fp.write(base64.b64decode(user["user"]["client-certificate-data"]))

        # prepare requests session object
        s = requests.Session()
        s.verify = ca_cert_file
        s.cert = (client_cert_file, client_key_file)
        return s

    def get_kwargs(self, **kwargs):
        kwargs["url"] = "{}/api/{}/namespaces/{}{}".format(
            self.url,
            self.version,
            self.namespace,
            kwargs.get("url", "")
        )
        return kwargs

    def request(self, *args, **kwargs):
        return self.session.request(*args, **self.get_kwargs(**kwargs))

    def get(self, *args, **kwargs):
        return self.session.get(*args, **self.get_kwargs(**kwargs))

    def options(self, *args, **kwargs):
        return self.session.options(*args, **self.get_kwargs(**kwargs))

    def head(self, *args, **kwargs):
        return self.session.head(*args, **self.get_kwargs(**kwargs))

    def post(self, *args, **kwargs):
        return self.session.post(*args, **self.get_kwargs(**kwargs))

    def put(self, *args, **kwargs):
        return self.session.put(*args, **self.get_kwargs(**kwargs))

    def patch(self, *args, **kwargs):
        return self.session.patch(*args, **self.get_kwargs(**kwargs))

    def delete(self, *args, **kwargs):
        return self.session.delete(*args, **self.get_kwargs(**kwargs))
<|endoftext|>


<commit_before>"""
Toolbox for statistical methods.  For all functions, this toolbox assumes that
the first dimension is the temporal sampling dimension.

Author: Andre Perkins
"""

import numpy as np
import numexpr as ne
import dask.array as da
import logging
from scipy.linalg import svd
from scipy.ndimage import convolve1d
from sklearn import linear_model

logger = logging.getLogger(__name__)

def detrend_data(data, output_arr=None):
    """
    Detrend data using a linear fit.

    Parameters
    ----------
    data: ndarray-like
        Input dataset to detrend.  Assumes leading axis is sampling dimension.
    output_arr: ndarray-like, optional
        Output array with same shape as data to store detrended data.
    """

    dummy_time = np.arange(data.shape[0])[:, None]
    model = linear_model.LinearRegression(fit_intercept=False, n_jobs=-1)
    model.fit(dummy_time, data)

    linfit = model.predict(dummy_time)
    detrended = data - linfit

    if output_arr is not None:
        output_arr[:] = detrended
    else:
        output_arr = detrended

    return output_arr

def dask_detrend_data(data, output_arr):
    """
    Detrend data using a linear fit.

    Parameters
    ----------
    data: dask.array
        Input dataset to detrend.  Assumes leading axis is sampling dimension.
    output_arr: ndarray-like
        Output array with same shape as data to store detrended data.

    Notes
    -----
    This is a very expensive operation if using a large dataset.  May slow down
    if forced to spill onto the disk cache
    """
    dummy_time = np.arange(data.shape[0])

    x = dummy_time[:, None]
    y = data
    ymean = data.mean(axis=0)
    xmean = dummy_time.mean()

    xanom = x - xmean
    yanom = y - ymean
    covxy = da.sum(xanom * yanom, axis=0)
    xvar = x.var(axis=0, ddof=1)
    beta = covxy/xvar
    intercept = ymean - xmean*beta

    predict = beta*x + intercept
    detrended = data - predict

    da.store(detrended, output_arr)

    return output_arr

def calc_anomaly(data, yrsize, climo=None, output_arr=None):
    """
    Caculate anomaly for the given data.  Right now it assumes sub-annual data
    input so that the climatology subtracts means for each month instead
    of the mean of the entire series.

    Note: May take yrsize argument out and leave it to user to format data
    as to take the desired anomaly.

    Parameters
    ----------
    data: ndarray
        Input data to calculate the anomaly from.  Leading dimension should be
        the temporal axis.
    yrsize: int
        Number of elements that compose a full year.  Used to reshape the data
        time axis to num years x size year for climatology purposes.
    climo: ndarray, optional
        User-provided climatology to subtract from the data.  Must be
        broadcastable over the time-dimension of data
    output_arr: ndarray-like, optional
        Array to place output of anomaly calculation that supports
        ndarray-like slicing.  This is required for dask array input.
    Returns
    -------
    anomaly: ndarray-like
        Data converted to its anomaly form.
    climo: ndarray
        The calculated climatology that was subtracted from the data
    """

    yrsize = int(yrsize)
    if not yrsize >= 1:
        raise ValueError('yrsize must be an integer >= 1')

    # Reshape to take monthly mean
    old_shp = data.shape
    new_shp = (old_shp[0]//yrsize, yrsize, old_shp[1])
    data = data.reshape(new_shp)

    # Use of data[:] should work for ndarray or ndarray-like
    if climo is None:
        climo = data.mean(axis=0)

    if is_dask_array(data):
        if output_arr is None:
            raise ValueError('calc_anomaly requires an output array keyword '
                             'argument when operating on a Dask array.')

        anomaly = data - climo
        old_shp_anom = anomaly.reshape(old_shp)
        da.store(old_shp_anom, output_arr)
        out_climo = climo.compute()
    else:
        if output_arr is not None:
            output_arr[:] = ne.evaluate('data - climo')
        else:
            output_arr = ne.evaluate('data - climo')
        output_arr = output_arr.reshape(old_shp)
        out_climo = climo

    return output_arr, out_climo

def calc_ce(fcast, trial_obs, obs):
    """
    Method to calculate the Coefficient of Efficiency as defined by Nash and
    Sutcliffe 1970.

    Parameters
    ----------
    fcast: ndarray
        Time series of forecast data. M x N where M is the temporal dimension.
    obs: ndarray
        Time series of observations. M x N

    Returns
    -------
    CE: ndarray
        Coefficient of efficiency for all locations over the time range.
    """

    assert(fcast.shape == trial_obs.shape)

    # Climatological variance
    cvar = obs.var(axis=0, ddof=1)

    # Error variance
    error = ne.evaluate('(trial_obs - fcast)**2')
    evar = error.sum(axis=0)/(len(error))

    return 1 - evar/cvar

def calc_eofs(data, num_eigs, ret_pcs=False, var_stats_dict=None):
    """
    Method to calculate the EOFs of given  dataset.  This assumes data comes in as
    an m x n matrix where m is the temporal dimension and n is the spatial
    dimension.

    Parameters
    ----------
    data: ndarray
        Dataset to calculate EOFs from
    num_eigs: int
        Number of eigenvalues/vectors to return.  Must be less than min(m, n).
    ret_pcs: bool, optional
        Return principal component matrix along with EOFs
    var_stats_dict: dict, optional
        Dictionary target to star some simple statistics about the EOF
        calculation.  Note: if this is provided for a dask array it prompts two
        SVD calculations for both the compressed and full singular values.

    Returns
    -------
    eofs: ndarray
        The eofs (as column vectors) of the data with dimensions n x k where
        k is the num_eigs.
    svals: ndarray
        Singular values from the svd decomposition.  Returned as a row vector
        in order from largest to smallest.
    """

    if is_dask_array(data):
        pcs, full_svals, eofs = da.linalg.svd_compressed(data, num_eigs)

        out_svals = np.zeros(num_eigs)
        out_eofs = np.zeros((num_eigs, data.shape[1]))
        out_pcs = np.zeros((data.shape[0], num_eigs))
        da.store([eofs, full_svals, pcs], [out_eofs, out_svals, out_pcs])

        out_eofs = out_eofs.T
        out_pcs = out_pcs.T

        if var_stats_dict is not None:
            logger.warning('Cannot currently provide variance statistics for '
                           'EOFs computed on a dask array.')

    else:
        eofs, full_svals, pcs = svd(data[:].T, full_matrices=False)
        out_eofs = eofs[:, :num_eigs]
        out_svals = full_svals[:num_eigs]
        out_pcs = pcs[:num_eigs]

    # variance stats
    if var_stats_dict is not None:
        try:
            nt = data.shape[0]
            ns = data.shape[1]
            eig_vals = (full_svals ** 2) / (nt * ns)
            total_var = eig_vals.sum()
            var_expl_by_mode = eig_vals / total_var
            var_expl_by_retained = var_expl_by_mode[0:num_eigs].sum()

            var_stats_dict['nt'] = nt
            var_stats_dict['ns'] = ns
            var_stats_dict['eigvals'] = eig_vals
            var_stats_dict['num_ret_modes'] = num_eigs
            var_stats_dict['total_var'] = total_var
            var_stats_dict['var_expl_by_mode'] = var_expl_by_mode
            var_stats_dict['var_expl_by_ret'] = var_expl_by_retained
        except TypeError as e:
            print 'Must past dictionary type to var_stats_dict in order to ' \
                  'output variance statistics.'
            print e

    if ret_pcs:
        return out_eofs, out_svals, out_pcs
    else:
        return out_eofs, out_svals

def calc_lac(fcast, obs):
    """
    Method to calculate the Local Anomaly Correlation (LAC).  Uses numexpr
    for speed over larger datasets.

    Note: If necessary (memory concerns) in the future, the numexpr statements
    can be extended to use pytable arrays.  Would need to provide means to
    function, as summing over the dataset is still very slow it seems.

    Parameters
    ----------
    fcast: ndarray
        Time series of forecast data. M x N where M is the temporal dimension.
    obs: ndarray
        Time series of observations. M x N

    Returns
    -------
    lac: ndarray
        Local anomaly corellations for all locations over the time range.
    """
    # Calculate means of data
    f_mean = fcast.mean(axis=0)
    o_mean = obs.mean(axis=0)

    # Calculate covariance between time series
    cov = ne.evaluate('(fcast - f_mean) * (obs - o_mean)')
    cov = cov.sum(axis=0)

    # Calculate standarization terms
    f_std = ne.evaluate('(fcast - f_mean)**2')
    f_std = np.sqrt(f_std.sum(axis=0))
    o_std = ne.evaluate('(obs - o_mean)**2')
    o_std = np.sqrt(o_std.sum(axis=0))
    std = f_std * o_std

    return cov / std

def calc_n_eff(data1, data2=None):
    """
    Calculate the effective degrees of freedom for data using lag-1
    autocorrelation.

    Parameters
    ----------
    data1: ndarray
        Dataset to calculate effective degrees of freedom for.  Assumes
        first dimension is the temporal dimension.
    data2: ndarray, optional
        A second dataset to calculate the effective degrees of freedom
        for covariances/correlations etc.

    Returns
    -------
    n_eff: ndarray
        Effective degrees of freedom for input data.
    """

    if data2 is not None:
        assert data1.shape == data2.shape,\
            'Data must have have same shape for combined n_eff calculation'

    # Lag-1 autocorrelation
    r1 = calc_lac(data1[0:-1], data1[1:])
    n = len(data1)

    if data2 is not None:
        r2 = calc_lac(data2[0:-1], data2[1:])
        n_eff = n*((1 - r1*r2)/(1+r1*r2))
    else:
        n_eff = n*((1-r1)/(1+r1))

    return n_eff

def run_mean(data, window_size, trim_edge=None, output_arr=None):
    """
    A function for calculating the running mean on data.

    Parameters
    ----------
    data: ndarray
        Data matrix to perform running mean over. Expected to be in time(row) x
        space(column) format. And that samples span full years.
    window_size: int
        Size of the window to compute the running mean over.
    trim_edge: int, optional
        Remove specified items from the start and end of the sampling
        dimension of the running mean.  Otherwise the window_size/2 items at
        the start and the end will have reflected padding effects.
    output_arr: ndarray-like, optional
        Array to place output of running mean that supports
        ndarray-like slicing.  This is required for dask array input.

    Returns
    -------
    result: ndarray
        Running mean result of given data.
    bot_edge: int
        Number of elements removed from beginning of the time series
    top_edge: int
        Number of elements removed from the ending of the time series
    """

    sample_len = data.shape[0]
    if sample_len < window_size:
        raise ValueError("Window size must be smaller than or equal to the "
                         "length of the time dimension of the data.")

    if trim_edge is not None:
        sample_len -= trim_edge*2

        if sample_len < 1:
            raise ValueError('Not enough data to trim edges. Please try with '
                             'trim_edge=None')

    weights = [1.0/float(window_size) for _ in xrange(window_size)]
    if is_dask_array(data):
        if output_arr is None:
            raise ValueError('calc_anomaly requires an output array keyword '
                             'argument when operating on a Dask array.')

        def _run_mean_block(block):
            return convolve1d(block, weights, axis=0)

        old_chunk_shape = data
        pad = window_size // 2
        ghost = da.ghost.ghost(data, depth={0: pad}, boundary={0: 'reflect'})
        filt = ghost.map_blocks(_run_mean_block)
        unpadded = da.ghost.trim_internal(filt, {0: pad})
        if trim_edge is not None:
            unpadded = unpadded[trim_edge:-trim_edge]

        da.store(unpadded, output_arr)
    else:
        res = convolve1d(data, weights, axis=0)
        if trim_edge:
            res = res[trim_edge:-trim_edge]

        if output_arr is not None:
            output_arr[:] = res
        else:
            output_arr = res

    return output_arr

def is_dask_array(arr):
    return hasattr(arr, 'dask')
<commit_msg>Dask detrend update<commit_after>"""
Toolbox for statistical methods.  For all functions, this toolbox assumes that
the first dimension is the temporal sampling dimension.

Author: Andre Perkins
"""

import numpy as np
import numexpr as ne
import dask.array as da
import logging
from scipy.linalg import svd
from scipy.ndimage import convolve1d
from sklearn import linear_model

logger = logging.getLogger(__name__)

def detrend_data(data, output_arr=None):
    """
    Detrend data using a linear fit.

    Parameters
    ----------
    data: ndarray-like
        Input dataset to detrend.  Assumes leading axis is sampling dimension.
    output_arr: ndarray-like, optional
        Output array with same shape as data to store detrended data.
    """

    dummy_time = np.arange(data.shape[0])[:, None]
    model = linear_model.LinearRegression(fit_intercept=False, n_jobs=-1)
    model.fit(dummy_time, data)

    linfit = model.predict(dummy_time)
    detrended = data - linfit

    if output_arr is not None:
        output_arr[:] = detrended
    else:
        output_arr = detrended

    return output_arr

def dask_detrend_data(data, output_arr):
    """
    Detrend data using a linear fit.

    Parameters
    ----------
    data: dask.array
        Input dataset to detrend.  Assumes leading axis is sampling dimension.
    output_arr: ndarray-like
        Output array with same shape as data to store detrended data.

    Notes
    -----
    This is a very expensive operation if using a large dataset.  May slow down
    if forced to spill onto the disk cache
    """

    dummy_time = np.arange(data.shape[0])[:, None]
    dummy_time = da.from_array(dummy_time, chunks=dummy_time.shape)

    rechunk_shp = (data.shape[0], 1)
    tmp_data = da.rechunk(data, chunks=rechunk_shp)

    coefs, resid, rank, s = da.linalg.lstsq(dummy_time, tmp_data)

    predict = da.dot(dummy_time, coefs)
    detrended = data - predict

    da.store(detrended, output_arr)

    return output_arr

def calc_anomaly(data, yrsize, climo=None, output_arr=None):
    """
    Caculate anomaly for the given data.  Right now it assumes sub-annual data
    input so that the climatology subtracts means for each month instead
    of the mean of the entire series.

    Note: May take yrsize argument out and leave it to user to format data
    as to take the desired anomaly.

    Parameters
    ----------
    data: ndarray
        Input data to calculate the anomaly from.  Leading dimension should be
        the temporal axis.
    yrsize: int
        Number of elements that compose a full year.  Used to reshape the data
        time axis to num years x size year for climatology purposes.
    climo: ndarray, optional
        User-provided climatology to subtract from the data.  Must be
        broadcastable over the time-dimension of data
    output_arr: ndarray-like, optional
        Array to place output of anomaly calculation that supports
        ndarray-like slicing.  This is required for dask array input.
    Returns
    -------
    anomaly: ndarray-like
        Data converted to its anomaly form.
    climo: ndarray
        The calculated climatology that was subtracted from the data
    """

    yrsize = int(yrsize)
    if not yrsize >= 1:
        raise ValueError('yrsize must be an integer >= 1')

    # Reshape to take monthly mean
    old_shp = data.shape
    new_shp = (old_shp[0]//yrsize, yrsize, old_shp[1])
    data = data.reshape(new_shp)

    # Use of data[:] should work for ndarray or ndarray-like
    if climo is None:
        climo = data.mean(axis=0)

    if is_dask_array(data):
        if output_arr is None:
            raise ValueError('calc_anomaly requires an output array keyword '
                             'argument when operating on a Dask array.')

        anomaly = data - climo
        old_shp_anom = anomaly.reshape(old_shp)
        da.store(old_shp_anom, output_arr)
        out_climo = climo.compute()
    else:
        if output_arr is not None:
            output_arr[:] = ne.evaluate('data - climo')
        else:
            output_arr = ne.evaluate('data - climo')
        output_arr = output_arr.reshape(old_shp)
        out_climo = climo

    return output_arr, out_climo

def calc_ce(fcast, trial_obs, obs):
    """
    Method to calculate the Coefficient of Efficiency as defined by Nash and
    Sutcliffe 1970.

    Parameters
    ----------
    fcast: ndarray
        Time series of forecast data. M x N where M is the temporal dimension.
    obs: ndarray
        Time series of observations. M x N

    Returns
    -------
    CE: ndarray
        Coefficient of efficiency for all locations over the time range.
    """

    assert(fcast.shape == trial_obs.shape)

    # Climatological variance
    cvar = obs.var(axis=0, ddof=1)

    # Error variance
    error = ne.evaluate('(trial_obs - fcast)**2')
    evar = error.sum(axis=0)/(len(error))

    return 1 - evar/cvar

def calc_eofs(data, num_eigs, ret_pcs=False, var_stats_dict=None):
    """
    Method to calculate the EOFs of given  dataset.  This assumes data comes in as
    an m x n matrix where m is the temporal dimension and n is the spatial
    dimension.

    Parameters
    ----------
    data: ndarray
        Dataset to calculate EOFs from
    num_eigs: int
        Number of eigenvalues/vectors to return.  Must be less than min(m, n).
    ret_pcs: bool, optional
        Return principal component matrix along with EOFs
    var_stats_dict: dict, optional
        Dictionary target to star some simple statistics about the EOF
        calculation.  Note: if this is provided for a dask array it prompts two
        SVD calculations for both the compressed and full singular values.

    Returns
    -------
    eofs: ndarray
        The eofs (as column vectors) of the data with dimensions n x k where
        k is the num_eigs.
    svals: ndarray
        Singular values from the svd decomposition.  Returned as a row vector
        in order from largest to smallest.
    """

    if is_dask_array(data):
        pcs, full_svals, eofs = da.linalg.svd_compressed(data, num_eigs)

        out_svals = np.zeros(num_eigs)
        out_eofs = np.zeros((num_eigs, data.shape[1]))
        out_pcs = np.zeros((data.shape[0], num_eigs))
        da.store([eofs, full_svals, pcs], [out_eofs, out_svals, out_pcs])

        out_eofs = out_eofs.T
        out_pcs = out_pcs.T

        if var_stats_dict is not None:
            logger.warning('Cannot currently provide variance statistics for '
                           'EOFs computed on a dask array.')

    else:
        eofs, full_svals, pcs = svd(data[:].T, full_matrices=False)
        out_eofs = eofs[:, :num_eigs]
        out_svals = full_svals[:num_eigs]
        out_pcs = pcs[:num_eigs]

    # variance stats
    if var_stats_dict is not None:
        try:
            nt = data.shape[0]
            ns = data.shape[1]
            eig_vals = (full_svals ** 2) / (nt * ns)
            total_var = eig_vals.sum()
            var_expl_by_mode = eig_vals / total_var
            var_expl_by_retained = var_expl_by_mode[0:num_eigs].sum()

            var_stats_dict['nt'] = nt
            var_stats_dict['ns'] = ns
            var_stats_dict['eigvals'] = eig_vals
            var_stats_dict['num_ret_modes'] = num_eigs
            var_stats_dict['total_var'] = total_var
            var_stats_dict['var_expl_by_mode'] = var_expl_by_mode
            var_stats_dict['var_expl_by_ret'] = var_expl_by_retained
        except TypeError as e:
            print 'Must past dictionary type to var_stats_dict in order to ' \
                  'output variance statistics.'
            print e

    if ret_pcs:
        return out_eofs, out_svals, out_pcs
    else:
        return out_eofs, out_svals

def calc_lac(fcast, obs):
    """
    Method to calculate the Local Anomaly Correlation (LAC).  Uses numexpr
    for speed over larger datasets.

    Note: If necessary (memory concerns) in the future, the numexpr statements
    can be extended to use pytable arrays.  Would need to provide means to
    function, as summing over the dataset is still very slow it seems.

    Parameters
    ----------
    fcast: ndarray
        Time series of forecast data. M x N where M is the temporal dimension.
    obs: ndarray
        Time series of observations. M x N

    Returns
    -------
    lac: ndarray
        Local anomaly corellations for all locations over the time range.
    """
    # Calculate means of data
    f_mean = fcast.mean(axis=0)
    o_mean = obs.mean(axis=0)

    # Calculate covariance between time series
    cov = ne.evaluate('(fcast - f_mean) * (obs - o_mean)')
    cov = cov.sum(axis=0)

    # Calculate standarization terms
    f_std = ne.evaluate('(fcast - f_mean)**2')
    f_std = np.sqrt(f_std.sum(axis=0))
    o_std = ne.evaluate('(obs - o_mean)**2')
    o_std = np.sqrt(o_std.sum(axis=0))
    std = f_std * o_std

    return cov / std

def calc_n_eff(data1, data2=None):
    """
    Calculate the effective degrees of freedom for data using lag-1
    autocorrelation.

    Parameters
    ----------
    data1: ndarray
        Dataset to calculate effective degrees of freedom for.  Assumes
        first dimension is the temporal dimension.
    data2: ndarray, optional
        A second dataset to calculate the effective degrees of freedom
        for covariances/correlations etc.

    Returns
    -------
    n_eff: ndarray
        Effective degrees of freedom for input data.
    """

    if data2 is not None:
        assert data1.shape == data2.shape,\
            'Data must have have same shape for combined n_eff calculation'

    # Lag-1 autocorrelation
    r1 = calc_lac(data1[0:-1], data1[1:])
    n = len(data1)

    if data2 is not None:
        r2 = calc_lac(data2[0:-1], data2[1:])
        n_eff = n*((1 - r1*r2)/(1+r1*r2))
    else:
        n_eff = n*((1-r1)/(1+r1))

    return n_eff

def run_mean(data, window_size, trim_edge=None, output_arr=None):
    """
    A function for calculating the running mean on data.

    Parameters
    ----------
    data: ndarray
        Data matrix to perform running mean over. Expected to be in time(row) x
        space(column) format. And that samples span full years.
    window_size: int
        Size of the window to compute the running mean over.
    trim_edge: int, optional
        Remove specified items from the start and end of the sampling
        dimension of the running mean.  Otherwise the window_size/2 items at
        the start and the end will have reflected padding effects.
    output_arr: ndarray-like, optional
        Array to place output of running mean that supports
        ndarray-like slicing.  This is required for dask array input.

    Returns
    -------
    result: ndarray
        Running mean result of given data.
    bot_edge: int
        Number of elements removed from beginning of the time series
    top_edge: int
        Number of elements removed from the ending of the time series
    """

    sample_len = data.shape[0]
    if sample_len < window_size:
        raise ValueError("Window size must be smaller than or equal to the "
                         "length of the time dimension of the data.")

    if trim_edge is not None:
        sample_len -= trim_edge*2

        if sample_len < 1:
            raise ValueError('Not enough data to trim edges. Please try with '
                             'trim_edge=None')

    weights = [1.0/float(window_size) for _ in xrange(window_size)]
    if is_dask_array(data):
        if output_arr is None:
            raise ValueError('calc_anomaly requires an output array keyword '
                             'argument when operating on a Dask array.')

        def _run_mean_block(block):
            return convolve1d(block, weights, axis=0)

        old_chunk_shape = data
        pad = window_size // 2
        ghost = da.ghost.ghost(data, depth={0: pad}, boundary={0: 'reflect'})
        filt = ghost.map_blocks(_run_mean_block)
        unpadded = da.ghost.trim_internal(filt, {0: pad})
        if trim_edge is not None:
            unpadded = unpadded[trim_edge:-trim_edge]

        da.store(unpadded, output_arr)
    else:
        res = convolve1d(data, weights, axis=0)
        if trim_edge:
            res = res[trim_edge:-trim_edge]

        if output_arr is not None:
            output_arr[:] = res
        else:
            output_arr = res

    return output_arr

def is_dask_array(arr):
    return hasattr(arr, 'dask')
<|endoftext|>


<commit_before>"""Test related functionality

Adds a Pylons plugin to `nose <http://www.somethingaboutorange.com/mrl/projects/nose/>`_
that loads the Pylons app *before* scanning for doc tests.

This can be configured in the projects :file:`setup.cfg` under a
``[nosetests]`` block:

.. code-block:: ini

    [nosetests]
    with-pylons=development.ini

Alternate ini files may be specified if the app should be loaded using
a different configuration.

"""
import os
import sys

import nose.plugins
import pkg_resources
from paste.deploy import loadapp

import pylons
from pylons.i18n.translation import _get_translator

pylonsapp = None

class PylonsPlugin(nose.plugins.Plugin):
    """Nose plugin extension

    For use with nose to allow a project to be configured before nose
    proceeds to scan the project for doc tests and unit tests. This
    prevents modules from being loaded without a configured Pylons
    environment.

    """
    enabled = False
    enableOpt = 'pylons_config'
    name = 'pylons'

    def add_options(self, parser, env=os.environ):
        """Add command-line options for this plugin"""
        env_opt = 'NOSE_WITH_%s' % self.name.upper()
        env_opt.replace('-', '_')

        parser.add_option("--with-%s" % self.name,
                          dest=self.enableOpt, type="string",
                          default="",
                          help="Setup Pylons environment with the config file"
                          " specified by ATTR [NOSE_ATTR]")

    def configure(self, options, conf):
        """Configure the plugin"""
        self.config_file = None
        self.conf = conf
        if hasattr(options, self.enableOpt):
            self.enabled = bool(getattr(options, self.enableOpt))
            self.config_file = getattr(options, self.enableOpt)

    def begin(self):
        """Called before any tests are collected or run

        Loads the application, and in turn its configuration.

        """
        global pylonsapp
        path = os.getcwd()
        sys.path.insert(0, path)
        pkg_resources.working_set.add_entry(path)
        self.app = pylonsapp = loadapp('config:' + self.config_file, relative_to=path)
<commit_msg>Add setup of default translator object so doctests can run when using _. Fixes #509.<commit_after>"""Test related functionality

Adds a Pylons plugin to `nose <http://www.somethingaboutorange.com/mrl/projects/nose/>`_
that loads the Pylons app *before* scanning for doc tests.

This can be configured in the projects :file:`setup.cfg` under a
``[nosetests]`` block:

.. code-block:: ini

    [nosetests]
    with-pylons=development.ini

Alternate ini files may be specified if the app should be loaded using
a different configuration.

"""
import os
import sys

import nose.plugins
import pkg_resources
from paste.deploy import loadapp

import pylons
from pylons.i18n.translation import _get_translator

pylonsapp = None

class PylonsPlugin(nose.plugins.Plugin):
    """Nose plugin extension

    For use with nose to allow a project to be configured before nose
    proceeds to scan the project for doc tests and unit tests. This
    prevents modules from being loaded without a configured Pylons
    environment.

    """
    enabled = False
    enableOpt = 'pylons_config'
    name = 'pylons'

    def add_options(self, parser, env=os.environ):
        """Add command-line options for this plugin"""
        env_opt = 'NOSE_WITH_%s' % self.name.upper()
        env_opt.replace('-', '_')

        parser.add_option("--with-%s" % self.name,
                          dest=self.enableOpt, type="string",
                          default="",
                          help="Setup Pylons environment with the config file"
                          " specified by ATTR [NOSE_ATTR]")

    def configure(self, options, conf):
        """Configure the plugin"""
        self.config_file = None
        self.conf = conf
        if hasattr(options, self.enableOpt):
            self.enabled = bool(getattr(options, self.enableOpt))
            self.config_file = getattr(options, self.enableOpt)

    def begin(self):
        """Called before any tests are collected or run

        Loads the application, and in turn its configuration.

        """
        global pylonsapp
        path = os.getcwd()
        sys.path.insert(0, path)
        pkg_resources.working_set.add_entry(path)
        self.app = pylonsapp = loadapp('config:' + self.config_file, relative_to=path)

        # For tests that utilize the i18n _ object, initialize a NullTranslator
        pylons.translator._push_object(_get_translator(pylons.config.get('lang')))
<|endoftext|>