<commit_before>#!/usr/bin/env python

from __future__ import absolute_import, print_function

import __main__

import sys
import os

from socket import socketpair, AF_INET, AF_UNIX, SOCK_STREAM, fromfd

import attr

from twisted.python.usage import Options
from twisted.internet.task import react
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.internet.endpoints import TCP4ServerEndpoint
from twisted.internet.protocol import Protocol, Factory, ProcessProtocol
from twisted.protocols.amp import AMP, Command, Descriptor
from twisted.internet.unix import Server as UNIXServer

from twisted.web.server import Site
from twisted.web.static import Data

STDIN = 0
STDOUT = 1
STDERR = 2
MAGIC_FILE_DESCRIPTOR = 7

def show(*args):
    """

    """
    print(*args)
    sys.stdout.flush()
    sys.stderr.flush()

class ManagerServerNotReallyProtocol(Protocol, object):
    """

    """

    def connectionMade(self):
        """

        """
        transport = self.transport
        transport.stopReading()
        transport.stopWriting()

        socketObject = transport.getHandle()

        sent = (self.factory.mpmManager
                .sendOutFileDescriptor(socketObject.fileno()))

        @sent.addBoth
        def inevitably(result):
            socketObject.close()
            return result

class ManagerServerFactory(Factory, object):
    """

    """

    def __init__(self, mpmManager):
        """

        """
        self.mpmManager = mpmManager

    protocol = ManagerServerNotReallyProtocol

class ManagerOptions(Options, object):
    """

    """

    @inlineCallbacks
    def go(self, reactor):
        """

        """
        mgr = MPMManager(reactor)
        endpoint = TCP4ServerEndpoint(reactor, 8123)
        msf = ManagerServerFactory(mgr)
        yield endpoint.listen(msf)
        mgr.newSubProcess()
        yield Deferred()

class Ping(Command, object):
    """

    """

class SendDescriptor(Command, object):
    """

    """
    arguments = [("descriptor", Descriptor())]
    response = []
    errors = []

class ConnectionFromManager(AMP, object):
    """

    """
    def __init__(self, reactor, factory):
        """

        """
        super(ConnectionFromManager, self).__init__()
        self.factory = factory
        self.reactor = reactor

    @SendDescriptor.responder
    def receiveDescriptor(self, descriptor):
        """

        """
        self.reactor.adoptStreamConnection(descriptor, AF_INET, self.factory)
        return {}

    @Ping.responder
    def pung(self):
        """

        """
        return {}

class WorkerOptions(Options, object):
    """

    """

    def go(self, reactor):
        """

        """
        data = Data("Hello world\n", "text/plain")
        data.putChild("", data)
        factory = Site(data)

        # TODO: adoptStreamConnection should really support AF_UNIX
        protocol = ConnectionFromManager(reactor, factory)
        skt = fromfd(MAGIC_FILE_DESCRIPTOR, AF_UNIX, SOCK_STREAM)
        os.close(MAGIC_FILE_DESCRIPTOR)
        serverTransport = UNIXServer(skt, protocol, None, None, 1234, reactor)
        protocol.makeConnection(serverTransport)
        serverTransport.startReading()
        return Deferred()

class CommandLineOptions(Options, object):
    """

    """
    synopsis = "Usage: warped_alloy [options]"

    subCommands = [
        # ['command-name', 'command-shortcut', ParserClass, documentation]
        ['manager', 'm', ManagerOptions, 'For managing'],
        ['worker', 'w', WorkerOptions, 'For workering'],
    ]

    defaultSubCommand = 'manager'

@attr.s
class MyProcessProtocol(ProcessProtocol, object):
    """

    """
    mpmManager = attr.ib()

    def outReceived(self, data):
        """

        """
        show("sub-out:", repr(data))

    def errReceived(self, data):
        """

        """
        show("sub-err:", repr(data))

    def processExited(self, reason):
        """

        """
        show("exit?", reason)

    def processEnded(self, reason):
        """

        """
        show("ended?", reason)

class OneWorkerProtocol(AMP, object):
    """

    """

    def sendFD(self, fileDescriptor):
        """

        """
        return self.callRemote(SendDescriptor, descriptor=fileDescriptor)

@attr.s
class MPMManager(object):
    """

    """
    reactor = attr.ib()
    openSubprocessConnections = attr.ib(default=attr.Factory(list))

    def sendOutFileDescriptor(self, fileDescriptor):
        if not self.openSubprocessConnections:
            self.newSubProcess()
        return self.openSubprocessConnections[0].sendFD(fileDescriptor)

    def newSubProcess(self):
        """

        """
        here, there = socketpair(AF_UNIX, SOCK_STREAM)
        owp = OneWorkerProtocol()
        serverTransport = UNIXServer(here, owp, None, None, 4321, self.reactor)
        owp.makeConnection(serverTransport)
        script = __main__.__file__
        argv = [sys.executable, script, b'w']
        self.reactor.spawnProcess(
            MyProcessProtocol(self), sys.executable,
            args=argv,
            env=os.environ.copy(),
            childFDs={STDIN: 'w', STDOUT: 'r', STDERR: 'r',
                      MAGIC_FILE_DESCRIPTOR: there.fileno()}
        )
        there.close()
        serverTransport.startReading()
        self.openSubprocessConnections.append(owp)

@react
def main(reactor):
    """

    """
    clo = CommandLineOptions()
    clo.parseOptions(sys.argv[1:])
    subCommandParser = clo.subOptions
    return subCommandParser.go(reactor)
<commit_msg>no longer need ping command<commit_after>#!/usr/bin/env python

from __future__ import absolute_import, print_function

import __main__

import sys
import os

from socket import socketpair, AF_INET, AF_UNIX, SOCK_STREAM, fromfd

import attr

from twisted.python.usage import Options
from twisted.internet.task import react
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.internet.endpoints import TCP4ServerEndpoint
from twisted.internet.protocol import Protocol, Factory, ProcessProtocol
from twisted.protocols.amp import AMP, Command, Descriptor
from twisted.internet.unix import Server as UNIXServer

from twisted.web.server import Site
from twisted.web.static import Data

STDIN = 0
STDOUT = 1
STDERR = 2
MAGIC_FILE_DESCRIPTOR = 7

def show(*args):
    """

    """
    print(*args)
    sys.stdout.flush()
    sys.stderr.flush()

class ManagerServerNotReallyProtocol(Protocol, object):
    """

    """

    def connectionMade(self):
        """

        """
        transport = self.transport
        transport.stopReading()
        transport.stopWriting()

        socketObject = transport.getHandle()

        sent = (self.factory.mpmManager
                .sendOutFileDescriptor(socketObject.fileno()))

        @sent.addBoth
        def inevitably(result):
            socketObject.close()
            return result

class ManagerServerFactory(Factory, object):
    """

    """

    def __init__(self, mpmManager):
        """

        """
        self.mpmManager = mpmManager

    protocol = ManagerServerNotReallyProtocol

class ManagerOptions(Options, object):
    """

    """

    @inlineCallbacks
    def go(self, reactor):
        """

        """
        mgr = MPMManager(reactor)
        endpoint = TCP4ServerEndpoint(reactor, 8123)
        msf = ManagerServerFactory(mgr)
        yield endpoint.listen(msf)
        mgr.newSubProcess()
        yield Deferred()

class SendDescriptor(Command, object):
    """

    """
    arguments = [("descriptor", Descriptor())]
    response = []
    errors = []

class ConnectionFromManager(AMP, object):
    """

    """
    def __init__(self, reactor, factory):
        """

        """
        super(ConnectionFromManager, self).__init__()
        self.factory = factory
        self.reactor = reactor

    @SendDescriptor.responder
    def receiveDescriptor(self, descriptor):
        """

        """
        self.reactor.adoptStreamConnection(descriptor, AF_INET, self.factory)
        return {}

class WorkerOptions(Options, object):
    """

    """

    def go(self, reactor):
        """

        """
        data = Data("Hello world\n", "text/plain")
        data.putChild("", data)
        factory = Site(data)

        # TODO: adoptStreamConnection should really support AF_UNIX
        protocol = ConnectionFromManager(reactor, factory)
        skt = fromfd(MAGIC_FILE_DESCRIPTOR, AF_UNIX, SOCK_STREAM)
        os.close(MAGIC_FILE_DESCRIPTOR)
        serverTransport = UNIXServer(skt, protocol, None, None, 1234, reactor)
        protocol.makeConnection(serverTransport)
        serverTransport.startReading()
        return Deferred()

class CommandLineOptions(Options, object):
    """

    """
    synopsis = "Usage: warped_alloy [options]"

    subCommands = [
        # ['command-name', 'command-shortcut', ParserClass, documentation]
        ['manager', 'm', ManagerOptions, 'For managing'],
        ['worker', 'w', WorkerOptions, 'For workering'],
    ]

    defaultSubCommand = 'manager'

@attr.s
class MyProcessProtocol(ProcessProtocol, object):
    """

    """
    mpmManager = attr.ib()

    def outReceived(self, data):
        """

        """
        show("sub-out:", repr(data))

    def errReceived(self, data):
        """

        """
        show("sub-err:", repr(data))

    def processExited(self, reason):
        """

        """
        show("exit?", reason)

    def processEnded(self, reason):
        """

        """
        show("ended?", reason)

class OneWorkerProtocol(AMP, object):
    """

    """

    def sendFD(self, fileDescriptor):
        """

        """
        return self.callRemote(SendDescriptor, descriptor=fileDescriptor)

@attr.s
class MPMManager(object):
    """

    """
    reactor = attr.ib()
    openSubprocessConnections = attr.ib(default=attr.Factory(list))

    def sendOutFileDescriptor(self, fileDescriptor):
        if not self.openSubprocessConnections:
            self.newSubProcess()
        return self.openSubprocessConnections[0].sendFD(fileDescriptor)

    def newSubProcess(self):
        """

        """
        here, there = socketpair(AF_UNIX, SOCK_STREAM)
        owp = OneWorkerProtocol()
        serverTransport = UNIXServer(here, owp, None, None, 4321, self.reactor)
        owp.makeConnection(serverTransport)
        script = __main__.__file__
        argv = [sys.executable, script, b'w']
        self.reactor.spawnProcess(
            MyProcessProtocol(self), sys.executable,
            args=argv,
            env=os.environ.copy(),
            childFDs={STDIN: 'w', STDOUT: 'r', STDERR: 'r',
                      MAGIC_FILE_DESCRIPTOR: there.fileno()}
        )
        there.close()
        serverTransport.startReading()
        self.openSubprocessConnections.append(owp)

@react
def main(reactor):
    """

    """
    clo = CommandLineOptions()
    clo.parseOptions(sys.argv[1:])
    subCommandParser = clo.subOptions
    return subCommandParser.go(reactor)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2012 The Pycroft Authors. See the AUTHORS file.
# This file is part of the Pycroft project and licensed under the terms of
# the Apache License, Version 2.0. See the LICENSE file for details.
"""
    web
    ~~~~~~~~~~~~~~

    This package contains the web interface based on flask

    :copyright: (c) 2012 by AG DSN.
"""

from flask import Flask
from blueprints import finance, infrastructure, rights, user

from pycroft.model.session import session

def make_app():
    """  Create and configure the main? Flask app object

    :return: The fully configured app object
    """
    app = Flask(__name__)

    #initialization code
    app.secret_key = r"eiNohfaefaig5Iek6oshai0eijuph4ohla6Eo1vi5bahnaeh3Bah7ohy1einuaxu"

    app.register_blueprint(user.bp, url_prefix="/user")
    app.register_blueprint(infrastructure.bp, url_prefix="/infrastructure")
    app.register_blueprint(rights.bp, url_prefix="/rights")
    app.register_blueprint(finance.bp, url_prefix="/finance")

    @app.teardown_request
    def shutdown_session(exception=None):
        session.remove()

    return app

from blueprints.user import *

<commit_msg>vix some debugging fallout<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2012 The Pycroft Authors. See the AUTHORS file.
# This file is part of the Pycroft project and licensed under the terms of
# the Apache License, Version 2.0. See the LICENSE file for details.
"""
    web
    ~~~~~~~~~~~~~~

    This package contains the web interface based on flask

    :copyright: (c) 2012 by AG DSN.
"""

from flask import Flask
from blueprints import finance, infrastructure, rights, user

from pycroft.model.session import session

def make_app():
    """  Create and configure the main? Flask app object

    :return: The fully configured app object
    """
    app = Flask(__name__)

    #initialization code
    app.secret_key = r"eiNohfaefaig5Iek6oshai0eijuph4ohla6Eo1vi5bahnaeh3Bah7ohy1einuaxu"

    app.register_blueprint(user.bp, url_prefix="/user")
    app.register_blueprint(infrastructure.bp, url_prefix="/infrastructure")
    app.register_blueprint(rights.bp, url_prefix="/rights")
    app.register_blueprint(finance.bp, url_prefix="/finance")

    @app.teardown_request
    def shutdown_session(exception=None):
        session.remove()

    return app

<|endoftext|>


<commit_before>from typing import List, Tuple, Union, Dict, Type, Callable
try:
    from typing import TypedDict
except ImportError:
    from typing_extensions import TypedDict
import contextlib
import itertools
import logging
import pool.proc
import pool.thread
import time
import tornado.httpclient
import tornado.web
import urllib
import util.exceptions
import util.func
import util.log
import util.net
from tornado.ioloop import IOLoop
from tornado.web import RequestHandler, Application
from tornado.httputil import HTTPServerRequest, HTTPFile
from tornado.simple_httpclient import HTTPTimeoutError

class Request(TypedDict):
    verb: str
    url: str
    path: str
    query: Dict[str, List[str]]
    body: Union[str, bytes]
    headers: Dict[str, Union[str, int]]
    args: List[str]
    files: Dict[str, List[HTTPFile]]
    kwargs: Dict[str, str]
    remote: str

class Response(TypedDict, total=False):
    code: int
    reason: str
    headers: Dict[str, str]
    body: Union[str, bytes]

def _try_decode(text):
    try:
        return text.decode('utf-8')
    except:
        return text

def _handler_function_to_tornado_handler_method(fn):
    name = util.func.name(fn)
    async def method(self, *a, **kw):
        req = _tornado_req_to_dict(self.request, a, kw)
        try:
            resp = await fn(req)
        except:
            logging.exception('uncaught exception in: %s', name)
            resp = {'code': 500}
        _update_handler_from_dict_resp(resp, self)
    method.fn = fn
    return method

def _verbs_dict_to_tornado_handler_class(verbs_or_handler: Union[Dict[str, Callable], Type[RequestHandler]]) -> Type[RequestHandler]:
    if isinstance(verbs_or_handler, type(RequestHandler)):
        return verbs_or_handler
    else:
        class Handler(RequestHandler):
            for verb, fn in verbs_or_handler.items(): # type: ignore
                locals()[verb.lower()] = _handler_function_to_tornado_handler_method(fn)
            del verb, fn
        return Handler

def _update_handler_from_dict_resp(resp: Response, handler: RequestHandler) -> None:
    body = resp.get('body')
    if body:
        handler.write(body)
    handler.set_status(resp.get('code') or 200, resp.get('reason') or '')
    for header, value in (resp.get('headers') or {}).items():
        handler.set_header(header, value)

def _parse_query_string(query: str) -> Dict[str, List[str]]:
    return {k: list(v) for k, v in urllib.parse.parse_qs(query, True).items()}

def _tornado_req_to_dict(obj: HTTPServerRequest, a: List[str], kw: Dict[str, str]) -> Request:
    return {
        'verb': (obj.method or '').lower(),
        'url': obj.uri or '',
        'path': obj.path,
        'query': _parse_query_string(obj.query),
        'body': _try_decode(obj.body),
        'headers': {k.lower(): v for k, v in dict(obj.headers).items()},
        'args': a,
        'kwargs': kw,
        'files': obj.files,
        'remote': obj.remote_ip,
    }

def _parse_route_str(route: str) -> str:
    return '/'.join([f'(?P<{x[1:]}>.*)' if x.startswith(':') else x for x in route.split('/')])

def app(routes: List[Tuple[str, Union[Type[RequestHandler], Dict[str, Callable]]]], debug: bool = False, **settings) -> Application:
    routes = [(_parse_route_str(route),
               _verbs_dict_to_tornado_handler_class(verbs))
              for route, verbs in routes]
    return Application(routes, debug=debug, **settings) # type: ignore

def wait_for_http(url, max_wait_seconds=5):
    import requests
    import requests.exceptions
    start = time.time()
    for i in itertools.count(1):
        assert time.time() - start < max_wait_seconds, 'timed out'
        try:
            assert requests.get(url).status_code != 599
            break
        except requests.exceptions.ConnectionError:
            time.sleep(.01 * 1)

@contextlib.contextmanager
def test(app, poll='/', context=None, use_thread=False):
    from unittest import mock
    context = context or (lambda: mock.patch.object(mock, '_fake_', create=True))
    port = util.net.free_port()
    url = f'http://0.0.0.0:{port}'
    def run():
        with context():
            util.log.setup()
            if isinstance(app, Application):
                app.listen(port)
            else:
                app().listen(port)
            if not use_thread:
                IOLoop.current().start()
    proc = (pool.thread.new if use_thread else pool.proc.new)(run)
    if poll:
        wait_for_http(url + poll)
    try:
        yield url
    except:
        raise
    finally:
        if not use_thread:
            proc.terminate()

with util.exceptions.ignore(ImportError):
    tornado.httpclient.AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
tornado.httpclient.AsyncHTTPClient.configure(None, max_clients=256)

class Blowup(Exception):
    def __init__(self, message, code, reason, body):
        super().__init__(message)
        self.code = code
        self.reason = reason
        self.body = _try_decode(body)

    def __str__(self):
        return f'{self.args[0] if self.args else ""}, code={self.code}, reason="{self.reason}"\n{self.body}'

async def _fetch(verb: str, url: str, **kw: dict) -> Response:
    url, timeout, blowup, kw = _process_fetch_kwargs(url, kw)
    kw['user_agent'] = kw.get('user_agent') or "Mozilla/5.0 (compatible; pycurl)" # type: ignore
    future = tornado.httpclient.AsyncHTTPClient().fetch(url, method=verb, raise_error=False, connect_timeout=timeout, request_timeout=timeout, **kw)
    resp = await future
    if blowup and resp.code != 200:
        raise Blowup(f'{verb} {url} did not return 200, returned {resp.code}',
                     resp.code,
                     resp.reason,
                     resp.body)
    return {'code': resp.code,
            'reason': resp.reason or '',
            'headers': {k.lower(): v for k, v in resp.headers.items()},
            'body': _try_decode(resp.body or b'')}

def _process_fetch_kwargs(url, kw):
    timeout = kw.pop('timeout', 10)
    blowup = kw.pop('blowup', False)
    if 'query' in kw:
        assert '?' not in url, f'you cannot user keyword arg query and have ? already in the url: {url}'
        url += '?' + '&'.join(f'{k}={tornado.escape.url_escape(v)}'
                              for k, v in kw.pop('query').items())
    return url, timeout, blowup, kw

def get(url, **kw):
    return _fetch('GET', url, **kw)

def post(url, body='', **kw):
    return _fetch('POST', url, body=body, **kw)

Timeout = HTTPTimeoutError
<commit_msg>use any for routes type<commit_after>from typing import List, Tuple, Union, Dict, Type, Callable, Any
try:
    from typing import TypedDict
except ImportError:
    from typing_extensions import TypedDict
import contextlib
import itertools
import logging
import pool.proc
import pool.thread
import time
import tornado.httpclient
import tornado.web
import urllib
import util.exceptions
import util.func
import util.log
import util.net
from tornado.ioloop import IOLoop
from tornado.web import RequestHandler, Application
from tornado.httputil import HTTPServerRequest, HTTPFile
from tornado.simple_httpclient import HTTPTimeoutError

class Request(TypedDict):
    verb: str
    url: str
    path: str
    query: Dict[str, List[str]]
    body: Union[str, bytes]
    headers: Dict[str, Union[str, int]]
    args: List[str]
    files: Dict[str, List[HTTPFile]]
    kwargs: Dict[str, str]
    remote: str

class Response(TypedDict, total=False):
    code: int
    reason: str
    headers: Dict[str, str]
    body: Union[str, bytes]

def _try_decode(text):
    try:
        return text.decode('utf-8')
    except:
        return text

def _handler_function_to_tornado_handler_method(fn):
    name = util.func.name(fn)
    async def method(self, *a, **kw):
        req = _tornado_req_to_dict(self.request, a, kw)
        try:
            resp = await fn(req)
        except:
            logging.exception('uncaught exception in: %s', name)
            resp = {'code': 500}
        _update_handler_from_dict_resp(resp, self)
    method.fn = fn
    return method

def _verbs_dict_to_tornado_handler_class(verbs_or_handler: Union[Dict[str, Callable], Type[RequestHandler]]) -> Type[RequestHandler]:
    if isinstance(verbs_or_handler, type(RequestHandler)):
        return verbs_or_handler
    else:
        class Handler(RequestHandler):
            for verb, fn in verbs_or_handler.items(): # type: ignore
                locals()[verb.lower()] = _handler_function_to_tornado_handler_method(fn)
            del verb, fn
        return Handler

def _update_handler_from_dict_resp(resp: Response, handler: RequestHandler) -> None:
    body = resp.get('body')
    if body:
        handler.write(body)
    handler.set_status(resp.get('code') or 200, resp.get('reason') or '')
    for header, value in (resp.get('headers') or {}).items():
        handler.set_header(header, value)

def _parse_query_string(query: str) -> Dict[str, List[str]]:
    return {k: list(v) for k, v in urllib.parse.parse_qs(query, True).items()}

def _tornado_req_to_dict(obj: HTTPServerRequest, a: List[str], kw: Dict[str, str]) -> Request:
    return {
        'verb': (obj.method or '').lower(),
        'url': obj.uri or '',
        'path': obj.path,
        'query': _parse_query_string(obj.query),
        'body': _try_decode(obj.body),
        'headers': {k.lower(): v for k, v in dict(obj.headers).items()},
        'args': a,
        'kwargs': kw,
        'files': obj.files,
        'remote': obj.remote_ip,
    }

def _parse_route_str(route: str) -> str:
    return '/'.join([f'(?P<{x[1:]}>.*)' if x.startswith(':') else x for x in route.split('/')])

def app(routes: List[Tuple[str, Any]], debug: bool = False, **settings) -> Application:
    routes = [(_parse_route_str(route),
               _verbs_dict_to_tornado_handler_class(verbs))
              for route, verbs in routes]
    return Application(routes, debug=debug, **settings) # type: ignore

def wait_for_http(url, max_wait_seconds=5):
    import requests
    import requests.exceptions
    start = time.time()
    for i in itertools.count(1):
        assert time.time() - start < max_wait_seconds, 'timed out'
        try:
            assert requests.get(url).status_code != 599
            break
        except requests.exceptions.ConnectionError:
            time.sleep(.01 * 1)

@contextlib.contextmanager
def test(app, poll='/', context=None, use_thread=False):
    from unittest import mock
    context = context or (lambda: mock.patch.object(mock, '_fake_', create=True))
    port = util.net.free_port()
    url = f'http://0.0.0.0:{port}'
    def run():
        with context():
            util.log.setup()
            if isinstance(app, Application):
                app.listen(port)
            else:
                app().listen(port)
            if not use_thread:
                IOLoop.current().start()
    proc = (pool.thread.new if use_thread else pool.proc.new)(run)
    if poll:
        wait_for_http(url + poll)
    try:
        yield url
    except:
        raise
    finally:
        if not use_thread:
            proc.terminate()

with util.exceptions.ignore(ImportError):
    tornado.httpclient.AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
tornado.httpclient.AsyncHTTPClient.configure(None, max_clients=256)

class Blowup(Exception):
    def __init__(self, message, code, reason, body):
        super().__init__(message)
        self.code = code
        self.reason = reason
        self.body = _try_decode(body)

    def __str__(self):
        return f'{self.args[0] if self.args else ""}, code={self.code}, reason="{self.reason}"\n{self.body}'

async def _fetch(verb: str, url: str, **kw: dict) -> Response:
    url, timeout, blowup, kw = _process_fetch_kwargs(url, kw)
    kw['user_agent'] = kw.get('user_agent') or "Mozilla/5.0 (compatible; pycurl)" # type: ignore
    future = tornado.httpclient.AsyncHTTPClient().fetch(url, method=verb, raise_error=False, connect_timeout=timeout, request_timeout=timeout, **kw)
    resp = await future
    if blowup and resp.code != 200:
        raise Blowup(f'{verb} {url} did not return 200, returned {resp.code}',
                     resp.code,
                     resp.reason,
                     resp.body)
    return {'code': resp.code,
            'reason': resp.reason or '',
            'headers': {k.lower(): v for k, v in resp.headers.items()},
            'body': _try_decode(resp.body or b'')}

def _process_fetch_kwargs(url, kw):
    timeout = kw.pop('timeout', 10)
    blowup = kw.pop('blowup', False)
    if 'query' in kw:
        assert '?' not in url, f'you cannot user keyword arg query and have ? already in the url: {url}'
        url += '?' + '&'.join(f'{k}={tornado.escape.url_escape(v)}'
                              for k, v in kw.pop('query').items())
    return url, timeout, blowup, kw

def get(url, **kw):
    return _fetch('GET', url, **kw)

def post(url, body='', **kw):
    return _fetch('POST', url, body=body, **kw)

Timeout = HTTPTimeoutError
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import re

def tokenFieldPredicate(field):
    """returns the tokens"""

    return tuple(field.split())

def commonIntegerPredicate(field):
    """"return any integers"""
    return tuple(re.findall("\d+", field))

def nearIntegersPredicate(field):
    """return any integers N, N+1, and N-1"""
    ints = sorted([int(i) for i in re.findall("\d+", field)])
    near_ints = set([])
    [near_ints.update((i - 1, i, i + 1)) for i in ints]
    return tuple(near_ints)

def ngrams(field, n):
    """ngrams returns all unique, contiguous sequences of n characters
    of a given field.

    :param field: the string to be
    :param n: the number of characters to be included in each gram

    usage:
    >>> from dedupe.dedupe.predicated import ngrams
    >>> ngrams("deduplicate", 3)
    ('ded', 'edu', 'dup', 'upl', 'pli', 'lic', 'ica', 'cat', 'ate')
    """
    return tuple([field[pos:pos + n] for pos in xrange(len(field) - n + 1)])

def commonFourGram(field):
    """return 4-grams"""
    return ngrams(field, 4)

def commonSixGram(field):
    """"return 6-grams"""
    return ngrams(field, 6)

def initials(field, n=None):
    """predicate which returns first a tuple containing
    the first n chars of a field if and only if the
    field contains at least n characters, or an empty
    tuple otherwise.

    :param field: the string
    :type n: int, default None

    usage:
    >>> initials("dedupe", 7)
    ()
    >>> initials("deduplication", 7)
    ('dedupli', )
    >>> initials("noslice")
    ('noslice', )
    """
    return (field[:n], ) if not n or len(field) > n-1 else ()

# Consider deprecating in favor of initials
def wholeFieldPredicate(field):
    """return the whole field"""
    return initials(field)

def sameThreeCharStartPredicate(field):
    """return first three characters"""
    return initials(field, 3)

def sameFiveCharStartPredicate(field):
    """return first five characters"""
    return initials(field, 5)

def sameSevenCharStartPredicate(field):
    """return first seven characters"""
    return initials(field, 7)
<commit_msg>reverted wholeFieldPredicate body re: @fgregg<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import re

def tokenFieldPredicate(field):
    """returns the tokens"""
    return tuple(field.split())

def commonIntegerPredicate(field):
    """"return any integers"""
    return tuple(re.findall("\d+", field))

def nearIntegersPredicate(field):
    """return any integers N, N+1, and N-1"""
    ints = sorted([int(i) for i in re.findall("\d+", field)])
    near_ints = set([])
    [near_ints.update((i - 1, i, i + 1)) for i in ints]
    return tuple(near_ints)

def ngrams(field, n):
    """ngrams returns all unique, contiguous sequences of n characters
    of a given field.

    :param field: the string to be
    :param n: the number of characters to be included in each gram

    usage:
    >>> from dedupe.dedupe.predicated import ngrams
    >>> ngrams("deduplicate", 3)
    ('ded', 'edu', 'dup', 'upl', 'pli', 'lic', 'ica', 'cat', 'ate')
    """
    return tuple([field[pos:pos + n] for pos in xrange(len(field) - n + 1)])

def commonFourGram(field):
    """return 4-grams"""
    return ngrams(field, 4)

def commonSixGram(field):
    """"return 6-grams"""
    return ngrams(field, 6)

def initials(field, n=None):
    """predicate which returns first a tuple containing
    the first n chars of a field if and only if the
    field contains at least n characters, or an empty
    tuple otherwise.

    :param field: the string
    :type n: int, default None

    usage:
    >>> initials("dedupe", 7)
    ()
    >>> initials("deduplication", 7)
    ('dedupli', )
    >>> initials("noslice")
    ('noslice', )
    """
    return (field[:n], ) if not n or len(field) > n-1 else ()

def wholeFieldPredicate(field):
    """return the whole field
    consider replacing with initials(field)
    """
    return (field, ) if field else ()

def sameThreeCharStartPredicate(field):
    """return first three characters"""
    return initials(field, 3)

def sameFiveCharStartPredicate(field):
    """return first five characters"""
    return initials(field, 5)

def sameSevenCharStartPredicate(field):
    """return first seven characters"""
    return initials(field, 7)
<|endoftext|>


<commit_before>__version__ = (0, 1, 0)

from .data import Delorean, datetime_timezone, localize, normalize
from .utils import capture<commit_msg>adding steps and days of week<commit_after>__version__ = (0, 1, 0)

from .utils import capture, stops
from .data import Delorean, datetime_timezone, localize, normalize
from dateutil.rrule import (YEARLY,
    MONTHLY, WEEKLY, DAILY, HOURLY,
    MINUTELY, SECONDLY, MO, TU, WE, TH, FR, SA, SU)
<|endoftext|>


<commit_before>
# This __about__.py file for storing project metadata is inspired by
# the warehouse project:
#   https://github.com/pypa/warehouse/blob/master/warehouse/__about__.py

__version__ = '0.5.0-rc2'
__version_info__ = __version__.replace('.', ' ').replace('-', ' ').split()

__title__ = 'pyDelphin'
__summary__ = 'Libraries and scripts for DELPH-IN data'
__uri__ = 'https://github.com/delph-in/pydelphin'

__author__ = 'Michael Wayne Goodman'
__email__ = 'goodman.m.w@gmail.com'

__license__ = 'MIT'
__copyright__ = '2013 %s <%s> and contributors' % (__author__, __email__)
<commit_msg>Release v0.5.0<commit_after>
# This __about__.py file for storing project metadata is inspired by
# the warehouse project:
#   https://github.com/pypa/warehouse/blob/master/warehouse/__about__.py

__version__ = '0.5.0'
__version_info__ = __version__.replace('.', ' ').replace('-', ' ').split()

__title__ = 'pyDelphin'
__summary__ = 'Libraries and scripts for DELPH-IN data'
__uri__ = 'https://github.com/delph-in/pydelphin'

__author__ = 'Michael Wayne Goodman'
__email__ = 'goodman.m.w@gmail.com'

__license__ = 'MIT'
__copyright__ = '2013 %s <%s> and contributors' % (__author__, __email__)
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import properties
from scipy.constants import mu_0

from SimPEG import Survey
from SimPEG import Problem
from SimPEG import Utils
from SimPEG import Maps
from SimPEG import Props
from SimPEG import Solver as SimpegSolver

__all__ = ['BaseEMProblem', 'BaseEMSurvey', 'BaseEMSrc']

###############################################################################
#                                                                             #
#                             Base EM Problem                                 #
#                                                                             #
###############################################################################

class BaseEMProblem(Problem.BaseProblem):

    sigma, sigmaMap, sigmaDeriv = Props.Invertible(
        "Electrical conductivity (S/m)"
    )

    rho, rhoMap, rhoDeriv = Props.Invertible(
        "Electrical resistivity (Ohm m)"
    )

    Props.Reciprocal(sigma, rho)

    mu = Props.PhysicalProperty(
        "Magnetic Permeability (H/m)",
        default=mu_0
    )
    mui = Props.PhysicalProperty(
        "Inverse Magnetic Permeability (m/H)"
    )

    Props.Reciprocal(mu, mui)

    surveyPair = Survey.BaseSurvey  #: The survey to pair with.
    dataPair = Survey.Data  #: The data to pair with.

    mapPair = Maps.IdentityMap  #: Type of mapping to pair with

    Solver = SimpegSolver  #: Type of solver to pair with
    solverOpts = {}  #: Solver options

    verbose = False

    ####################################################
    # Make A Symmetric
    ####################################################
    @property
    def _makeASymmetric(self):
        if getattr(self, '__makeASymmetric', None) is None:
            self.__makeASymmetric = True
        return self.__makeASymmetric

    ####################################################
    # Mass Matrices
    ####################################################
    @property
    def deleteTheseOnModelUpdate(self):
        toDelete = []
        if self.sigmaMap is not None or self.rhoMap is not None:
            toDelete += ['_MeSigma', '_MeSigmaI', '_MfRho', '_MfRhoI']

        if hasattr(self, 'muMap') or hasattr(self, 'muiMap'):
            if self.muMap is not None or self.muiMap is not None:
                toDelete += ['_MeMu', '_MeMuI', '_MfMui', '_MfMuiI']

        return toDelete

    @property
    def Me(self):
        """
            Edge inner product matrix
        """
        if getattr(self, '_Me', None) is None:
            self._Me = self.mesh.getEdgeInnerProduct()
        return self._Me

    @property
    def MeI(self):
        """
            Edge inner product matrix
        """
        if getattr(self, '_MeI', None) is None:
            self._MeI = self.mesh.getEdgeInnerProduct(invMat=True)
        return self._MeI

    @property
    def Mf(self):
        """
            Face inner product matrix
        """
        if getattr(self, '_Mf', None) is None:
            self._Mf = self.mesh.getFaceInnerProduct()
        return self._Mf

    @property
    def MfI(self):
        """
            Face inner product matrix
        """
        if getattr(self, '_MfI', None) is None:
            self._MfI = self.mesh.getFaceInnerProduct(invMat=True)
        return self._MfI

    @property
    def Vol(self):
        if getattr(self, '_Vol', None) is None:
            self._Vol = Utils.sdiag(self.mesh.vol)
        return self._Vol

    ####################################################
    # Magnetic Permeability
    ####################################################
    @property
    def MfMui(self):
        """
        Face inner product matrix for \\(\\mu^{-1}\\).
        Used in the E-B formulation
        """
        if getattr(self, '_MfMui', None) is None:
            self._MfMui = self.mesh.getFaceInnerProduct(self.mui)
        return self._MfMui

    def MfMuiDeriv(self, u):
        """
        Derivative of :code:`MfMui` with respect to the model.
        """
        if self.muiMap is None:
            return Utils.Zero()

        return (
            self.mesh.getFaceInnerProductDeriv(self.mui)(u) * self.muiDeriv
        )

    @property
    def MfMuiI(self):
        """
        Inverse of :code:`MfMui`.
        """
        if getattr(self, '_MfMuiI', None) is None:
            self._MfMuiI = self.mesh.getFaceInnerProduct(self.mui, invMat=True)
        return self._MfMuiI

    # TODO: This should take a vector
    def MfMuiIDeriv(self, u):
        """
        Derivative of :code:`MfMui` with respect to the model
        """

        if self.muiMap is None:
            return Utils.Zero()

        if len(self.mui.shape) > 1:
            if self.mui.shape[1] > self.mesh.dim:
                raise NotImplementedError(
                        "Full anisotropy is not implemented for MfMuiIDeriv."
                )

        dMfMuiI_dI = -self.MfMuiI**2
        dMf_dmui = self.mesh.getEdgeInnerProductDeriv(self.mui)(u)
        return dMfMuiI_dI * (dMf_dmui * self.muiDeriv)

    @property
    def MeMu(self):
        """
        Edge inner product matrix for \\(\\mu\\).
        Used in the H-J formulation
        """
        if getattr(self, '_MeMu', None) is None:
            self._MeMu = self.mesh.getEdgeInnerProduct(self.mu)
        return self._MeMu

    def MeMuDeriv(self, u):
        """
        Derivative of :code:`MeMu` with respect to the model.
        """
        if self.muMap is None:
            return Utils.Zero()

        return (
            self.mesh.getEdgeInnerProductDeriv(self.mu)(u) * self.muDeriv
        )

    @property
    def MeMuI(self):
        """
            Inverse of :code:`MeMu`
        """
        if getattr(self, '_MeMuI', None) is None:
            self._MeMuI = self.mesh.getEdgeInnerProduct(self.mu, invMat=True)
        return self._MeMuI

    # TODO: This should take a vector
    def MeMuIDeriv(self, u):
        """
        Derivative of :code:`MeMuI` with respect to the model
        """

        if self.muMap is None:
            return Utils.Zero()

        if len(self.mu.shape) > 1:
            if self.mu.shape[1] > self.mesh.dim:
                raise NotImplementedError(
                    "Full anisotropy is not implemented for MeMuIDeriv."
                )

        dMeMuI_dI = -self.MeMuI**2
        dMe_dmu = self.mesh.getEdgeInnerProductDeriv(self.mu)(u)
        return dMeMuI_dI * (dMe_dmu * self.muDeriv)

    ####################################################
    # Electrical Conductivity
    ####################################################
    @property
    def MeSigma(self):
        """
        Edge inner product matrix for \\(\\sigma\\).
        Used in the E-B formulation
        """
        if getattr(self, '_MeSigma', None) is None:
            self._MeSigma = self.mesh.getEdgeInnerProduct(self.sigma)
        return self._MeSigma

    # TODO: This should take a vector
    def MeSigmaDeriv(self, u):
        """
        Derivative of MeSigma with respect to the model
        """
        if self.sigmaMap is None:
            return Utils.Zero()

        return (
            self.mesh.getEdgeInnerProductDeriv(self.sigma)(u) *
            self.sigmaDeriv
        )

    @property
    def MeSigmaI(self):
        """
        Inverse of the edge inner product matrix for \\(\\sigma\\).
        """
        if getattr(self, '_MeSigmaI', None) is None:
            self._MeSigmaI = self.mesh.getEdgeInnerProduct(
                self.sigma, invMat=True
            )
        return self._MeSigmaI

    # TODO: This should take a vector
    def MeSigmaIDeriv(self, u):
        """
        Derivative of :code:`MeSigmaI` with respect to the model
        """
        if self.sigmaMap is None:
            return Utils.Zero()

        if len(self.sigma.shape) > 1:
            if self.sigma.shape[1] > self.mesh.dim:
                raise NotImplementedError(
                    "Full anisotropy is not implemented for MeSigmaIDeriv."
                )

        dMeSigmaI_dI = -self.MeSigmaI**2
        dMe_dsig = self.mesh.getEdgeInnerProductDeriv(self.sigma)(u)
        return dMeSigmaI_dI * (dMe_dsig * self.sigmaDeriv)

    @property
    def MfRho(self):
        """
        Face inner product matrix for \\(\\rho\\). Used in the H-J
        formulation
        """
        if getattr(self, '_MfRho', None) is None:
            self._MfRho = self.mesh.getFaceInnerProduct(self.rho)
        return self._MfRho

    # TODO: This should take a vector
    def MfRhoDeriv(self, u):
        """
        Derivative of :code:`MfRho` with respect to the model.
        """
        if self.rhoMap is None:
            return Utils.Zero()

        return (
            self.mesh.getFaceInnerProductDeriv(self.rho)(u) * self.rhoDeriv
        )

    @property
    def MfRhoI(self):
        """
        Inverse of :code:`MfRho`
        """
        if getattr(self, '_MfRhoI', None) is None:
            self._MfRhoI = self.mesh.getFaceInnerProduct(self.rho, invMat=True)
        return self._MfRhoI

    # TODO: This should take a vector
    def MfRhoIDeriv(self, u):
        """
            Derivative of :code:`MfRhoI` with respect to the model.
        """
        if self.rhoMap is None:
            return Utils.Zero()

        if len(self.rho.shape) > 1:
            if self.rho.shape[1] > self.mesh.dim:
                raise NotImplementedError(
                    "Full anisotropy is not implemented for MfRhoIDeriv."
                )

        dMfRhoI_dI = -self.MfRhoI**2
        dMf_drho = self.mesh.getFaceInnerProductDeriv(self.rho)(u)
        return dMfRhoI_dI * (dMf_drho * self.rhoDeriv)

###############################################################################
#                                                                             #
#                             Base EM Survey                                  #
#                                                                             #
###############################################################################

class BaseEMSurvey(Survey.BaseSurvey):

    def __init__(self, srcList, **kwargs):
        # Sort these by frequency
        self.srcList = srcList
        Survey.BaseSurvey.__init__(self, **kwargs)

    def eval(self, f):
        """Project fields to receiver locations

        :param Fields u: fields object
        :rtype: numpy.ndarray
        :return: data
        """
        data = Survey.Data(self)
        for src in self.srcList:
            for rx in src.rxList:
                data[src, rx] = rx.eval(src, self.mesh, f)
        return data

    def evalDeriv(self, f):
        raise Exception('Use Receivers to project fields deriv.')

###############################################################################
#                                                                             #
#                             Base EM Source                                  #
#                                                                             #
###############################################################################

class BaseEMSrc(Survey.BaseSrc):

    integrate = properties.Bool("integrate the source term?", default=False)

    def eval(self, prob):
        """
        - :math:`s_m` : magnetic source term
        - :math:`s_e` : electric source term

        :param BaseFDEMProblem prob: FDEM Problem
        :rtype: tuple
        :return: tuple with magnetic source term and electric source term
        """
        s_m = self.s_m(prob)
        s_e = self.s_e(prob)
        return s_m, s_e

    def evalDeriv(self, prob, v=None, adjoint=False):
        """
        Derivatives of the source terms with respect to the inversion model
        - :code:`s_mDeriv` : derivative of the magnetic source term
        - :code:`s_eDeriv` : derivative of the electric source term

        :param BaseFDEMProblem prob: FDEM Problem
        :param numpy.ndarray v: vector to take product with
        :param bool adjoint: adjoint?
        :rtype: tuple
        :return: tuple with magnetic source term and electric source term
            derivatives times a vector

        """
        if v is not None:
            return (
                self.s_mDeriv(prob, v, adjoint),
                self.s_eDeriv(prob, v, adjoint)
            )
        else:
            return (
                lambda v: self.s_mDeriv(prob, v, adjoint),
                lambda v: self.s_eDeriv(prob, v, adjoint)
            )

    def s_m(self, prob):
        """
        Magnetic source term

        :param BaseFDEMProblem prob: FDEM Problem
        :rtype: numpy.ndarray
        :return: magnetic source term on mesh
        """
        return Utils.Zero()

    def s_e(self, prob):
        """
        Electric source term

        :param BaseFDEMProblem prob: FDEM Problem
        :rtype: numpy.ndarray
        :return: electric source term on mesh
        """
        return Utils.Zero()

    def s_mDeriv(self, prob, v, adjoint=False):
        """
        Derivative of magnetic source term with respect to the inversion model

        :param BaseFDEMProblem prob: FDEM Problem
        :param numpy.ndarray v: vector to take product with
        :param bool adjoint: adjoint?
        :rtype: numpy.ndarray
        :return: product of magnetic source term derivative with a vector
        """

        return Utils.Zero()

    def s_eDeriv(self, prob, v, adjoint=False):
        """
        Derivative of electric source term with respect to the inversion model

        :param BaseFDEMProblem prob: FDEM Problem
        :param numpy.ndarray v: vector to take product with
        :param bool adjoint: adjoint?
        :rtype: numpy.ndarray
        :return: product of electric source term derivative with a vector
        """
        return Utils.Zero()
<commit_msg>clear mu mats and sigma mats if the fixed variables sigma, mu are updated<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import properties
from scipy.constants import mu_0

from SimPEG import Survey
from SimPEG import Problem
from SimPEG import Utils
from SimPEG import Maps
from SimPEG import Props
from SimPEG import Solver as SimpegSolver

__all__ = ['BaseEMProblem', 'BaseEMSurvey', 'BaseEMSrc']

###############################################################################
#                                                                             #
#                             Base EM Problem                                 #
#                                                                             #
###############################################################################

class BaseEMProblem(Problem.BaseProblem):

    sigma, sigmaMap, sigmaDeriv = Props.Invertible(
        "Electrical conductivity (S/m)"
    )

    rho, rhoMap, rhoDeriv = Props.Invertible(
        "Electrical resistivity (Ohm m)"
    )

    Props.Reciprocal(sigma, rho)

    mu = Props.PhysicalProperty(
        "Magnetic Permeability (H/m)",
        default=mu_0
    )
    mui = Props.PhysicalProperty(
        "Inverse Magnetic Permeability (m/H)"
    )

    Props.Reciprocal(mu, mui)

    surveyPair = Survey.BaseSurvey  #: The survey to pair with.
    dataPair = Survey.Data  #: The data to pair with.

    mapPair = Maps.IdentityMap  #: Type of mapping to pair with

    Solver = SimpegSolver  #: Type of solver to pair with
    solverOpts = {}  #: Solver options

    verbose = False

    ####################################################
    # Make A Symmetric
    ####################################################
    @property
    def _makeASymmetric(self):
        if getattr(self, '__makeASymmetric', None) is None:
            self.__makeASymmetric = True
        return self.__makeASymmetric

    ####################################################
    # Mass Matrices
    ####################################################

    @property
    def _clear_on_mu_update(self):
        return ['_MeMu', '_MeMuI', '_MfMui', '_MfMuiI']

    @property
    def _clear_on_sigma_update(self):
        return ['_MeSigma', '_MeSigmaI', '_MfRho', '_MfRhoI']

    @property
    def deleteTheseOnModelUpdate(self):
        toDelete = []
        if self.sigmaMap is not None or self.rhoMap is not None:
            toDelete += self._clear_on_sigma_update

        if hasattr(self, 'muMap') or hasattr(self, 'muiMap'):
            if self.muMap is not None or self.muiMap is not None:
                toDelete += self._clear_on_mu_update
        return toDelete

    @properties.observer('mu')
    def _clear_mu_mats_on_mu_update(self, change):
        for mat in self._clear_on_mu_update:
            if hasattr(self, mat):
                delattr(self, mat)

    @properties.observer('mui')
    def _clear_mu_mats_on_mui_update(self, change):
        for mat in self._clear_on_mu_update:
            if hasattr(self, mat):
                delattr(self, mat)

    @properties.observer('sigma')
    def _clear_sigma_mats_on_sigma_update(self, change):
        for mat in self._clear_on_sigma_update:
            if hasattr(self, mat):
                delattr(self, mat)

    @properties.observer('rho')
    def _clear_sigma_mats_on_rho_update(self, change):
        for mat in self._clear_on_sigma_update:
            if hasattr(self, mat):
                delattr(self, mat)

    @property
    def Me(self):
        """
            Edge inner product matrix
        """
        if getattr(self, '_Me', None) is None:
            self._Me = self.mesh.getEdgeInnerProduct()
        return self._Me

    @property
    def MeI(self):
        """
            Edge inner product matrix
        """
        if getattr(self, '_MeI', None) is None:
            self._MeI = self.mesh.getEdgeInnerProduct(invMat=True)
        return self._MeI

    @property
    def Mf(self):
        """
            Face inner product matrix
        """
        if getattr(self, '_Mf', None) is None:
            self._Mf = self.mesh.getFaceInnerProduct()
        return self._Mf

    @property
    def MfI(self):
        """
            Face inner product matrix
        """
        if getattr(self, '_MfI', None) is None:
            self._MfI = self.mesh.getFaceInnerProduct(invMat=True)
        return self._MfI

    @property
    def Vol(self):
        if getattr(self, '_Vol', None) is None:
            self._Vol = Utils.sdiag(self.mesh.vol)
        return self._Vol

    ####################################################
    # Magnetic Permeability
    ####################################################
    @property
    def MfMui(self):
        """
        Face inner product matrix for \\(\\mu^{-1}\\).
        Used in the E-B formulation
        """
        if getattr(self, '_MfMui', None) is None:
            self._MfMui = self.mesh.getFaceInnerProduct(self.mui)
        return self._MfMui

    def MfMuiDeriv(self, u):
        """
        Derivative of :code:`MfMui` with respect to the model.
        """
        if self.muiMap is None:
            return Utils.Zero()

        return (
            self.mesh.getFaceInnerProductDeriv(self.mui)(u) * self.muiDeriv
        )

    @property
    def MfMuiI(self):
        """
        Inverse of :code:`MfMui`.
        """
        if getattr(self, '_MfMuiI', None) is None:
            self._MfMuiI = self.mesh.getFaceInnerProduct(self.mui, invMat=True)
        return self._MfMuiI

    # TODO: This should take a vector
    def MfMuiIDeriv(self, u):
        """
        Derivative of :code:`MfMui` with respect to the model
        """

        if self.muiMap is None:
            return Utils.Zero()

        if len(self.mui.shape) > 1:
            if self.mui.shape[1] > self.mesh.dim:
                raise NotImplementedError(
                        "Full anisotropy is not implemented for MfMuiIDeriv."
                )

        dMfMuiI_dI = -self.MfMuiI**2
        dMf_dmui = self.mesh.getEdgeInnerProductDeriv(self.mui)(u)
        return dMfMuiI_dI * (dMf_dmui * self.muiDeriv)

    @property
    def MeMu(self):
        """
        Edge inner product matrix for \\(\\mu\\).
        Used in the H-J formulation
        """
        if getattr(self, '_MeMu', None) is None:
            self._MeMu = self.mesh.getEdgeInnerProduct(self.mu)
        return self._MeMu

    def MeMuDeriv(self, u):
        """
        Derivative of :code:`MeMu` with respect to the model.
        """
        if self.muMap is None:
            return Utils.Zero()

        return (
            self.mesh.getEdgeInnerProductDeriv(self.mu)(u) * self.muDeriv
        )

    @property
    def MeMuI(self):
        """
            Inverse of :code:`MeMu`
        """
        if getattr(self, '_MeMuI', None) is None:
            self._MeMuI = self.mesh.getEdgeInnerProduct(self.mu, invMat=True)
        return self._MeMuI

    # TODO: This should take a vector
    def MeMuIDeriv(self, u):
        """
        Derivative of :code:`MeMuI` with respect to the model
        """

        if self.muMap is None:
            return Utils.Zero()

        if len(self.mu.shape) > 1:
            if self.mu.shape[1] > self.mesh.dim:
                raise NotImplementedError(
                    "Full anisotropy is not implemented for MeMuIDeriv."
                )

        dMeMuI_dI = -self.MeMuI**2
        dMe_dmu = self.mesh.getEdgeInnerProductDeriv(self.mu)(u)
        return dMeMuI_dI * (dMe_dmu * self.muDeriv)

    ####################################################
    # Electrical Conductivity
    ####################################################
    @property
    def MeSigma(self):
        """
        Edge inner product matrix for \\(\\sigma\\).
        Used in the E-B formulation
        """
        if getattr(self, '_MeSigma', None) is None:
            self._MeSigma = self.mesh.getEdgeInnerProduct(self.sigma)
        return self._MeSigma

    # TODO: This should take a vector
    def MeSigmaDeriv(self, u):
        """
        Derivative of MeSigma with respect to the model
        """
        if self.sigmaMap is None:
            return Utils.Zero()

        return (
            self.mesh.getEdgeInnerProductDeriv(self.sigma)(u) *
            self.sigmaDeriv
        )

    @property
    def MeSigmaI(self):
        """
        Inverse of the edge inner product matrix for \\(\\sigma\\).
        """
        if getattr(self, '_MeSigmaI', None) is None:
            self._MeSigmaI = self.mesh.getEdgeInnerProduct(
                self.sigma, invMat=True
            )
        return self._MeSigmaI

    # TODO: This should take a vector
    def MeSigmaIDeriv(self, u):
        """
        Derivative of :code:`MeSigmaI` with respect to the model
        """
        if self.sigmaMap is None:
            return Utils.Zero()

        if len(self.sigma.shape) > 1:
            if self.sigma.shape[1] > self.mesh.dim:
                raise NotImplementedError(
                    "Full anisotropy is not implemented for MeSigmaIDeriv."
                )

        dMeSigmaI_dI = -self.MeSigmaI**2
        dMe_dsig = self.mesh.getEdgeInnerProductDeriv(self.sigma)(u)
        return dMeSigmaI_dI * (dMe_dsig * self.sigmaDeriv)

    @property
    def MfRho(self):
        """
        Face inner product matrix for \\(\\rho\\). Used in the H-J
        formulation
        """
        if getattr(self, '_MfRho', None) is None:
            self._MfRho = self.mesh.getFaceInnerProduct(self.rho)
        return self._MfRho

    # TODO: This should take a vector
    def MfRhoDeriv(self, u):
        """
        Derivative of :code:`MfRho` with respect to the model.
        """
        if self.rhoMap is None:
            return Utils.Zero()

        return (
            self.mesh.getFaceInnerProductDeriv(self.rho)(u) * self.rhoDeriv
        )

    @property
    def MfRhoI(self):
        """
        Inverse of :code:`MfRho`
        """
        if getattr(self, '_MfRhoI', None) is None:
            self._MfRhoI = self.mesh.getFaceInnerProduct(self.rho, invMat=True)
        return self._MfRhoI

    # TODO: This should take a vector
    def MfRhoIDeriv(self, u):
        """
            Derivative of :code:`MfRhoI` with respect to the model.
        """
        if self.rhoMap is None:
            return Utils.Zero()

        if len(self.rho.shape) > 1:
            if self.rho.shape[1] > self.mesh.dim:
                raise NotImplementedError(
                    "Full anisotropy is not implemented for MfRhoIDeriv."
                )

        dMfRhoI_dI = -self.MfRhoI**2
        dMf_drho = self.mesh.getFaceInnerProductDeriv(self.rho)(u)
        return dMfRhoI_dI * (dMf_drho * self.rhoDeriv)

###############################################################################
#                                                                             #
#                             Base EM Survey                                  #
#                                                                             #
###############################################################################

class BaseEMSurvey(Survey.BaseSurvey):

    def __init__(self, srcList, **kwargs):
        # Sort these by frequency
        self.srcList = srcList
        Survey.BaseSurvey.__init__(self, **kwargs)

    def eval(self, f):
        """Project fields to receiver locations

        :param Fields u: fields object
        :rtype: numpy.ndarray
        :return: data
        """
        data = Survey.Data(self)
        for src in self.srcList:
            for rx in src.rxList:
                data[src, rx] = rx.eval(src, self.mesh, f)
        return data

    def evalDeriv(self, f):
        raise Exception('Use Receivers to project fields deriv.')

###############################################################################
#                                                                             #
#                             Base EM Source                                  #
#                                                                             #
###############################################################################

class BaseEMSrc(Survey.BaseSrc):

    integrate = properties.Bool("integrate the source term?", default=False)

    def eval(self, prob):
        """
        - :math:`s_m` : magnetic source term
        - :math:`s_e` : electric source term

        :param BaseFDEMProblem prob: FDEM Problem
        :rtype: tuple
        :return: tuple with magnetic source term and electric source term
        """
        s_m = self.s_m(prob)
        s_e = self.s_e(prob)
        return s_m, s_e

    def evalDeriv(self, prob, v=None, adjoint=False):
        """
        Derivatives of the source terms with respect to the inversion model
        - :code:`s_mDeriv` : derivative of the magnetic source term
        - :code:`s_eDeriv` : derivative of the electric source term

        :param BaseFDEMProblem prob: FDEM Problem
        :param numpy.ndarray v: vector to take product with
        :param bool adjoint: adjoint?
        :rtype: tuple
        :return: tuple with magnetic source term and electric source term
            derivatives times a vector

        """
        if v is not None:
            return (
                self.s_mDeriv(prob, v, adjoint),
                self.s_eDeriv(prob, v, adjoint)
            )
        else:
            return (
                lambda v: self.s_mDeriv(prob, v, adjoint),
                lambda v: self.s_eDeriv(prob, v, adjoint)
            )

    def s_m(self, prob):
        """
        Magnetic source term

        :param BaseFDEMProblem prob: FDEM Problem
        :rtype: numpy.ndarray
        :return: magnetic source term on mesh
        """
        return Utils.Zero()

    def s_e(self, prob):
        """
        Electric source term

        :param BaseFDEMProblem prob: FDEM Problem
        :rtype: numpy.ndarray
        :return: electric source term on mesh
        """
        return Utils.Zero()

    def s_mDeriv(self, prob, v, adjoint=False):
        """
        Derivative of magnetic source term with respect to the inversion model

        :param BaseFDEMProblem prob: FDEM Problem
        :param numpy.ndarray v: vector to take product with
        :param bool adjoint: adjoint?
        :rtype: numpy.ndarray
        :return: product of magnetic source term derivative with a vector
        """

        return Utils.Zero()

    def s_eDeriv(self, prob, v, adjoint=False):
        """
        Derivative of electric source term with respect to the inversion model

        :param BaseFDEMProblem prob: FDEM Problem
        :param numpy.ndarray v: vector to take product with
        :param bool adjoint: adjoint?
        :rtype: numpy.ndarray
        :return: product of electric source term derivative with a vector
        """
        return Utils.Zero()
<|endoftext|>


<commit_before>from pytest import yield_fixture, raises
from subprocess import check_call
from os import devnull, remove, listdir
from string import ascii_uppercase
from random import choice
from time import strftime
from pygiftgrab import Codec, VideoTargetFactory, VideoFrame, ColourSpace
from giftgrab.utils import inspection

# for easily removing created files when done
tmp_file_prefix = 'tmp_GiftGrab_test_'
factory = None
target = None
frame = None
frame_with_colour_mismatch = None
file_name = None
frame_rate = 60

def __file_ext(codec):
    if codec == Codec.HEVC:
        return 'mp4'
    elif codec == Codec.Xvid:
        return 'avi'
    elif codec == Codec.VP9:
        return 'webm'

def __codec2str(codec):
    if codec == Codec.HEVC:
        return 'HEVC'
    elif codec == Codec.Xvid:
        return 'Xvid'
    elif codec == Codec.VP9:
        return 'VP9'

@yield_fixture(autouse=True)
def peri_test(codec, colour_space):
    # This section runs before each test

    global frame
    frame = VideoFrame(colour_space, 400, 640)

    global frame_with_colour_mismatch
    if colour_space == ColourSpace.BGRA:
        colour_space_with_mismatch = ColourSpace.I420
    else:
        colour_space_with_mismatch = ColourSpace.BGRA
    frame_with_colour_mismatch = VideoFrame(colour_space_with_mismatch,
                                            400, 640)

    global factory
    factory = VideoTargetFactory.get_instance()
    assert factory is not None

    global file_name
    file_name = '{}_{}-{}.{}'.format(
        tmp_file_prefix,
        strftime('%Y-%m-%d-%H-%M-%S'),
        ''.join(choice(ascii_uppercase) for _ in range(5)),
        __file_ext(codec)
    )

    global target, frame_rate
    target = None
    target = factory.create_file_writer(codec,
                                        file_name,
                                        frame_rate)
    assert target is not None

    try:
        file_null = open(devnull, 'w')
        check_call(['ffprobe', '--help'],
                   stdout=file_null, stderr=file_null)
    except BaseException:
        raise RuntimeError(
            'FFmpeg does not appear to be installed on your system,\n'
            'GiftGrab tests need FFmpeg.')

    # Run test
    yield

    # This section runs after each test

    global tmp_file_prefix
    for f in listdir('.'):
        if str(f).startswith(tmp_file_prefix):
            remove(f)

def test_append_with_colour_mismatch(codec):
    if codec != Codec.Xvid:
        return

    with raises(RuntimeError):
        target.append(frame_with_colour_mismatch)
    target.finalise()

def test_frame_rate():
    global target, file_name, frame_rate
    for i in range(10):
        target.append(frame)
    target.finalise()
    assert inspection.frame_rate(file_name) == frame_rate

def test_resolution(colour_space):
    rows = 1080
    cols = 1920
    frame1920x1080 = VideoFrame(colour_space, cols, rows)
    global target, file_name
    for i in range(10):
        target.append(frame1920x1080)
    target.finalise()
    assert inspection.resolution(file_name) == (cols, rows)

def test_num_frames(codec):
    global target, file_name
    num_frames = 3 * frame_rate
    for i in range(num_frames):
        target.append(frame)
    target.finalise()
    assert inspection.duration(file_name) * frame_rate == num_frames

def test_filetype_checked(codec):
    file_ext = 'rrr'
    assert not file_ext == __file_ext(codec)
    file_name = '%sfiletype_checked.%s'\
                % (tmp_file_prefix, file_ext)
    target = None
    global factory
    with raises(RuntimeError):
        target = factory.create_file_writer(codec,
                                            file_name,
                                            frame_rate)
    assert target is None
<commit_msg>Issue #122: revised test_num_frames for global specifiers and removed unused args<commit_after>from pytest import yield_fixture, raises
from subprocess import check_call
from os import devnull, remove, listdir
from string import ascii_uppercase
from random import choice
from time import strftime
from pygiftgrab import Codec, VideoTargetFactory, VideoFrame, ColourSpace
from giftgrab.utils import inspection

# for easily removing created files when done
tmp_file_prefix = 'tmp_GiftGrab_test_'
factory = None
target = None
frame = None
frame_with_colour_mismatch = None
file_name = None
frame_rate = 60

def __file_ext(codec):
    if codec == Codec.HEVC:
        return 'mp4'
    elif codec == Codec.Xvid:
        return 'avi'
    elif codec == Codec.VP9:
        return 'webm'

def __codec2str(codec):
    if codec == Codec.HEVC:
        return 'HEVC'
    elif codec == Codec.Xvid:
        return 'Xvid'
    elif codec == Codec.VP9:
        return 'VP9'

@yield_fixture(autouse=True)
def peri_test(codec, colour_space):
    # This section runs before each test

    global frame
    frame = VideoFrame(colour_space, 400, 640)

    global frame_with_colour_mismatch
    if colour_space == ColourSpace.BGRA:
        colour_space_with_mismatch = ColourSpace.I420
    else:
        colour_space_with_mismatch = ColourSpace.BGRA
    frame_with_colour_mismatch = VideoFrame(colour_space_with_mismatch,
                                            400, 640)

    global factory
    factory = VideoTargetFactory.get_instance()
    assert factory is not None

    global file_name
    file_name = '{}_{}-{}.{}'.format(
        tmp_file_prefix,
        strftime('%Y-%m-%d-%H-%M-%S'),
        ''.join(choice(ascii_uppercase) for _ in range(5)),
        __file_ext(codec)
    )

    global target, frame_rate
    target = None
    target = factory.create_file_writer(codec,
                                        file_name,
                                        frame_rate)
    assert target is not None

    try:
        file_null = open(devnull, 'w')
        check_call(['ffprobe', '--help'],
                   stdout=file_null, stderr=file_null)
    except BaseException:
        raise RuntimeError(
            'FFmpeg does not appear to be installed on your system,\n'
            'GiftGrab tests need FFmpeg.')

    # Run test
    yield

    # This section runs after each test

    global tmp_file_prefix
    for f in listdir('.'):
        if str(f).startswith(tmp_file_prefix):
            remove(f)

def test_append_with_colour_mismatch(codec):
    if codec != Codec.Xvid:
        return

    with raises(RuntimeError):
        target.append(frame_with_colour_mismatch)
    target.finalise()

def test_frame_rate():
    global target, file_name, frame_rate
    for i in range(10):
        target.append(frame)
    target.finalise()
    assert inspection.frame_rate(file_name) == frame_rate

def test_resolution(colour_space):
    rows = 1080
    cols = 1920
    frame1920x1080 = VideoFrame(colour_space, cols, rows)
    global target, file_name
    for i in range(10):
        target.append(frame1920x1080)
    target.finalise()
    assert inspection.resolution(file_name) == (cols, rows)

def test_num_frames():
    global target, file_name, frame, frame_rate
    num_frames = 3 * frame_rate
    for i in range(num_frames):
        target.append(frame)
    target.finalise()
    assert inspection.duration(file_name) * frame_rate == num_frames

def test_filetype_checked(codec):
    file_ext = 'rrr'
    assert not file_ext == __file_ext(codec)
    file_name = '%sfiletype_checked.%s'\
                % (tmp_file_prefix, file_ext)
    target = None
    global factory
    with raises(RuntimeError):
        target = factory.create_file_writer(codec,
                                            file_name,
                                            frame_rate)
    assert target is None
<|endoftext|>


<commit_before>#! /usr/bin/env python
"""
StandardName class to hold a CSDMS standard name.
"""

import re

class Error(Exception):
    """Base class for exceptions from this module."""
    pass

class BadNameError(Error):
    """
    Error to indicate a poorly-formed standard name.
    """
    def __init__(self, name):
        super(BadNameError, self).__init__()
        self._name = name

    def __str__(self):
        return self._name

_PREFIX_REGEX = '^[a-z]([a-z0-9-]|_(?!_))*'
#_PREFIX_REGEX = '^[a-z0-9]([a-z]|_(?!_))*'
_SUFFIX_REGEX = '[a-z0-9]([a-z0-9-]|_(?!_))*[a-z0-9]$'
STANDARD_NAME_REGEX = re.compile(
    _PREFIX_REGEX + '(__)' + _SUFFIX_REGEX
)
#    '^[a-z][a-z0-9_]*[a-z0-9](__)[a-z0-9][a-z0-9_]*[a-z0-9]$'

def is_valid_name(name):
    """
    Check if a string is a valid standard name.

    :name: Standard name as a string

    :returns: True if the string is a valid standard name
    """
    return bool(STANDARD_NAME_REGEX.match(name))

class StandardName(str):
    """
    A CSDMS standard name.
    """
    def __init__(self, name):
        """
        Initialize a standard name object from the string, *name*.
        """
        if not is_valid_name(name):
            raise BadNameError(name)

        super(StandardName, self).__init__(name)

        self._name = name

        (self._object,
         self._quantity,
         self._operators) = StandardName.decompose_name(name)

    @staticmethod
    def decompose_name(name):
        """
        Decompose the *name* standard name string into it's constituent
        parts (object, quantity, and operator). Returns a tuple of
        (object, quantity, operator) where object and quantitiy are strings,
        and operator is itself a tuple of strings (or empty).
        """
        try:
            (object_part, quantity_clause) = name.split('__')
        except ValueError:
            raise BadNameError(name)

        (operators, quantity_part) = StandardName.decompose_quantity(
            quantity_clause)

        return object_part, quantity_part, operators

    def _compose_name(self):
        """
        Create a string from the parts of StandardName.
        """
        operator = '_of_'.join(self._operators)
        if len(operator) > 0:
            quantity = '_of_'.join([operator, self._quantity])
        else:
            quantity = self._quantity

        return self._object + '__' + quantity

    @staticmethod
    def decompose_quantity(quantity_clause):
        """
        Decompose the *quantity_clause* string into operator and base
        quantity constituents. Because multiple operators can act on a
        quantity, the operators are given as a tuple regardless of the
        number of operators actually present. Returns the parts of the
        quantity as a tuple of (operators, base_quantity)
        """
        quantity_parts = quantity_clause.split('_of_')
        quantity = quantity_parts[-1]
        operators = tuple(quantity_parts[:-1])

        return operators, quantity

    @property
    def name(self):
        """
        The full standard name as a string.
        """
        #return self._name
        return str(self)

    #@name.setter
    #def name(self, value):
    #    self._name = value

    @property
    def object(self):
        """
        The object part of the standard name.

        :returns: Object name as a string
        """
        return self._object

    @object.setter
    def object(self, value):
        self._object = value

    @property
    def quantity(self):
        """
        The quantity part of the standard name.

        :returns: Quantity name as a string
        """
        return self._quantity

    @quantity.setter
    def quantity(self, value):
        self._quantity = value

    @property
    def operators(self):
        """
        The operator part of the standard name.

        :returns: Operator name as a string
        """
        return self._operators

    @operators.setter
    def operators(self, value):
        self._operators = value

    def __repr__(self):
        return 'StandardName(%r)' % self.name

    def __str__(self):
        return self._compose_name()

    def __eq__(self, that):
        return self.name == str(that)

    def __ne__(self, that):
        return self.name != str(that)

    def __cmp__(self, that):
        return self.name == str(that)

    def __hash__(self):
        return hash(self.name)
<commit_msg>Added capital letters to object regex.<commit_after>#! /usr/bin/env python
"""
StandardName class to hold a CSDMS standard name.
"""

import re

class Error(Exception):
    """Base class for exceptions from this module."""
    pass

class BadNameError(Error):
    """
    Error to indicate a poorly-formed standard name.
    """
    def __init__(self, name):
        super(BadNameError, self).__init__()
        self._name = name

    def __str__(self):
        return self._name

_PREFIX_REGEX = '^[a-z]([a-zA-Z0-9-]|_(?!_))*'
#_PREFIX_REGEX = '^[a-z0-9]([a-z]|_(?!_))*'
_SUFFIX_REGEX = '[a-z0-9]([a-z0-9-]|_(?!_))*[a-z0-9]$'
STANDARD_NAME_REGEX = re.compile(
    _PREFIX_REGEX + '(__)' + _SUFFIX_REGEX
)
#    '^[a-z][a-z0-9_]*[a-z0-9](__)[a-z0-9][a-z0-9_]*[a-z0-9]$'

def is_valid_name(name):
    """
    Check if a string is a valid standard name.

    :name: Standard name as a string

    :returns: True if the string is a valid standard name
    """
    return bool(STANDARD_NAME_REGEX.match(name))

class StandardName(str):
    """
    A CSDMS standard name.
    """
    def __init__(self, name):
        """
        Initialize a standard name object from the string, *name*.
        """
        if not is_valid_name(name):
            raise BadNameError(name)

        super(StandardName, self).__init__(name)

        self._name = name

        (self._object,
         self._quantity,
         self._operators) = StandardName.decompose_name(name)

    @staticmethod
    def decompose_name(name):
        """
        Decompose the *name* standard name string into it's constituent
        parts (object, quantity, and operator). Returns a tuple of
        (object, quantity, operator) where object and quantitiy are strings,
        and operator is itself a tuple of strings (or empty).
        """
        try:
            (object_part, quantity_clause) = name.split('__')
        except ValueError:
            raise BadNameError(name)

        (operators, quantity_part) = StandardName.decompose_quantity(
            quantity_clause)

        return object_part, quantity_part, operators

    def _compose_name(self):
        """
        Create a string from the parts of StandardName.
        """
        operator = '_of_'.join(self._operators)
        if len(operator) > 0:
            quantity = '_of_'.join([operator, self._quantity])
        else:
            quantity = self._quantity

        return self._object + '__' + quantity

    @staticmethod
    def decompose_quantity(quantity_clause):
        """
        Decompose the *quantity_clause* string into operator and base
        quantity constituents. Because multiple operators can act on a
        quantity, the operators are given as a tuple regardless of the
        number of operators actually present. Returns the parts of the
        quantity as a tuple of (operators, base_quantity)
        """
        quantity_parts = quantity_clause.split('_of_')
        quantity = quantity_parts[-1]
        operators = tuple(quantity_parts[:-1])

        return operators, quantity

    @property
    def name(self):
        """
        The full standard name as a string.
        """
        #return self._name
        return str(self)

    #@name.setter
    #def name(self, value):
    #    self._name = value

    @property
    def object(self):
        """
        The object part of the standard name.

        :returns: Object name as a string
        """
        return self._object

    @object.setter
    def object(self, value):
        self._object = value

    @property
    def quantity(self):
        """
        The quantity part of the standard name.

        :returns: Quantity name as a string
        """
        return self._quantity

    @quantity.setter
    def quantity(self, value):
        self._quantity = value

    @property
    def operators(self):
        """
        The operator part of the standard name.

        :returns: Operator name as a string
        """
        return self._operators

    @operators.setter
    def operators(self, value):
        self._operators = value

    def __repr__(self):
        return 'StandardName(%r)' % self.name

    def __str__(self):
        return self._compose_name()

    def __eq__(self, that):
        return self.name == str(that)

    def __ne__(self, that):
        return self.name != str(that)

    def __cmp__(self, that):
        return self.name == str(that)

    def __hash__(self):
        return hash(self.name)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import io

class Source(object):
    def __init__(self, url_or_file, type_):
        self.source = url_or_file
        self.type = type_

        if self.type == 'file':
            self.checkFiles()

    def isUrl(self):
        return 'url' in self.type

    def isFile(self, path=None):
        # dirty hack to check where file is opened with codecs module
        # (because it returns 'instance' type when encoding is specified
        if path:
            return isinstance(path, io.IOBase) or path.__class__.__name__ == 'StreamReaderWriter'
        else:
            return 'file' in self.type

    def checkFiles(self):
        if isinstance(self.source, list):
            for path in self.source:
                if not os.path.exists(path):
                    raise IOError('No such file: %s' % path)
        else:
            if not hasattr(self.source, 'read') and not os.path.exists(self.source):
                raise IOError('No such file: %s' % self.source)

    def isString(self):
        return 'string' in self.type

    def isFileObj(self):
        return hasattr(self.source, 'read')

    def to_s(self):
        return self.source
<commit_msg>Force decode input string to unicode on python 2<commit_after># -*- coding: utf-8 -*-
import os
import io
try:
    # Python 2.x and 3.x support for checking string types
    assert basestring
    assert unicode
except NameError:
    basestring = str
    unicode = str

class Source(object):
    def __init__(self, url_or_file, type_):
        self.source = url_or_file
        self.type = type_

        if self.type == 'file':
            self.checkFiles()

    def isUrl(self):
        return 'url' in self.type

    def isFile(self, path=None):
        # dirty hack to check where file is opened with codecs module
        # (because it returns 'instance' type when encoding is specified
        if path:
            return isinstance(path, io.IOBase) or path.__class__.__name__ == 'StreamReaderWriter'
        else:
            return 'file' in self.type

    def checkFiles(self):
        if isinstance(self.source, list):
            for path in self.source:
                if not os.path.exists(path):
                    raise IOError('No such file: %s' % path)
        else:
            if not hasattr(self.source, 'read') and not os.path.exists(self.source):
                raise IOError('No such file: %s' % self.source)

    def isString(self):
        return 'string' in self.type

    def isFileObj(self):
        return hasattr(self.source, 'read')

    def to_s(self):
        # String should be in unicode(python2)/str(python3) type since we will
        # later encode it to utf-8 bytes array to pipe into subprocess
        # With some charachters on python 2 it sets to str type (bytes) which is wrong
        # and cant later encode properly, this is a workaround for this.
        # See issue #42
        if isinstance(self.source, unicode):
            return self.source
        else:
            return unicode(self.source, 'utf-8')
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import ctypes

_LIB = ctypes.CDLL('libletmecreate_click.so')

def enable():
    ret = _LIB.proximity_click_enable()
    if ret < 0:
        raise Exception("proximity click enable failed")

def get_measure():
    measure = ctypes.c_uint16(0)
    ret = _LIB.proximity_click_get_measure(ctypes.byref(measure))
    if ret < 0:
        raise Exception("proximity click get measure failed")
    return measure.value

def disable():
    ret = _LIB.proximity_click_disable()
    if ret < 0:
        raise Exception("proximity click disable failed")
<commit_msg>proximity: add docstring<commit_after>#!/usr/bin/env python3
"""Python binding of Proximity Click wrapper of LetMeCreate library.

You must configure and select the right I2C bus before using any of these
functions.
"""
import ctypes

_LIB = ctypes.CDLL('libletmecreate_click.so')

def enable():
    """Enable the Proximity Click.

    Note: An exception is thrown if it fails to enable the sensor.
    """
    ret = _LIB.proximity_click_enable()
    if ret < 0:
        raise Exception("proximity click enable failed")

def get_measure():
    """Returns a measure read from Proximity Click.

    Note: An exception is thrown if it fails to get a measure from the
    Proximity Click.
    """
    measure = ctypes.c_uint16(0)
    ret = _LIB.proximity_click_get_measure(ctypes.byref(measure))
    if ret < 0:
        raise Exception("proximity click get measure failed")
    return measure.value

def disable():
    """Disables the Proximity Click.

    Note: An exception is thrown if it fails to disable the sensor.
    """
    ret = _LIB.proximity_click_disable()
    if ret < 0:
        raise Exception("proximity click disable failed")
<|endoftext|>


<commit_before>#!/bin/env python

from __future__ import print_function

import argparse
import click
import munge
import munge.codec
import munge.codec.all
import os
import peeringdb
from peeringdb import client
from peeringdb.localdb import LocalDB
import pip
from pkg_resources import resource_stream, Requirement
import re
import sys

def install_deps(deps, quiet=True):
    for pkg in deps:
        if pip.main(['install', pkg, '--quiet']):
            # run again to display output
            if not quiet:
                pip.main(['install', pkg])
            raise RuntimeError("failed to install %s" % (pkg,))

def get_deps(typ):
    """ get deps from requirement file of specified type """
    deps=[]
    filename = 'facsimile/requirements-%s.txt' % typ
    with resource_stream(Requirement.parse("peeringdb"), filename) as req:
        for line in req:
            deps.append(line.strip())
    return deps

def dict_prompt(data, key, default=''):
    data[key] = click.prompt(key, default=data.get(key, default))

def db_prompt(data):
    for k in ('host', 'port', 'name', 'user', 'password'):
        dict_prompt(data, k)

def cb_list_codecs(ctx, param, value):
    click.echo(', '.join(munge.codec.list_codecs()))
    ctx.exit()

@click.group()
@click.version_option()
@click.option('--list-codecs', is_flag=True,
    callback=cb_list_codecs,
    expose_value=False, is_eager=True)
def cli():
    """
    PeeringDB
    """
    pass

@cli.command()
@click.option('--config', envvar='PEERINGDB_HOME', default='~/.peeringdb')
def conf_write(config):
    """ write config file with defaults """
    cfg = peeringdb.config.get_config(config)
    peeringdb.config.write_config(cfg, config)
    return 0

@cli.command()
@click.option('--config', help='config directory',
    envvar='PEERINGDB_HOME',
    default='~/.peeringdb', prompt='config directory')
@click.option('--database', help='database type for local sync',
    type=click.Choice(['none', 'mysql', 'sqlite3']),
    default='none',
    prompt='database type for local sync (mysql,sqlite3)')
def configure(config, database):
    """ configure peeringdb """
    cfg = peeringdb.config.get_config(config)
    db = {}
    if database != 'none':
        print("enter the database config, blank for engine defaults")
        db['engine'] = database
        if database == 'mysql':
            db_prompt(db)
        elif database == 'sqlite3':
            db['name'] = click.prompt('sqlite filename', default='peeringdb.sqlite3')

    cfg['database'] = db
    peeringdb.config.write_config(cfg, config)
    return 0

@cli.command()
@click.option('--config', envvar='PEERINGDB_HOME', default='~/.peeringdb')
def depcheck(config):
    """ check for dependencies, install if necessary """
    cfg = peeringdb.config.get_config(config)
    engine = cfg.get('database', {}).get('engine')
#    if engine == 'mysql':
#    elif engine == 'sqlite3':
    install_deps(get_deps(engine))
    return 0

@cli.command()
@click.option('--config', envvar='PEERINGDB_HOME', default='~/.peeringdb')
@click.option('--output-format', default='yaml', help='output data format')
@click.argument('poids', nargs=-1)
def get(config, output_format, poids):
    """ get an object from peeringdb """
    pdb = client.PeeringDB()
    codec = munge.get_codec(output_format)()

    for poid in poids:
        res = parse_objid(poid)
        data = pdb.get(res[0], res[1])
        codec.dump(data, sys.stdout)

@cli.command()
@click.option('--config', envvar='PEERINGDB_HOME', default='~/.peeringdb')
def sync(config):
    """ synchronize to a local database """
    cfg = peeringdb.config.get_config(config)
    db = LocalDB(cfg)
    db.sync()
    return 0

def parse_objid(objid):
    match = re.search('(?P<typ>\w+)[^\d]*(?P<id>\d+)', objid)
    if not match:
        return None
    return (match.group('typ'), match.group('id'))

def try_dump_obj(args):
    res = parse_objid(args.cmd)
    if not res:
        return False

    pdb = client.PeeringDB()
    data = pdb.get(res[0], res[1])
    codec = munge.get_codec(args.output_format)()
    codec.dump(data, sys.stdout)
    return True

<commit_msg>fix list codecs<commit_after>#!/bin/env python

from __future__ import print_function

import argparse
import click
import munge
import munge.codec
import munge.codec.all
import os
import peeringdb
from peeringdb import client
from peeringdb.localdb import LocalDB
import pip
from pkg_resources import resource_stream, Requirement
import re
import sys

def install_deps(deps, quiet=True):
    for pkg in deps:
        if pip.main(['install', pkg, '--quiet']):
            # run again to display output
            if not quiet:
                pip.main(['install', pkg])
            raise RuntimeError("failed to install %s" % (pkg,))

def get_deps(typ):
    """ get deps from requirement file of specified type """
    deps=[]
    filename = 'facsimile/requirements-%s.txt' % typ
    with resource_stream(Requirement.parse("peeringdb"), filename) as req:
        for line in req:
            deps.append(line.strip())
    return deps

def dict_prompt(data, key, default=''):
    data[key] = click.prompt(key, default=data.get(key, default))

def db_prompt(data):
    for k in ('host', 'port', 'name', 'user', 'password'):
        dict_prompt(data, k)

def cb_list_codecs(ctx, param, value):
    if not value or ctx.resilient_parsing:
        return
    click.echo(', '.join(munge.codec.list_codecs()))
    ctx.exit()

@click.group()
@click.version_option()
@click.option('--list-codecs', is_flag=True,
    callback=cb_list_codecs,
    expose_value=False, is_eager=True)
def cli():
    """
    PeeringDB
    """
    pass

@cli.command()
@click.option('--config', envvar='PEERINGDB_HOME', default='~/.peeringdb')
def conf_write(config):
    """ write config file with defaults """
    cfg = peeringdb.config.get_config(config)
    peeringdb.config.write_config(cfg, config)
    return 0

@cli.command()
@click.option('--config', help='config directory',
    envvar='PEERINGDB_HOME',
    default='~/.peeringdb', prompt='config directory')
@click.option('--database', help='database type for local sync',
    type=click.Choice(['none', 'mysql', 'sqlite3']),
    default='none',
    prompt='database type for local sync (mysql,sqlite3)')
def configure(config, database):
    """ configure peeringdb """
    cfg = peeringdb.config.get_config(config)
    db = {}
    if database != 'none':
        print("enter the database config, blank for engine defaults")
        db['engine'] = database
        if database == 'mysql':
            db_prompt(db)
        elif database == 'sqlite3':
            db['name'] = click.prompt('sqlite filename', default='peeringdb.sqlite3')

    cfg['database'] = db
    peeringdb.config.write_config(cfg, config)
    return 0

@cli.command()
@click.option('--config', envvar='PEERINGDB_HOME', default='~/.peeringdb')
def depcheck(config):
    """ check for dependencies, install if necessary """
    cfg = peeringdb.config.get_config(config)
    engine = cfg.get('database', {}).get('engine')
#    if engine == 'mysql':
#    elif engine == 'sqlite3':
    install_deps(get_deps(engine))
    return 0

@cli.command()
@click.option('--config', envvar='PEERINGDB_HOME', default='~/.peeringdb')
@click.option('--output-format', default='yaml', help='output data format')
@click.argument('poids', nargs=-1)
def get(config, output_format, poids):
    """ get an object from peeringdb """
    pdb = client.PeeringDB()
    codec = munge.get_codec(output_format)()

    for poid in poids:
        res = parse_objid(poid)
        data = pdb.get(res[0], res[1])
        codec.dump(data, sys.stdout)

@cli.command()
@click.option('--config', envvar='PEERINGDB_HOME', default='~/.peeringdb')
def sync(config):
    """ synchronize to a local database """
    cfg = peeringdb.config.get_config(config)
    db = LocalDB(cfg)
    db.sync()
    return 0

def parse_objid(objid):
    match = re.search('(?P<typ>\w+)[^\d]*(?P<id>\d+)', objid)
    if not match:
        return None
    return (match.group('typ'), match.group('id'))

def try_dump_obj(args):
    res = parse_objid(args.cmd)
    if not res:
        return False

    pdb = client.PeeringDB()
    data = pdb.get(res[0], res[1])
    codec = munge.get_codec(args.output_format)()
    codec.dump(data, sys.stdout)
    return True

<|endoftext|>