<commit_before># -*- coding: utf-8 -*-
# OpenID relying party library
# Copyright Martin v. Löwis, 2009
# Licensed under the Academic Free License, version 3

# This library implements OpenID Authentication 2.0,
# in the role of a relying party
# It has the following assumptions and limitations:
# - service discovery requires YADIS (HTML discovery not implemented)
# - only provider-directed mode (identifier_select) is supported
# - direct requests require https
# - as a signature algorithm, HMAC-SHA1 is requested

import urllib, BeautifulSoup, xml.etree.ElementTree
import cStringIO, base64, hmac, sha, datetime

# Don't use urllib2, since it breaks in 2.5
# for https://login.launchpad.net//+xrds

def parse_response(s):
    '''Parse a key-value form (OpenID section 4.1.1) into a dictionary'''
    res = {}
    for line in s.splitlines():
        k,v = line.split(':', 1)
        res[k] = v
    return res

def discover(url):
    '''Perform service discovery on the OP URL.
    Return list of service types, and the auth/2.0 URL,
    or None if discovery fails.'''
    opener = urllib.FancyURLopener()
    opener.addheader('Accept', "text/html; q=0.3, "+
                     "application/xhtml+xml; q=0.5, "+
                     "application/xrds+xml")
    res = opener.open(url)

    content_type = res.headers.gettype()

    # Yadis 6.2.5 option 2 and 3: header includes x-xrds-location
    xrds_loc = res.headers.get('x-xrds-location')
    if xrds_loc and content_type != 'application/xrds+xml':
        res.close()
        return discover(xrds_loc)

    if content_type == 'text/html':
        data = res.read()
        res.close()
        soup = BeautifulSoup.BeautifulSoup(data)
        # Yadis 6.2.5 option 1: meta tag
        meta = soup.findAll('meta', {'http-equiv':lambda v:v.lower()=='x-xrds-location'})
        if meta:
            xrds_loc = meta.attrs['content']
            return discover(xrds_loc)
        # OpenID 7.3, item 3: attempt html based discovery
        # TODO
        return None

    if content_type == 'application/xrds+xml':
        # Yadis 6.2.5 option 4
        doc = xml.etree.ElementTree.parse(res)
        res.close()
        for svc in doc.findall(".//{xri://$xrd*($v*2.0)}Service"):
            services = [x.text for x in svc.findall("{xri://$xrd*($v*2.0)}Type")]
            if not 'http://specs.openid.net/auth/2.0/server' in services:
                # We don't support discovery of Claimed Identifiers, only OP Identifiers
                continue
            uri = svc.find("{xri://$xrd*($v*2.0)}URI")
            if uri is None:
                return None # invalid service
            break
        else:
            return None # No OpenID 2.0 service found
    return services, uri.text

def associate(url):
    '''Create an association (OpenID section 8) between RP and OP.
    Return response as a dictionary.'''
    assert url.startswith('https') # we only support no-encryption sessions
    data = {
        'openid.ns':"http://specs.openid.net/auth/2.0",
        'openid.mode':"associate",
        'openid.assoc_type':"HMAC-SHA1",
        'openid.session_type':"no-encryption",
        }
    res = urllib.urlopen(url, urllib.urlencode(data))
    data = parse_response(res.read())
    return data

def request_authentication(services, url, assoc_handle, return_to):
    '''Request authentication (OpenID section 9).
    services is the list of discovered service types,
    url the OP service URL, assoc_handle the established session
    dictionary, and return_to the return URL.

    The return_to URL will also be passed as realm, and the
    OP may perform RP discovery on it; always request these
    data through SREG 1.0 as well.

    If AX or SREG 1.1 are supported, request email address,
    first/last name, or nickname.

    Return the URL that the browser should be redirected to.'''
    data = {
        'openid.ns':"http://specs.openid.net/auth/2.0",
        'openid.mode':"checkid_setup",
        'openid.assoc_handle':assoc_handle,
        'openid.return_to':return_to,
        'openid.claimed_id':"http://specs.openid.net/auth/2.0/identifier_select",
        'openid.identity':"http://specs.openid.net/auth/2.0/identifier_select",
        'openid.realm':return_to,
        'openid.ns.sreg':"http://openid.net/sreg/1.0",
        'openid.sreg.required':'nickname,email',
        }
    if "http://openid.net/srv/ax/1.0" in services:
        data.update({
            'openid.ns.ax':"http://openid.net/srv/ax/1.0",
            'openid.ax.mode':'fetch_request',
            'openid.ax.required':'email,first,last',
            'openid.ax.type.email':'http://axschema.org/contact/email',
            'openid.ax.type.first':"http://axschema.org/namePerson/first",
            'openid.ax.type.last':"http://axschema.org/namePerson/last",
            })
    if "http://openid.net/extensions/sreg/1.1" in services:
        data.update({
            'openid.ns.sreg11':"http://openid.net/extensions/sreg/1.1",
            'openid.sreg11.required':'nickname,email'
            })
    return url+"?"+urllib.urlencode(data)

class NotAuthenticated(Exception):
    pass

def authenticate(session, response):
    '''Process an authentication response.
    session must be the established session (minimally including
    assoc_handle and mac_key), response is the query string as parse
    by cgi.parse_qs.
    If authentication succeeds, return None.
    If the user was not authenticated, NotAuthenticated is raised.
    If the HTTP request is invalid (missing parameters, failure to
    validate signature), different exceptions will be raised, typically
    ValueError.

    Callers must check openid.response_nonce for replay attacks.
    '''

    if response['openid.ns'][0] != 'http://specs.openid.net/auth/2.0':
        raise ValueError('missing openid.ns')
    if session['assoc_handle'] != response['openid.assoc_handle'][0]:
        raise ValueError('incorrect session')
    if response['openid.mode'][0] == 'cancel':
        raise NotAuthenticated('provider did not authenticate user (cancelled)')
    if response['openid.mode'][0] != 'id_res':
        raise ValueError('invalid openid.mode')
    if  'openid.claimed_id' not in response:
        raise ValueError('missing openid.claimed_id')

    # Won't check nonce value - caller must verify this is not a replay

    signed = response['openid.signed'][0].split(',')
    if not (set(['op_endpoint', 'return_to', 'response_nonce',
                 'assoc_handle', 'claimed_id']) < set(signed)):
        raise ValueError("signature failed to sign important fields")
    query = []
    for name in signed:
        value = response['openid.'+name][0]
        query.append('%s:%s\n' % (name, value))
    query = ''.join(query)

    mac_key = base64.decodestring(session['mac_key'])
    transmitted_sig = base64.decodestring(response['openid.sig'][0])
    computed_sig = hmac.new(mac_key, query, sha).digest()

    if transmitted_sig != computed_sig:
        raise ValueError('Invalid signature')

def parse_nonce(nonce):
    '''Split a nonce into a (timestamp, ID) pair'''
    stamp = nonce.split('Z', 1)[0]
    stamp = datetime.datetime.strptime(stamp,"%Y-%m-%dT%H:%M:%S")
    return stamp

def get_namespaces(resp):
    res = {}
    for k, v in resp.items():
        if k.startswith('openid.ns.'):
            k = k.rsplit('.', 1)[1]
            res[v[0]] = k
    return res

def get_ax(resp, ns, validated):
    if "http://openid.net/srv/ax/1.0" not in ns:
        return {}
    ax = ns["http://openid.net/srv/ax/1.0"]+"."
    oax = "openid."+ax
    res = {}
    for k, v in resp.items():
        if k.startswith(oax+"type."):
            k = k.rsplit('.',1)[1]
            value_name = oax+"value."+k
            if ax+"value."+k not in validated:
                continue
            res[v[0]] = resp[value_name][0]
    return res

def get_email(resp):
    "Return the email address embedded response, or None."

    validated = resp['openid.signed'][0]

    # SREG 1.0; doesn't require namespace, as the protocol doesn't
    # specify one
    if 'openid.sreg.email' in resp and \
       'sreg.email' in validated:
        return resp['openid.sreg.email'][0]

    ns = get_namespaces(resp)

    ax = get_ax(resp, ns, validated)
    if "http://axschema.org/contact/email" in ax:
        return ax["http://axschema.org/contact/email"]

    # TODO: SREG 1.1
    return None

def get_username(resp):
    "Return either nickname or (first, last) or None."

    validated = resp['openid.signed'][0]
    if 'openid.sreg.nickname' in resp and \
       'sreg.nickname' in validated:
        return resp['openid.sreg.nickname'][0]

    ns = get_namespaces(resp)

    ax = get_ax(resp, ns, validated)
    if "http://axschema.org/namePerson/first" in ax and \
       "http://axschema.org/namePerson/last" in ax:
        return (ax["http://axschema.org/namePerson/first"],
                ax["http://axschema.org/namePerson/last"])

    # TODO: SREG 1.1
    return

################ Test Server #################################

import BaseHTTPServer, cgi

# supported providers
providers = (
    ('Google', 'http://www.google.com/favicon.ico', 'https://www.google.com/accounts/o8/id'),
    ('Yahoo', 'http://www.yahoo.com/favicon.ico', 'http://yahoo.com/'),
    # Verisigns service URL is not https
    #('Verisign', 'https://pip.verisignlabs.com/favicon.ico', 'https://pip.verisignlabs.com')
    ('myOpenID', 'https://www.myopenid.com/favicon.ico', 'https://www.myopenid.com/'),
    ('Launchpad', 'https://login.launchpad.net/favicon.ico', 'https://login.launchpad.net/')
    )

sessions = []
class Handler(BaseHTTPServer.BaseHTTPRequestHandler):

    def write(self, payload, type):
        self.send_response(200)
        self.send_header("Content-type", type)
        self.send_header("Content-length", str(len(payload)))
        self.end_headers()
        self.wfile.write(payload)

    def do_GET(self):
        if self.path == '/':
            return self.root()
        path = self.path
        i = path.rfind('?')
        if i >= 0:
            query = cgi.parse_qs(path[i+1:])
            path = path[:i]
        else:
            query = {}
        if path == '/':
            if 'provider' in query:
                prov = [p for p in providers if p[0]  == query['provider'][0]]
                if len(prov) != 1:
                    return self.not_found()
                prov = prov[0]
                services, url = discover(prov[2])
                session = associate(url)
                print session
                sessions.append(session)
                self.send_response(307) # temporary redirect - do not cache
                self.send_header("Location", request_authentication
                                 (services, url, session['assoc_handle'],
                                  self.base_url+"?returned=1"))
                self.end_headers()
                return
            if 'returned' in query:
                if 'openid.ns' not in query:
                    return self.rp_discovery()
                handle = query['openid.assoc_handle'][0]
                for session in sessions:
                    if session['assoc_handle'] == handle:
                        break
                else:
                    session = None
                if not session:
                    return self.error('Not authenticated (no session)')
                print query
                try:
                    authenticate(session, query)
                except Exception, e:
                    self.error("Authentication failed"+repr(e))
                claimed = query['openid.claimed_id'][0]
                payload = "Hello "+claimed+"\n"
                email = get_email(query)
                if email:
                    payload += 'Your email is '+email+"\n"
                else:
                    payload += 'No email address is known\n'
                username = get_username(query)
                if isinstance(username, tuple):
                    username = " ".join(username)
                if username:
                    payload += 'Your nickname is '+username+'\n'
                else:
                    payload += 'No nickname is known\n'
                return self.write(payload, "text/plain")

        return self.not_found()

    def debug(self, value):
        payload = repr(value)
        self.write(payload, "text/plain")

    def error(self, text):
        self.write(text, "text/plain")

    def root(self):
        payload = "<title>OpenID login</title>"

        for name, icon, provider in providers:
            payload += "

" % (
                self.base_url, name, icon, name)
        payload += ""
        self.write(payload, "text/html")

    def rp_discovery(self):
        payload = '''<xrds:XRDS
                xmlns:xrds="xri://$xrds"
                xmlns="xri://$xrd*($v*2.0)">
                <XRD>
                     <Service priority="1">
                              <Type>http://specs.openid.net/auth/2.0/return_to</Type>
                              <URI>%s</URI>
                     </Service>
                </XRD>
                </xrds:XRDS>
        ''' % (self.base_url+"/?returned=1")
        self.write(payload, 'application/xrds+xml')

    def not_found(self):
        self.send_response(404)
        self.end_headers()

# OpenID providers often attempt relying-party discovery
# This requires the test server to use a globally valid URL
# If Python cannot correctly determine the base URL, you
# can pass it as command line argument
def test_server():
    import socket, sys
    if len(sys.argv) > 1:
        base_url = sys.argv[1]
    else:
        base_url = "http://" + socket.getfqdn() + "/"
    Handler.base_url = base_url
    BaseHTTPServer.HTTPServer.address_family = socket.AF_INET6
    httpd = BaseHTTPServer.HTTPServer(('', 8000), Handler)
    httpd.serve_forever()

if __name__ == '__main__':
    test_server()
<commit_msg>Use httplib instead of urllib.<commit_after># -*- coding: utf-8 -*-
# OpenID relying party library
# Copyright Martin v. Löwis, 2009
# Licensed under the Academic Free License, version 3

# This library implements OpenID Authentication 2.0,
# in the role of a relying party
# It has the following assumptions and limitations:
# - service discovery requires YADIS (HTML discovery not implemented)
# - only provider-directed mode (identifier_select) is supported
# - direct requests require https
# - as a signature algorithm, HMAC-SHA1 is requested

import urlparse, httplib, BeautifulSoup, xml.etree.ElementTree
import cStringIO, base64, hmac, sha, datetime

# Don't use urllib2, since it breaks in 2.5
# for https://login.launchpad.net//+xrds

# Don't use urllib, since it sometimes selects HTTP/1.1 (e.g. in PyPI)
# and then fails to parse chunked responses.

def parse_response(s):
    '''Parse a key-value form (OpenID section 4.1.1) into a dictionary'''
    res = {}
    for line in s.splitlines():
        k,v = line.split(':', 1)
        res[k] = v
    return res

def discover(url):
    '''Perform service discovery on the OP URL.
    Return list of service types, and the auth/2.0 URL,
    or None if discovery fails.'''
    scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
    assert not query and not fragment
    assert scheme == 'https'
    conn = httplib.HTTPSConnection(netloc)
    conn.putrequest("GET", path)
    conn.putheader('Accept', "text/html; q=0.3, "+
                   "application/xhtml+xml; q=0.5, "+
                   "application/xrds+xml")
    conn.endheaders()

    res = conn.getresponse()
    data = res.read()
    conn.close()

    content_type = res.msg.gettype()

    # Yadis 6.2.5 option 2 and 3: header includes x-xrds-location
    xrds_loc = res.msg.get('x-xrds-location')
    if xrds_loc and content_type != 'application/xrds+xml':
        return discover(xrds_loc)

    if content_type == 'text/html':
        soup = BeautifulSoup.BeautifulSoup(data)
        # Yadis 6.2.5 option 1: meta tag
        meta = soup.findAll('meta', {'http-equiv':lambda v:v.lower()=='x-xrds-location'})
        if meta:
            xrds_loc = meta.attrs['content']
            return discover(xrds_loc)
        # OpenID 7.3, item 3: attempt html based discovery
        # TODO
        return None

    if content_type == 'application/xrds+xml':
        # Yadis 6.2.5 option 4
        doc = xml.etree.ElementTree.fromstring(data)
        for svc in doc.findall(".//{xri://$xrd*($v*2.0)}Service"):
            services = [x.text for x in svc.findall("{xri://$xrd*($v*2.0)}Type")]
            if not 'http://specs.openid.net/auth/2.0/server' in services:
                # We don't support discovery of Claimed Identifiers, only OP Identifiers
                continue
            uri = svc.find("{xri://$xrd*($v*2.0)}URI")
            if uri is None:
                return None # invalid service
            break
        else:
            return None # No OpenID 2.0 service found
    return services, uri.text

def associate(url):
    '''Create an association (OpenID section 8) between RP and OP.
    Return response as a dictionary.'''
    assert url.startswith('https') # we only support no-encryption sessions
    data = {
        'openid.ns':"http://specs.openid.net/auth/2.0",
        'openid.mode':"associate",
        'openid.assoc_type':"HMAC-SHA1",
        'openid.session_type':"no-encryption",
        }
    res = urllib.urlopen(url, urllib.urlencode(data))
    data = parse_response(res.read())
    return data

def request_authentication(services, url, assoc_handle, return_to):
    '''Request authentication (OpenID section 9).
    services is the list of discovered service types,
    url the OP service URL, assoc_handle the established session
    dictionary, and return_to the return URL.

    The return_to URL will also be passed as realm, and the
    OP may perform RP discovery on it; always request these
    data through SREG 1.0 as well.

    If AX or SREG 1.1 are supported, request email address,
    first/last name, or nickname.

    Return the URL that the browser should be redirected to.'''
    data = {
        'openid.ns':"http://specs.openid.net/auth/2.0",
        'openid.mode':"checkid_setup",
        'openid.assoc_handle':assoc_handle,
        'openid.return_to':return_to,
        'openid.claimed_id':"http://specs.openid.net/auth/2.0/identifier_select",
        'openid.identity':"http://specs.openid.net/auth/2.0/identifier_select",
        'openid.realm':return_to,
        'openid.ns.sreg':"http://openid.net/sreg/1.0",
        'openid.sreg.required':'nickname,email',
        }
    if "http://openid.net/srv/ax/1.0" in services:
        data.update({
            'openid.ns.ax':"http://openid.net/srv/ax/1.0",
            'openid.ax.mode':'fetch_request',
            'openid.ax.required':'email,first,last',
            'openid.ax.type.email':'http://axschema.org/contact/email',
            'openid.ax.type.first':"http://axschema.org/namePerson/first",
            'openid.ax.type.last':"http://axschema.org/namePerson/last",
            })
    if "http://openid.net/extensions/sreg/1.1" in services:
        data.update({
            'openid.ns.sreg11':"http://openid.net/extensions/sreg/1.1",
            'openid.sreg11.required':'nickname,email'
            })
    return url+"?"+urllib.urlencode(data)

class NotAuthenticated(Exception):
    pass

def authenticate(session, response):
    '''Process an authentication response.
    session must be the established session (minimally including
    assoc_handle and mac_key), response is the query string as parse
    by cgi.parse_qs.
    If authentication succeeds, return None.
    If the user was not authenticated, NotAuthenticated is raised.
    If the HTTP request is invalid (missing parameters, failure to
    validate signature), different exceptions will be raised, typically
    ValueError.

    Callers must check openid.response_nonce for replay attacks.
    '''

    if response['openid.ns'][0] != 'http://specs.openid.net/auth/2.0':
        raise ValueError('missing openid.ns')
    if session['assoc_handle'] != response['openid.assoc_handle'][0]:
        raise ValueError('incorrect session')
    if response['openid.mode'][0] == 'cancel':
        raise NotAuthenticated('provider did not authenticate user (cancelled)')
    if response['openid.mode'][0] != 'id_res':
        raise ValueError('invalid openid.mode')
    if  'openid.claimed_id' not in response:
        raise ValueError('missing openid.claimed_id')

    # Won't check nonce value - caller must verify this is not a replay

    signed = response['openid.signed'][0].split(',')
    if not (set(['op_endpoint', 'return_to', 'response_nonce',
                 'assoc_handle', 'claimed_id']) < set(signed)):
        raise ValueError("signature failed to sign important fields")
    query = []
    for name in signed:
        value = response['openid.'+name][0]
        query.append('%s:%s\n' % (name, value))
    query = ''.join(query)

    mac_key = base64.decodestring(session['mac_key'])
    transmitted_sig = base64.decodestring(response['openid.sig'][0])
    computed_sig = hmac.new(mac_key, query, sha).digest()

    if transmitted_sig != computed_sig:
        raise ValueError('Invalid signature')

def parse_nonce(nonce):
    '''Split a nonce into a (timestamp, ID) pair'''
    stamp = nonce.split('Z', 1)[0]
    stamp = datetime.datetime.strptime(stamp,"%Y-%m-%dT%H:%M:%S")
    return stamp

def get_namespaces(resp):
    res = {}
    for k, v in resp.items():
        if k.startswith('openid.ns.'):
            k = k.rsplit('.', 1)[1]
            res[v[0]] = k
    return res

def get_ax(resp, ns, validated):
    if "http://openid.net/srv/ax/1.0" not in ns:
        return {}
    ax = ns["http://openid.net/srv/ax/1.0"]+"."
    oax = "openid."+ax
    res = {}
    for k, v in resp.items():
        if k.startswith(oax+"type."):
            k = k.rsplit('.',1)[1]
            value_name = oax+"value."+k
            if ax+"value."+k not in validated:
                continue
            res[v[0]] = resp[value_name][0]
    return res

def get_email(resp):
    "Return the email address embedded response, or None."

    validated = resp['openid.signed'][0]

    # SREG 1.0; doesn't require namespace, as the protocol doesn't
    # specify one
    if 'openid.sreg.email' in resp and \
       'sreg.email' in validated:
        return resp['openid.sreg.email'][0]

    ns = get_namespaces(resp)

    ax = get_ax(resp, ns, validated)
    if "http://axschema.org/contact/email" in ax:
        return ax["http://axschema.org/contact/email"]

    # TODO: SREG 1.1
    return None

def get_username(resp):
    "Return either nickname or (first, last) or None."

    validated = resp['openid.signed'][0]
    if 'openid.sreg.nickname' in resp and \
       'sreg.nickname' in validated:
        return resp['openid.sreg.nickname'][0]

    ns = get_namespaces(resp)

    ax = get_ax(resp, ns, validated)
    if "http://axschema.org/namePerson/first" in ax and \
       "http://axschema.org/namePerson/last" in ax:
        return (ax["http://axschema.org/namePerson/first"],
                ax["http://axschema.org/namePerson/last"])

    # TODO: SREG 1.1
    return

################ Test Server #################################

import BaseHTTPServer, cgi

# supported providers
providers = (
    ('Google', 'http://www.google.com/favicon.ico', 'https://www.google.com/accounts/o8/id'),
    ('Yahoo', 'http://www.yahoo.com/favicon.ico', 'http://yahoo.com/'),
    # Verisigns service URL is not https
    #('Verisign', 'https://pip.verisignlabs.com/favicon.ico', 'https://pip.verisignlabs.com')
    ('myOpenID', 'https://www.myopenid.com/favicon.ico', 'https://www.myopenid.com/'),
    ('Launchpad', 'https://login.launchpad.net/favicon.ico', 'https://login.launchpad.net/')
    )

sessions = []
class Handler(BaseHTTPServer.BaseHTTPRequestHandler):

    def write(self, payload, type):
        self.send_response(200)
        self.send_header("Content-type", type)
        self.send_header("Content-length", str(len(payload)))
        self.end_headers()
        self.wfile.write(payload)

    def do_GET(self):
        if self.path == '/':
            return self.root()
        path = self.path
        i = path.rfind('?')
        if i >= 0:
            query = cgi.parse_qs(path[i+1:])
            path = path[:i]
        else:
            query = {}
        if path == '/':
            if 'provider' in query:
                prov = [p for p in providers if p[0]  == query['provider'][0]]
                if len(prov) != 1:
                    return self.not_found()
                prov = prov[0]
                services, url = discover(prov[2])
                session = associate(url)
                print session
                sessions.append(session)
                self.send_response(307) # temporary redirect - do not cache
                self.send_header("Location", request_authentication
                                 (services, url, session['assoc_handle'],
                                  self.base_url+"?returned=1"))
                self.end_headers()
                return
            if 'returned' in query:
                if 'openid.ns' not in query:
                    return self.rp_discovery()
                handle = query['openid.assoc_handle'][0]
                for session in sessions:
                    if session['assoc_handle'] == handle:
                        break
                else:
                    session = None
                if not session:
                    return self.error('Not authenticated (no session)')
                print query
                try:
                    authenticate(session, query)
                except Exception, e:
                    self.error("Authentication failed"+repr(e))
                claimed = query['openid.claimed_id'][0]
                payload = "Hello "+claimed+"\n"
                email = get_email(query)
                if email:
                    payload += 'Your email is '+email+"\n"
                else:
                    payload += 'No email address is known\n'
                username = get_username(query)
                if isinstance(username, tuple):
                    username = " ".join(username)
                if username:
                    payload += 'Your nickname is '+username+'\n'
                else:
                    payload += 'No nickname is known\n'
                return self.write(payload, "text/plain")

        return self.not_found()

    def debug(self, value):
        payload = repr(value)
        self.write(payload, "text/plain")

    def error(self, text):
        self.write(text, "text/plain")

    def root(self):
        payload = "<title>OpenID login</title>"

        for name, icon, provider in providers:
            payload += "

" % (
                self.base_url, name, icon, name)
        payload += ""
        self.write(payload, "text/html")

    def rp_discovery(self):
        payload = '''<xrds:XRDS
                xmlns:xrds="xri://$xrds"
                xmlns="xri://$xrd*($v*2.0)">
                <XRD>
                     <Service priority="1">
                              <Type>http://specs.openid.net/auth/2.0/return_to</Type>
                              <URI>%s</URI>
                     </Service>
                </XRD>
                </xrds:XRDS>
        ''' % (self.base_url+"/?returned=1")
        self.write(payload, 'application/xrds+xml')

    def not_found(self):
        self.send_response(404)
        self.end_headers()

# OpenID providers often attempt relying-party discovery
# This requires the test server to use a globally valid URL
# If Python cannot correctly determine the base URL, you
# can pass it as command line argument
def test_server():
    import socket, sys
    if len(sys.argv) > 1:
        base_url = sys.argv[1]
    else:
        base_url = "http://" + socket.getfqdn() + "/"
    Handler.base_url = base_url
    BaseHTTPServer.HTTPServer.address_family = socket.AF_INET6
    httpd = BaseHTTPServer.HTTPServer(('', 8000), Handler)
    httpd.serve_forever()

if __name__ == '__main__':
    test_server()
<|endoftext|>


<commit_before>#! /usr/bin/env python
# encoding: utf-8

def configure(conf):
    def get_param(varname,default):
        return getattr(Options.options,varname,'')or default

    openmp_fragment="#include <omp.h>\nint main() { omp_set_num_threads(10);}"

    flags=['-fopenmp','-openmp']
    if conf.options.openmp_flag:
        flags=[conf.options.openmp_flag]
    found_openmp=False
    for flag in flags:
        try:
            conf.check_cxx(msg="Checking OpenMP flag " + flag,
                           fragment=openmp_fragment,
                           cxxflags=flag, linkflags=flag, uselib_store='openmp')
        except conf.errors.ConfigurationError:
            continue
        else:
            found_openmp=True
            break
    if not found_openmp:
        conf.fatal('Could not find OpenMP flag')

def options(opt):
    openmp=opt.add_option_group('OpenMP Options')
    openmp.add_option('--openmp-flag',
                     help='Flag to enable OpenMP')
<commit_msg>Add help to openmp option<commit_after>#! /usr/bin/env python
# encoding: utf-8

def configure(conf):
    def get_param(varname,default):
        return getattr(Options.options,varname,'')or default

    openmp_fragment="#include <omp.h>\nint main() { omp_set_num_threads(10);}"

    flags=['-fopenmp','-openmp']
    if conf.options.openmp_flag:
        flags=[conf.options.openmp_flag]
    found_openmp=False
    for flag in flags:
        try:
            conf.check_cxx(msg="Checking OpenMP flag " + flag,
                           fragment=openmp_fragment,
                           cxxflags=flag, linkflags=flag, uselib_store='openmp')
        except conf.errors.ConfigurationError:
            continue
        else:
            found_openmp=True
            break
    if not found_openmp:
        conf.fatal('Could not find OpenMP flag')

def options(opt):
    openmp=opt.add_option_group('OpenMP Options')
    openmp.add_option('--openmp-flag', help='Flag to enable OpenMP (e.g. -fopenmp')
<|endoftext|>


<commit_before>import os

def file_existing_and_readable(filename):
    if os.path.exists(filename):

        # Try to open it

        try:

            with open(filename):

                pass

        except:

            return False

        else:

            return True

    else:

        return False

def sanitize_filename(filename):
    return os.path.expandvars(os.path.expanduser(filename))
<commit_msg>Sanitize the file before checking it<commit_after>import os

def file_existing_and_readable(filename):

    sanitized_filename = sanitize_filename(filename)

    if os.path.exists(sanitized_filename):

        # Try to open it

        try:

            with open(sanitized_filename):

                pass

        except:

            return False

        else:

            return True

    else:

        return False

def sanitize_filename(filename):
    return os.path.expandvars(os.path.expanduser(filename))
<|endoftext|>


<commit_before>__version__ = '0.4.834'
<commit_msg>Automatic patch number increase [ci skip]<commit_after>__version__ = '0.4.835'
<|endoftext|>


<commit_before># Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

try:
    from paddle_api_config import *
    import os.path
    import platform

    system = platform.system().lower()
    is_osx = (system == 'darwin')
    is_win = (system == 'windows')
    is_lin = (system == 'linux')

    if is_lin:
        whole_start = "-Wl,--whole-archive"
        whole_end = "-Wl,--no-whole-archive"
    elif is_osx:
        whole_start = ""
        whole_end = ""

    LIB_DIRS = [
        "math", 'function', 'utils', 'parameter', "gserver", "api", "cuda",
        "pserver", "trainer"
    ]
    PARENT_LIB_DIRS = ['proto']

    class PaddleLDFlag(object):
        def __init__(self):
            self.paddle_build_dir = PADDLE_BUILD_DIR
            self.paddle_build_dir = os.path.abspath(self.paddle_build_dir)
            self.with_gpu = PaddleLDFlag.cmake_bool(WITH_GPU)
            self.protolib = PROTOBUF_LIBRARY
            self.zlib = ZLIB_LIBRARIES
            self.thread = CMAKE_THREAD_LIB
            self.dl_libs = CMAKE_DL_LIBS
            self.with_python = PaddleLDFlag.cmake_bool(WITH_PYTHON)
            self.python_libs = PYTHON_LIBRARIES

            self.glog_libs = GLOG_LIBRARIES

            self.with_coverage = PaddleLDFlag.cmake_bool(WITH_COVERALLS)
            self.gflags_location = GFLAGS_LOCATION
            self.cblas_libs = CBLAS_LIBRARIES
            self.curt = CUDA_LIBRARIES

        def ldflag_str(self):
            return " ".join(
                [self.libs_dir_str(), self.parent_dir_str(), self.libs_str()])

        def libs_dir_str(self):
            libdirs = LIB_DIRS
            return " ".join(
                map(lambda x: "-L" + os.path.join(self.paddle_build_dir, x),
                    libdirs))

        def parent_dir_str(self):
            libdirs = PARENT_LIB_DIRS
            return " ".join(
                map(lambda x: "-L" + os.path.join(self.paddle_build_dir, '..', x),
                    libdirs))

        def libs_str(self):
            libs = [
                whole_start,
                "-lpaddle_gserver",
                "-lpaddle_function",
                whole_end,
                "-lpaddle_pserver",
                "-lpaddle_trainer_lib",
                "-lpaddle_network",
                '-lpaddle_parameter',
                "-lpaddle_math",
                '-lpaddle_utils',
                "-lpaddle_proto",
                "-lpaddle_cuda",
                "-lpaddle_api",
                self.normalize_flag(self.protolib),
                self.normalize_flag(self.glog_libs),
                self.normalize_flag("gflags"),
                self.normalize_flag(self.zlib),
                self.normalize_flag(self.thread),
                self.normalize_flag(self.dl_libs),
                self.normalize_flag(self.cblas_libs),
            ]

            if self.with_python:
                libs.append(self.normalize_flag(self.python_libs))
            if self.with_gpu:
                libs.append(self.normalize_flag(self.curt))
            if self.with_coverage:
                libs.append("-fprofile-arcs")
            return " ".join(filter(lambda l: len(l) != 0, libs))

        def normalize_flag(self, cmake_flag):
            """
            CMake flag string to ld flag
            :type cmake_flag: str
            """
            if ";" in cmake_flag:
                return " ".join(map(self.normalize_flag, cmake_flag.split(";")))
            if cmake_flag.startswith("/"):  # is a path
                return cmake_flag
            elif cmake_flag.startswith("-l"):  # normal link command
                return cmake_flag
            elif cmake_flag == "gflags":  # special for gflags
                assert PaddleLDFlag.cmake_bool(self.gflags_location)
                return self.gflags_location
            elif len(cmake_flag) != 0:
                return "".join(["-l", cmake_flag])
            else:
                return ""

        @staticmethod
        def cmake_bool(cmake_str):
            """
            CMake bool string to bool
            :param cmake_str: cmake boolean string
            :type cmake_str: str
            :rtype: bool
            """
            if cmake_str in ["FALSE", "OFF", "NO"] or cmake_str.endswith(
                    "-NOTFOUND"):
                return False
            else:
                return True

        def c_flag(self):
            if self.with_coverage:
                return [
                    "-fprofile-arcs", "-ftest-coverage", "-O0", "-g",
                    "-std=c++11"
                ]
            else:
                return ["-std=c++11"]
except ImportError:

    class PaddleLDFlag(object):
        def ldflag_str(self):
            pass

        def c_flag(self):
            pass
<commit_msg>Revert unchanged files<commit_after># Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

try:
    from paddle_api_config import *
    import os.path
    import platform

    system = platform.system().lower()
    is_osx = (system == 'darwin')
    is_win = (system == 'windows')
    is_lin = (system == 'linux')

    if is_lin:
        whole_start = "-Wl,--whole-archive"
        whole_end = "-Wl,--no-whole-archive"
    elif is_osx:
        whole_start = ""
        whole_end = ""

    LIB_DIRS = [
        "math", 'function', 'utils', 'parameter', "gserver", "api", "cuda",
        "pserver", "trainer"
    ]
    PARENT_LIB_DIRS = ['proto']

    class PaddleLDFlag(object):
        def __init__(self):
            self.paddle_build_dir = PADDLE_BUILD_DIR
            self.paddle_build_dir = os.path.abspath(self.paddle_build_dir)
            self.with_gpu = PaddleLDFlag.cmake_bool(WITH_GPU)
            self.protolib = PROTOBUF_LIBRARY
            self.zlib = ZLIB_LIBRARIES
            self.thread = CMAKE_THREAD_LIB
            self.dl_libs = CMAKE_DL_LIBS
            self.with_python = PaddleLDFlag.cmake_bool(WITH_PYTHON)
            self.python_libs = PYTHON_LIBRARIES

            self.glog_libs = GLOG_LIBRARIES

            self.with_coverage = PaddleLDFlag.cmake_bool(WITH_COVERALLS)
            self.gflags_libs = GFLAGS_LIBRARIES
            self.gflags_location = GFLAGS_LOCATION
            self.cblas_libs = CBLAS_LIBRARIES
            self.curt = CUDA_LIBRARIES

        def ldflag_str(self):
            return " ".join(
                [self.libs_dir_str(), self.parent_dir_str(), self.libs_str()])

        def libs_dir_str(self):
            libdirs = LIB_DIRS
            return " ".join(
                map(lambda x: "-L" + os.path.join(self.paddle_build_dir, x),
                    libdirs))

        def parent_dir_str(self):
            libdirs = PARENT_LIB_DIRS
            return " ".join(
                map(lambda x: "-L" + os.path.join(self.paddle_build_dir, '..', x),
                    libdirs))

        def libs_str(self):
            libs = [
                whole_start,
                "-lpaddle_gserver",
                "-lpaddle_function",
                whole_end,
                "-lpaddle_pserver",
                "-lpaddle_trainer_lib",
                "-lpaddle_network",
                '-lpaddle_parameter',
                "-lpaddle_math",
                '-lpaddle_utils',
                "-lpaddle_proto",
                "-lpaddle_cuda",
                "-lpaddle_api",
                self.normalize_flag(self.protolib),
                self.normalize_flag(self.glog_libs),
                self.normalize_flag(self.gflags_libs),
                self.normalize_flag(self.zlib),
                self.normalize_flag(self.thread),
                self.normalize_flag(self.dl_libs),
                self.normalize_flag(self.cblas_libs),
            ]

            if self.with_python:
                libs.append(self.normalize_flag(self.python_libs))
            if self.with_gpu:
                libs.append(self.normalize_flag(self.curt))
            if self.with_coverage:
                libs.append("-fprofile-arcs")
            return " ".join(filter(lambda l: len(l) != 0, libs))

        def normalize_flag(self, cmake_flag):
            """
            CMake flag string to ld flag
            :type cmake_flag: str
            """
            if ";" in cmake_flag:
                return " ".join(map(self.normalize_flag, cmake_flag.split(";")))
            if cmake_flag.startswith("/"):  # is a path
                return cmake_flag
            elif cmake_flag.startswith("-l"):  # normal link command
                return cmake_flag
            elif cmake_flag in [
                    "gflags-shared", "gflags-static", "gflags_nothreads-shared",
                    "gflags_nothreads-static"
            ]:  # special for gflags
                assert PaddleLDFlag.cmake_bool(self.gflags_location)
                return self.gflags_location
            elif len(cmake_flag) != 0:
                return "".join(["-l", cmake_flag])
            else:
                return ""

        @staticmethod
        def cmake_bool(cmake_str):
            """
            CMake bool string to bool
            :param cmake_str: cmake boolean string
            :type cmake_str: str
            :rtype: bool
            """
            if cmake_str in ["FALSE", "OFF", "NO"] or cmake_str.endswith(
                    "-NOTFOUND"):
                return False
            else:
                return True

        def c_flag(self):
            if self.with_coverage:
                return [
                    "-fprofile-arcs", "-ftest-coverage", "-O0", "-g",
                    "-std=c++11"
                ]
            else:
                return ["-std=c++11"]
except ImportError:

    class PaddleLDFlag(object):
        def ldflag_str(self):
            pass

        def c_flag(self):
            pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# thumbor imaging service
# https://github.com/thumbor/thumbor/wiki

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 globo.com timehome@corp.globo.com

import re

from thumbor.filters import BaseFilter, filter_method, PHASE_PRE_LOAD
from thumbor.point import FocalPoint

class Filter(BaseFilter):
    phase = PHASE_PRE_LOAD

    focal_regex = re.compile("(\d+)x(\d+):(\d+)x(\d+)")

    @filter_method(BaseFilter.String)
    def focal(self, focal_string):
        left, top, right, bottom = self.focal_regex.match(focal_string).groups()
        left, top, right, bottom = int(left), int(top), int(right), int(bottom)
        width = right - left
        height = bottom - top

        if width and height:
            self.context.request.focal_points.append(
                FocalPoint.from_square(left, top, width, height, origin="Explicit")
            )
<commit_msg>lint fix<commit_after># -*- coding: utf-8 -*-

# thumbor imaging service
# https://github.com/thumbor/thumbor/wiki

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 globo.com timehome@corp.globo.com

import re

from thumbor.filters import BaseFilter, filter_method, PHASE_PRE_LOAD
from thumbor.point import FocalPoint

class Filter(BaseFilter):
    phase = PHASE_PRE_LOAD

    focal_regex = re.compile("(\d+)x(\d+):(\d+)x(\d+)")

    @filter_method(BaseFilter.String)
    def focal(self, focal_string):
        left, top, right, bottom = self.focal_regex.match(focal_string).groups()
        left, top, right, bottom = int(left), int(top), int(right), int(bottom)
        width = right - left
        height = bottom - top

        if width and height:
            self.context.request.focal_points.append(
                FocalPoint.from_square(left, top, width, height, origin="Explicit")
            )
<|endoftext|>


<commit_before>__version__ = '0.4.931'
<commit_msg>Automatic patch number increase [ci skip]<commit_after>__version__ = '0.4.932'
<|endoftext|>


<commit_before>__version__ = '0.4.645'
<commit_msg>Automatic patch number increase [ci skip]<commit_after>__version__ = '0.4.647'
<|endoftext|>


<commit_before>import unittest
import nose
from datetime import datetime

import pandas as pd
import pandas.io.data as web
from pandas.util.testing import (network, assert_frame_equal,
                                 assert_series_equal,
                                 assert_almost_equal)
from numpy.testing.decorators import slow

import urllib2

class TestYahoo(unittest.TestCase):

    @slow
    @network
    def test_yahoo(self):
        # asserts that yahoo is minimally working and that it throws
        # an excecption when DataReader can't get a 200 response from
        # yahoo
        start = datetime(2010, 1, 1)
        end = datetime(2013, 01, 27)

        try:
            self.assertEquals(
                web.DataReader("F", 'yahoo', start, end)['Close'][-1],
                13.68)

            self.assertRaises(
                Exception,
                lambda: web.DataReader("NON EXISTENT TICKER", 'yahoo',
                                      start, end))
        except urllib2.URLError:
            try:
                urllib2.urlopen('http://www.google.com')
            except urllib2.URLError:
                raise nose.SkipTest
            else:
                raise

    @slow
    @network
    def test_get_quote(self):
        df = web.get_quote_yahoo(pd.Series(['GOOG', 'AAPL', 'GOOG']))
        assert_series_equal(df.ix[0], df.ix[2])

    @slow
    @network
    def test_get_components(self):

        df = web.get_components_yahoo('^DJI') #Dow Jones
        assert isinstance(df, pd.DataFrame)
        assert len(df) == 30

        df = web.get_components_yahoo('^GDAXI') #DAX
        assert isinstance(df, pd.DataFrame)
        assert len(df) == 30
        assert df[df.name.str.contains('adidas', case=False)].index == 'ADS.DE'

        df = web.get_components_yahoo('^NDX') #NASDAQ-100
        assert isinstance(df, pd.DataFrame)
        #assert len(df) == 100
        #Usual culprits, should be around for a while
        assert 'AAPL' in df.index
        assert 'GOOG' in df.index
        assert 'AMZN' in df.index

    @slow
    @network
    def test_get_data(self):
        import numpy as np
        #single symbol
        #http://finance.yahoo.com/q/hp?s=GOOG&a=09&b=08&c=2010&d=09&e=10&f=2010&g=d
        df = web.get_data_yahoo('GOOG')
        assert df.Volume.ix['OCT-08-2010'] == 2859200

        sl = ['AAPL', 'AMZN', 'GOOG']
        pan = web.get_data_yahoo(sl, '2012')
        ts = pan.Close.GOOG.index[pan.Close.AAPL > pan.Close.GOOG]
        assert ts[0].dayofyear == 96

        #dfi = web.get_components_yahoo('^DJI')
        #pan = web.get_data_yahoo(dfi, 'JAN-01-12', 'JAN-31-12')
        pan = web.get_data_yahoo(['GE', 'MSFT', 'INTC'], 'JAN-01-12', 'JAN-31-12')
        expected = [19.02, 28.23, 25.39]
        result = pan.Close.ix['01-18-12'][['GE', 'MSFT', 'INTC']].tolist()
        assert result == expected

        # sanity checking
        t= np.array(result)
        assert     np.issubdtype(t.dtype, np.floating)
        assert     t.shape == (3,)

        expected = [[ 18.99,  28.4 ,  25.18],
                    [ 18.58,  28.31,  25.13],
                    [ 19.03,  28.16,  25.52],
                    [ 18.81,  28.82,  25.87]]
        result = pan.Open.ix['Jan-15-12':'Jan-20-12'][['GE', 'MSFT', 'INTC']].values
        assert (result == expected).all()

        #Check ret_index
        pan = web.get_data_yahoo(['GE', 'INTC', 'IBM'], '1977', '1987',
                                 ret_index=True)
        tstamp = pan.Ret_Index.INTC.first_valid_index()
        result = pan.Ret_Index.ix[tstamp]['INTC']
        expected = 1.0
        assert result == expected

        # sanity checking
        t= np.array(pan)
        assert     np.issubdtype(t.dtype, np.floating)

if __name__ == '__main__':
    nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
                   exit=False)
<commit_msg>Tag yahoo data tests as @network only<commit_after>import unittest
import nose
from datetime import datetime

import pandas as pd
import pandas.io.data as web
from pandas.util.testing import (network, assert_frame_equal,
                                 assert_series_equal,
                                 assert_almost_equal)
from numpy.testing.decorators import slow

import urllib2

class TestYahoo(unittest.TestCase):

    @network
    def test_yahoo(self):
        # asserts that yahoo is minimally working and that it throws
        # an excecption when DataReader can't get a 200 response from
        # yahoo
        start = datetime(2010, 1, 1)
        end = datetime(2013, 01, 27)

        try:
            self.assertEquals(
                web.DataReader("F", 'yahoo', start, end)['Close'][-1],
                13.68)

            self.assertRaises(
                Exception,
                lambda: web.DataReader("NON EXISTENT TICKER", 'yahoo',
                                      start, end))
        except urllib2.URLError:
            try:
                urllib2.urlopen('http://www.google.com')
            except urllib2.URLError:
                raise nose.SkipTest
            else:
                raise

    @network
    def test_get_quote(self):
        df = web.get_quote_yahoo(pd.Series(['GOOG', 'AAPL', 'GOOG']))
        assert_series_equal(df.ix[0], df.ix[2])

    @network
    def test_get_components(self):

        df = web.get_components_yahoo('^DJI') #Dow Jones
        assert isinstance(df, pd.DataFrame)
        assert len(df) == 30

        df = web.get_components_yahoo('^GDAXI') #DAX
        assert isinstance(df, pd.DataFrame)
        assert len(df) == 30
        assert df[df.name.str.contains('adidas', case=False)].index == 'ADS.DE'

        df = web.get_components_yahoo('^NDX') #NASDAQ-100
        assert isinstance(df, pd.DataFrame)
        #assert len(df) == 100
        #Usual culprits, should be around for a while
        assert 'AAPL' in df.index
        assert 'GOOG' in df.index
        assert 'AMZN' in df.index

    @network
    def test_get_data(self):
        import numpy as np
        #single symbol
        #http://finance.yahoo.com/q/hp?s=GOOG&a=09&b=08&c=2010&d=09&e=10&f=2010&g=d
        df = web.get_data_yahoo('GOOG')
        assert df.Volume.ix['OCT-08-2010'] == 2859200

        sl = ['AAPL', 'AMZN', 'GOOG']
        pan = web.get_data_yahoo(sl, '2012')
        ts = pan.Close.GOOG.index[pan.Close.AAPL > pan.Close.GOOG]
        assert ts[0].dayofyear == 96

        #dfi = web.get_components_yahoo('^DJI')
        #pan = web.get_data_yahoo(dfi, 'JAN-01-12', 'JAN-31-12')
        pan = web.get_data_yahoo(['GE', 'MSFT', 'INTC'], 'JAN-01-12', 'JAN-31-12')
        expected = [19.02, 28.23, 25.39]
        result = pan.Close.ix['01-18-12'][['GE', 'MSFT', 'INTC']].tolist()
        assert result == expected

        # sanity checking
        t= np.array(result)
        assert     np.issubdtype(t.dtype, np.floating)
        assert     t.shape == (3,)

        expected = [[ 18.99,  28.4 ,  25.18],
                    [ 18.58,  28.31,  25.13],
                    [ 19.03,  28.16,  25.52],
                    [ 18.81,  28.82,  25.87]]
        result = pan.Open.ix['Jan-15-12':'Jan-20-12'][['GE', 'MSFT', 'INTC']].values
        assert (result == expected).all()

        #Check ret_index
        pan = web.get_data_yahoo(['GE', 'INTC', 'IBM'], '1977', '1987',
                                 ret_index=True)
        tstamp = pan.Ret_Index.INTC.first_valid_index()
        result = pan.Ret_Index.ix[tstamp]['INTC']
        expected = 1.0
        assert result == expected

        # sanity checking
        t= np.array(pan)
        assert     np.issubdtype(t.dtype, np.floating)

if __name__ == '__main__':
    nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
                   exit=False)
<|endoftext|>


<commit_before>from psycopg2.extras import RealDictCursor
from tilequeue.postgresql import DBAffinityConnectionsNoLimit
from tilequeue.tile import coord_to_mercator_bounds
import sys

def generate_query(start_zoom, template, bounds, zoom):
    if zoom < start_zoom:
        return None
    query = template.render(bounds=bounds, zoom=zoom)
    return query

class JinjaQueryGenerator(object):

    def __init__(self, template, start_zoom):
        self.template = template
        self.start_zoom = start_zoom

    def __call__(self, bounds, zoom):
        return generate_query(self.start_zoom, self.template, bounds, zoom)

class DevJinjaQueryGenerator(object):

    def __init__(self, environment, template_name, start_zoom):
        self.environment = environment
        self.template_name = template_name
        self.start_zoom = start_zoom

    def __call__(self, bounds, zoom):
        template = self.environment.get_template(self.template_name)
        return generate_query(self.start_zoom, template, bounds, zoom)

def jinja_filter_geometry(value):
    return 'ST_AsBinary(%s)' % value

def jinja_filter_bbox_filter(bounds, geometry_col_name, srid=900913):
    min_point = 'ST_MakePoint(%.12f, %.12f)' % (bounds[0], bounds[1])
    max_point = 'ST_MakePoint(%.12f, %.12f)' % (bounds[2], bounds[3])
    bbox_no_srid = 'ST_MakeBox2D(%s, %s)' % (min_point, max_point)
    bbox = 'ST_SetSrid(%s, %d)' % (bbox_no_srid, srid)
    bbox_filter = '%s && %s' % (geometry_col_name, bbox)
    return bbox_filter

def jinja_filter_bbox_intersection(bounds, geometry_col_name, srid=900913):
    min_point = 'ST_MakePoint(%.12f, %.12f)' % (bounds[0], bounds[1])
    max_point = 'ST_MakePoint(%.12f, %.12f)' % (bounds[2], bounds[3])
    bbox_no_srid = 'ST_MakeBox2D(%s, %s)' % (min_point, max_point)
    bbox = 'ST_SetSrid(%s, %d)' % (bbox_no_srid, srid)
    bbox_intersection = 'st_intersection(%s, %s)' % (geometry_col_name, bbox)
    return bbox_intersection

def jinja_filter_bbox(bounds, srid=900913):
    min_point = 'ST_MakePoint(%.12f, %.12f)' % (bounds[0], bounds[1])
    max_point = 'ST_MakePoint(%.12f, %.12f)' % (bounds[2], bounds[3])
    bbox_no_srid = 'ST_MakeBox2D(%s, %s)' % (min_point, max_point)
    bbox = 'ST_SetSrid(%s, %d)' % (bbox_no_srid, srid)
    return bbox

def build_feature_queries(unpadded_bounds, layer_data, zoom):
    queries_to_execute = []
    for layer_datum in layer_data:
        query_bounds_pad_fn = layer_datum['query_bounds_pad_fn']
        if query_bounds_pad_fn:
            padded_bounds = query_bounds_pad_fn(unpadded_bounds)
        else:
            padded_bounds = unpadded_bounds
        query_generator = layer_datum['query_generator']
        query = query_generator(padded_bounds, zoom)
        queries_to_execute.append((layer_datum, query, padded_bounds))
    return queries_to_execute

def execute_query(conn, query, layer_datum, padded_bounds):
    try:
        cursor = conn.cursor(cursor_factory=RealDictCursor)
        cursor.execute(query)
        rows = list(cursor.fetchall())
        return rows, layer_datum, padded_bounds
    except:
        # If any exception occurs during query execution, close the
        # connection to ensure it is not in an invalid state. The
        # connection pool knows to create new connections to replace
        # those that are closed
        try:
            conn.close()
        except:
            pass
        raise

def trim_layer_datum(layer_datum):
    layer_datum_result = dict(
        [(k, v) for k, v in layer_datum.items()
         if k not in ('query_generator', 'query_bounds_pad_fn')])
    return layer_datum_result

def enqueue_queries(sql_conns, thread_pool, layer_data, zoom, unpadded_bounds):

    queries_to_execute = build_feature_queries(
        unpadded_bounds, layer_data, zoom)

    empty_results = []
    async_results = []
    for (layer_datum, query, padded_bounds), sql_conn in zip(
            queries_to_execute, sql_conns):
        layer_datum = trim_layer_datum(layer_datum)
        if query is None:
            empty_feature_layer = dict(
                name=layer_datum['name'],
                features=[],
                layer_datum=layer_datum,
                padded_bounds=padded_bounds,
            )
            empty_results.append(empty_feature_layer)
        else:
            async_result = thread_pool.apply_async(
                execute_query, (sql_conn, query, layer_datum, padded_bounds))
            async_results.append(async_result)

    return empty_results, async_results

class DataFetcher(object):

    def __init__(self, conn_info, layer_data, io_pool, n_conn):
        self.conn_info = dict(conn_info)
        self.layer_data = layer_data
        self.io_pool = io_pool

        self.dbnames = self.conn_info.pop('dbnames')
        self.dbnames_query_index = 0
        self.sql_conn_pool = DBAffinityConnectionsNoLimit(
            self.dbnames, n_conn, self.conn_info)

    def __call__(self, coord, layer_data=None):
        if layer_data is None:
            layer_data = self.layer_data
        zoom = coord.zoom
        unpadded_bounds = coord_to_mercator_bounds(coord)

        sql_conns, conn_info = self.sql_conn_pool.get_conns()
        try:
            # the padded bounds are used here in order to only have to
            # issue a single set of queries to the database for all
            # formats
            empty_results, async_results = enqueue_queries(
                sql_conns, self.io_pool, layer_data, zoom, unpadded_bounds)

            feature_layers = []
            async_exception = None
            for async_result in async_results:
                try:
                    rows, layer_datum, padded_bounds = async_result.get()
                except:
                    exc_type, exc_value, exc_traceback = sys.exc_info()
                    async_exception = exc_value
                    # iterate through all async results to give others
                    # a chance to close any connections that yielded
                    # exceptions
                    continue

                # don't continue processing if an error occurred on
                # any results
                if async_exception is not None:
                    continue

                # read the bytes out of each row, otherwise the pickle
                # will fail because the geometry is a read buffer
                read_rows = []
                for row in rows:
                    geometry = row.pop('__geometry__')
                    if geometry is None:
                        # there are cases when the geometry comes back as None
                        # eg when they are unioned together
                        continue
                    geometry_bytes = bytes(geometry)
                    row['__geometry__'] = geometry_bytes
                    read_rows.append(row)

                feature_layer = dict(
                    name=layer_datum['name'], features=read_rows,
                    layer_datum=layer_datum,
                    padded_bounds=padded_bounds,
                )
                feature_layers.append(feature_layer)

            # bail if an error occurred
            if async_exception is not None:
                raise async_exception

            feature_layers.extend(empty_results)

            return dict(
                feature_layers=feature_layers,
                unpadded_bounds=unpadded_bounds,
                padded_bounds=padded_bounds,
            )

        finally:
            self.sql_conn_pool.put_conns(sql_conns)
<commit_msg>Remove unnecessary branch condition<commit_after>from psycopg2.extras import RealDictCursor
from tilequeue.postgresql import DBAffinityConnectionsNoLimit
from tilequeue.tile import coord_to_mercator_bounds
import sys

def generate_query(start_zoom, template, bounds, zoom):
    if zoom < start_zoom:
        return None
    query = template.render(bounds=bounds, zoom=zoom)
    return query

class JinjaQueryGenerator(object):

    def __init__(self, template, start_zoom):
        self.template = template
        self.start_zoom = start_zoom

    def __call__(self, bounds, zoom):
        return generate_query(self.start_zoom, self.template, bounds, zoom)

class DevJinjaQueryGenerator(object):

    def __init__(self, environment, template_name, start_zoom):
        self.environment = environment
        self.template_name = template_name
        self.start_zoom = start_zoom

    def __call__(self, bounds, zoom):
        template = self.environment.get_template(self.template_name)
        return generate_query(self.start_zoom, template, bounds, zoom)

def jinja_filter_geometry(value):
    return 'ST_AsBinary(%s)' % value

def jinja_filter_bbox_filter(bounds, geometry_col_name, srid=900913):
    min_point = 'ST_MakePoint(%.12f, %.12f)' % (bounds[0], bounds[1])
    max_point = 'ST_MakePoint(%.12f, %.12f)' % (bounds[2], bounds[3])
    bbox_no_srid = 'ST_MakeBox2D(%s, %s)' % (min_point, max_point)
    bbox = 'ST_SetSrid(%s, %d)' % (bbox_no_srid, srid)
    bbox_filter = '%s && %s' % (geometry_col_name, bbox)
    return bbox_filter

def jinja_filter_bbox_intersection(bounds, geometry_col_name, srid=900913):
    min_point = 'ST_MakePoint(%.12f, %.12f)' % (bounds[0], bounds[1])
    max_point = 'ST_MakePoint(%.12f, %.12f)' % (bounds[2], bounds[3])
    bbox_no_srid = 'ST_MakeBox2D(%s, %s)' % (min_point, max_point)
    bbox = 'ST_SetSrid(%s, %d)' % (bbox_no_srid, srid)
    bbox_intersection = 'st_intersection(%s, %s)' % (geometry_col_name, bbox)
    return bbox_intersection

def jinja_filter_bbox(bounds, srid=900913):
    min_point = 'ST_MakePoint(%.12f, %.12f)' % (bounds[0], bounds[1])
    max_point = 'ST_MakePoint(%.12f, %.12f)' % (bounds[2], bounds[3])
    bbox_no_srid = 'ST_MakeBox2D(%s, %s)' % (min_point, max_point)
    bbox = 'ST_SetSrid(%s, %d)' % (bbox_no_srid, srid)
    return bbox

def build_feature_queries(unpadded_bounds, layer_data, zoom):
    queries_to_execute = []
    for layer_datum in layer_data:
        query_bounds_pad_fn = layer_datum['query_bounds_pad_fn']
        padded_bounds = query_bounds_pad_fn(unpadded_bounds)
        query_generator = layer_datum['query_generator']
        query = query_generator(padded_bounds, zoom)
        queries_to_execute.append((layer_datum, query, padded_bounds))
    return queries_to_execute

def execute_query(conn, query, layer_datum, padded_bounds):
    try:
        cursor = conn.cursor(cursor_factory=RealDictCursor)
        cursor.execute(query)
        rows = list(cursor.fetchall())
        return rows, layer_datum, padded_bounds
    except:
        # If any exception occurs during query execution, close the
        # connection to ensure it is not in an invalid state. The
        # connection pool knows to create new connections to replace
        # those that are closed
        try:
            conn.close()
        except:
            pass
        raise

def trim_layer_datum(layer_datum):
    layer_datum_result = dict(
        [(k, v) for k, v in layer_datum.items()
         if k not in ('query_generator', 'query_bounds_pad_fn')])
    return layer_datum_result

def enqueue_queries(sql_conns, thread_pool, layer_data, zoom, unpadded_bounds):

    queries_to_execute = build_feature_queries(
        unpadded_bounds, layer_data, zoom)

    empty_results = []
    async_results = []
    for (layer_datum, query, padded_bounds), sql_conn in zip(
            queries_to_execute, sql_conns):
        layer_datum = trim_layer_datum(layer_datum)
        if query is None:
            empty_feature_layer = dict(
                name=layer_datum['name'],
                features=[],
                layer_datum=layer_datum,
                padded_bounds=padded_bounds,
            )
            empty_results.append(empty_feature_layer)
        else:
            async_result = thread_pool.apply_async(
                execute_query, (sql_conn, query, layer_datum, padded_bounds))
            async_results.append(async_result)

    return empty_results, async_results

class DataFetcher(object):

    def __init__(self, conn_info, layer_data, io_pool, n_conn):
        self.conn_info = dict(conn_info)
        self.layer_data = layer_data
        self.io_pool = io_pool

        self.dbnames = self.conn_info.pop('dbnames')
        self.dbnames_query_index = 0
        self.sql_conn_pool = DBAffinityConnectionsNoLimit(
            self.dbnames, n_conn, self.conn_info)

    def __call__(self, coord, layer_data=None):
        if layer_data is None:
            layer_data = self.layer_data
        zoom = coord.zoom
        unpadded_bounds = coord_to_mercator_bounds(coord)

        sql_conns, conn_info = self.sql_conn_pool.get_conns()
        try:
            # the padded bounds are used here in order to only have to
            # issue a single set of queries to the database for all
            # formats
            empty_results, async_results = enqueue_queries(
                sql_conns, self.io_pool, layer_data, zoom, unpadded_bounds)

            feature_layers = []
            async_exception = None
            for async_result in async_results:
                try:
                    rows, layer_datum, padded_bounds = async_result.get()
                except:
                    exc_type, exc_value, exc_traceback = sys.exc_info()
                    async_exception = exc_value
                    # iterate through all async results to give others
                    # a chance to close any connections that yielded
                    # exceptions
                    continue

                # don't continue processing if an error occurred on
                # any results
                if async_exception is not None:
                    continue

                # read the bytes out of each row, otherwise the pickle
                # will fail because the geometry is a read buffer
                read_rows = []
                for row in rows:
                    geometry = row.pop('__geometry__')
                    if geometry is None:
                        # there are cases when the geometry comes back as None
                        # eg when they are unioned together
                        continue
                    geometry_bytes = bytes(geometry)
                    row['__geometry__'] = geometry_bytes
                    read_rows.append(row)

                feature_layer = dict(
                    name=layer_datum['name'], features=read_rows,
                    layer_datum=layer_datum,
                    padded_bounds=padded_bounds,
                )
                feature_layers.append(feature_layer)

            # bail if an error occurred
            if async_exception is not None:
                raise async_exception

            feature_layers.extend(empty_results)

            return dict(
                feature_layers=feature_layers,
                unpadded_bounds=unpadded_bounds,
                padded_bounds=padded_bounds,
            )

        finally:
            self.sql_conn_pool.put_conns(sql_conns)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2007-2014 Parisson SARL
# Copyright (c) 2006-2014 Guillaume Pellerin <pellerin@parisson.com>
# Copyright (c) 2010-2014 Paul Brossier <piem@piem.org>
# Copyright (c) 2013-2014 Thomas Fillon <thomas@parisson.com>

# This file is part of TimeSide.

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from timeside.core import Processor, implements, interfacedoc, abstract
from timeside.core.api import IEncoder
from .tools.gstutils import numpy_array_to_gst_buffer, MainloopThread

import gi
gi.require_version('Gst', '1.0')
gi.require_version('GLib', '2.0')
from gi.repository import GLib, Gst
Gst.init(None)

import threading

# Streaming queue configuration
QUEUE_SIZE = 10
GST_APPSINK_MAX_BUFFERS = 10

class GstEncoder(Processor):
    implements(IEncoder)
    abstract()

    type = 'encoder'

    def __init__(self, output, streaming=False, overwrite=False):

        super(GstEncoder, self).__init__()

        if isinstance(output, str):
            import os.path
            if os.path.isdir(output):
                raise IOError("Encoder output must be a file, not a directory")
            elif os.path.isfile(output) and not overwrite:
                raise IOError(
                    "Encoder output %s exists, but overwrite set to False")
            self.filename = output
        else:
            self.filename = None
        self.streaming = streaming

        if not self.filename and not self.streaming:
            raise Exception('Must give an output')

        self.end_cond = threading.Condition(threading.Lock())
        self.eod = False
        self.metadata = None
        self.num_samples = 0

        self._chunk_len = 0

    @interfacedoc
    def release(self):
        if hasattr(self, 'eod') and hasattr(self, 'mainloopthread'):
            self.end_cond.acquire()
            while not hasattr(self, 'end_reached'):
                self.end_cond.wait()
            self.end_cond.release()
        if hasattr(self, 'error_msg'):
            raise IOError(self.error_msg)

    def __del__(self):
        self.release()

    def start_pipeline(self, channels, samplerate):
        self.pipeline = Gst.parse_launch(self.pipe)
        # store a pointer to appsrc in our encoder object
        self.src = self.pipeline.get_by_name('src')

        if self.streaming:
            try: # py3
                import queue
            except: # py2
                import Queue as queue
            self._streaming_queue = queue.Queue(QUEUE_SIZE)
            # store a pointer to appsink in our encoder object
            self.app = self.pipeline.get_by_name('app')
            self.app.set_property('max-buffers', GST_APPSINK_MAX_BUFFERS)
            self.app.set_property("drop", False)
            self.app.set_property('emit-signals', True)
            self.app.connect("new-buffer", self._on_new_buffer_streaming)
            #self.app.connect('new-preroll', self._on_new_preroll_streaming)

        srccaps = Gst.caps_from_string("""audio/x-raw-float,
            format=F32LE,
            layout=interleaved,
            channels=(int)%s,
            rate=(int)%d""" % (int(channels), int(samplerate)))
        self.src.set_property("caps", srccaps)
        self.src.set_property('emit-signals', True)
        self.src.set_property('num-buffers', -1)
        self.src.set_property('block', False)
        self.src.set_property('do-timestamp', True)

        self.bus = self.pipeline.get_bus()
        self.bus.add_signal_watch()
        self.bus.connect("message", self._on_message_cb)

        self.mainloop = GLib.MainLoop()
        self.mainloopthread = MainloopThread(self.mainloop)
        self.mainloopthread.start()

        # start pipeline
        self.pipeline.set_state(Gst.State.PLAYING)

    def _on_message_cb(self, bus, message):
        t = message.type
        if t == Gst.MessageType.EOS:
            self.end_cond.acquire()
            if self.streaming:
                self._streaming_queue.put(Gst.MessageType.EOS)

            self.pipeline.set_state(Gst.State.NULL)
            self.mainloop.quit()
            self.end_reached = True
            self.end_cond.notify()
            self.end_cond.release()

        elif t == Gst.MessageType.ERROR:
            self.end_cond.acquire()
            self.pipeline.set_state(Gst.State.NULL)
            self.mainloop.quit()
            self.end_reached = True
            err, debug = message.parse_error()
            self.error_msg = "Error: %s" % err, debug
            self.end_cond.notify()
            self.end_cond.release()

    def _on_new_buffer_streaming(self, appsink):
        # print('pull-buffer')
        chunk = appsink.emit('pull-buffer')
        self._streaming_queue.put(chunk)

    def _on_new_preroll_streaming(self, appsink):
        # print('preroll')
        chunk = appsink.emit('pull-preroll')
        self._streaming_queue.put(chunk)

    @interfacedoc
    def set_metadata(self, metadata):
        self.metadata = metadata

    @interfacedoc
    def process(self, frames, eod=False):
        self.eod = eod
        if eod:
            self.num_samples += frames.shape[0]
        else:
            self.num_samples += self.blocksize()

        buf = numpy_array_to_gst_buffer(frames, frames.shape[0],
                                        self.num_samples, self.samplerate())

        self.src.emit('push-buffer', buf)
        if self.eod:
            self.src.emit('end-of-stream')

        return frames, eod

    def get_stream_chunk(self):
        if self.streaming:
            chunk = self._streaming_queue.get(block=True)
            if chunk == Gst.MessageType.EOS:
                return None
            else:
                self._streaming_queue.task_done()
                return chunk

        else:
            raise TypeError('function only available in streaming mode')

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<commit_msg>[core] use sample signals in encoder.py<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2007-2014 Parisson SARL
# Copyright (c) 2006-2014 Guillaume Pellerin <pellerin@parisson.com>
# Copyright (c) 2010-2014 Paul Brossier <piem@piem.org>
# Copyright (c) 2013-2014 Thomas Fillon <thomas@parisson.com>

# This file is part of TimeSide.

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from timeside.core import Processor, implements, interfacedoc, abstract
from timeside.core.api import IEncoder
from .tools.gstutils import numpy_array_to_gst_buffer, MainloopThread

import gi
gi.require_version('Gst', '1.0')
gi.require_version('GLib', '2.0')
from gi.repository import GLib, Gst
Gst.init(None)

import threading

# Streaming queue configuration
QUEUE_SIZE = 10
GST_APPSINK_MAX_BUFFERS = 10

class GstEncoder(Processor):
    implements(IEncoder)
    abstract()

    type = 'encoder'

    def __init__(self, output, streaming=False, overwrite=False):

        super(GstEncoder, self).__init__()

        if isinstance(output, str):
            import os.path
            if os.path.isdir(output):
                raise IOError("Encoder output must be a file, not a directory")
            elif os.path.isfile(output) and not overwrite:
                raise IOError(
                    "Encoder output %s exists, but overwrite set to False")
            self.filename = output
        else:
            self.filename = None
        self.streaming = streaming

        if not self.filename and not self.streaming:
            raise Exception('Must give an output')

        self.end_cond = threading.Condition(threading.Lock())
        self.eod = False
        self.metadata = None
        self.num_samples = 0

        self._chunk_len = 0

    @interfacedoc
    def release(self):
        if hasattr(self, 'eod') and hasattr(self, 'mainloopthread'):
            self.end_cond.acquire()
            while not hasattr(self, 'end_reached'):
                self.end_cond.wait()
            self.end_cond.release()
        if hasattr(self, 'error_msg'):
            raise IOError(self.error_msg)

    def __del__(self):
        self.release()

    def start_pipeline(self, channels, samplerate):
        self.pipeline = Gst.parse_launch(self.pipe)
        # store a pointer to appsrc in our encoder object
        self.src = self.pipeline.get_by_name('src')

        if self.streaming:
            try: # py3
                import queue
            except: # py2
                import Queue as queue
            self._streaming_queue = queue.Queue(QUEUE_SIZE)
            # store a pointer to appsink in our encoder object
            self.app = self.pipeline.get_by_name('app')
            self.app.set_property('max-buffers', GST_APPSINK_MAX_BUFFERS)
            self.app.set_property("drop", False)
            self.app.set_property('emit-signals', True)
            self.app.connect("new-sample", self._on_new_sample_streaming)
            #self.app.connect('new-preroll', self._on_new_preroll_streaming)

        srccaps = Gst.caps_from_string("""audio/x-raw-float,
            format=F32LE,
            layout=interleaved,
            channels=(int)%s,
            rate=(int)%d""" % (int(channels), int(samplerate)))
        self.src.set_property("caps", srccaps)
        self.src.set_property('emit-signals', True)
        self.src.set_property('num-buffers', -1)
        self.src.set_property('block', False)
        self.src.set_property('do-timestamp', True)

        self.bus = self.pipeline.get_bus()
        self.bus.add_signal_watch()
        self.bus.connect("message", self._on_message_cb)

        self.mainloop = GLib.MainLoop()
        self.mainloopthread = MainloopThread(self.mainloop)
        self.mainloopthread.start()

        # start pipeline
        self.pipeline.set_state(Gst.State.PLAYING)

    def _on_message_cb(self, bus, message):
        t = message.type
        if t == Gst.MessageType.EOS:
            self.end_cond.acquire()
            if self.streaming:
                self._streaming_queue.put(Gst.MessageType.EOS)

            self.pipeline.set_state(Gst.State.NULL)
            self.mainloop.quit()
            self.end_reached = True
            self.end_cond.notify()
            self.end_cond.release()

        elif t == Gst.MessageType.ERROR:
            self.end_cond.acquire()
            self.pipeline.set_state(Gst.State.NULL)
            self.mainloop.quit()
            self.end_reached = True
            err, debug = message.parse_error()
            self.error_msg = "Error: %s" % err, debug
            self.end_cond.notify()
            self.end_cond.release()

    def _on_new_sample_streaming(self, appsink):
        # print('pull-sample')
        chunk = appsink.emit('pull-sample')
        self._streaming_queue.put(chunk)

    def _on_new_preroll_streaming(self, appsink):
        # print('preroll')
        chunk = appsink.emit('pull-preroll')
        self._streaming_queue.put(chunk)

    @interfacedoc
    def set_metadata(self, metadata):
        self.metadata = metadata

    @interfacedoc
    def process(self, frames, eod=False):
        self.eod = eod
        if eod:
            self.num_samples += frames.shape[0]
        else:
            self.num_samples += self.blocksize()

        buf = numpy_array_to_gst_buffer(frames, frames.shape[0],
                                        self.num_samples, self.samplerate())

        self.src.emit('push-buffer', buf)
        if self.eod:
            self.src.emit('end-of-stream')

        return frames, eod

    def get_stream_chunk(self):
        if self.streaming:
            chunk = self._streaming_queue.get(block=True)
            if chunk == Gst.MessageType.EOS:
                return None
            else:
                self._streaming_queue.task_done()
                return chunk

        else:
            raise TypeError('function only available in streaming mode')

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<|endoftext|>


<commit_before>from bs4 import BeautifulSoup
from park_api.util import convert_date
from park_api.geodata import GeoData
import datetime

# This loads the geodata for this city if <city>.geojson exists in the same directory as this file.
# No need to remove this if there's no geodata (yet), everything will still work.
geodata = GeoData(__file__)

# This function is called by the scraper and given the data of the page specified as source in geojson above.
# It's supposed to return a dictionary containing everything the current spec expects. Tests will fail if it doesn't ;)
def parse_html(html):

    # BeautifulSoup is a great and easy way to parse the html and find the bits and pieces we're looking for.
    soup = BeautifulSoup(html, "html.parser")

    # last_updated is the date when the data on the page was last updated, it should be listed on most pages
    #   suche:
zuletzt aktualisiert: 28.05.2019 15.30 Uhr

    updated = soup.find( "p", class_="updateinfo")
    last_updated = datetime.datetime.strptime(updated.text, 'zuletzt aktualisiert: %d.%m.%Y %H.%M Uhr')

    data = {
        # convert_date is a utility function you can use to turn this date into the correct string format
        "last_updated": convert_date(last_updated, "%d.%m.%Y %H:%M Uhr"),
        # URL for the page where the scraper can gather the data
        "lots": []
    }

    parking_lots = soup.find_all("div", class_="accordeon parkmoeglichkeit")
    for one_lot in parking_lots :
        parking_name = one_lot.find("h3").text
        lot = geodata.lot(parking_name)

        parking_state = 'open'
        parking_free = 0
        try :
            parking_belegung=one_lot.find("div", class_="belegung")
            if (parking_belegung != None ) :
                parking_free=int(parking_belegung.find("strong").text)
        except :
            parking_state='nodata'

        data["lots"].append({
            "name": lot.name,
            "free": parking_free,
            "total": lot.total,
            "address": lot.address,
            "coords": lot.coords,
            "state": parking_state,
            "lot_type": lot.type,
            "id": lot.id,
            "forecast": False,
        })

    return data
<commit_msg>Update Regensburg.py<commit_after>from bs4 import BeautifulSoup
from park_api.util import convert_date
from park_api.geodata import GeoData
import datetime

# This loads the geodata for this city if <city>.geojson exists in the same directory as this file.
# No need to remove this if there's no geodata (yet), everything will still work.
geodata = GeoData(__file__)

# This function is called by the scraper and given the data of the page specified as source in geojson above.
# It's supposed to return a dictionary containing everything the current spec expects. Tests will fail if it doesn't ;)
def parse_html(html):

    # BeautifulSoup is a great and easy way to parse the html and find the bits and pieces we're looking for.
    soup = BeautifulSoup(html, "html.parser")

    # last_updated is the date when the data on the page was last updated, it should be listed on most pages
    #   suche:
zuletzt aktualisiert: 28.05.2019 15.30 Uhr

    updated = soup.find( "p", class_="updateinfo")
    last_updated = datetime.datetime.strptime(updated.text, 'zuletzt aktualisiert: %d.%m.%Y %H.%M Uhr')

    data = {
        "last_updated": last_updated,
        # URL for the page where the scraper can gather the data
        "lots": []
    }

    parking_lots = soup.find_all("div", class_="accordeon parkmoeglichkeit")
    for one_lot in parking_lots :
        parking_name = one_lot.find("h3").text
        lot = geodata.lot(parking_name)

        parking_state = 'open'
        parking_free = 0
        try :
            parking_belegung=one_lot.find("div", class_="belegung")
            if (parking_belegung != None ) :
                parking_free=int(parking_belegung.find("strong").text)
        except :
            parking_state='nodata'

        data["lots"].append({
            "name": lot.name,
            "free": parking_free,
            "total": lot.total,
            "address": lot.address,
            "coords": lot.coords,
            "state": parking_state,
            "lot_type": lot.type,
            "id": lot.id,
            "forecast": False,
        })

    return data
<|endoftext|>