<commit_before>#!/usr/bin/python

import numpy as np

def logit(x):
    """ Computes logit function

    Parameters
    ----------
    x : {float, int}

    Returns
    -------
    out : {float, int}
        Logit value
    """
    if x > 0:
        out = np.log(1.*x/(1-x))
        return out

def complement(x):
    """ Computes complement of a probability/parameter

    Parameters
    ----------
    x : {float, int}

    Returns
    -------
    out : {float, int}
        Complement
    """
    out = 1-x
    return out

class NaiveBayes:

    def __init__(self,prior=[],conditional=[]):
        self.model_prior = prior
        self.model_conditional = conditional
        # Parameters from fitted model: prior & class conditional

    def fit(self,X,Y):
        """Fits Naive Bayes generative model according to training data.

        Parameters
        ----------
        X : {array}, shape (n_samples, n_features)

        Y : array-like, shape (n_samples,)

        Returns
        -------
        self : object
            Returns self.
        """
        K = len(np.unique(Y))
        p = np.shape(X)[1]
        n = np.shape(X)[0]
        conditional = []
        # Class conditional distribution parameters (Laplace smoothing)
        prior = []
        # Class prior distribution parameters (MLE)
        for label in xrange(K):
            indices = np.where(Y==label+1)[0]
            temp_split = X[indices,:]
            temp_count = np.shape(temp_split)[0]
            prior.append(1.*temp_count/n)
            temp_sum = np.apply_along_axis(sum,0,temp_split.toarray())
            conditional.append(1.*(1+1.*temp_sum)/(2+temp_count))
        self.model_prior = prior
        self.model_conditional = conditional
        return self

    def predict(self,params,test):
        """Makes predictions on test data based on Naive Bayes model

        Parameters
        ----------
        test: {array}, shape (n_samples, n_features)

        Returns
        -------
        preds : list
            Returns predicted class for each test point.
        """
        #
        #
        vect_logit = np.vectorize(logit)
        vect_comp = np.vectorize(complement)
        n_test = np.shape(test)[0]
        prior = self.model_prior
        conditional = self.model_conditional
        K = len(prior)
        p = np.shape(conditional)[1]
        test_p = np.shape(test)[1]
        if p != test_p:
            print "Error: Number of training and testing features differ"
            return
        preds = []
        weight = vect_logit(conditional)
        # Stores weight matrix
        condition_comp = vect_comp(conditional)
        intercept = np.log(prior) + np.sum(np.apply_along_axis(np.log,1,condition_comp),axis=1)
        # Stores intercept vector
        weight = weight.transpose()
        classifier = test.dot(weight) + intercept
        preds = np.argmax(classifier,axis=1) + 1
        # Add 1 because the classes have a 1-based index
        preds = preds.reshape(n_test,1)
        return preds
<commit_msg>Update style<commit_after>#!/usr/bin/python

import numpy as np

def logit(x):
    """ Computes logit function

    Parameters
    ----------
    x : {float, int}

    Returns
    -------
    out : {float, int}
        Logit value
    """
    if x > 0:
        out = np.log(1. * x / (1 - x))
        return out

def complement(x):
    """ Computes complement of a probability/parameter

    Parameters
    ----------
    x : {float, int}

    Returns
    -------
    out : {float, int}
        Complement
    """
    out = 1 - x
    return out

class NaiveBayes:

    def __init__(self, prior=[], conditional=[]):
        self.model_prior = prior
        self.model_conditional = conditional
        # Parameters from fitted model: prior & class conditional

    def fit(self, X, Y):
        """Fits Naive Bayes generative model according to training data.

        Parameters
        ----------
        X : {array}, shape (n_samples, n_features)

        Y : array-like, shape (n_samples,)

        Returns
        -------
        self : object
            Returns self.
        """
        K = len(np.unique(Y))
        p = np.shape(X)[1]
        n = np.shape(X)[0]
        conditional = []
        # Class conditional distribution parameters (Laplace smoothing)
        prior = []
        # Class prior distribution parameters (MLE)
        for label in xrange(K):
            indices = np.where(Y == label + 1)[0]
            temp_split = X[indices, :]
            temp_count = np.shape(temp_split)[0]
            prior.append(1. * temp_count / n)
            temp_sum = np.apply_along_axis(sum, 0, temp_split.toarray())
            conditional.append(1. * (1 + 1. * temp_sum) / (2 + temp_count))
        self.model_prior = prior
        self.model_conditional = conditional
        return self

    def predict(self, params, test):
        """Makes predictions on test data based on Naive Bayes model

        Parameters
        ----------
        test: {array}, shape (n_samples, n_features)

        Returns
        -------
        preds : list
            Returns predicted class for each test point.
        """
        vect_logit = np.vectorize(logit)
        vect_comp = np.vectorize(complement)
        n_test = np.shape(test)[0]
        prior = self.model_prior
        conditional = self.model_conditional
        K = len(prior)
        p = np.shape(conditional)[1]
        test_p = np.shape(test)[1]
        if p != test_p:
            print "Error: Number of training and testing features differ"
            return
        preds = []
        weight = vect_logit(conditional)
        # Stores weight matrix
        condition_comp = vect_comp(conditional)
        intercept = np.log(
            prior) + np.sum(np.apply_along_axis(np.log, 1, condition_comp), axis=1)
        # Stores intercept vector
        weight = weight.transpose()
        classifier = test.dot(weight) + intercept
        preds = np.argmax(classifier, axis=1) + 1
        # Add 1 because the classes have a 1-based index
        preds = preds.reshape(n_test, 1)
        return preds
<|endoftext|>


<commit_before>#   Copyright (c) 2009 Canonical Ltd.
#   Copyright (c) 2009 Mikeal Rogers <mikeal.rogers@gmail.com>
#   Copyright (c) 2009 Domen Kozar <domen@dev.si>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
#   Contributor: Anthony Lenton <anthony.lenton@canonical.com>

"""
    This module provides an SSL-enabled HTTP server, WindmillChooserApplication
    and WindmillProxyApplication that are drop-in replacements for the standard
    non-ssl-enabled ones.
"""
import time
import socket
import select
import urllib
import SocketServer
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from StringIO import StringIO
from proxy import WindmillProxyApplication
from httplib import HTTPConnection, HTTPException
import traceback
import sys
import windmill
if not sys.version.startswith('2.4'):
    from urlparse import urlparse, urlunparse
else:
    # python 2.4
    from windmill.tools.urlparse_25 import urlparse, urlunparse

import logging
logger = logging.getLogger(__name__)

try:
    import ssl # python 2.6
    _ssl_wrap_socket = ssl.wrap_socket
    _socket_create_connection = socket.create_connection
except ImportError:
    # python 2.5
    if windmill.has_ssl:
        from OpenSSL import SSL
    from httplib import FakeSocket

    class BetterFakeSocket(FakeSocket):
        """ A FakeSocket that implements sendall and
            handles exceptions better
        """
        class _closedsocket:
            def __nonzero__(self):
                return False
            def __getattr__(self, name):
                if name == '__nonzero__':
                    return self.__nonzero__
                raise HTTPException(9, 'Bad file descriptor')

        def __init__(self, sock, ssl):
            FakeSocket.__init__(self, sock, ssl)
            self.ok = True

        def sendall(self, data):
            if self.ok:
                try:
                    self._ssl.sendall(data)
                except SSL.SysCallError, err:
                    self.ok = False
                    print err

        def recv(self, len = 1024, flags = 0):
            if self.ok:
                try:
                    result = self._ssl.read(len)
                except SSL.SysCallError, err:
                    self.ok = False
                    print err
                return result

    def _ssl_verify_peer(conection, certificate, errnum, depth, ok):
        return True

    def _ssl_wrap_socket(sock, certfile=None,
                server_side=False, ca_certs=None,
                do_handshake_on_connect=True,
                suppress_ragged_eofs=True):
        ctx = SSL.Context(SSL.SSLv23_METHOD)
        if certfile is not None:
            ctx.use_privatekey_file(certfile)
            try:
                ctx.use_certificate_file(certfile)
            except: pass
            ctx.load_verify_locations(certfile)
        ctx.set_verify(SSL.VERIFY_NONE, _ssl_verify_peer)
        ssl_sock = SSL.Connection(ctx, sock)
        if server_side:
            ssl_sock.set_accept_state()
        else:
            ssl_sock.set_connect_state()
        return BetterFakeSocket(sock, ssl_sock)

    def _socket_create_connection(address, timeout=None):
        if timeout is None:
            timeout = socket.getdefaulttimeout()
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        oldtimeout = socket.getdefaulttimeout()
        socket.setdefaulttimeout(timeout)
        sock.connect(address)
        socket.setdefaulttimeout(oldtimeout)
        return sock

class WindmillHTTPRequestHandler(SocketServer.ThreadingMixIn, BaseHTTPRequestHandler):
    def __init__(self, request, client_address, server):
        self.headers_set = []
        self.headers_sent = []
        self.header_buffer = ''
        BaseHTTPRequestHandler.__init__(self, request, client_address,
                                             server)

    def _sock_connect_to(self, netloc, soc):
        """Parse netloc string and establish connection on socket."""
        host_port = netloc.split(':', 1)
        if len(host_port) == 1:
            host_port.append(80)

        # establish connection or else write 404 and fail the function
        try:
            soc.connect(host_port)
        except socket.error, msg:
            self.send_error(404, msg)
            return False

        return True

    def do_CONNECT(self):
        """ Handle CONNECT commands.  Just set up SSL and restart. """
        request = None
        try:
            try:
                self.log_request(200)
                self.wfile.write(self.protocol_version +
                                 ' 200 Connection established\r\n')
                self.wfile.write('Proxy-agent: %s\r\n' % self.version_string())
                self.wfile.write('\r\n')
                if not windmill.has_ssl:
                    return
                request = self.connection
                connstream = _ssl_wrap_socket(self.connection,
                               server_side=True,
                               certfile=self.server.cert_creator[self.path].certfile)

                self.request = connstream
                # And here we go again!
                # setup...
                self.base_path = 'https://' + self.path
                if self.base_path.endswith(':443'):
                    self.base_path = self.base_path[:-4]
                self.connection = self.request
                self.rfile = socket._fileobject(self.connection, 'rb', self.rbufsize)
                self.wfile = socket._fileobject(self.connection, 'wb', self.wbufsize)
                # handle...
                try:
                    self.handle()
                    self.finish()
                finally:
                    sys.exc_traceback = None    # Help garbage collection
                    #self.connection.close()
            except socket.error, err:
                logger.debug("%s while serving (%s) %s" % (err,
                                                  self.command, self.path))
        finally:
            if request is not None:
                request.close()

    def handle_ALL(self):
        namespaces = self.server.namespaces
        proxy = self.server.proxy
        found = None
        path = self.path.split('?', 1)[0]
        for key in namespaces:
            if path.find('/'+key+'/') is not -1:
                found = key
                environ = self.get_environ()
                result = namespaces[found](environ, self.start_response)
                break
        else:
            found = None
            environ = self.get_environ()
            result = proxy(environ, self.start_response)
        # == Old blocking code ==
        # out = list(result)
        # # send data back to browser
        # try:
        #     self.write(''.join(out))
        # except socket.error, err:
        #     logger.debug("%s while serving (%s) %s" % (err,
        #                                       self.command, self.path))

        # == New non-blocking code ==
        try:
            for out in result:
                self.write(out)
        except socket.error, err:
            logger.debug("%s while serving (%s) %s" % (err,self.command, self.path))

        self.wfile.flush()
        self.connection.close()

    do_GET = handle_ALL
    do_POST = handle_ALL

    def start_response(self, status, headers, exc_info=None):
        if exc_info:
            try:
                if self.headers_sent:
                    raise exc_info[0], exc_info[1], exc_info[2]
            finally:
                exc_info = None
        elif self.headers_set:
            raise AssertionError("Headers already set!")

        self.headers_set[:] = [status, headers]
        status, response_headers = self.headers_sent[:] = self.headers_set
        code, message = status.split(' ', 1)
        self.send_response(int(code), message)

        self.send_headers(response_headers)

        return self.write

    def send_headers(self, response_headers):
        for header in response_headers:
            self.send_header(header[0], header[1])
        try:
            self.wfile.write(self.header_buffer+'\r\n')
        except socket.error, e:
            if len(e.args) is 2 and e.args[0] is 32:
                logger.debug("Client severed connection prematurely.")
            else:
                raise e

    def send_header(self, keyword, value):
        """Send a MIME header."""
        if self.request_version != 'HTTP/0.9':
            self.header_buffer += "%s: %s\r\n" % (keyword, value)
        if keyword.lower() == 'connection':
            if value.lower() == 'close':
                self.close_connection = 1
            elif value.lower() == 'keep-alive':
                self.close_connection = 0

    def send_response(self, code, message=None):
        """Send the response header and log the response code.
            Also send two standard headers with the server software
            version and the current date.
            """
        self.log_request(code)
        if message is None:
            if code in self.responses:
                message = self.responses[code][0]
            else:
                message = ''
        if self.request_version != 'HTTP/0.9':
            self.header_buffer += "%s %d %s\r\n" % (self.protocol_version, code, message)
            # print (self.protocol_version, code, message)

    def write(self, data):
        if not self.headers_set:
            raise AssertionError("write() before start_response()")

        # elif not self.headers_sent:
        #     # Before the first output, send the stored headers
        #     status, response_headers = self.headers_sent[:] = self.headers_set
        #     code, message = status.split(' ', 1)
        #     self.send_response(int(code), message)
        #     for header in response_headers:
        #         self.send_header(header[0], header[1])

        self.wfile.write(data)
        # self.wfile.flush()

    def get_environ(self):
        """ Put together a wsgi environment """
        if hasattr(self, 'base_path'):
            self.path = self.base_path + self.path
        env = self.server.base_environ.copy()
        env['SERVER_PROTOCOL'] = self.request_version
        env['REQUEST_METHOD'] = self.command
        if '?' in self.path:
            path, query = self.path.split('?', 1)
        else:
            path, query = self.path,''
        env['PATH_INFO'] = urllib.unquote(path)
        env['QUERY_STRING'] = query

        host = self.address_string()
        if host != self.client_address[0]:
            env['REMOTE_HOST'] = host
        env['REMOTE_ADDR'] = self.client_address[0]

        if self.headers.typeheader is not None:
            env['CONTENT_TYPE'] = self.headers.typeheader

        length = self.headers.getheader('content-length')
        if length:
            env['CONTENT_LENGTH'] = length

        for header in self.headers.headers:
            key, value = header.split(':', 1)
            key = key.replace('-', '_').upper()
            value = value.strip()
            if key in env:
                continue  # skip content length, type,etc.
            if 'HTTP_' + key in env:
                env['HTTP_' + key] += ',' + value
            else:
                env['HTTP_' + key] = value
        env['wsgi.url_scheme'] = urlparse(self.path).scheme

        clen = self.headers.getheader('content-length')
        if clen is not None and int(clen) > 0:
            i = self.rfile.read(int(clen))
            env['wsgi.input'] = StringIO(i)
        else:
            env['wsgi.input'] = self.rfile
        self.reconstruct_url(env)
        return env

    def reconstruct_url(self, environ):
        """ This can be done much faster and in a tidier way."""
        url = environ['wsgi.url_scheme']+'://'
        if environ.get('HTTP_HOST'):
            url += environ['HTTP_HOST']
        else:
            url += environ['SERVER_NAME']
            if environ['wsgi.url_scheme'] == 'https':
                if environ['SERVER_PORT'] != '443':
                    url += ':' + environ['SERVER_PORT']
            else:
                if environ['SERVER_PORT'] != '80':
                    url += ':' + environ['SERVER_PORT']
        url += environ.get('SCRIPT_NAME','')
        if '://' in self.path:
            url = self.path
        else:
            url += self.path
        environ['reconstructed_url'] = url
        return url

    def log_message(self, format, *args):
        logger.debug(format % args)

class WindmillHTTPServer(SocketServer.ThreadingMixIn, HTTPServer):
    def __init__(self, address, handler, cert_creator, apps, proxy):
        # all we want from this method is to register a pem file
        # $ openssl req -x509 -nodes -days 365 -newkey rsa:1024
        #                     -keyout mycert.pem -out mycert.pem
        self.cert_creator = cert_creator
        self.namespaces = dict([ (arg.ns, arg) for arg in apps ])
        self.proxy = proxy

        # the rest is the same
        HTTPServer.__init__(self, address, handler)
        self.setup_environ()

    def setup_environ(self):
        # Set up base environment
        env = self.base_environ = {}
        env['SERVER_NAME'] = self.server_name
        env['GATEWAY_INTERFACE'] = 'CGI/1.1'
        env['SERVER_PORT'] = self.server_port
        env['REMOTE_HOST'] = ''
        env['CONTENT_LENGTH'] = ''
        env['SCRIPT_NAME'] = ''

    ready = False

    def start(self):
        WindmillHTTPServer.ready = True
        #self.current_request = 0
        while self.ready:
            #self.current_request += 1
            self.handle_request()
        # print "Attempting to shut down..."
        self.server_close()

    def server_close(self):
        try:
            self.socket.shutdown(socket.SHUT_RDWR)
            self.socket.close()
        except socket.error, e:
            if len(e.args) is 2 and e.args[0] is 57:
                logger.debug("Server was killed, socket didn't shutdown perfectly.")
            else:
                raise e

    def add_namespace(self, name, application):
        """Add an application to a specific url namespace in windmill"""
        self.namespaces[name] = application

    def stop(self):
        self.xmlrpc_methods_instance.stop_runserver()
        WindmillHTTPServer.ready = False
        try:
            s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
            s.connect(self.server_address)
            s.send("\n")
        except socket.error:
            pass

    def handle_error(self, request, client_address):
        try:
            args = sys.exec_info()
        except:
            return

        print '-' * 40
        print 'Exception happened during processing of request from',
        print client_address
        # traceback doesn't appear to be always be thread safe
        try:
            traceback.print_exception(*args)
        except TypeError:
            print "Traceback cannot be printed, probably do to a thread safety issue."
        print '-' * 40

class WindmillConnection(HTTPConnection):
    """ Decide on the run if we should do HTTP or HTTPS """
    def __init__(self, scheme, host, port=None, strict=None):
        self.scheme = scheme
        if scheme == 'https':
            self.default_port = 443
        HTTPConnection.__init__(self, host, port, strict)
        self.key_file = None
        self.cert_file = None
        # Python 2.5 support:
        if not hasattr(self, 'timeout'):
            self.timeout = None

    def connect(self):
        "Connect to a host on a given (SSL) port."
        if self.scheme == 'http':
            HTTPConnection.connect(self)
            return

        sock = _socket_create_connection((self.host, self.port), self.timeout)
        self.sock = _ssl_wrap_socket(sock, self.key_file, self.cert_file)

class WindmillHTTPSProxyApplication(WindmillProxyApplication):
    ConnectionClass = WindmillConnection

    def get_connection(self, url):
        return self.ConnectionClass(url.scheme, url.netloc)
<commit_msg>Using daemon threads in the threadingmixin gets rid of odd exceptions we're seeing during garbage collection.<commit_after>#   Copyright (c) 2009 Canonical Ltd.
#   Copyright (c) 2009 Mikeal Rogers <mikeal.rogers@gmail.com>
#   Copyright (c) 2009 Domen Kozar <domen@dev.si>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
#   Contributor: Anthony Lenton <anthony.lenton@canonical.com>

"""
    This module provides an SSL-enabled HTTP server, WindmillChooserApplication
    and WindmillProxyApplication that are drop-in replacements for the standard
    non-ssl-enabled ones.
"""
import time
import socket
import select
import urllib
import SocketServer
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from StringIO import StringIO
from proxy import WindmillProxyApplication
from httplib import HTTPConnection, HTTPException
import traceback
import sys
import windmill
if not sys.version.startswith('2.4'):
    from urlparse import urlparse, urlunparse
else:
    # python 2.4
    from windmill.tools.urlparse_25 import urlparse, urlunparse

import logging
logger = logging.getLogger(__name__)

try:
    import ssl # python 2.6
    _ssl_wrap_socket = ssl.wrap_socket
    _socket_create_connection = socket.create_connection
except ImportError:
    # python 2.5
    if windmill.has_ssl:
        from OpenSSL import SSL
    from httplib import FakeSocket

    class BetterFakeSocket(FakeSocket):
        """ A FakeSocket that implements sendall and
            handles exceptions better
        """
        class _closedsocket:
            def __nonzero__(self):
                return False
            def __getattr__(self, name):
                if name == '__nonzero__':
                    return self.__nonzero__
                raise HTTPException(9, 'Bad file descriptor')

        def __init__(self, sock, ssl):
            FakeSocket.__init__(self, sock, ssl)
            self.ok = True

        def sendall(self, data):
            if self.ok:
                try:
                    self._ssl.sendall(data)
                except SSL.SysCallError, err:
                    self.ok = False
                    print err

        def recv(self, len = 1024, flags = 0):
            if self.ok:
                try:
                    result = self._ssl.read(len)
                except SSL.SysCallError, err:
                    self.ok = False
                    print err
                return result

    def _ssl_verify_peer(conection, certificate, errnum, depth, ok):
        return True

    def _ssl_wrap_socket(sock, certfile=None,
                server_side=False, ca_certs=None,
                do_handshake_on_connect=True,
                suppress_ragged_eofs=True):
        ctx = SSL.Context(SSL.SSLv23_METHOD)
        if certfile is not None:
            ctx.use_privatekey_file(certfile)
            try:
                ctx.use_certificate_file(certfile)
            except: pass
            ctx.load_verify_locations(certfile)
        ctx.set_verify(SSL.VERIFY_NONE, _ssl_verify_peer)
        ssl_sock = SSL.Connection(ctx, sock)
        if server_side:
            ssl_sock.set_accept_state()
        else:
            ssl_sock.set_connect_state()
        return BetterFakeSocket(sock, ssl_sock)

    def _socket_create_connection(address, timeout=None):
        if timeout is None:
            timeout = socket.getdefaulttimeout()
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        oldtimeout = socket.getdefaulttimeout()
        socket.setdefaulttimeout(timeout)
        sock.connect(address)
        socket.setdefaulttimeout(oldtimeout)
        return sock

class WindmillHTTPRequestHandler(SocketServer.ThreadingMixIn, BaseHTTPRequestHandler):
    def __init__(self, request, client_address, server):
        self.headers_set = []
        self.headers_sent = []
        self.header_buffer = ''
        BaseHTTPRequestHandler.__init__(self, request, client_address,
                                             server)

    def _sock_connect_to(self, netloc, soc):
        """Parse netloc string and establish connection on socket."""
        host_port = netloc.split(':', 1)
        if len(host_port) == 1:
            host_port.append(80)

        # establish connection or else write 404 and fail the function
        try:
            soc.connect(host_port)
        except socket.error, msg:
            self.send_error(404, msg)
            return False

        return True

    def do_CONNECT(self):
        """ Handle CONNECT commands.  Just set up SSL and restart. """
        request = None
        try:
            try:
                self.log_request(200)
                self.wfile.write(self.protocol_version +
                                 ' 200 Connection established\r\n')
                self.wfile.write('Proxy-agent: %s\r\n' % self.version_string())
                self.wfile.write('\r\n')
                if not windmill.has_ssl:
                    return
                request = self.connection
                connstream = _ssl_wrap_socket(self.connection,
                               server_side=True,
                               certfile=self.server.cert_creator[self.path].certfile)

                self.request = connstream
                # And here we go again!
                # setup...
                self.base_path = 'https://' + self.path
                if self.base_path.endswith(':443'):
                    self.base_path = self.base_path[:-4]
                self.connection = self.request
                self.rfile = socket._fileobject(self.connection, 'rb', self.rbufsize)
                self.wfile = socket._fileobject(self.connection, 'wb', self.wbufsize)
                # handle...
                try:
                    self.handle()
                    self.finish()
                finally:
                    sys.exc_traceback = None    # Help garbage collection
                    #self.connection.close()
            except socket.error, err:
                logger.debug("%s while serving (%s) %s" % (err,
                                                  self.command, self.path))
        finally:
            if request is not None:
                request.close()

    def handle_ALL(self):
        namespaces = self.server.namespaces
        proxy = self.server.proxy
        found = None
        path = self.path.split('?', 1)[0]
        for key in namespaces:
            if path.find('/'+key+'/') is not -1:
                found = key
                environ = self.get_environ()
                result = namespaces[found](environ, self.start_response)
                break
        else:
            found = None
            environ = self.get_environ()
            result = proxy(environ, self.start_response)
        # == Old blocking code ==
        # out = list(result)
        # # send data back to browser
        # try:
        #     self.write(''.join(out))
        # except socket.error, err:
        #     logger.debug("%s while serving (%s) %s" % (err,
        #                                       self.command, self.path))

        # == New non-blocking code ==
        try:
            for out in result:
                self.write(out)
        except socket.error, err:
            logger.debug("%s while serving (%s) %s" % (err,self.command, self.path))

        self.wfile.flush()
        self.connection.close()

    do_GET = handle_ALL
    do_POST = handle_ALL

    def start_response(self, status, headers, exc_info=None):
        if exc_info:
            try:
                if self.headers_sent:
                    raise exc_info[0], exc_info[1], exc_info[2]
            finally:
                exc_info = None
        elif self.headers_set:
            raise AssertionError("Headers already set!")

        self.headers_set[:] = [status, headers]
        status, response_headers = self.headers_sent[:] = self.headers_set
        code, message = status.split(' ', 1)
        self.send_response(int(code), message)

        self.send_headers(response_headers)

        return self.write

    def send_headers(self, response_headers):
        for header in response_headers:
            self.send_header(header[0], header[1])
        try:
            self.wfile.write(self.header_buffer+'\r\n')
        except socket.error, e:
            if len(e.args) is 2 and e.args[0] is 32:
                logger.debug("Client severed connection prematurely.")
            else:
                raise e

    def send_header(self, keyword, value):
        """Send a MIME header."""
        if self.request_version != 'HTTP/0.9':
            self.header_buffer += "%s: %s\r\n" % (keyword, value)
        if keyword.lower() == 'connection':
            if value.lower() == 'close':
                self.close_connection = 1
            elif value.lower() == 'keep-alive':
                self.close_connection = 0

    def send_response(self, code, message=None):
        """Send the response header and log the response code.
            Also send two standard headers with the server software
            version and the current date.
            """
        self.log_request(code)
        if message is None:
            if code in self.responses:
                message = self.responses[code][0]
            else:
                message = ''
        if self.request_version != 'HTTP/0.9':
            self.header_buffer += "%s %d %s\r\n" % (self.protocol_version, code, message)
            # print (self.protocol_version, code, message)

    def write(self, data):
        if not self.headers_set:
            raise AssertionError("write() before start_response()")

        # elif not self.headers_sent:
        #     # Before the first output, send the stored headers
        #     status, response_headers = self.headers_sent[:] = self.headers_set
        #     code, message = status.split(' ', 1)
        #     self.send_response(int(code), message)
        #     for header in response_headers:
        #         self.send_header(header[0], header[1])

        self.wfile.write(data)
        # self.wfile.flush()

    def get_environ(self):
        """ Put together a wsgi environment """
        if hasattr(self, 'base_path'):
            self.path = self.base_path + self.path
        env = self.server.base_environ.copy()
        env['SERVER_PROTOCOL'] = self.request_version
        env['REQUEST_METHOD'] = self.command
        if '?' in self.path:
            path, query = self.path.split('?', 1)
        else:
            path, query = self.path,''
        env['PATH_INFO'] = urllib.unquote(path)
        env['QUERY_STRING'] = query

        host = self.address_string()
        if host != self.client_address[0]:
            env['REMOTE_HOST'] = host
        env['REMOTE_ADDR'] = self.client_address[0]

        if self.headers.typeheader is not None:
            env['CONTENT_TYPE'] = self.headers.typeheader

        length = self.headers.getheader('content-length')
        if length:
            env['CONTENT_LENGTH'] = length

        for header in self.headers.headers:
            key, value = header.split(':', 1)
            key = key.replace('-', '_').upper()
            value = value.strip()
            if key in env:
                continue  # skip content length, type,etc.
            if 'HTTP_' + key in env:
                env['HTTP_' + key] += ',' + value
            else:
                env['HTTP_' + key] = value
        env['wsgi.url_scheme'] = urlparse(self.path).scheme

        clen = self.headers.getheader('content-length')
        if clen is not None and int(clen) > 0:
            i = self.rfile.read(int(clen))
            env['wsgi.input'] = StringIO(i)
        else:
            env['wsgi.input'] = self.rfile
        self.reconstruct_url(env)
        return env

    def reconstruct_url(self, environ):
        """ This can be done much faster and in a tidier way."""
        url = environ['wsgi.url_scheme']+'://'
        if environ.get('HTTP_HOST'):
            url += environ['HTTP_HOST']
        else:
            url += environ['SERVER_NAME']
            if environ['wsgi.url_scheme'] == 'https':
                if environ['SERVER_PORT'] != '443':
                    url += ':' + environ['SERVER_PORT']
            else:
                if environ['SERVER_PORT'] != '80':
                    url += ':' + environ['SERVER_PORT']
        url += environ.get('SCRIPT_NAME','')
        if '://' in self.path:
            url = self.path
        else:
            url += self.path
        environ['reconstructed_url'] = url
        return url

    def log_message(self, format, *args):
        logger.debug(format % args)

class WindmillHTTPServer(SocketServer.ThreadingMixIn, HTTPServer):
    def __init__(self, address, handler, cert_creator, apps, proxy):
        # all we want from this method is to register a pem file
        # $ openssl req -x509 -nodes -days 365 -newkey rsa:1024
        #                     -keyout mycert.pem -out mycert.pem
        self.cert_creator = cert_creator
        self.namespaces = dict([ (arg.ns, arg) for arg in apps ])
        self.proxy = proxy

        # the rest is the same
        HTTPServer.__init__(self, address, handler)
        self.setup_environ()

    daemon_threads = True

    def setup_environ(self):
        # Set up base environment
        env = self.base_environ = {}
        env['SERVER_NAME'] = self.server_name
        env['GATEWAY_INTERFACE'] = 'CGI/1.1'
        env['SERVER_PORT'] = self.server_port
        env['REMOTE_HOST'] = ''
        env['CONTENT_LENGTH'] = ''
        env['SCRIPT_NAME'] = ''

    ready = False

    def start(self):
        WindmillHTTPServer.ready = True
        #self.current_request = 0
        while self.ready:
            #self.current_request += 1
            self.handle_request()
        # print "Attempting to shut down..."
        self.server_close()

    def server_close(self):
        try:
            self.socket.shutdown(socket.SHUT_RDWR)
            self.socket.close()
        except socket.error, e:
            if len(e.args) is 2 and e.args[0] is 57:
                logger.debug("Server was killed, socket didn't shutdown perfectly.")
            else:
                raise e

    def add_namespace(self, name, application):
        """Add an application to a specific url namespace in windmill"""
        self.namespaces[name] = application

    def stop(self):
        self.xmlrpc_methods_instance.stop_runserver()
        WindmillHTTPServer.ready = False
        try:
            s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
            s.connect(self.server_address)
            s.send("\n")
        except socket.error:
            pass

    def handle_error(self, request, client_address):
        try:
            args = sys.exec_info()
        except:
            return

        print '-' * 40
        print 'Exception happened during processing of request from',
        print client_address
        # traceback doesn't appear to be always be thread safe
        try:
            traceback.print_exception(*args)
        except TypeError:
            print "Traceback cannot be printed, probably do to a thread safety issue."
        print '-' * 40

class WindmillConnection(HTTPConnection):
    """ Decide on the run if we should do HTTP or HTTPS """
    def __init__(self, scheme, host, port=None, strict=None):
        self.scheme = scheme
        if scheme == 'https':
            self.default_port = 443
        HTTPConnection.__init__(self, host, port, strict)
        self.key_file = None
        self.cert_file = None
        # Python 2.5 support:
        if not hasattr(self, 'timeout'):
            self.timeout = None

    def connect(self):
        "Connect to a host on a given (SSL) port."
        if self.scheme == 'http':
            HTTPConnection.connect(self)
            return

        sock = _socket_create_connection((self.host, self.port), self.timeout)
        self.sock = _ssl_wrap_socket(sock, self.key_file, self.cert_file)

class WindmillHTTPSProxyApplication(WindmillProxyApplication):
    ConnectionClass = WindmillConnection

    def get_connection(self, url):
        return self.ConnectionClass(url.scheme, url.netloc)
<|endoftext|>


<commit_before>#   Copyright (c) 2006-2007 Open Source Applications Foundation
#   Copyright (c) 2008-2009 Mikeal Rogers <mikeal.rogers@gmail.com>
#   Copyright (c) 2009 Canonical Ltd.
#   Copyright (c) 2009 Domen Kozar <domen@dev.si>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import windmill

from httplib import HTTPConnection
import copy
import sys
import logging
import urllib
logger = logging.getLogger(__name__)
from forwardmanager import ForwardManager
if not sys.version.startswith('2.4'):
    from urlparse import urlparse
else:
    # python 2.4
    from windmill.tools.urlparse_25 import urlparse

first_forward_domains = []
exclude_from_retry = ['http://sb-ssl.google.com',
                      'https://sb-ssl.google.com',
                      'http://en-us.fxfeeds.mozilla.com',
                      ]

# Note that hoppish conntains proxy-connection, which is pre-HTTP-1.1 and
# is somewhat nebulous
_hoppish = {
    'connection':1, 'keep-alive':1, 'proxy-authenticate':1,
    'proxy-authorization':1, 'te':1, 'trailers':1, 'transfer-encoding':1,
    'upgrade':1, 'proxy-connection':1,
    'p3p':1 #Not actually a hop-by-hop header, just really annoying
    }

# Cache stopping headers
cache_headers = {'Pragma':'no-cache', 'Cache-Control': 'post-check=0, pre-check=0',
                 'Cache-Control': 'no-store, no-cache, must-revalidate, max-age=0',
                 'Expires': '-1'}

cache_removal = [k.lower() for k in cache_headers.keys()]
cache_additions = cache_headers.items()

def is_hop_by_hop(header):
    """check if the given header is hop_by_hop"""
    return _hoppish.has_key(header.lower())

class IterativeResponse(object):
    def __init__(self, response_instance):
        self.response_instance = response_instance

    def __iter__(self):
        if self.response_instance.chunked:
            yield self.response_instance.read(1024)
            while self.response_instance.chunk_left is not None:
                if self.response_instance.chunk_left < 1024:
                    yield self.response_instance.read()
                    self.response_instance.chunk_left = None
                else:
                    yield self.response_instance.read(1024)
        else:
            yield self.response_instance.read()

def get_wsgi_response(response):
    if type(response) is str:
        return [response]
    if response.length > 1000:
        return IterativeResponse(response)
    else:
        return [response.read()]

def conditions_pass(e):
    for c in windmill.server.forwarding_conditions:
        if c(e) is False:
            return False
    return True

def proxy_post_redirect_form(environ, action):
    body = environ['wsgi.input'].read(int(environ['CONTENT_LENGTH']))
    parameters = body.split('&')
    inputs = []
    for parameter in parameters:
        parts = parameter.split('=', 1)
        if len(parts) == 1:
            continue
        parts = tuple(unicode(urllib.unquote(part), 'utf-8') for part in parts)
        inputs.append('<input type="hidden" name="%s" value="%s" />' % parts)
    form = """<title>There is no spoon.</title>

  <form id="redirect" action="%s" method="POST">%s</form>
""" % (action, '\n'.join(inputs))
    return form.encode('utf-8')

forward_forms = {}

class WindmillProxyApplication(object):
    """Application to handle requests that need to be proxied"""

    def __init__(self):
        self.fmgr = None
        proxyInstances.append(self)

    ConnectionClass = HTTPConnection

    def handler(self, environ, start_response):
        """Proxy for requests to the actual http server"""
        url = urlparse(environ['reconstructed_url'])
        referer = environ.get('HTTP_REFERER', None)
        test_url = windmill.settings['FORWARDING_TEST_URL']
        if self.fmgr is None and windmill.settings['FORWARDING_TEST_URL'] is not None:
            # Be lazy at creating the forward manager to give
            # FORWARDING_TEST_URL a chance to be set
            self.fmgr = ForwardManager(test_url)
        # Once FORWARDING_TEST_URL is set we should check for cross-domain
        # forward but we must disable for localhost as redirects to localhost
        # will cause the browser to error.
        if windmill.settings['FORWARDING_TEST_URL'] is not None and (
                       not url.netloc.startswith('localhost') ) and (
                       not url.netloc.startswith('127.0.0.1') ) and (
                       conditions_pass(environ) ):
            # Do our domain change magic
            url = urlparse(environ['reconstructed_url'])
            test_netloc = urlparse(test_url).netloc

            if (self.fmgr.is_static_forwarded(url)):
                environ = self.fmgr.forward(url, environ)
                url = self.fmgr.forward_map(url)

            elif ( url.netloc != test_netloc ):
                # if the url's network address is not the test URL that has
                # been set we need to return a forward
                environ = self.fmgr.forward(url, environ)
                redirect_url = self.fmgr.forward_map(url).geturl()
                if environ['REQUEST_METHOD'] == 'POST':
                    form = proxy_post_redirect_form(environ, redirect_url)
                    forward_forms[redirect_url] = form
                response = 'Windmill is forwarding you to a new url at the proper test domain'
                length = str(len(response))
                start_response("302 Found", [('Content-Type', 'text/plain',),
                                             ('Content-Length', length,),
                                             ('Location', redirect_url),
                                             ]+cache_additions)
                logger.debug('Domain change, forwarded to ' + redirect_url)
                return [response]
            elif url.geturl() in forward_forms:
                response = forward_forms[url.geturl()]
                length = str(len(response))
                start_response("200 Ok", [('Content-Type', 'text/html',),
                                          ('Content-Length', length,),
                                         ]+cache_additions)
                del forward_forms[url.geturl()]
                return [response]
            elif (self.fmgr.is_forward_mapped(url)):
                orig_url = self.fmgr.forward_unmap(url)
                environ = self.fmgr.change_environ_domain(url, orig_url,
                                                          environ)
                url = orig_url
            elif (not self.fmgr.is_forward_mapped(url) and
               referer is not None and
               self.fmgr.is_forward_mapped(urlparse(referer))):
                # This handles the case that the referer is a url we've already
                # done a cross-domain request for
                orig_referer = self.fmgr.forward_unmap(urlparse(referer))
                orig_url = self.fmgr.forward_to(url, orig_referer)
                environ = self.fmgr.change_environ_domain(url, orig_url, environ)
                url = orig_url
                self.fmgr.forward(orig_url, {}) # Take note of the forwarding
        def make_remote_connection(url, environ):
            # Create connection object
            try:
                connection = self.get_connection(url)
                # Build path
                path = url.geturl().replace(url.scheme+'://'+url.netloc, '')
            except Exception, e:
                logger.exception('Could not Connect')
                return [("501 Gateway error", [('Content-Type', 'text/html')],),
                        '
Could not connect:

%s
' % (str(e),)]

            # Read in request body if it exists
            body = None
            if environ.has_key('body'):
                body = environ['body']
            elif environ.get('CONTENT_LENGTH'):
                length = int(environ['CONTENT_LENGTH'])
                body = environ['wsgi.input'].read(length)
                environ['body'] = body

            # Build headers
            headers = {}
            logger.debug('Environ ; %s' % str(environ))
            for key in environ.keys():
                # Keys that start with HTTP_ are all headers
                if key.startswith('HTTP_'):
                    # This is a hacky way of getting the header names right
                    value = environ[key]
                    key = key.replace('HTTP_', '', 1).swapcase().replace('_', '-')
                    if is_hop_by_hop(key) is False:
                        headers[key] = value
                    if key.lower() == 'location':
                        # There should never be a legitimate redirect
                        # to /windmill-serv from a remote site
                        if '/windmill-serv' in value:
                            value = value.split('/windmill-serv')[0]

            # Handler headers that aren't HTTP_ in environ
            if environ.get('CONTENT_TYPE'):
                headers['content-type'] = environ['CONTENT_TYPE']

            # Add our host if one isn't defined
            if not headers.has_key('host'):
                headers['host'] = environ['SERVER_NAME']

            # Make the remote request
            try:

                logger.debug('%s %s %s' % (environ['REQUEST_METHOD'], path,
                                           str(headers)))
                connection.request(environ['REQUEST_METHOD'], path, body=body,
                                   headers=headers)
                connection.url = url
                return connection
            except Exception, e:
                # We need extra exception handling in the case the server
                # fails in mid connection, it's an edge case but I've seen it
                return [("501 Gateway error", [('Content-Type', 'text/html')],),
                    '
Could not make request:

%s
' % (str(e),)]

        def retry_known_hosts(url, environ):
            # retry the given request against all the hosts the current session
            # has run against
            if self.fmgr is None:
                return
            for host in self.fmgr.known_hosts():
                orig_url = self.fmgr.forward_to(url, host)
                new_environ = self.fmgr.change_environ_domain(
                                        self.fmgr.forward_map(orig_url),
                                        orig_url, environ)
                connection = make_remote_connection(orig_url, new_environ)
                if isinstance(connection, HTTPConnection):
                    new_response = connection.getresponse()
                    if new_response.status > 199 and new_response.status < 399:
                        logger.info('Retry success, ' + url.geturl() + ' to ' +
                                    host.geturl())
                        new_response.url = connection.url
                        return new_response
        connection = make_remote_connection(url, environ)
        # This following code is ugly.  It should be refactored in to some
        # elegant way to decide when to retry, and which URLs to retry.
        # Maybe hand that responsability to the ForwardManager?
        if isinstance(connection, HTTPConnection):
            response = connection.getresponse()
            response.url = connection.url

        if environ['REQUEST_METHOD'] == 'POST':
            threshold = 399
        else:
            threshold = 200

        if not isinstance(connection, HTTPConnection) or \
            response.status > threshold:
            # if it's not an HTTPConnection object then the request failed
            # so we should retry
            new_response = retry_known_hosts(url, environ)
            if new_response is not None:
                response = new_response
                logger.info('Found after retry: %s' % (url.geturl(),))
            elif not isinstance(connection, HTTPConnection):
                status = connection[0][0]
                headers = connection[0][1]
                body = connection[1]
                for header in copy.copy(headers):
                    if header[0].lower() in cache_removal:
                        headers.remove(header)
                start_response(status, headers+cache_additions)
                return get_wsgi_response(body)

        # Remove hop by hop headers
        headers = self.parse_headers(response)
        if response.status == 404:
            logger.info('Could not fullfill proxy request to ' + url.geturl())

        for header in copy.copy(headers):
            if header[0].lower() in cache_removal:
                headers.remove(header)

        start_response(response.status.__str__()+' '+response.reason,
                       headers+cache_additions)
        return get_wsgi_response(response)

    def parse_headers(self, response):
        headers = [(x.lower(), y) for x, y in [z.split(':', 1) for z in
                             str(response.msg).splitlines() if ':' in z]]
        if self.fmgr is not None:
            self.fmgr.parse_headers(headers, response.url.netloc)
        for header in headers:
            if is_hop_by_hop(header[0]):
                headers.remove(header)
            elif header[0] == 'location':
                # There should never be a legitimate redirect to /windmill-serv
                # from a remote site
                if '/windmill-serv' in header[1]:
                    i = headers.index(header)
                    location = header[1]
                    headers.remove(header)
                    headers.insert(i, ('location',
                                   location.split('/windmill-serv')[0],))
        return headers

    def get_connection(self, url):
        """ Factory method for connections """
        connection = self.ConnectionClass(url.netloc)
        return connection

    def clearForwardingRegistry(self):
        if self.fmgr is not None:
            self.fmgr.clear()

    def __call__(self, environ, start_response):
        return self.handler(environ, start_response)

proxyInstances = []
def clearForwardingRegistry():
    for p in proxyInstances:
        p.clearForwardingRegistry()
<commit_msg>Fix for [ticket:285], server failure in retry loop<commit_after>#   Copyright (c) 2006-2007 Open Source Applications Foundation
#   Copyright (c) 2008-2009 Mikeal Rogers <mikeal.rogers@gmail.com>
#   Copyright (c) 2009 Canonical Ltd.
#   Copyright (c) 2009 Domen Kozar <domen@dev.si>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import windmill

from httplib import HTTPConnection
import copy
import sys
import logging
import urllib
logger = logging.getLogger(__name__)
from forwardmanager import ForwardManager
if not sys.version.startswith('2.4'):
    from urlparse import urlparse
else:
    # python 2.4
    from windmill.tools.urlparse_25 import urlparse

first_forward_domains = []
exclude_from_retry = ['http://sb-ssl.google.com',
                      'https://sb-ssl.google.com',
                      'http://en-us.fxfeeds.mozilla.com',
                      ]

# Note that hoppish conntains proxy-connection, which is pre-HTTP-1.1 and
# is somewhat nebulous
_hoppish = {
    'connection':1, 'keep-alive':1, 'proxy-authenticate':1,
    'proxy-authorization':1, 'te':1, 'trailers':1, 'transfer-encoding':1,
    'upgrade':1, 'proxy-connection':1,
    'p3p':1 #Not actually a hop-by-hop header, just really annoying
    }

# Cache stopping headers
cache_headers = {'Pragma':'no-cache', 'Cache-Control': 'post-check=0, pre-check=0',
                 'Cache-Control': 'no-store, no-cache, must-revalidate, max-age=0',
                 'Expires': '-1'}

cache_removal = [k.lower() for k in cache_headers.keys()]
cache_additions = cache_headers.items()

def is_hop_by_hop(header):
    """check if the given header is hop_by_hop"""
    return _hoppish.has_key(header.lower())

class IterativeResponse(object):
    def __init__(self, response_instance):
        self.response_instance = response_instance

    def __iter__(self):
        if self.response_instance.chunked:
            yield self.response_instance.read(1024)
            while self.response_instance.chunk_left is not None:
                if self.response_instance.chunk_left < 1024:
                    yield self.response_instance.read()
                    self.response_instance.chunk_left = None
                else:
                    yield self.response_instance.read(1024)
        else:
            yield self.response_instance.read()

def get_wsgi_response(response):
    if type(response) is str:
        return [response]
    if response.length > 1000:
        return IterativeResponse(response)
    else:
        return [response.read()]

def conditions_pass(e):
    for c in windmill.server.forwarding_conditions:
        if c(e) is False:
            return False
    return True

def proxy_post_redirect_form(environ, action):
    body = environ['wsgi.input'].read(int(environ['CONTENT_LENGTH']))
    parameters = body.split('&')
    inputs = []
    for parameter in parameters:
        parts = parameter.split('=', 1)
        if len(parts) == 1:
            continue
        parts = tuple(unicode(urllib.unquote(part), 'utf-8') for part in parts)
        inputs.append('<input type="hidden" name="%s" value="%s" />' % parts)
    form = """<title>There is no spoon.</title>

  <form id="redirect" action="%s" method="POST">%s</form>
""" % (action, '\n'.join(inputs))
    return form.encode('utf-8')

forward_forms = {}

class WindmillProxyApplication(object):
    """Application to handle requests that need to be proxied"""

    def __init__(self):
        self.fmgr = None
        proxyInstances.append(self)

    ConnectionClass = HTTPConnection

    def handler(self, environ, start_response):
        """Proxy for requests to the actual http server"""
        url = urlparse(environ['reconstructed_url'])
        referer = environ.get('HTTP_REFERER', None)
        test_url = windmill.settings['FORWARDING_TEST_URL']
        if self.fmgr is None and windmill.settings['FORWARDING_TEST_URL'] is not None:
            # Be lazy at creating the forward manager to give
            # FORWARDING_TEST_URL a chance to be set
            self.fmgr = ForwardManager(test_url)
        # Once FORWARDING_TEST_URL is set we should check for cross-domain
        # forward but we must disable for localhost as redirects to localhost
        # will cause the browser to error.
        if windmill.settings['FORWARDING_TEST_URL'] is not None and (
                       not url.netloc.startswith('localhost') ) and (
                       not url.netloc.startswith('127.0.0.1') ) and (
                       conditions_pass(environ) ):
            # Do our domain change magic
            url = urlparse(environ['reconstructed_url'])
            test_netloc = urlparse(test_url).netloc

            if (self.fmgr.is_static_forwarded(url)):
                environ = self.fmgr.forward(url, environ)
                url = self.fmgr.forward_map(url)

            elif ( url.netloc != test_netloc ):
                # if the url's network address is not the test URL that has
                # been set we need to return a forward
                environ = self.fmgr.forward(url, environ)
                redirect_url = self.fmgr.forward_map(url).geturl()
                if environ['REQUEST_METHOD'] == 'POST':
                    form = proxy_post_redirect_form(environ, redirect_url)
                    forward_forms[redirect_url] = form
                response = 'Windmill is forwarding you to a new url at the proper test domain'
                length = str(len(response))
                start_response("302 Found", [('Content-Type', 'text/plain',),
                                             ('Content-Length', length,),
                                             ('Location', redirect_url),
                                             ]+cache_additions)
                logger.debug('Domain change, forwarded to ' + redirect_url)
                return [response]
            elif url.geturl() in forward_forms:
                response = forward_forms[url.geturl()]
                length = str(len(response))
                start_response("200 Ok", [('Content-Type', 'text/html',),
                                          ('Content-Length', length,),
                                         ]+cache_additions)
                del forward_forms[url.geturl()]
                return [response]
            elif (self.fmgr.is_forward_mapped(url)):
                orig_url = self.fmgr.forward_unmap(url)
                environ = self.fmgr.change_environ_domain(url, orig_url,
                                                          environ)
                url = orig_url
            elif (not self.fmgr.is_forward_mapped(url) and
               referer is not None and
               self.fmgr.is_forward_mapped(urlparse(referer))):
                # This handles the case that the referer is a url we've already
                # done a cross-domain request for
                orig_referer = self.fmgr.forward_unmap(urlparse(referer))
                orig_url = self.fmgr.forward_to(url, orig_referer)
                environ = self.fmgr.change_environ_domain(url, orig_url, environ)
                url = orig_url
                self.fmgr.forward(orig_url, {}) # Take note of the forwarding
        def make_remote_connection(url, environ):
            # Create connection object
            try:
                connection = self.get_connection(url)
                # Build path
                path = url.geturl().replace(url.scheme+'://'+url.netloc, '')
            except Exception, e:
                logger.exception('Could not Connect')
                return [("501 Gateway error", [('Content-Type', 'text/html')],),
                        '
Could not connect:

%s
' % (str(e),)]

            # Read in request body if it exists
            body = None
            if environ.has_key('body'):
                body = environ['body']
            elif environ.get('CONTENT_LENGTH'):
                length = int(environ['CONTENT_LENGTH'])
                body = environ['wsgi.input'].read(length)
                environ['body'] = body

            # Build headers
            headers = {}
            logger.debug('Environ ; %s' % str(environ))
            for key in environ.keys():
                # Keys that start with HTTP_ are all headers
                if key.startswith('HTTP_'):
                    # This is a hacky way of getting the header names right
                    value = environ[key]
                    key = key.replace('HTTP_', '', 1).swapcase().replace('_', '-')
                    if is_hop_by_hop(key) is False:
                        headers[key] = value
                    if key.lower() == 'location':
                        # There should never be a legitimate redirect
                        # to /windmill-serv from a remote site
                        if '/windmill-serv' in value:
                            value = value.split('/windmill-serv')[0]

            # Handler headers that aren't HTTP_ in environ
            if environ.get('CONTENT_TYPE'):
                headers['content-type'] = environ['CONTENT_TYPE']

            # Add our host if one isn't defined
            if not headers.has_key('host'):
                headers['host'] = environ['SERVER_NAME']

            # Make the remote request
            try:

                logger.debug('%s %s %s' % (environ['REQUEST_METHOD'], path,
                                           str(headers)))
                connection.request(environ['REQUEST_METHOD'], path, body=body,
                                   headers=headers)
                connection.url = url
                return connection
            except Exception, e:
                # We need extra exception handling in the case the server
                # fails in mid connection, it's an edge case but I've seen it
                return [("501 Gateway error", [('Content-Type', 'text/html')],),
                    '
Could not make request:

%s
' % (str(e),)]

        def retry_known_hosts(url, environ):
            # retry the given request against all the hosts the current session
            # has run against
            if self.fmgr is None:
                return
            for host in self.fmgr.known_hosts():
                orig_url = self.fmgr.forward_to(url, host)
                new_environ = self.fmgr.change_environ_domain(
                                        self.fmgr.forward_map(orig_url),
                                        orig_url, environ)
                connection = make_remote_connection(orig_url, new_environ)
                if isinstance(connection, HTTPConnection):
                    try:
                        new_response = connection.getresponse()
                    except:
                        return
                    if new_response.status > 199 and new_response.status < 399:
                        logger.info('Retry success, ' + url.geturl() + ' to ' +
                                    host.geturl())
                        new_response.url = connection.url
                        return new_response
        connection = make_remote_connection(url, environ)
        # This following code is ugly.  It should be refactored in to some
        # elegant way to decide when to retry, and which URLs to retry.
        # Maybe hand that responsability to the ForwardManager?
        if isinstance(connection, HTTPConnection):
            response = connection.getresponse()
            response.url = connection.url

        if environ['REQUEST_METHOD'] == 'POST':
            threshold = 399
        else:
            threshold = 200

        if not isinstance(connection, HTTPConnection) or \
            response.status > threshold:
            # if it's not an HTTPConnection object then the request failed
            # so we should retry
            new_response = retry_known_hosts(url, environ)
            if new_response is not None:
                response = new_response
                logger.info('Found after retry: %s' % (url.geturl(),))
            elif not isinstance(connection, HTTPConnection):
                status = connection[0][0]
                headers = connection[0][1]
                body = connection[1]
                for header in copy.copy(headers):
                    if header[0].lower() in cache_removal:
                        headers.remove(header)
                start_response(status, headers+cache_additions)
                return get_wsgi_response(body)

        # Remove hop by hop headers
        headers = self.parse_headers(response)
        if response.status == 404:
            logger.info('Could not fullfill proxy request to ' + url.geturl())

        for header in copy.copy(headers):
            if header[0].lower() in cache_removal:
                headers.remove(header)

        start_response(response.status.__str__()+' '+response.reason,
                       headers+cache_additions)
        return get_wsgi_response(response)

    def parse_headers(self, response):
        headers = [(x.lower(), y) for x, y in [z.split(':', 1) for z in
                             str(response.msg).splitlines() if ':' in z]]
        if self.fmgr is not None:
            self.fmgr.parse_headers(headers, response.url.netloc)
        for header in headers:
            if is_hop_by_hop(header[0]):
                headers.remove(header)
            elif header[0] == 'location':
                # There should never be a legitimate redirect to /windmill-serv
                # from a remote site
                if '/windmill-serv' in header[1]:
                    i = headers.index(header)
                    location = header[1]
                    headers.remove(header)
                    headers.insert(i, ('location',
                                   location.split('/windmill-serv')[0],))
        return headers

    def get_connection(self, url):
        """ Factory method for connections """
        connection = self.ConnectionClass(url.netloc)
        return connection

    def clearForwardingRegistry(self):
        if self.fmgr is not None:
            self.fmgr.clear()

    def __call__(self, environ, start_response):
        return self.handler(environ, start_response)

proxyInstances = []
def clearForwardingRegistry():
    for p in proxyInstances:
        p.clearForwardingRegistry()
<|endoftext|>


<commit_before>#   Copyright (c) 2006-2007 Open Source Applications Foundation
#   Copyright (c) 2008-2009 Mikeal Rogers <mikeal.rogers@gmail.com>
#   Copyright (c) 2009 Canonical Ltd.
#   Copyright (c) 2009 Domen Kozar <domen@dev.si>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import windmill

from httplib import HTTPConnection
import copy
import sys
import logging
import urllib
logger = logging.getLogger(__name__)
from forwardmanager import ForwardManager
if not sys.version.startswith('2.4'):
    from urlparse import urlparse
else:
    # python 2.4
    from windmill.tools.urlparse_25 import urlparse

first_forward_domains = []
exclude_from_retry = ['http://sb-ssl.google.com',
                      'https://sb-ssl.google.com',
                      'http://en-us.fxfeeds.mozilla.com',
                      ]

# Note that hoppish conntains proxy-connection, which is pre-HTTP-1.1 and
# is somewhat nebulous
_hoppish = {
    'connection':1, 'keep-alive':1, 'proxy-authenticate':1,
    'proxy-authorization':1, 'te':1, 'trailers':1, 'transfer-encoding':1,
    'upgrade':1, 'proxy-connection':1,
    'p3p':1 #Not actually a hop-by-hop header, just really annoying
    }

# Cache stopping headers
cache_headers = {'Pragma':'no-cache', 'Cache-Control': 'post-check=0, pre-check=0',
                 'Cache-Control': 'no-store, no-cache, must-revalidate, max-age=0',
                 'Expires': '-1'}

cache_removal = [k.lower() for k in cache_headers.keys()]
cache_additions = cache_headers.items()

def is_hop_by_hop(header):
    """check if the given header is hop_by_hop"""
    return _hoppish.has_key(header.lower())

class IterativeResponse(object):
    def __init__(self, response_instance):
        self.response_instance = response_instance

    def __iter__(self):
        if self.response_instance.chunked:
            yield self.response_instance.read(1024)
            while self.response_instance.chunk_left is not None:
                if self.response_instance.chunk_left < 1024:
                    yield self.response_instance.read()
                    self.response_instance.chunk_left = None
                else:
                    yield self.response_instance.read(1024)
        else:
            yield self.response_instance.read()

def get_wsgi_response(response):
    if type(response) is str:
        return [response]
    if response.length > 1000:
        return IterativeResponse(response)
    else:
        return [response.read()]

def conditions_pass(e):
    for c in windmill.server.forwarding_conditions:
        if c(e) is False:
            return False
    return True

def proxy_post_redirect_form(environ, action):
    body = environ['wsgi.input'].read(int(environ['CONTENT_LENGTH']))
    parameters = body.split('&')
    inputs = []
    for parameter in parameters:
        parts = parameter.split('=', 1)
        if len(parts) == 1:
            continue
        parts = tuple(unicode(urllib.unquote(part), 'utf-8') for part in parts)
        inputs.append('<input type="hidden" name="%s" value="%s" />' % parts)
    form = """<title>There is no spoon.</title>

  <form id="redirect" action="%s" method="POST">%s</form>
""" % (action, '\n'.join(inputs))
    return form.encode('utf-8')

forward_forms = {}

class WindmillProxyApplication(object):
    """Application to handle requests that need to be proxied"""

    def __init__(self):
        self.fmgr = None
        proxyInstances.append(self)

    ConnectionClass = HTTPConnection

    def handler(self, environ, start_response):
        """Proxy for requests to the actual http server"""
        url = urlparse(environ['reconstructed_url'])
        referer = environ.get('HTTP_REFERER', None)
        test_url = windmill.settings['FORWARDING_TEST_URL']
        if self.fmgr is None and windmill.settings['FORWARDING_TEST_URL'] is not None:
            # Be lazy at creating the forward manager to give
            # FORWARDING_TEST_URL a chance to be set
            self.fmgr = ForwardManager(test_url)
        # Once FORWARDING_TEST_URL is set we should check for cross-domain
        # forward but we must disable for localhost as redirects to localhost
        # will cause the browser to error.
        if windmill.settings['FORWARDING_TEST_URL'] is not None and (
                       not url.netloc.startswith('localhost') ) and (
                       not url.netloc.startswith('127.0.0.1') ) and (
                       conditions_pass(environ) ):
            # Do our domain change magic
            url = urlparse(environ['reconstructed_url'])
            test_target = urlparse(test_url)
            #test_netloc = urlparse(test_url).netloc

            if (self.fmgr.is_static_forwarded(url)):
                environ = self.fmgr.forward(url, environ)
                url = self.fmgr.forward_map(url)

            elif ( url.scheme+"://"+url.netloc != test_target.scheme+"://"+test_target.netloc ):
                # if the url's network address is not the test URL that has
                # been set we need to return a forward
                environ = self.fmgr.forward(url, environ)
                redirect_url = self.fmgr.forward_map(url).geturl()
                if environ['REQUEST_METHOD'] == 'POST':
                    form = proxy_post_redirect_form(environ, redirect_url)
                    forward_forms[redirect_url] = form
                response = 'Windmill is forwarding you to a new url at the proper test domain'
                length = str(len(response))
                start_response("302 Found", [('Content-Type', 'text/plain',),
                                             ('Content-Length', length,),
                                             ('Location', redirect_url),
                                             ]+cache_additions)
                logger.debug('Domain change, forwarded to ' + redirect_url)
                return [response]
            elif url.geturl() in forward_forms:
                response = forward_forms[url.geturl()]
                length = str(len(response))
                start_response("200 Ok", [('Content-Type', 'text/html',),
                                          ('Content-Length', length,),
                                         ]+cache_additions)
                del forward_forms[url.geturl()]
                return [response]
            elif (self.fmgr.is_forward_mapped(url)):
                orig_url = self.fmgr.forward_unmap(url)
                environ = self.fmgr.change_environ_domain(url, orig_url,
                                                          environ)
                url = orig_url
            elif (not self.fmgr.is_forward_mapped(url) and
               referer is not None and
               self.fmgr.is_forward_mapped(urlparse(referer))):
                # This handles the case that the referer is a url we've already
                # done a cross-domain request for
                orig_referer = self.fmgr.forward_unmap(urlparse(referer))
                orig_url = self.fmgr.forward_to(url, orig_referer)
                environ = self.fmgr.change_environ_domain(url, orig_url, environ)
                url = orig_url
                self.fmgr.forward(orig_url, {}) # Take note of the forwarding
        def make_remote_connection(url, environ):
            # Create connection object
            try:
                connection = self.get_connection(url)
                # Build path
                path = url.geturl().replace(url.scheme+'://'+url.netloc, '')
            except Exception, e:
                logger.exception('Could not Connect')
                return [("501 Gateway error", [('Content-Type', 'text/html')],),
                        '
Could not connect:

%s
' % (str(e),)]

            # Read in request body if it exists
            body = None
            if environ.has_key('body'):
                body = environ['body']
            elif environ.get('CONTENT_LENGTH'):
                length = int(environ['CONTENT_LENGTH'])
                body = environ['wsgi.input'].read(length)
                environ['body'] = body

            # Build headers
            headers = {}
            logger.debug('Environ ; %s' % str(environ))
            for key in environ.keys():
                # Keys that start with HTTP_ are all headers
                if key.startswith('HTTP_'):
                    # This is a hacky way of getting the header names right
                    value = environ[key]
                    key = key.replace('HTTP_', '', 1).swapcase().replace('_', '-')
                    if is_hop_by_hop(key) is False:
                        headers[key] = value
                    if key.lower() == 'location':
                        # There should never be a legitimate redirect
                        # to /windmill-serv from a remote site
                        if '/windmill-serv' in value:
                            value = value.split('/windmill-serv')[0]

            # Handler headers that aren't HTTP_ in environ
            if environ.get('CONTENT_TYPE'):
                headers['content-type'] = environ['CONTENT_TYPE']

            # Add our host if one isn't defined
            if not headers.has_key('host'):
                headers['host'] = environ['SERVER_NAME']

            # Make the remote request
            try:

                logger.debug('%s %s %s' % (environ['REQUEST_METHOD'], path,
                                           str(headers)))
                connection.request(environ['REQUEST_METHOD'], path, body=body,
                                   headers=headers)
                connection.url = url
                return connection
            except Exception, e:
                # We need extra exception handling in the case the server
                # fails in mid connection, it's an edge case but I've seen it
                return [("501 Gateway error", [('Content-Type', 'text/html')],),
                    '
Could not make request:

%s
' % (str(e),)]

        def retry_known_hosts(url, environ):
            # retry the given request against all the hosts the current session
            # has run against
            if self.fmgr is None:
                return
            for host in self.fmgr.known_hosts():
                orig_url = self.fmgr.forward_to(url, host)
                new_environ = self.fmgr.change_environ_domain(
                                        self.fmgr.forward_map(orig_url),
                                        orig_url, environ)
                connection = make_remote_connection(orig_url, new_environ)
                if isinstance(connection, HTTPConnection):
                    try:
                        new_response = connection.getresponse()
                    except:
                        return
                    if new_response.status > 199 and new_response.status < 399:
                        logger.info('Retry success, ' + url.geturl() + ' to ' +
                                    host.geturl())
                        new_response.url = connection.url
                        return new_response
        connection = make_remote_connection(url, environ)
        # This following code is ugly.  It should be refactored in to some
        # elegant way to decide when to retry, and which URLs to retry.
        # Maybe hand that responsability to the ForwardManager?
        if isinstance(connection, HTTPConnection):
            response = connection.getresponse()
            response.url = connection.url

        if environ['REQUEST_METHOD'] == 'POST':
            threshold = 399
        else:
            threshold = 200

        if not isinstance(connection, HTTPConnection) or \
            response.status > threshold:
            # if it's not an HTTPConnection object then the request failed
            # so we should retry
            new_response = retry_known_hosts(url, environ)
            if new_response is not None:
                response = new_response
                logger.info('Found after retry: %s' % (url.geturl(),))
            elif not isinstance(connection, HTTPConnection):
                status = connection[0][0]
                headers = connection[0][1]
                body = connection[1]
                for header in copy.copy(headers):
                    if header[0].lower() in cache_removal:
                        headers.remove(header)
                start_response(status, headers+cache_additions)
                return get_wsgi_response(body)

        # Remove hop by hop headers
        headers = self.parse_headers(response)
        if response.status == 404:
            logger.info('Could not fullfill proxy request to ' + url.geturl())

        for header in copy.copy(headers):
            if header[0].lower() in cache_removal:
                headers.remove(header)

        start_response(response.status.__str__()+' '+response.reason,
                       headers+cache_additions)
        return get_wsgi_response(response)

    def parse_headers(self, response):
        headers = [(x.lower(), y) for x, y in [z.split(':', 1) for z in
                             str(response.msg).splitlines() if ':' in z]]
        if self.fmgr is not None:
            self.fmgr.parse_headers(headers, response.url.netloc)
        for header in headers:
            if is_hop_by_hop(header[0]):
                headers.remove(header)
            elif header[0] == 'location':
                # There should never be a legitimate redirect to /windmill-serv
                # from a remote site
                if '/windmill-serv' in header[1]:
                    i = headers.index(header)
                    location = header[1]
                    headers.remove(header)
                    headers.insert(i, ('location',
                                   location.split('/windmill-serv')[0],))
        return headers

    def get_connection(self, url):
        """ Factory method for connections """
        connection = self.ConnectionClass(url.netloc)
        return connection

    def clearForwardingRegistry(self):
        if self.fmgr is not None:
            self.fmgr.clear()

    def __call__(self, environ, start_response):
        return self.handler(environ, start_response)

proxyInstances = []
def clearForwardingRegistry():
    for p in proxyInstances:
        p.clearForwardingRegistry()
<commit_msg>Another fix, breakage loading bbc.co.uk<commit_after>#   Copyright (c) 2006-2007 Open Source Applications Foundation
#   Copyright (c) 2008-2009 Mikeal Rogers <mikeal.rogers@gmail.com>
#   Copyright (c) 2009 Canonical Ltd.
#   Copyright (c) 2009 Domen Kozar <domen@dev.si>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import windmill

from httplib import HTTPConnection
import copy
import sys
import logging
import urllib
logger = logging.getLogger(__name__)
from forwardmanager import ForwardManager
if not sys.version.startswith('2.4'):
    from urlparse import urlparse
else:
    # python 2.4
    from windmill.tools.urlparse_25 import urlparse

first_forward_domains = []
exclude_from_retry = ['http://sb-ssl.google.com',
                      'https://sb-ssl.google.com',
                      'http://en-us.fxfeeds.mozilla.com',
                      ]

# Note that hoppish conntains proxy-connection, which is pre-HTTP-1.1 and
# is somewhat nebulous
_hoppish = {
    'connection':1, 'keep-alive':1, 'proxy-authenticate':1,
    'proxy-authorization':1, 'te':1, 'trailers':1, 'transfer-encoding':1,
    'upgrade':1, 'proxy-connection':1,
    'p3p':1 #Not actually a hop-by-hop header, just really annoying
    }

# Cache stopping headers
cache_headers = {'Pragma':'no-cache', 'Cache-Control': 'post-check=0, pre-check=0',
                 'Cache-Control': 'no-store, no-cache, must-revalidate, max-age=0',
                 'Expires': '-1'}

cache_removal = [k.lower() for k in cache_headers.keys()]
cache_additions = cache_headers.items()

def is_hop_by_hop(header):
    """check if the given header is hop_by_hop"""
    return _hoppish.has_key(header.lower())

class IterativeResponse(object):
    def __init__(self, response_instance):
        self.response_instance = response_instance

    def __iter__(self):
        if self.response_instance.chunked:
            yield self.response_instance.read(1024)
            while self.response_instance.chunk_left is not None:
                if self.response_instance.chunk_left < 1024:
                    yield self.response_instance.read()
                    self.response_instance.chunk_left = None
                else:
                    yield self.response_instance.read(1024)
        else:
            yield self.response_instance.read()

def get_wsgi_response(response):
    if type(response) is str:
        return [response]
    if response.length > 1000:
        return IterativeResponse(response)
    else:
        return [response.read()]

def conditions_pass(e):
    for c in windmill.server.forwarding_conditions:
        if c(e) is False:
            return False
    return True

def proxy_post_redirect_form(environ, action):
    body = environ['wsgi.input'].read(int(environ['CONTENT_LENGTH']))
    parameters = body.split('&')
    inputs = []
    for parameter in parameters:
        parts = parameter.split('=', 1)
        if len(parts) == 1:
            continue
        parts = tuple(unicode(urllib.unquote(part), 'utf-8') for part in parts)
        inputs.append('<input type="hidden" name="%s" value="%s" />' % parts)
    form = """<title>There is no spoon.</title>

  <form id="redirect" action="%s" method="POST">%s</form>
""" % (action, '\n'.join(inputs))
    return form.encode('utf-8')

forward_forms = {}

class WindmillProxyApplication(object):
    """Application to handle requests that need to be proxied"""

    def __init__(self):
        self.fmgr = None
        proxyInstances.append(self)

    ConnectionClass = HTTPConnection

    def handler(self, environ, start_response):
        """Proxy for requests to the actual http server"""
        url = urlparse(environ['reconstructed_url'])
        referer = environ.get('HTTP_REFERER', None)
        test_url = windmill.settings['FORWARDING_TEST_URL']
        if self.fmgr is None and windmill.settings['FORWARDING_TEST_URL'] is not None:
            # Be lazy at creating the forward manager to give
            # FORWARDING_TEST_URL a chance to be set
            self.fmgr = ForwardManager(test_url)
        # Once FORWARDING_TEST_URL is set we should check for cross-domain
        # forward but we must disable for localhost as redirects to localhost
        # will cause the browser to error.
        if windmill.settings['FORWARDING_TEST_URL'] is not None and (
                       not url.netloc.startswith('localhost') ) and (
                       not url.netloc.startswith('127.0.0.1') ) and (
                       conditions_pass(environ) ):
            # Do our domain change magic
            url = urlparse(environ['reconstructed_url'])
            test_target = urlparse(test_url)
            #test_netloc = urlparse(test_url).netloc

            if (self.fmgr.is_static_forwarded(url)):
                environ = self.fmgr.forward(url, environ)
                url = self.fmgr.forward_map(url)

            elif ( url.scheme+"://"+url.netloc != test_target.scheme+"://"+test_target.netloc ):
                # if the url's network address is not the test URL that has
                # been set we need to return a forward
                environ = self.fmgr.forward(url, environ)
                redirect_url = self.fmgr.forward_map(url).geturl()
                if environ['REQUEST_METHOD'] == 'POST':
                    form = proxy_post_redirect_form(environ, redirect_url)
                    forward_forms[redirect_url] = form
                response = 'Windmill is forwarding you to a new url at the proper test domain'
                length = str(len(response))
                start_response("302 Found", [('Content-Type', 'text/plain',),
                                             ('Content-Length', length,),
                                             ('Location', redirect_url),
                                             ]+cache_additions)
                logger.debug('Domain change, forwarded to ' + redirect_url)
                return [response]
            elif url.geturl() in forward_forms:
                response = forward_forms[url.geturl()]
                length = str(len(response))
                start_response("200 Ok", [('Content-Type', 'text/html',),
                                          ('Content-Length', length,),
                                         ]+cache_additions)
                del forward_forms[url.geturl()]
                return [response]
            elif (self.fmgr.is_forward_mapped(url)):
                orig_url = self.fmgr.forward_unmap(url)
                environ = self.fmgr.change_environ_domain(url, orig_url,
                                                          environ)
                url = orig_url
            elif (not self.fmgr.is_forward_mapped(url) and
               referer is not None and
               self.fmgr.is_forward_mapped(urlparse(referer))):
                # This handles the case that the referer is a url we've already
                # done a cross-domain request for
                orig_referer = self.fmgr.forward_unmap(urlparse(referer))
                orig_url = self.fmgr.forward_to(url, orig_referer)
                environ = self.fmgr.change_environ_domain(url, orig_url, environ)
                url = orig_url
                self.fmgr.forward(orig_url, {}) # Take note of the forwarding
        def make_remote_connection(url, environ):
            # Create connection object
            try:
                connection = self.get_connection(url)
                # Build path
                path = url.geturl().replace(url.scheme+'://'+url.netloc, '')
            except Exception, e:
                logger.exception('Could not Connect')
                return [("501 Gateway error", [('Content-Type', 'text/html')],),
                        '
Could not connect:

%s
' % (str(e),)]

            # Read in request body if it exists
            body = None
            if environ.has_key('body'):
                body = environ['body']
            elif environ.get('CONTENT_LENGTH'):
                length = int(environ['CONTENT_LENGTH'])
                body = environ['wsgi.input'].read(length)
                environ['body'] = body

            # Build headers
            headers = {}
            logger.debug('Environ ; %s' % str(environ))
            for key in environ.keys():
                # Keys that start with HTTP_ are all headers
                if key.startswith('HTTP_'):
                    # This is a hacky way of getting the header names right
                    value = environ[key]
                    key = key.replace('HTTP_', '', 1).swapcase().replace('_', '-')
                    if is_hop_by_hop(key) is False:
                        headers[key] = value
                    if key.lower() == 'location':
                        # There should never be a legitimate redirect
                        # to /windmill-serv from a remote site
                        if '/windmill-serv' in value:
                            value = value.split('/windmill-serv')[0]

            # Handler headers that aren't HTTP_ in environ
            if environ.get('CONTENT_TYPE'):
                headers['content-type'] = environ['CONTENT_TYPE']

            # Add our host if one isn't defined
            if not headers.has_key('host'):
                headers['host'] = environ['SERVER_NAME']

            # Make the remote request
            try:

                logger.debug('%s %s %s' % (environ['REQUEST_METHOD'], path,
                                           str(headers)))
                connection.request(environ['REQUEST_METHOD'], path, body=body,
                                   headers=headers)
                connection.url = url
                return connection
            except Exception, e:
                # We need extra exception handling in the case the server
                # fails in mid connection, it's an edge case but I've seen it
                return [("501 Gateway error", [('Content-Type', 'text/html')],),
                    '
Could not make request:

%s
' % (str(e),)]

        def retry_known_hosts(url, environ):
            # retry the given request against all the hosts the current session
            # has run against
            if self.fmgr is None:
                return
            for host in self.fmgr.known_hosts():
                orig_url = self.fmgr.forward_to(url, host)
                new_environ = self.fmgr.change_environ_domain(
                                        self.fmgr.forward_map(orig_url),
                                        orig_url, environ)
                connection = make_remote_connection(orig_url, new_environ)
                if isinstance(connection, HTTPConnection):
                    try:
                        new_response = connection.getresponse()
                    except:
                        return
                    if new_response.status > 199 and new_response.status < 399:
                        logger.info('Retry success, ' + url.geturl() + ' to ' +
                                    host.geturl())
                        new_response.url = connection.url
                        return new_response
        connection = make_remote_connection(url, environ)
        # This following code is ugly.  It should be refactored in to some
        # elegant way to decide when to retry, and which URLs to retry.
        # Maybe hand that responsability to the ForwardManager?
        if isinstance(connection, HTTPConnection):
            response = connection.getresponse()
            response.url = connection.url

        if environ['REQUEST_METHOD'] == 'POST':
            threshold = 399
        else:
            threshold = 200

        if not isinstance(connection, HTTPConnection) or \
            response.status > threshold:
            # if it's not an HTTPConnection object then the request failed
            # so we should retry
            new_response = retry_known_hosts(url, environ)
            if new_response is not None:
                response = new_response
                logger.info('Found after retry: %s' % (url.geturl(),))
            elif not isinstance(connection, HTTPConnection):
                status = connection[0][0]
                headers = connection[0][1]
                body = connection[1]
                for header in copy.copy(headers):
                    if header[0].lower() in cache_removal:
                        headers.remove(header)
                start_response(status, headers+cache_additions)
                return get_wsgi_response(body)

        # Remove hop by hop headers
        headers = self.parse_headers(response)
        if response.status == 404:
            logger.info('Could not fullfill proxy request to ' + url.geturl())

        for header in copy.copy(headers):
            if header[0].lower() in cache_removal:
                headers.remove(header)

        start_response(response.status.__str__()+' '+response.reason,
                       headers+cache_additions)
        return get_wsgi_response(response)

    def parse_headers(self, response):
        headers = [(x.lower(), y) for x, y in [z.split(':', 1) for z in
                             str(response.msg).splitlines() if ':' in z]]
        #all this does is cookie management, which we currently turned off
        #if self.fmgr is not None:
            #self.fmgr.parse_headers(headers, response.url.netloc)
        for header in headers:
            if is_hop_by_hop(header[0]):
                headers.remove(header)
            elif header[0] == 'location':
                # There should never be a legitimate redirect to /windmill-serv
                # from a remote site
                if '/windmill-serv' in header[1]:
                    i = headers.index(header)
                    location = header[1]
                    headers.remove(header)
                    headers.insert(i, ('location',
                                   location.split('/windmill-serv')[0],))
        return headers

    def get_connection(self, url):
        """ Factory method for connections """
        connection = self.ConnectionClass(url.netloc)
        return connection

    def clearForwardingRegistry(self):
        if self.fmgr is not None:
            self.fmgr.clear()

    def __call__(self, environ, start_response):
        return self.handler(environ, start_response)

proxyInstances = []
def clearForwardingRegistry():
    for p in proxyInstances:
        p.clearForwardingRegistry()
<|endoftext|>