<commit_before># -*-coding:utf-8-*-

import socket
import select
from karlooper.logger.logger import init_logger
from karlooper.web import IOModel
from karlooper.web.__async_core_server import EchoServer, asyncore
from karlooper.web.http_connection import HttpConnection
from karlooper.web.http_io_buffer import HttpIOBuffer
from karlooper.web.http_io_routine_pool import HttpIORoutinePool
from karlooper.http_parser.http_parser import HttpParser
from karlooper.config import get_cli_data, set_cli_data
from karlooper.config.config import SOCKET_RECEIVE_SIZE, DEFAULT_PORT, CLIENT_CONNECT_TO_SERVER_NUM

__author__ = 'karlvorndoenitz@gmail.com'

class Application(object):
    def __init__(self, handlers, settings=None, **kwargs):
        """

        :param handlers: handlers mapping, dict type
        :param settings: settings mapping, dict type
        :param kwargs: options

        """
        self.settings = settings
        self.handlers = handlers
        set_cli_data(self.settings)
        set_cli_data(kwargs)
        cli_data = get_cli_data()
        self.port = int(cli_data.get("port", DEFAULT_PORT))
        log_conf = self.settings.get("log_conf", None) if self.settings else kwargs.get("log_conf", None)
        self.logger = init_logger(config_path=log_conf)
        self.EOL1 = b'\n\n'
        self.EOL2 = b'\n\r\n'
        self.response = ""

    def listen(self, port):
        """listen port

        :param port: port that application listened
        :return: None

        """
        self.port = int(port)

    def __run_epoll(self):
        """
        run the application use epoll
        """
        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        server_socket.bind(('0.0.0.0', self.port))
        server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM)  # the number of client that connect to server
        server_socket.setblocking(0)  # set 0 not block other block
        server_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
        epoll = select.epoll()
        epoll.register(server_socket.fileno(), select.EPOLLIN)
        try:
            http_connection = HttpConnection()
            http_io_buffer = HttpIOBuffer()
            http_io_routine_pool = HttpIORoutinePool()
            events_buf = []
            while True:
                events = epoll.poll(1) + events_buf
                events_buf = []
                for fileno, event in events:
                    try:
                        if fileno == server_socket.fileno():  # if request come
                            connection, address = server_socket.accept()  # waiting income connection
                            connection.setblocking(0)  # none block
                            epoll.register(connection.fileno(), select.EPOLLIN)  # register socket read event to epoll
                            http_connection.add_connection(connection.fileno(), connection)
                            http_io_buffer.add_request(connection.fileno(), b'')
                            http_io_buffer.add_response(connection.fileno(), self.response)
                        elif event & select.EPOLLIN:  # when data in os's read buffer area
                            http_parser = http_io_routine_pool.get(file_no=fileno)
                            if http_parser:
                                data = http_parser.parse()
                                if isinstance(data, str) or isinstance(data, unicode):
                                    http_io_buffer.add_response(
                                        fileno,
                                        http_io_buffer.get_response(fileno) + data
                                    )
                                    epoll.modify(fileno, select.EPOLLOUT)  # change file number to epoll out mode
                                    http_io_routine_pool.remove(fileno)
                                else:  # if coroutine
                                    http_io_routine_pool.add(fileno, http_parser)
                                    events_buf.append((fileno, event))
                            else:
                                http_request_buffer = http_connection.get_connection(fileno).recv(SOCKET_RECEIVE_SIZE)
                                http_io_buffer.add_request(
                                    fileno,
                                    http_io_buffer.get_request(fileno) + http_request_buffer
                                )
                                if self.EOL1 in http_io_buffer.get_request(fileno) \
                                        or self.EOL2 in http_io_buffer.get_request(fileno):
                                    request_data = http_io_buffer.get_request(fileno)[:-2] \
                                        if http_io_buffer.get_request(fileno).endswith("\r\n") \
                                        else http_io_buffer.get_request(fileno)
                                    http_parser = HttpParser(
                                        request_data,
                                        self.handlers,
                                        settings=self.settings
                                    )
                                    data = http_parser.parse()
                                    if isinstance(data, str) or isinstance(data, unicode):
                                        http_io_buffer.add_response(
                                            fileno,
                                            http_io_buffer.get_response(fileno) + data
                                        )
                                        epoll.modify(fileno, select.EPOLLOUT)  # change file number to epoll out mode
                                        http_io_routine_pool.remove(fileno)
                                    else:  # if coroutine
                                        http_io_routine_pool.add(fileno, http_parser)
                                        events_buf.append((fileno, event))
                        elif event & select.EPOLLOUT:  # if out mode
                            bytes_written = http_connection.get_connection(fileno).send(
                                http_io_buffer.get_response(fileno)
                            )
                            http_io_buffer.add_response(fileno, http_io_buffer.get_response(fileno)[bytes_written:])
                            if len(http_io_buffer.get_response(fileno)) == 0:  # if file sent
                                http_connection.get_connection(fileno).shutdown(socket.SHUT_RDWR)
                                epoll.modify(fileno, select.EPOLLHUP)
                        elif event & select.EPOLLHUP:  # if message sent and file number in epoll is hup
                            epoll.unregister(fileno)  # remove file number from epoll
                            http_connection.get_connection(fileno).close()  # close connection
                            http_connection.remove_connection(fileno)  # delete connection from connections dict
                    except Exception, e:
                        self.logger.error("error in __run_epoll: %s", str(e))
                        continue
        finally:
            epoll.unregister(server_socket.fileno())
            epoll.close()
            server_socket.close()

    def __run_kqueue(self):
        """
        run server use kqueue
        """
        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_socket.bind(('0.0.0.0', self.port))
        server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM)
        kq = select.kqueue()
        http_connection = HttpConnection()
        index = 1
        events = [select.kevent(server_socket.fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD)]
        while True:
            try:
                event_list = kq.control(events, 1)
            except select.error as e:
                self.logger.error("error in __run_kqueue: %s", str(e))
                break
            if event_list:
                for each in event_list:
                    if each.ident == server_socket.fileno():
                        index += 1
                        conn, addr = server_socket.accept()
                        http_connection.add_connection(index, conn)
                        events.append(
                            select.kevent(
                                http_connection.get_connection(index).fileno(),
                                select.KQ_FILTER_READ,
                                select.KQ_EV_ADD,
                                udata=index
                            )
                        )
                    else:
                        try:
                            if each.udata >= 1 and each.flags == select.KQ_EV_ADD \
                                    and each.filter == select.KQ_FILTER_READ:
                                conn = http_connection.get_connection(each.udata)
                                request_data = conn.recv(SOCKET_RECEIVE_SIZE)
                                request_data = request_data[:-2] if request_data.endswith("\r\n") else request_data
                                data = HttpParser(
                                    request_data,
                                    handlers=self.handlers,
                                    settings=self.settings
                                ).parse()
                                conn.send(data)
                        except Exception, e:
                            self.logger.error("error in __run_kqueue event list: %s", str(e))
                        finally:
                            events.remove(select.kevent(
                                http_connection.get_connection(each.udata).fileno(),
                                select.KQ_FILTER_READ,
                                select.KQ_EV_ADD,
                                udata=each.udata)
                            )
                            http_connection.remove_connection(each.udata)
                            conn.close()
        server_socket.close()

    def __run_poll(self):
        """
        run server use poll, I will modify __run_poll and __run_epoll in the future
        """
        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        server_socket.bind(('0.0.0.0', self.port))
        server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM)  # the number of client that connect to server
        server_socket.setblocking(0)  # set 0 not block other block
        server_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
        poll = select.poll()
        poll.register(server_socket.fileno(), select.POLLIN)
        try:
            http_connection = HttpConnection()
            http_io_buffer = HttpIOBuffer()
            http_io_routine_pool = HttpIORoutinePool()
            events_buf = []
            while True:
                events = poll.poll(1) + events_buf
                events_buf = []
                for fileno, event in events:
                    try:
                        if fileno == server_socket.fileno():  # if request come
                            connection, address = server_socket.accept()  # waiting income connection
                            connection.setblocking(0)  # none block
                            poll.register(connection.fileno(), select.POLLIN)  # register socket read event to poll
                            http_connection.add_connection(connection.fileno(), connection)
                            http_io_buffer.add_request(connection.fileno(), b'')
                            http_io_buffer.add_response(connection.fileno(), self.response)
                        elif event & select.POLLIN:  # when data in os's read buffer area
                            http_parser = http_io_routine_pool.get(file_no=fileno)
                            if http_parser:
                                data = http_parser.parse()
                                if isinstance(data, str) or isinstance(data, unicode):
                                    http_io_buffer.add_response(
                                        fileno,
                                        http_io_buffer.get_response(fileno) + data
                                    )
                                    poll.modify(fileno, select.POLLOUT)  # change file number to epoll out mode
                                    http_io_routine_pool.remove(fileno)
                                else:  # if coroutine
                                    http_io_routine_pool.add(fileno, http_parser)
                                    events_buf.append((fileno, event))
                            else:
                                http_request_buffer = http_connection.get_connection(fileno).recv(SOCKET_RECEIVE_SIZE)
                                http_io_buffer.add_request(
                                    fileno,
                                    http_io_buffer.get_request(fileno) + http_request_buffer
                                )
                                if self.EOL1 in http_io_buffer.get_request(fileno) \
                                        or self.EOL2 in http_io_buffer.get_request(fileno):
                                    request_data = http_io_buffer.get_request(fileno)[:-2] \
                                        if http_io_buffer.get_request(fileno).endswith("\r\n") \
                                        else http_io_buffer.get_request(fileno)
                                    http_parser = HttpParser(
                                        request_data,
                                        self.handlers,
                                        settings=self.settings
                                    )
                                    data = http_parser.parse()
                                    if isinstance(data, str) or isinstance(data, unicode):
                                        http_io_buffer.add_response(
                                            fileno,
                                            http_io_buffer.get_response(fileno) + data
                                        )
                                        poll.modify(fileno, select.POLLOUT)  # change file number to epoll out mode
                                        http_io_routine_pool.remove(fileno)
                                    else:  # if coroutine
                                        http_io_routine_pool.add(fileno, http_parser)
                                        events_buf.append((fileno, event))
                        elif event & select.POLLOUT:  # if out mode
                            bytes_written = http_connection.get_connection(fileno).send(
                                http_io_buffer.get_response(fileno)
                            )
                            http_io_buffer.add_response(fileno, http_io_buffer.get_response(fileno)[bytes_written:])
                            if len(http_io_buffer.get_response(fileno)) == 0:  # if file sent
                                http_connection.get_connection(fileno).shutdown(socket.SHUT_RDWR)
                                poll.modify(fileno, select.POLLHUP)
                        elif event & select.POLLHUP:  # if message sent and file number in poll is hup
                            poll.unregister(fileno)  # remove file number from poll
                            http_connection.get_connection(fileno).close()  # close connection
                            http_connection.remove_connection(fileno)  # delete connection from connections dict
                    except Exception, e:
                        self.logger.error("error in __run_poll: %s", str(e))
                        continue
        finally:
            poll.unregister(server_socket.fileno())
            poll.close()
            server_socket.close()

    def __run_async_io(self):
        """
        run server use asyncore
        """
        EchoServer('0.0.0.0', self.port, self.handlers, self.settings)
        asyncore.loop()

    def run(self, io_model=None):
        """run the web server

        :param io_model: os io model
        :return: None

        """
        print "server run on port: %d" % self.port
        self.logger.info("server run on port: %d" % self.port)
        if io_model:
            if io_model == IOModel.EPOLL and hasattr(select, "epoll"):
                print "run with epoll"
                self.logger.info("run with epoll")
                self.__run_epoll()
            elif io_model == IOModel.KQUEUE and hasattr(select, "kqueue"):
                print "run with kqueue"
                self.logger.info("run with kqueue")
                self.__run_kqueue()
            elif io_model == IOModel.POLL and hasattr(select, "poll"):
                print "run with poll"
                self.logger.info("run with poll")
                self.__run_poll()
        else:
            if hasattr(select, "epoll"):
                print "run with epoll"
                self.logger.info("run with epoll")
                self.__run_epoll()
            elif hasattr(select, "kqueue"):
                print "run with kqueue"
                self.logger.info("run with kqueue")
                self.__run_kqueue()
            elif hasattr(select, "poll"):
                print "run with poll"
                self.logger.info("run with poll")
                self.__run_poll()
            else:
                print "run with asyncore"
                self.logger.info("run with asyncore")
                self.__run_async_io()
<commit_msg>modified kqueue run method<commit_after># -*-coding:utf-8-*-

import socket
import select
from karlooper.logger.logger import init_logger
from karlooper.web import IOModel
from karlooper.web.__async_core_server import EchoServer, asyncore
from karlooper.web.http_connection import HttpConnection
from karlooper.web.http_io_buffer import HttpIOBuffer
from karlooper.web.http_io_routine_pool import HttpIORoutinePool
from karlooper.http_parser.http_parser import HttpParser
from karlooper.config import get_cli_data, set_cli_data
from karlooper.config.config import SOCKET_RECEIVE_SIZE, DEFAULT_PORT, CLIENT_CONNECT_TO_SERVER_NUM

__author__ = 'karlvorndoenitz@gmail.com'

class Application(object):
    def __init__(self, handlers, settings=None, **kwargs):
        """

        :param handlers: handlers mapping, dict type
        :param settings: settings mapping, dict type
        :param kwargs: options

        """
        self.settings = settings
        self.handlers = handlers
        set_cli_data(self.settings)
        set_cli_data(kwargs)
        cli_data = get_cli_data()
        self.port = int(cli_data.get("port", DEFAULT_PORT))
        log_conf = self.settings.get("log_conf", None) if self.settings else kwargs.get("log_conf", None)
        self.logger = init_logger(config_path=log_conf)
        self.EOL1 = b'\n\n'
        self.EOL2 = b'\n\r\n'
        self.response = ""

    def listen(self, port):
        """listen port

        :param port: port that application listened
        :return: None

        """
        self.port = int(port)

    def __run_epoll(self):
        """
        run the application use epoll
        """
        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        server_socket.bind(('0.0.0.0', self.port))
        server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM)  # the number of client that connect to server
        server_socket.setblocking(0)  # set 0 not block other block
        server_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
        epoll = select.epoll()
        epoll.register(server_socket.fileno(), select.EPOLLIN)
        try:
            http_connection = HttpConnection()
            http_io_buffer = HttpIOBuffer()
            http_io_routine_pool = HttpIORoutinePool()
            events_buf = []
            while True:
                events = epoll.poll(1) + events_buf
                events_buf = []
                for fileno, event in events:
                    try:
                        if fileno == server_socket.fileno():  # if request come
                            connection, address = server_socket.accept()  # waiting income connection
                            connection.setblocking(0)  # none block
                            epoll.register(connection.fileno(), select.EPOLLIN)  # register socket read event to epoll
                            http_connection.add_connection(connection.fileno(), connection)
                            http_io_buffer.add_request(connection.fileno(), b'')
                            http_io_buffer.add_response(connection.fileno(), self.response)
                        elif event & select.EPOLLIN:  # when data in os's read buffer area
                            http_parser = http_io_routine_pool.get(file_no=fileno)
                            if http_parser:
                                data = http_parser.parse()
                                if isinstance(data, str) or isinstance(data, unicode):
                                    http_io_buffer.add_response(
                                        fileno,
                                        http_io_buffer.get_response(fileno) + data
                                    )
                                    epoll.modify(fileno, select.EPOLLOUT)  # change file number to epoll out mode
                                    http_io_routine_pool.remove(fileno)
                                else:  # if coroutine
                                    http_io_routine_pool.add(fileno, http_parser)
                                    events_buf.append((fileno, event))
                            else:
                                http_request_buffer = http_connection.get_connection(fileno).recv(SOCKET_RECEIVE_SIZE)
                                http_io_buffer.add_request(
                                    fileno,
                                    http_io_buffer.get_request(fileno) + http_request_buffer
                                )
                                if self.EOL1 in http_io_buffer.get_request(fileno) \
                                        or self.EOL2 in http_io_buffer.get_request(fileno):
                                    request_data = http_io_buffer.get_request(fileno)[:-2] \
                                        if http_io_buffer.get_request(fileno).endswith("\r\n") \
                                        else http_io_buffer.get_request(fileno)
                                    http_parser = HttpParser(
                                        request_data,
                                        self.handlers,
                                        settings=self.settings
                                    )
                                    data = http_parser.parse()
                                    if isinstance(data, str) or isinstance(data, unicode):
                                        http_io_buffer.add_response(
                                            fileno,
                                            http_io_buffer.get_response(fileno) + data
                                        )
                                        epoll.modify(fileno, select.EPOLLOUT)  # change file number to epoll out mode
                                        http_io_routine_pool.remove(fileno)
                                    else:  # if coroutine
                                        http_io_routine_pool.add(fileno, http_parser)
                                        events_buf.append((fileno, event))
                        elif event & select.EPOLLOUT:  # if out mode
                            bytes_written = http_connection.get_connection(fileno).send(
                                http_io_buffer.get_response(fileno)
                            )
                            http_io_buffer.add_response(fileno, http_io_buffer.get_response(fileno)[bytes_written:])
                            if len(http_io_buffer.get_response(fileno)) == 0:  # if file sent
                                http_connection.get_connection(fileno).shutdown(socket.SHUT_RDWR)
                                epoll.modify(fileno, select.EPOLLHUP)
                        elif event & select.EPOLLHUP:  # if message sent and file number in epoll is hup
                            epoll.unregister(fileno)  # remove file number from epoll
                            http_connection.get_connection(fileno).close()  # close connection
                            http_connection.remove_connection(fileno)  # delete connection from connections dict
                    except Exception, e:
                        self.logger.error("error in __run_epoll: %s", str(e))
                        continue
        finally:
            epoll.unregister(server_socket.fileno())
            epoll.close()
            server_socket.close()

    def __run_kqueue(self):
        """
        run server use kqueue
        """
        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_socket.bind(('0.0.0.0', self.port))
        server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM)
        kq = select.kqueue()
        http_connection = HttpConnection()
        index = 1
        events = [select.kevent(server_socket.fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD)]
        while True:
            try:
                event_list = kq.control(events, 1)
            except select.error as e:
                self.logger.error("error in __run_kqueue: %s", str(e))
                break
            if event_list:
                for each in event_list:
                    if each.ident == server_socket.fileno():
                        index += 1
                        conn, addr = server_socket.accept()
                        http_connection.add_connection(index, conn)
                        events.append(
                            select.kevent(
                                http_connection.get_connection(index).fileno(),
                                select.KQ_FILTER_READ,
                                select.KQ_EV_ADD,
                                udata=index
                            )
                        )
                    else:
                        try:
                            if each.udata >= 1 and each.flags == select.KQ_EV_ADD \
                                    and each.filter == select.KQ_FILTER_READ:
                                conn = http_connection.get_connection(each.udata)
                                request_data = conn.recv(SOCKET_RECEIVE_SIZE)
                                request_data = request_data[:-2] if request_data.endswith("\r\n") else request_data
                                data = HttpParser(
                                    request_data,
                                    handlers=self.handlers,
                                    settings=self.settings
                                ).parse()
                                conn.send(data)
                                events.remove(select.kevent(
                                    http_connection.get_connection(each.udata).fileno(),
                                    select.KQ_FILTER_READ,
                                    select.KQ_EV_ADD,
                                    udata=each.udata)
                                )
                                http_connection.remove_connection(each.udata)
                                conn.close()
                        except Exception, e:
                            self.logger.error("error in __run_kqueue event list: %s", str(e))
        server_socket.close()

    def __run_poll(self):
        """
        run server use poll, I will modify __run_poll and __run_epoll in the future
        """
        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        server_socket.bind(('0.0.0.0', self.port))
        server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM)  # the number of client that connect to server
        server_socket.setblocking(0)  # set 0 not block other block
        server_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
        poll = select.poll()
        poll.register(server_socket.fileno(), select.POLLIN)
        try:
            http_connection = HttpConnection()
            http_io_buffer = HttpIOBuffer()
            http_io_routine_pool = HttpIORoutinePool()
            events_buf = []
            while True:
                events = poll.poll(1) + events_buf
                events_buf = []
                for fileno, event in events:
                    try:
                        if fileno == server_socket.fileno():  # if request come
                            connection, address = server_socket.accept()  # waiting income connection
                            connection.setblocking(0)  # none block
                            poll.register(connection.fileno(), select.POLLIN)  # register socket read event to poll
                            http_connection.add_connection(connection.fileno(), connection)
                            http_io_buffer.add_request(connection.fileno(), b'')
                            http_io_buffer.add_response(connection.fileno(), self.response)
                        elif event & select.POLLIN:  # when data in os's read buffer area
                            http_parser = http_io_routine_pool.get(file_no=fileno)
                            if http_parser:
                                data = http_parser.parse()
                                if isinstance(data, str) or isinstance(data, unicode):
                                    http_io_buffer.add_response(
                                        fileno,
                                        http_io_buffer.get_response(fileno) + data
                                    )
                                    poll.modify(fileno, select.POLLOUT)  # change file number to epoll out mode
                                    http_io_routine_pool.remove(fileno)
                                else:  # if coroutine
                                    http_io_routine_pool.add(fileno, http_parser)
                                    events_buf.append((fileno, event))
                            else:
                                http_request_buffer = http_connection.get_connection(fileno).recv(SOCKET_RECEIVE_SIZE)
                                http_io_buffer.add_request(
                                    fileno,
                                    http_io_buffer.get_request(fileno) + http_request_buffer
                                )
                                if self.EOL1 in http_io_buffer.get_request(fileno) \
                                        or self.EOL2 in http_io_buffer.get_request(fileno):
                                    request_data = http_io_buffer.get_request(fileno)[:-2] \
                                        if http_io_buffer.get_request(fileno).endswith("\r\n") \
                                        else http_io_buffer.get_request(fileno)
                                    http_parser = HttpParser(
                                        request_data,
                                        self.handlers,
                                        settings=self.settings
                                    )
                                    data = http_parser.parse()
                                    if isinstance(data, str) or isinstance(data, unicode):
                                        http_io_buffer.add_response(
                                            fileno,
                                            http_io_buffer.get_response(fileno) + data
                                        )
                                        poll.modify(fileno, select.POLLOUT)  # change file number to epoll out mode
                                        http_io_routine_pool.remove(fileno)
                                    else:  # if coroutine
                                        http_io_routine_pool.add(fileno, http_parser)
                                        events_buf.append((fileno, event))
                        elif event & select.POLLOUT:  # if out mode
                            bytes_written = http_connection.get_connection(fileno).send(
                                http_io_buffer.get_response(fileno)
                            )
                            http_io_buffer.add_response(fileno, http_io_buffer.get_response(fileno)[bytes_written:])
                            if len(http_io_buffer.get_response(fileno)) == 0:  # if file sent
                                http_connection.get_connection(fileno).shutdown(socket.SHUT_RDWR)
                                poll.modify(fileno, select.POLLHUP)
                        elif event & select.POLLHUP:  # if message sent and file number in poll is hup
                            poll.unregister(fileno)  # remove file number from poll
                            http_connection.get_connection(fileno).close()  # close connection
                            http_connection.remove_connection(fileno)  # delete connection from connections dict
                    except Exception, e:
                        self.logger.error("error in __run_poll: %s", str(e))
                        continue
        finally:
            poll.unregister(server_socket.fileno())
            poll.close()
            server_socket.close()

    def __run_async_io(self):
        """
        run server use asyncore
        """
        EchoServer('0.0.0.0', self.port, self.handlers, self.settings)
        asyncore.loop()

    def run(self, io_model=None):
        """run the web server

        :param io_model: os io model
        :return: None

        """
        print "server run on port: %d" % self.port
        self.logger.info("server run on port: %d" % self.port)
        if io_model:
            if io_model == IOModel.EPOLL and hasattr(select, "epoll"):
                print "run with epoll"
                self.logger.info("run with epoll")
                self.__run_epoll()
            elif io_model == IOModel.KQUEUE and hasattr(select, "kqueue"):
                print "run with kqueue"
                self.logger.info("run with kqueue")
                self.__run_kqueue()
            elif io_model == IOModel.POLL and hasattr(select, "poll"):
                print "run with poll"
                self.logger.info("run with poll")
                self.__run_poll()
        else:
            if hasattr(select, "epoll"):
                print "run with epoll"
                self.logger.info("run with epoll")
                self.__run_epoll()
            elif hasattr(select, "kqueue"):
                print "run with kqueue"
                self.logger.info("run with kqueue")
                self.__run_kqueue()
            elif hasattr(select, "poll"):
                print "run with poll"
                self.logger.info("run with poll")
                self.__run_poll()
            else:
                print "run with asyncore"
                self.logger.info("run with asyncore")
                self.__run_async_io()
        print "server start failed!"
        self.logger.info("server start failed!")
<|endoftext|>


<commit_before>#!/usr/bin/env python
""" pandas_finance.

Usage:
    pandas_finance.py STOCKS START_DATE END_DATE

Arguments:
    STOCKS     comma separated stock ticker symbols, e.g. FB,TWTR
    START_DATE start date, yyyy/mm/dd
    END_DATE   end date, yyyy/mm/dd
"""
import datetime
import sqlite3

import docopt
import pandas.io.data as web
import pandas.io.sql as sql

def get_stock(stock, start, end):
    """
    Return data frame of Yahoo! Finance data for stock.

    Takes stock name and start and end dates as datetimes.
    """
    return web.DataReader(stock, 'yahoo', start, end)

def scrape_stock(stock, start, end):
    """
    Write SQLite table of stock data from Yahoo! Finance

    Take a stock name and start and end dates as datetimes.
    """
    sqlite_db.execute("drop table if exists {};".format(stock))
    frame = (get_stock(stock, start, end))
    # make Date not an index so it appears in table
    frame = frame.reset_index()
    # force Date datetime to string
    frame[['Date']] = frame[['Date']].applymap(lambda x: x.isoformat())
    sql.write_frame(frame, stock, sqlite_db)

def parse_wanted_stocks(stocks_string):
    """
    Take comma-separated string of stocks; return list of strings.
    >>> parse_wanted_stocks('TWTR,FB')
    ['TWTR', 'FB']
    >>> parse_wanted_stocks('TWTR')
    ['TWTR']
    """
    return stocks_string.split(',')

def get_arguments():
    """ Get arguments from command line. """
    return docopt.docopt(__doc__)

def main():
    """
    Save stock ticker data from Yahoo! Finance to sqlite.
    """
    arguments = get_arguments()
    global sqlite_db
    sqlite_db = sqlite3.connect("scraperwiki.sqlite")
    # arbitrary start
    start = datetime.datetime(2014, 3, 1)
    end = datetime.datetime.today()
    for ticker in parse_wanted_stocks(arguments['STOCKS']):
        scrape_stock(ticker, start, end)

if __name__ == '__main__':
    main()
<commit_msg>Get start and end dates from args.<commit_after>#!/usr/bin/env python
""" pandas_finance.

Usage:
    pandas_finance.py STOCKS START_DATE END_DATE

Arguments:
    STOCKS     comma separated stock ticker symbols, e.g. FB,TWTR
    START_DATE start date, yyyy/mm/dd
    END_DATE   end date, yyyy/mm/dd
"""
import datetime
import sqlite3

import docopt
import pandas.io.data as web
import pandas.io.sql as sql

def get_stock(stock, start, end):
    """
    Return data frame of Yahoo! Finance data for stock.

    Takes stock name and start and end dates as datetimes.
    """
    return web.DataReader(stock, 'yahoo', start, end)

def scrape_stock(stock, start, end):
    """
    Write SQLite table of stock data from Yahoo! Finance

    Take a stock name and start and end dates as datetimes.
    """
    sqlite_db.execute("drop table if exists {};".format(stock))
    frame = (get_stock(stock, start, end))
    # make Date not an index so it appears in table
    frame = frame.reset_index()
    # force Date datetime to string
    frame[['Date']] = frame[['Date']].applymap(lambda x: x.isoformat())
    sql.write_frame(frame, stock, sqlite_db)

def parse_wanted_stocks(stocks_string):
    """
    Take comma-separated string of stocks; return list of strings.
    >>> parse_wanted_stocks('TWTR,FB')
    ['TWTR', 'FB']
    >>> parse_wanted_stocks('TWTR')
    ['TWTR']
    """
    return stocks_string.split(',')

def get_arguments():
    """ Get arguments from command line. """
    return docopt.docopt(__doc__)

def convert_date_string_to_datetime(date_string):
    """ Take date string as YYYY/MM/DD and return as datetime object.
    >>> convert_date_string_to_datetime('2014/03/02')
    datetime.datetime(2014, 3, 2, 0, 0)
    """
    return datetime.datetime.strptime(date_string, '%Y/%m/%d')

def main():
    """
    Save stock ticker data from Yahoo! Finance to sqlite.
    """
    arguments = get_arguments()
    start = convert_date_string_to_datetime(arguments['START_DATE'])
    end = convert_date_string_to_datetime(arguments['END_DATE'])
    global sqlite_db
    sqlite_db = sqlite3.connect("scraperwiki.sqlite")
    for ticker in parse_wanted_stocks(arguments['STOCKS']):
        scrape_stock(ticker, start, end)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import multiprocessing, os, pickle, select, socket, sys, time
from common.safeprint import safeprint
from common.bounty import *

if os.name != "nt":
    import fcntl
    import struct

    def get_interface_ip(ifname):
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        return socket.inet_ntoa(fcntl.ioctl(s.fileno(), 0x8915, struct.pack('256s',
                                ifname[:15]))[20:24])

def get_lan_ip():
    if sys.version_info[0] <= 3:
        ip = socket.gethostbyname(socket.gethostname())
        if ip.startswith("127.") and os.name != "nt":
            interfaces = ["eth0","eth1","eth2","wlan0","wlan1","wifi0","ath0","ath1","ppp0",]
            for ifname in interfaces:
                try:
                    ip = get_interface_ip(ifname)
                    break
                except IOError:
                    pass
        return ip
    else:
        return "127.0.0.1"

seedlist = ["127.0.0.1:44565", "localhost:44565", "10.132.80.128:44565"]
peerlist = [get_lan_ip() + ":44565"]
remove   = []
bounties = []

#constants
peers_file      = "data" + os.sep + "peerlist.pickle"
close_signal    = "Close Signal......."
peer_request    = "Requesting Peers..."
incoming_bounty = "Incoming Bounty...."
valid_signal    = "Bounty was valid..."
invalid_signal  = "Bounty was invalid."

Alive = True

def getFromFile():
  if os.path.exists(peers_file):
    peerlist = pickle.load(open(peers_file,"rb"))

def saveToFile():
  if not os.path.exists(peers_file.split(os.sep)[0]):
    os.mkdir(peers_file.split(os.sep)[0])
  pickle.dump(peerlist,open(peers_file,"wb"))

def getFromSeeds():
  for seed in seedlist:
    safeprint(seed)
    peerlist.extend(requestPeerlist(seed))
    time.sleep(1)

def requestPeerlist(address):
  con = socket.socket()
  con.settimeout(5)
  safeprint(address)
  try:
    safeprint(address.split(":")[0] + ":" + address.split(":")[1])
    con.connect((address.split(":")[0],int(address.split(":")[1])))
    con.send(peer_request)
    connected = True
    s = ""
    while connected:
      a = con.recv(64)
      safeprint(a)
      if not a == close_signal:
        s += a
      else:
        con.close()
        connected = False
    return pickle.loads(s)
  except Exception as e:
    safeprint("Failed:" + str(type(e)))
    safeprint(e)
    remove.extend([address])
    return []

def sendPeerlist(address):
  safeprint("currently unsupported")
  s = pickle.dumps(peerlist)
  #send list

def initializePeerConnections():
  getFromFile()
  safeprint("peers fetched from file")
  getFromSeeds()
  safeprint("peers fetched from seedlist")
  trimPeers()
  if len(peerlist) < 12:
    safeprint(len(peerlist))
    newlist = []
    for peer in peerlist:
      newlist.extend(requestPeerlist(peer))
    peerlist.extend(newlist)
  trimPeers()
  safeprint("peer network extended")
  saveToFile()
  safeprint("peer network saved to file")

def trimPeers():
  temp = list(set(peerlist))
  for peer in remove:
    try:
      del temp[temp.index(peer)]
    except:
      continue
  del remove[:]
  del peerlist[:]
  peerlist.extend(temp)

def listen(port, ID):
  server = socket.socket()
  server.bind(("0.0.0.0",port))
  server.listen(10)
  while True:
    safeprint("listening on " + str(get_lan_ip()) + ":" + str(port))
    try:
      a, addr = server.accept()
      safeprint("connection accepted")
      b = a.recv(len(peer_request))
      safeprint("Received: " + str(b))
      if b == peer_request:
        a.send(pickle.dumps(peerlist + [get_lan_ip()+":"+str(port)]))
        time.sleep(0.1)
      elif b == incoming_bounty:
        connected = True
        s = ""
        while connected:
          c = a.recv(len(close_signal))
          safeprint(c)
          if not c == close_signal:
            s += c
          else:
            con.close()
            connected = False
          if (verify(s)):
            Bounty.bountyList.append(bount)
            a.send(valid_signal)
          else:
            a.send(invalid_signal)
      a.send(close_signal)
      time.sleep(1)
      a.close()
      safeprint("connection closed")
    except Exception as e:
      safeprint("Failed: " + str(type(e)))
      safeprint(e)

class listener(multiprocessing.Process):
  def __init__(self, threadID, port):
    multiprocessing.Process.__init__(self)
    self.threadID = threadID
    self.port = port
  def run(self):
    safeprint("listener started")
    listen(self.port,self.threadID)
<commit_msg>Python3 LAN ip fix (requires trust)<commit_after>import multiprocessing, os, pickle, select, socket, sys, time
from common.safeprint import safeprint
from common.bounty import *

if os.name != "nt":
    import fcntl
    import struct

    def get_interface_ip(ifname):
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        return socket.inet_ntoa(fcntl.ioctl(s.fileno(), 0x8915, struct.pack('256s',
                                ifname[:15]))[20:24])

def get_lan_ip():
    if sys.version_info[0] <= 3:
        ip = socket.gethostbyname(socket.gethostname())
        if ip.startswith("127.") and os.name != "nt":
            interfaces = ["eth0","eth1","eth2","wlan0","wlan1","wifi0","ath0","ath1","ppp0",]
            for ifname in interfaces:
                try:
                    ip = get_interface_ip(ifname)
                    break
                except IOError:
                    pass
        return ip
    else:
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.connect(('8.8.8.8', 0))
        return s.getsockname()[0]

seedlist = ["127.0.0.1:44565", "localhost:44565", "10.132.80.128:44565"]
peerlist = [get_lan_ip() + ":44565"]
remove   = []
bounties = []

#constants
peers_file      = "data" + os.sep + "peerlist.pickle"
close_signal    = "Close Signal......."
peer_request    = "Requesting Peers..."
incoming_bounty = "Incoming Bounty...."
valid_signal    = "Bounty was valid..."
invalid_signal  = "Bounty was invalid."

Alive = True

def getFromFile():
  if os.path.exists(peers_file):
    peerlist = pickle.load(open(peers_file,"rb"))

def saveToFile():
  if not os.path.exists(peers_file.split(os.sep)[0]):
    os.mkdir(peers_file.split(os.sep)[0])
  pickle.dump(peerlist,open(peers_file,"wb"))

def getFromSeeds():
  for seed in seedlist:
    safeprint(seed)
    peerlist.extend(requestPeerlist(seed))
    time.sleep(1)

def requestPeerlist(address):
  con = socket.socket()
  con.settimeout(5)
  safeprint(address)
  try:
    safeprint(address.split(":")[0] + ":" + address.split(":")[1])
    con.connect((address.split(":")[0],int(address.split(":")[1])))
    con.send(peer_request)
    connected = True
    s = ""
    while connected:
      a = con.recv(64)
      safeprint(a)
      if not a == close_signal:
        s += a
      else:
        con.close()
        connected = False
    return pickle.loads(s)
  except Exception as e:
    safeprint("Failed:" + str(type(e)))
    safeprint(e)
    remove.extend([address])
    return []

def sendPeerlist(address):
  safeprint("currently unsupported")
  s = pickle.dumps(peerlist)
  #send list

def initializePeerConnections():
  getFromFile()
  safeprint("peers fetched from file")
  getFromSeeds()
  safeprint("peers fetched from seedlist")
  trimPeers()
  if len(peerlist) < 12:
    safeprint(len(peerlist))
    newlist = []
    for peer in peerlist:
      newlist.extend(requestPeerlist(peer))
    peerlist.extend(newlist)
  trimPeers()
  safeprint("peer network extended")
  saveToFile()
  safeprint("peer network saved to file")

def trimPeers():
  temp = list(set(peerlist))
  for peer in remove:
    try:
      del temp[temp.index(peer)]
    except:
      continue
  del remove[:]
  del peerlist[:]
  peerlist.extend(temp)

def listen(port, ID):
  server = socket.socket()
  server.bind(("0.0.0.0",port))
  server.listen(10)
  while True:
    safeprint("listening on " + str(get_lan_ip()) + ":" + str(port))
    try:
      a, addr = server.accept()
      safeprint("connection accepted")
      b = a.recv(len(peer_request))
      safeprint("Received: " + str(b))
      if b == peer_request:
        a.send(pickle.dumps(peerlist + [get_lan_ip()+":"+str(port)]))
        time.sleep(0.1)
      elif b == incoming_bounty:
        connected = True
        s = ""
        while connected:
          c = a.recv(len(close_signal))
          safeprint(c)
          if not c == close_signal:
            s += c
          else:
            con.close()
            connected = False
          if (verify(s)):
            Bounty.bountyList.append(bount)
            a.send(valid_signal)
          else:
            a.send(invalid_signal)
      a.send(close_signal)
      time.sleep(1)
      a.close()
      safeprint("connection closed")
    except Exception as e:
      safeprint("Failed: " + str(type(e)))
      safeprint(e)

class listener(multiprocessing.Process):
  def __init__(self, threadID, port):
    multiprocessing.Process.__init__(self)
    self.threadID = threadID
    self.port = port
  def run(self):
    safeprint("listener started")
    listen(self.port,self.threadID)
<|endoftext|>


<commit_before>#!/usr/bin/python2.4
# Authors: Jared Kuolt <me@superjared.com>, Mark Roach <mrroach@google.com>

"""Connect to and interact with a REST server and its objects."""

import new
import re
import sys
import urllib
import urlparse
from string import Template
from pyactiveresource import connection
from pyactiveresource import util

VALID_NAME = re.compile('[a-z_]\w*')  # Valid python attribute names

class Error(Exception):
    """A general error derived from Exception."""
    pass

class ResourceMeta(type):
    """A metaclass to handle singular/plural attributes."""

    def __new__(mcs, name, bases, new_attrs):
        """Create a new class.

        Args:
            mcs: The metaclass.
            name: The name of the class.
            bases: List of base classes from which mcs inherits.
            new_attrs: The class attribute dictionary.
        """
        if '_singular' not in new_attrs or not new_attrs['_singular']:
            # Convert CamelCase to lower_underscore
            singular = re.sub(r'\B((?<=[a-z])[A-Z]|[A-Z](?=[a-z]))',
                              r'_\1', name).lower()
            new_attrs['_singular'] = singular

        if '_plural' not in new_attrs or not new_attrs['_plural']:
            new_attrs['_plural'] = util.pluralize(new_attrs['_singular'])

        klass = type.__new__(mcs, name, bases, new_attrs)
        return klass

class ActiveResource(object):
    """Represents an activeresource object."""

    __metaclass__ = ResourceMeta

    _site = ''
    _user = ''
    _password = ''
    _connection_obj = None
    _headers = None
    _timeout = 5

    def __init__(self, attributes, prefix_options=None):
        """Initialize a new ActiveResource object.

        Args:
            attributes: A dictionary of attributes which represent this object.
            prefix_options: A dict of prefixes to add to the request for
                            nested URLs.
        """
        self.attributes = {}
        if prefix_options:
            self._prefix_options = prefix_options
        else:
            self._prefix_options = {}
        self._update(attributes)
        self._initialized = True

    # Public class methods which act as factory functions
    @classmethod
    def find(cls, id_=None, from_=None, **kwargs):
        """Core method for finding resources.

        Args:
            id_: A specific resource to retrieve.
            from_: The path that resources will be fetched from.
            kwargs: any keyword arguments for query.

        Returns:
            An ActiveResource object.
        Raises:
            connection.Error: On any communications errors.
            Error: On any other errors.
        """
        if id_:
            return cls._find_single(id_, **kwargs)

        return cls._find_every(from_=from_, **kwargs)

    @classmethod
    def find_one(cls, from_, **kwargs):
        """Get a single resource from a specific URL.

        Args:
            from_: The path that resources will be fetched from.
            kwargs: Any keyword arguments for query.
        Returns:
            An ActiveResource object.
        Raises:
            connection.Error: On any communications errors.
            Error: On any other errors.
        """
        return cls._find_one(from_, kwargs)

    @classmethod
    def exists(cls, id_, **kwargs):
        """Check whether a resource exists.

        Args:
            id_: The id or other key which specifies a unique object.
            kwargs: Any keyword arguments for query.
        Returns:
            True if the resource is found, False otherwise.
        """
        prefix_options, query_options = cls._split_options(kwargs)
        path = cls._element_path(id_, prefix_options, query_options)
        try:
            _ = cls._connection().head(path, cls._headers)
            return True
        except connection.Error:
            return False

    # Non-public class methods to support the above
    @classmethod
    def _split_options(cls, options):
        """Split prefix options and query options.

        Args:
            options: A dictionary of prefix and/or query options.
        Returns:
            A tuple containing (prefix_options, query_options)
        """
        #TODO(mrroach): figure out prefix_options
        prefix_options = {}
        query_options = {}
        for key, value in options.items():
            if key in cls._prefix_parameters():
                prefix_options[key] = value
            else:
                query_options[key] = value
        return [prefix_options, query_options]

    @classmethod
    def _find_single(cls, id_, **kwargs):
        """Get a single object from the default URL.

        Args:
            id_: The id or other key which specifies a unique object.
            kwargs: Any keyword arguments for the query.
        Returns:
            An ActiveResource object.
        Raises:
            ConnectionError: On any error condition.
        """
        prefix_options, query_options = cls._split_options(kwargs)
        path = cls._element_path(id_, prefix_options, query_options)
        return cls._build_object(cls._connection().get(path, cls._headers))

    @classmethod
    def _find_one(cls, from_, query_options):
        """Find a single resource from a one-off URL.

        Args:
            from_: The path from which to retrieve the resource.
            query_options: Any keyword arguments for the query.
        Returns:
            An ActiveResource object.
        Raises:
            connection.ConnectionError: On any error condition.
        """
        #TODO(mrroach): allow from_ to be a string-generating function
        path = from_ + cls._query_string(query_options)
        return cls._build_object(cls._connection().get(path, cls._headers))

    @classmethod
    def _find_every(cls, from_=None, **kwargs):
        """Get all resources.

        Args:
            from_: (optional) The path from which to retrieve the resource.
            kwargs: Any keyword arguments for the query.
        Returns:
            A list of resources.
        """
        if from_:
            path = from_ + cls._query_string(kwargs)
            prefix_options = None
        else:
            prefix_options, query_options = cls._split_options(kwargs)
            path = cls._collection_path(prefix_options, query_options)
        return cls._build_list(cls._connection().get(path, cls._headers),
                               prefix_options)

    @classmethod
    def _build_object(cls, xml, prefix_options=None):
        """Create an object or objects for the given xml string.

        Args:
            xml: An xml string containing the object definition.
        Returns:
            An ActiveResource object.
        """
        element_type, attributes = util.xml_to_dict(
                xml, saveroot=True).items()[0]
        return cls(attributes, prefix_options=prefix_options)

    @classmethod
    def _build_list(cls, xml, prefix_options=None):
        """Create a list of objects for the given xml string.

        Args:
            xml: An xml string containing multiple object definitions.
        Returns:
            A list of ActiveResource objects.
        """
        resources = []
        collection_type, collection = util.xml_to_dict(
                xml, saveroot=True).items()[0]
        element_type, elements = collection.items()[0]

        for element in elements:
            resources.append(cls(element, prefix_options))
        return elements

    @classmethod
    def _query_string(cls, query_options):
        """Return a query string for the given options.

        Args:
            query_options: A dictionary of query keys/values.
        Returns:
            A string containing the encoded query.
        """
        if query_options:
            return '?' + urllib.urlencode(query_options)
        else:
            return ''

    @classmethod
    def _element_path(cls, id_, prefix_options=None, query_options=None):
        """Get the element path for the given id.

        Examples:
            Comment.element_path(1, {'post_id': 5}) -> /posts/5/act
        Args:
            id_: The id of the object to retrieve.
            prefix_options: A dict of prefixes to add to the request for
                            nested URLs.
            query_options: A dict of items to add to the query string for
                           the request.
        Returns:
            The path (relative to site) to the element formatted with the query.
        """
        return '%(prefix)s/%(plural)s/%(id)s.%(format)s%(query)s' % {
                'prefix': cls._prefix(prefix_options),
                'plural': cls._plural,
                'id': id_,
                'format': 'xml',
                'query': cls._query_string(query_options)}

    @classmethod
    def _collection_path(cls, prefix_options=None, query_options=None):
        """Get the collection path for this object type.

        Examples:
            Comment.collection_path() -> /comments.xml
            Comment.collection_path(query_options={'active': 1})
                -> /comments.xml?active=1
            Comment.collection_path({'posts': 5})
                -> /posts/5/comments.xml
        Args:
            prefix_options: A dict of prefixes to add to the request for
                            nested URLs
            query_options: A dict of items to add to the query string for
                           the request.
        Returns:
            The path (relative to site) to this type of collection.
        """
        return '%(prefix)s/%(plural)s.%(format)s%(query)s' % {
                'prefix': cls._prefix(prefix_options),
                'plural': cls._plural,
                'format': 'xml',
                'query': cls._query_string(query_options)}

    @classmethod
    def _prefix_parameters(cls):
        """Return a list of the parameters used in the site prefix.

        e.g. /objects/$object_id would yield ['object_id']
             /objects/${object_id}/people/$person_id/ would yield
             ['object_id', 'person_id']
        Args:
            None
        Returns:
            A set of named parameters.
        """
        path = urlparse.urlsplit(cls._site)[2]
        template = Template(path)
        keys = set()
        for match in template.pattern.finditer(path):
            for match_type in 'braced', 'named':
                if match.groupdict()[match_type]:
                    keys.add(match.groupdict()[match_type])
        return keys

    @classmethod
    def _prefix(cls, options=None):
        """Return the prefix for this object type.

        Args:
            options: A dictionary containing additional prefixes to prepend.
        Returns:
            A string containing the path to this element.
        """
        path = re.sub('/$', '', urlparse.urlsplit(cls._site)[2])
        template = Template(path)
        keys = cls._prefix_parameters()
        options = dict([(k, options.get(k, '')) for k in keys])
        return template.safe_substitute(options)

    @classmethod
    def _connection(cls):
        """Return a connection object which handles HTTP requests."""
        if not cls._connection_obj:
            cls._connection_obj = connection.Connection(
                    cls._site, cls._user, cls._password, cls._timeout)
        return cls._connection_obj

    @classmethod
    def _scrub_name(cls, name):
        """Remove invalid characters from attribute names.

        Args:
            name: the string to scrub
        Returns:
            The part of the string that is a valid name, or None if unscrubbable
        """
        name = name.lower().replace('-', '_')
        match = VALID_NAME.search(name)
        if match:
            return match.group(0)
        return None

    def to_dict(self):
        """Convert the object to a dictionary."""
        values = {}
        for key, value in self.attributes.iteritems():
            if isinstance(value, list):
                values[key] = [i.to_dict() for i in value]
            elif isinstance(value, ActiveResource):
                values[key] = value.to_dict()
            else:
                values[key] = value
        return values

    # Public instance methods
    def to_xml(self, root=None, header=True, pretty=False):
        """Convert the object to an xml string.

        Args:
            root: The name of the root element for xml output.
            header: Whether to include the xml header.
        Returns:
            An xml string.
        """
        if not root:
            root = self._singular
        return util.to_xml(self.to_dict(), root=root,
                                header=header, pretty=pretty)

    def save(self):
        """Save the object to the server.

        Args:
            None
        Returns:
            None.
        Raises:
            connection.Error: On any communications problems.
        """
        if self.id:
            response = self._connection().put(
                    self._element_path(self.id, self._prefix_options),
                    self._headers,
                    data=self.to_xml())
        else:
            response = self._connection().post(
                    self._collection_path(self._prefix_options),
                    self._headers,
                    data=self.to_xml())
        try:
            attributes = util.xml_to_dict(response)
        except Error:
            return
        self._update(attributes)
        return response

    def destroy(self):
        """Deletes the resource from the remote service.

        Args:
            None
        Returns:
            None
        """
        self._connection().delete(
                self._element_path(self.id, self._prefix_options),
                self.__class__._headers)

    def __getattr__(self, name):
        """Retrieve the requested attribute if it exists.

        Args:
            name: The attribute name.
        Returns:
            The attribute's value.
        Raises:
            AttributeError: if no such attribute exists.
        """
        #TODO(mrroach): Use descriptors instead of __getattr__
        if name == 'id':
            # id should always be getattrable
            return self.attributes.get('id')
        if name in self.attributes:
            return self.attributes[name]
        raise AttributeError(name)

    def __setattr__(self, name, value):
        """Set the named attributes.

        Args:
            name: The attribute name.
            value: The attribute's value.
        Returns:
            None
        """
        if '_initialized' in self.__dict__:
            if name in self.__dict__:
                # Update a normal attribute
                self.__dict__[name] = value
            elif name in self.__class__.__dict__:
                # Update a class attribute
                self.__class__.__dict__[name] = value
            else:
                # Add/update an attribute
                self.attributes[name] = value
        self.__dict__[name] = value

    def __repr__(self):
        return '%s(%s)' % (self._singular, self.id)

    def __cmp__(self, other):
        if isinstance(other, self.__class__):
            return cmp(self.id, other.id)
        else:
            return cmp(self.id, other)

    def _update(self, attributes):
        """Update the object with the given attributes.

        Args:
            attributes: A dictionary of attributes.
        Returns:
            None
        """
        self.attributes = {}
        # Add all the tags in the element as attributes
        for key, value in attributes.items():
            if isinstance(value, dict):
                klass = self._find_class_for(key)
                attr = klass(value)
            elif isinstance(value, list):
                klass = self._find_class_for(util.singularize(key))
                attr = [klass(child) for child in value]
            else:
                attr = value
            # Store the actual value in the attributes dictionary
            self.attributes[key] = attr
            attr_name = self._scrub_name(key)
            if attr_name != key:
                # key is not a valid attribute name,
                # access via self.attributes[key]
                continue

    def _find_class_for(self, element_name):
        """Look in the parent modules for classes matching the element name.

        Args:
            element_name: The name of the element type.
        Returns:
            A Resource class.
        """
        module_path = self.__module__.split('.')
        class_name = util.camelize(element_name)
        for depth in range(len(module_path), 0, -1):
            try:
                __import__('.'.join(module_path[:depth]))
                module = sys.modules['.'.join(module_path[:depth])]
            except ImportError:
                continue
            try:
                klass = getattr(module, class_name)
                return klass
            except AttributeError:
                try:
                    __import__('.'.join([module.__name__, element_name]))
                    submodule = sys.modules['.'.join([module.__name__,
                                                      element_name])]
                except ImportError:
                    continue
                try:
                    klass = getattr(submodule, class_name)
                    return klass
                except AttributeError:
                    continue

        # If we made it this far, no such class was found
        return new.classobj(class_name, (self.__class__,),
                            {'__module__': self.__module__})
<commit_msg>Make __setattr__ use object.__setattr__ where possible Allow passing one/both of class_name/element_name to find_class_for<commit_after>#!/usr/bin/python2.4
# Authors: Jared Kuolt <me@superjared.com>, Mark Roach <mrroach@google.com>

"""Connect to and interact with a REST server and its objects."""

import new
import re
import sys
import urllib
import urlparse
from string import Template
from pyactiveresource import connection
from pyactiveresource import util

VALID_NAME = re.compile('[a-z_]\w*')  # Valid python attribute names

class Error(Exception):
    """A general error derived from Exception."""
    pass

class ResourceMeta(type):
    """A metaclass to handle singular/plural attributes."""

    def __new__(mcs, name, bases, new_attrs):
        """Create a new class.

        Args:
            mcs: The metaclass.
            name: The name of the class.
            bases: List of base classes from which mcs inherits.
            new_attrs: The class attribute dictionary.
        """
        if '_singular' not in new_attrs or not new_attrs['_singular']:
            # Convert CamelCase to lower_underscore
            singular = re.sub(r'\B((?<=[a-z])[A-Z]|[A-Z](?=[a-z]))',
                              r'_\1', name).lower()
            new_attrs['_singular'] = singular

        if '_plural' not in new_attrs or not new_attrs['_plural']:
            new_attrs['_plural'] = util.pluralize(new_attrs['_singular'])

        klass = type.__new__(mcs, name, bases, new_attrs)
        return klass

class ActiveResource(object):
    """Represents an activeresource object."""

    __metaclass__ = ResourceMeta

    _site = ''
    _user = ''
    _password = ''
    _connection_obj = None
    _headers = None
    _timeout = 5

    def __init__(self, attributes, prefix_options=None):
        """Initialize a new ActiveResource object.

        Args:
            attributes: A dictionary of attributes which represent this object.
            prefix_options: A dict of prefixes to add to the request for
                            nested URLs.
        """
        self.attributes = {}
        if prefix_options:
            self._prefix_options = prefix_options
        else:
            self._prefix_options = {}
        self._update(attributes)
        self._initialized = True

    # Public class methods which act as factory functions
    @classmethod
    def find(cls, id_=None, from_=None, **kwargs):
        """Core method for finding resources.

        Args:
            id_: A specific resource to retrieve.
            from_: The path that resources will be fetched from.
            kwargs: any keyword arguments for query.

        Returns:
            An ActiveResource object.
        Raises:
            connection.Error: On any communications errors.
            Error: On any other errors.
        """
        if id_:
            return cls._find_single(id_, **kwargs)

        return cls._find_every(from_=from_, **kwargs)

    @classmethod
    def find_one(cls, from_, **kwargs):
        """Get a single resource from a specific URL.

        Args:
            from_: The path that resources will be fetched from.
            kwargs: Any keyword arguments for query.
        Returns:
            An ActiveResource object.
        Raises:
            connection.Error: On any communications errors.
            Error: On any other errors.
        """
        return cls._find_one(from_, kwargs)

    @classmethod
    def exists(cls, id_, **kwargs):
        """Check whether a resource exists.

        Args:
            id_: The id or other key which specifies a unique object.
            kwargs: Any keyword arguments for query.
        Returns:
            True if the resource is found, False otherwise.
        """
        prefix_options, query_options = cls._split_options(kwargs)
        path = cls._element_path(id_, prefix_options, query_options)
        try:
            _ = cls._connection().head(path, cls._headers)
            return True
        except connection.Error:
            return False

    # Non-public class methods to support the above
    @classmethod
    def _split_options(cls, options):
        """Split prefix options and query options.

        Args:
            options: A dictionary of prefix and/or query options.
        Returns:
            A tuple containing (prefix_options, query_options)
        """
        #TODO(mrroach): figure out prefix_options
        prefix_options = {}
        query_options = {}
        for key, value in options.items():
            if key in cls._prefix_parameters():
                prefix_options[key] = value
            else:
                query_options[key] = value
        return [prefix_options, query_options]

    @classmethod
    def _find_single(cls, id_, **kwargs):
        """Get a single object from the default URL.

        Args:
            id_: The id or other key which specifies a unique object.
            kwargs: Any keyword arguments for the query.
        Returns:
            An ActiveResource object.
        Raises:
            ConnectionError: On any error condition.
        """
        prefix_options, query_options = cls._split_options(kwargs)
        path = cls._element_path(id_, prefix_options, query_options)
        return cls._build_object(cls._connection().get(path, cls._headers))

    @classmethod
    def _find_one(cls, from_, query_options):
        """Find a single resource from a one-off URL.

        Args:
            from_: The path from which to retrieve the resource.
            query_options: Any keyword arguments for the query.
        Returns:
            An ActiveResource object.
        Raises:
            connection.ConnectionError: On any error condition.
        """
        #TODO(mrroach): allow from_ to be a string-generating function
        path = from_ + cls._query_string(query_options)
        return cls._build_object(cls._connection().get(path, cls._headers))

    @classmethod
    def _find_every(cls, from_=None, **kwargs):
        """Get all resources.

        Args:
            from_: (optional) The path from which to retrieve the resource.
            kwargs: Any keyword arguments for the query.
        Returns:
            A list of resources.
        """
        if from_:
            path = from_ + cls._query_string(kwargs)
            prefix_options = None
        else:
            prefix_options, query_options = cls._split_options(kwargs)
            path = cls._collection_path(prefix_options, query_options)
        return cls._build_list(cls._connection().get(path, cls._headers),
                               prefix_options)

    @classmethod
    def _build_object(cls, xml, prefix_options=None):
        """Create an object or objects for the given xml string.

        Args:
            xml: An xml string containing the object definition.
            prefix_options: A dict of prefixes to add to the request for
                            nested URLs.
        Returns:
            An ActiveResource object.
        """
        element_type, attributes = util.xml_to_dict(
                xml, saveroot=True).items()[0]
        return cls(attributes, prefix_options)

    @classmethod
    def _build_list(cls, xml, prefix_options=None):
        """Create a list of objects for the given xml string.

        Args:
            xml: An xml string containing multiple object definitions.
            prefix_options: A dict of prefixes to add to the request for
                            nested URLs.
        Returns:
            A list of ActiveResource objects.
        """
        resources = []
        collection_type, collection = util.xml_to_dict(
                xml, saveroot=True).items()[0]
        element_type, elements = collection.items()[0]

        for element in elements:
            resources.append(cls(element, prefix_options))
        return elements

    @classmethod
    def _query_string(cls, query_options):
        """Return a query string for the given options.

        Args:
            query_options: A dictionary of query keys/values.
        Returns:
            A string containing the encoded query.
        """
        if query_options:
            return '?' + urllib.urlencode(query_options)
        else:
            return ''

    @classmethod
    def _element_path(cls, id_, prefix_options=None, query_options=None):
        """Get the element path for the given id.

        Examples:
            Comment.element_path(1, {'post_id': 5}) -> /posts/5/act
        Args:
            id_: The id of the object to retrieve.
            prefix_options: A dict of prefixes to add to the request for
                            nested URLs.
            query_options: A dict of items to add to the query string for
                           the request.
        Returns:
            The path (relative to site) to the element formatted with the query.
        """
        return '%(prefix)s/%(plural)s/%(id)s.%(format)s%(query)s' % {
                'prefix': cls._prefix(prefix_options),
                'plural': cls._plural,
                'id': id_,
                'format': 'xml',
                'query': cls._query_string(query_options)}

    @classmethod
    def _collection_path(cls, prefix_options=None, query_options=None):
        """Get the collection path for this object type.

        Examples:
            Comment.collection_path() -> /comments.xml
            Comment.collection_path(query_options={'active': 1})
                -> /comments.xml?active=1
            Comment.collection_path({'posts': 5})
                -> /posts/5/comments.xml
        Args:
            prefix_options: A dict of prefixes to add to the request for
                            nested URLs
            query_options: A dict of items to add to the query string for
                           the request.
        Returns:
            The path (relative to site) to this type of collection.
        """
        return '%(prefix)s/%(plural)s.%(format)s%(query)s' % {
                'prefix': cls._prefix(prefix_options),
                'plural': cls._plural,
                'format': 'xml',
                'query': cls._query_string(query_options)}

    @classmethod
    def _prefix_parameters(cls):
        """Return a list of the parameters used in the site prefix.

        e.g. /objects/$object_id would yield ['object_id']
             /objects/${object_id}/people/$person_id/ would yield
             ['object_id', 'person_id']
        Args:
            None
        Returns:
            A set of named parameters.
        """
        path = urlparse.urlsplit(cls._site)[2]
        template = Template(path)
        keys = set()
        for match in template.pattern.finditer(path):
            for match_type in 'braced', 'named':
                if match.groupdict()[match_type]:
                    keys.add(match.groupdict()[match_type])
        return keys

    @classmethod
    def _prefix(cls, options=None):
        """Return the prefix for this object type.

        Args:
            options: A dictionary containing additional prefixes to prepend.
        Returns:
            A string containing the path to this element.
        """
        path = re.sub('/$', '', urlparse.urlsplit(cls._site)[2])
        template = Template(path)
        keys = cls._prefix_parameters()
        options = dict([(k, options.get(k, '')) for k in keys])
        return template.safe_substitute(options)

    @classmethod
    def _connection(cls):
        """Return a connection object which handles HTTP requests."""
        if not cls._connection_obj:
            cls._connection_obj = connection.Connection(
                    cls._site, cls._user, cls._password, cls._timeout)
        return cls._connection_obj

    @classmethod
    def _scrub_name(cls, name):
        """Remove invalid characters from attribute names.

        Args:
            name: the string to scrub
        Returns:
            The part of the string that is a valid name, or None if unscrubbable
        """
        name = name.lower().replace('-', '_')
        match = VALID_NAME.search(name)
        if match:
            return match.group(0)
        return None

    def to_dict(self):
        """Convert the object to a dictionary."""
        values = {}
        for key, value in self.attributes.iteritems():
            if isinstance(value, list):
                values[key] = [i.to_dict() for i in value]
            elif isinstance(value, ActiveResource):
                values[key] = value.to_dict()
            else:
                values[key] = value
        return values

    # Public instance methods
    def to_xml(self, root=None, header=True, pretty=False):
        """Convert the object to an xml string.

        Args:
            root: The name of the root element for xml output.
            header: Whether to include the xml header.
        Returns:
            An xml string.
        """
        if not root:
            root = self._singular
        return util.to_xml(self.to_dict(), root=root,
                                header=header, pretty=pretty)

    def save(self):
        """Save the object to the server.

        Args:
            None
        Returns:
            None.
        Raises:
            connection.Error: On any communications problems.
        """
        if self.id:
            response = self._connection().put(
                    self._element_path(self.id, self._prefix_options),
                    self._headers,
                    data=self.to_xml())
        else:
            response = self._connection().post(
                    self._collection_path(self._prefix_options),
                    self._headers,
                    data=self.to_xml())
        try:
            attributes = util.xml_to_dict(response)
        except Error:
            return
        self._update(attributes)
        return response

    def destroy(self):
        """Deletes the resource from the remote service.

        Args:
            None
        Returns:
            None
        """
        self._connection().delete(
                self._element_path(self.id, self._prefix_options),
                self.__class__._headers)

    def __getattr__(self, name):
        """Retrieve the requested attribute if it exists.

        Args:
            name: The attribute name.
        Returns:
            The attribute's value.
        Raises:
            AttributeError: if no such attribute exists.
        """
        #TODO(mrroach): Use descriptors instead of __getattr__
        if name == 'id':
            # id should always be getattrable
            return self.attributes.get('id')
        if name in self.attributes:
            return self.attributes[name]
        raise AttributeError(name)

    def __setattr__(self, name, value):
        """Set the named attributes.

        Args:
            name: The attribute name.
            value: The attribute's value.
        Returns:
            None
        """
        if '_initialized' in self.__dict__:
            if name in self.__dict__ or name in self.__class__.__dict__:
                # Update a normal attribute
                object.__setattr__(self, name, value)
            else:
                # Add/update an attribute
                self.attributes[name] = value
        object.__setattr__(self, name, value)

    def __repr__(self):
        return '%s(%s)' % (self._singular, self.id)

    def __cmp__(self, other):
        if isinstance(other, self.__class__):
            return cmp(self.id, other.id)
        else:
            return cmp(self.id, other)

    def _update(self, attributes):
        """Update the object with the given attributes.

        Args:
            attributes: A dictionary of attributes.
        Returns:
            None
        """
        self.attributes = {}
        # Add all the tags in the element as attributes
        for key, value in attributes.items():
            if isinstance(value, dict):
                klass = self._find_class_for(key)
                attr = klass(value)
            elif isinstance(value, list):
                klass = self._find_class_for(util.singularize(key))
                attr = [klass(child) for child in value]
            else:
                attr = value
            # Store the actual value in the attributes dictionary
            self.attributes[key] = attr
            attr_name = self._scrub_name(key)
            if attr_name != key:
                # key is not a valid attribute name,
                # access via self.attributes[key]
                continue

    def _find_class_for(self, element_name=None, class_name=None):
        """Look in the parent modules for classes matching the element name.

        One, or both of element/class name must be specified.

        Args:
            element_name: The name of the element type.
            class_name: The class name of the element type.
        Returns:
            A Resource class.
        """
        if not element_name and not class_name:
            raise Error('One of element_name,class_name must be specified.')
        elif not element_name:
            element_name = util.underscore(class_name)
        elif not class_name:
            class_name = util.camelize(element_name)

        module_path = self.__module__.split('.')
        for depth in range(len(module_path), 0, -1):
            try:
                __import__('.'.join(module_path[:depth]))
                module = sys.modules['.'.join(module_path[:depth])]
            except ImportError:
                continue
            try:
                klass = getattr(module, class_name)
                return klass
            except AttributeError:
                try:
                    __import__('.'.join([module.__name__, element_name]))
                    submodule = sys.modules['.'.join([module.__name__,
                                                      element_name])]
                except ImportError:
                    continue
                try:
                    klass = getattr(submodule, class_name)
                    return klass
                except AttributeError:
                    continue

        # If we made it this far, no such class was found
        return new.classobj(class_name, (self.__class__,),
                            {'__module__': self.__module__})
<|endoftext|>


<commit_before># Eloipool - Python Bitcoin pool server
# Copyright (C) 2011-2012  Luke Dashjr <luke-jr+eloipool@utopios.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from binascii import a2b_hex, b2a_hex
from bitcoin.script import countSigOps
from bitcoin.txn import Txn
from collections import deque
from queue import Queue
import jsonrpc
import logging
from merkletree import MerkleTree
from struct import pack
import threading
from time import sleep, time
import traceback

_makeCoinbase = [0, 0]

class merkleMaker(threading.Thread):
	def __init__(self, *a, **k):
		super().__init__(*a, **k)
		self.daemon = True
		self.logger = logging.getLogger('merkleMaker')
		self.CoinbasePrefix = b''
		self.CoinbaseAux = {}
		self.isOverflowed = False
		self.overflowed = 0

	def _prepare(self):
		self.access = jsonrpc.ServiceProxy(self.UpstreamURI)

		self.currentBlock = (None, None)
		self.currentMerkleTree = None
		self.merkleRoots = deque(maxlen=self.WorkQueueSizeRegular[1])
		self.LowestMerkleRoots = self.WorkQueueSizeRegular[1]
		self.clearMerkleTree = MerkleTree([self.clearCoinbaseTxn])
		self.clearMerkleRoots = Queue(self.WorkQueueSizeLongpoll[1])
		self.LowestClearMerkleRoots = self.WorkQueueSizeLongpoll[1]

		self.nextMerkleUpdate = 0
		global now
		now = time()
		self.updateMerkleTree()

	def updateBlock(self, newBlock, bits = None, _HBH = None):
		if newBlock == self.currentBlock[0]:
			return

		if bits is None:
			bits = self.currentBlock[1]
		if _HBH is None:
			_HBH = (b2a_hex(newBlock[::-1]).decode('utf8'), b2a_hex(bits[::-1]).decode('utf8'))
		self.logger.debug('New block: %s (bits: %s)' % _HBH)
		self.merkleRoots.clear()
		self.currentMerkleTree = self.clearMerkleTree
		self.lastBlock = self.currentBlock
		self.currentBlock = (newBlock, bits)
		self.needMerkle = 2
		self.onBlockChange()

	def updateMerkleTree(self):
		global now
		self.logger.debug('Polling bitcoind for memorypool')
		self.nextMerkleUpdate = now + self.TxnUpdateRetryWait
		MP = self.access.getmemorypool()
		prevBlock = a2b_hex(MP['previousblockhash'])[::-1]
		bits = a2b_hex(MP['bits'])[::-1]
		if (prevBlock, bits) != self.currentBlock:
			self.updateBlock(prevBlock, bits, _HBH=(MP['previousblockhash'], MP['bits']))
		# TODO: cache Txn or at least txid from previous merkle roots?
		txnlist = [a for a in map(a2b_hex, MP['transactions'])]

		t = self.makeCoinbaseTxn(MP['coinbasevalue'])
		t.setCoinbase(b'\0\0')
		t.assemble()
		txnlist.insert(0, t.data)

		txnlistsz = sum(map(len, txnlist))
		while txnlistsz > 934464:  # TODO: 1 "MB" limit - 64 KB breathing room
			self.logger.debug('Trimming transaction for size limit')
			txnlistsz -= len(txnlist.pop())

		txnlistsz = sum(map(countSigOps, txnlist))
		while txnlistsz > 19488:  # TODO: 20k limit - 0x200 breathing room
			self.logger.debug('Trimming transaction for SigOp limit')
			txnlistsz -= countSigOps(txnlist.pop())

		txnlist = [a for a in map(Txn, txnlist[1:])]
		txnlist.insert(0, t)
		txnlist = list(txnlist)
		newMerkleTree = MerkleTree(txnlist)
		if newMerkleTree.merkleRoot() != self.currentMerkleTree.merkleRoot():
			self.logger.debug('Updating merkle tree')
			self.currentMerkleTree = newMerkleTree
		self.nextMerkleUpdate = now + self.MinimumTxnUpdateWait

		if self.needMerkle == 2:
			self.needMerkle = 1

	def makeCoinbase(self):
		now = int(time())
		if now > _makeCoinbase[0]:
			_makeCoinbase[0] = now
			_makeCoinbase[1] = 0
		else:
			_makeCoinbase[1] += 1
		rv = self.CoinbasePrefix
		rv += pack('>L', now) + pack('>Q', _makeCoinbase[1]).lstrip(b'\0')
		for v in self.CoinbaseAux.values():
			rv += v
		if len(rv) > 100:
			t = time()
			if self.overflowed < t - 300:
				self.logger.warning('Overflowing coinbase data! %d bytes long' % (len(rv),))
				self.overflowed = t
				self.isOverflowed = True
			rv = rv[:100]
		else:
			self.isOverflowed = False
		return rv

	def makeMerkleRoot(self, merkleTree):
		t = merkleTree.data[0]
		cb = self.makeCoinbase()
		t.setCoinbase(cb)
		t.assemble()
		merkleRoot = merkleTree.merkleRoot()
		return (merkleRoot, merkleTree, cb)

	_doing_last = None
	def _doing(self, what):
		if self._doing_last == what:
			self._doing_i += 1
			return
		global now
		if self._doing_last:
			self.logger.debug("Switching from (%4dx in %5.3f seconds) %s => %s" % (self._doing_i, now - self._doing_s, self._doing_last, what))
		self._doing_last = what
		self._doing_i = 1
		self._doing_s = now

	def merkleMaker_I(self):
		global now

		# First, update merkle tree if we haven't for a while and aren't crunched for time
		now = time()
		if self.nextMerkleUpdate <= now and self.clearMerkleRoots.qsize() > self.WorkQueueSizeLongpoll[0] and len(self.merkleRoots) > self.WorkQueueSizeRegular[0]:
			self.updateMerkleTree()
		# Next, fill up the longpoll queue first, since it can be used as a failover for the main queue
		elif not self.clearMerkleRoots.full():
			self._doing('blank merkle roots')
			self.clearMerkleRoots.put(self.makeMerkleRoot(self.clearMerkleTree))
		# Next, fill up the main queue (until they're all current)
		elif len(self.merkleRoots) < self.WorkQueueSizeRegular[1] or self.merkleRoots[0][1] != self.currentMerkleTree:
			self._doing('regular merkle roots')
			self.merkleRoots.append(self.makeMerkleRoot(self.currentMerkleTree))
		else:
			if self.needMerkle == 1:
				self.onBlockUpdate()
				self.needMerkle = False
			self._doing('idle')
			# TODO: rather than sleepspin, block until MinimumTxnUpdateWait expires or threading.Condition(?)
			sleep(self.IdleSleepTime)

	def run(self):
		while True:
			try:
				self.merkleMaker_I()
			except:
				self.logger.critical(traceback.format_exc())

	def start(self, *a, **k):
		self._prepare()
		super().start(*a, **k)

	def getMRD(self):
		(prevBlock, bits) = self.currentBlock
		try:
			MRD = self.merkleRoots.pop()
			self.LowestMerkleRoots = min(len(self.merkleRoots), self.LowestMerkleRoots)
			rollPrevBlk = False
		except IndexError:
			qsz = self.clearMerkleRoots.qsize()
			if qsz < 0x10:
				self.logger.warning('clearMerkleRoots running out! only %d left' % (qsz,))
			MRD = self.clearMerkleRoots.get()
			self.LowestClearMerkleRoots = min(self.clearMerkleRoots.qsize(), self.LowestClearMerkleRoots)
			rollPrevBlk = True
		(merkleRoot, merkleTree, cb) = MRD
		return (merkleRoot, merkleTree, cb, prevBlock, bits, rollPrevBlk)
<commit_msg>Promote new block from debug to info<commit_after># Eloipool - Python Bitcoin pool server
# Copyright (C) 2011-2012  Luke Dashjr <luke-jr+eloipool@utopios.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from binascii import a2b_hex, b2a_hex
from bitcoin.script import countSigOps
from bitcoin.txn import Txn
from collections import deque
from queue import Queue
import jsonrpc
import logging
from merkletree import MerkleTree
from struct import pack
import threading
from time import sleep, time
import traceback

_makeCoinbase = [0, 0]

class merkleMaker(threading.Thread):
	def __init__(self, *a, **k):
		super().__init__(*a, **k)
		self.daemon = True
		self.logger = logging.getLogger('merkleMaker')
		self.CoinbasePrefix = b''
		self.CoinbaseAux = {}
		self.isOverflowed = False
		self.overflowed = 0

	def _prepare(self):
		self.access = jsonrpc.ServiceProxy(self.UpstreamURI)

		self.currentBlock = (None, None)
		self.currentMerkleTree = None
		self.merkleRoots = deque(maxlen=self.WorkQueueSizeRegular[1])
		self.LowestMerkleRoots = self.WorkQueueSizeRegular[1]
		self.clearMerkleTree = MerkleTree([self.clearCoinbaseTxn])
		self.clearMerkleRoots = Queue(self.WorkQueueSizeLongpoll[1])
		self.LowestClearMerkleRoots = self.WorkQueueSizeLongpoll[1]

		self.nextMerkleUpdate = 0
		global now
		now = time()
		self.updateMerkleTree()

	def updateBlock(self, newBlock, bits = None, _HBH = None):
		if newBlock == self.currentBlock[0]:
			return

		if bits is None:
			bits = self.currentBlock[1]
		if _HBH is None:
			_HBH = (b2a_hex(newBlock[::-1]).decode('utf8'), b2a_hex(bits[::-1]).decode('utf8'))
		self.logger.info('New block: %s (bits: %s)' % _HBH)
		self.merkleRoots.clear()
		self.currentMerkleTree = self.clearMerkleTree
		self.lastBlock = self.currentBlock
		self.currentBlock = (newBlock, bits)
		self.needMerkle = 2
		self.onBlockChange()

	def updateMerkleTree(self):
		global now
		self.logger.debug('Polling bitcoind for memorypool')
		self.nextMerkleUpdate = now + self.TxnUpdateRetryWait
		MP = self.access.getmemorypool()
		prevBlock = a2b_hex(MP['previousblockhash'])[::-1]
		bits = a2b_hex(MP['bits'])[::-1]
		if (prevBlock, bits) != self.currentBlock:
			self.updateBlock(prevBlock, bits, _HBH=(MP['previousblockhash'], MP['bits']))
		# TODO: cache Txn or at least txid from previous merkle roots?
		txnlist = [a for a in map(a2b_hex, MP['transactions'])]

		t = self.makeCoinbaseTxn(MP['coinbasevalue'])
		t.setCoinbase(b'\0\0')
		t.assemble()
		txnlist.insert(0, t.data)

		txnlistsz = sum(map(len, txnlist))
		while txnlistsz > 934464:  # TODO: 1 "MB" limit - 64 KB breathing room
			self.logger.debug('Trimming transaction for size limit')
			txnlistsz -= len(txnlist.pop())

		txnlistsz = sum(map(countSigOps, txnlist))
		while txnlistsz > 19488:  # TODO: 20k limit - 0x200 breathing room
			self.logger.debug('Trimming transaction for SigOp limit')
			txnlistsz -= countSigOps(txnlist.pop())

		txnlist = [a for a in map(Txn, txnlist[1:])]
		txnlist.insert(0, t)
		txnlist = list(txnlist)
		newMerkleTree = MerkleTree(txnlist)
		if newMerkleTree.merkleRoot() != self.currentMerkleTree.merkleRoot():
			self.logger.debug('Updating merkle tree')
			self.currentMerkleTree = newMerkleTree
		self.nextMerkleUpdate = now + self.MinimumTxnUpdateWait

		if self.needMerkle == 2:
			self.needMerkle = 1

	def makeCoinbase(self):
		now = int(time())
		if now > _makeCoinbase[0]:
			_makeCoinbase[0] = now
			_makeCoinbase[1] = 0
		else:
			_makeCoinbase[1] += 1
		rv = self.CoinbasePrefix
		rv += pack('>L', now) + pack('>Q', _makeCoinbase[1]).lstrip(b'\0')
		for v in self.CoinbaseAux.values():
			rv += v
		if len(rv) > 100:
			t = time()
			if self.overflowed < t - 300:
				self.logger.warning('Overflowing coinbase data! %d bytes long' % (len(rv),))
				self.overflowed = t
				self.isOverflowed = True
			rv = rv[:100]
		else:
			self.isOverflowed = False
		return rv

	def makeMerkleRoot(self, merkleTree):
		t = merkleTree.data[0]
		cb = self.makeCoinbase()
		t.setCoinbase(cb)
		t.assemble()
		merkleRoot = merkleTree.merkleRoot()
		return (merkleRoot, merkleTree, cb)

	_doing_last = None
	def _doing(self, what):
		if self._doing_last == what:
			self._doing_i += 1
			return
		global now
		if self._doing_last:
			self.logger.debug("Switching from (%4dx in %5.3f seconds) %s => %s" % (self._doing_i, now - self._doing_s, self._doing_last, what))
		self._doing_last = what
		self._doing_i = 1
		self._doing_s = now

	def merkleMaker_I(self):
		global now

		# First, update merkle tree if we haven't for a while and aren't crunched for time
		now = time()
		if self.nextMerkleUpdate <= now and self.clearMerkleRoots.qsize() > self.WorkQueueSizeLongpoll[0] and len(self.merkleRoots) > self.WorkQueueSizeRegular[0]:
			self.updateMerkleTree()
		# Next, fill up the longpoll queue first, since it can be used as a failover for the main queue
		elif not self.clearMerkleRoots.full():
			self._doing('blank merkle roots')
			self.clearMerkleRoots.put(self.makeMerkleRoot(self.clearMerkleTree))
		# Next, fill up the main queue (until they're all current)
		elif len(self.merkleRoots) < self.WorkQueueSizeRegular[1] or self.merkleRoots[0][1] != self.currentMerkleTree:
			self._doing('regular merkle roots')
			self.merkleRoots.append(self.makeMerkleRoot(self.currentMerkleTree))
		else:
			if self.needMerkle == 1:
				self.onBlockUpdate()
				self.needMerkle = False
			self._doing('idle')
			# TODO: rather than sleepspin, block until MinimumTxnUpdateWait expires or threading.Condition(?)
			sleep(self.IdleSleepTime)

	def run(self):
		while True:
			try:
				self.merkleMaker_I()
			except:
				self.logger.critical(traceback.format_exc())

	def start(self, *a, **k):
		self._prepare()
		super().start(*a, **k)

	def getMRD(self):
		(prevBlock, bits) = self.currentBlock
		try:
			MRD = self.merkleRoots.pop()
			self.LowestMerkleRoots = min(len(self.merkleRoots), self.LowestMerkleRoots)
			rollPrevBlk = False
		except IndexError:
			qsz = self.clearMerkleRoots.qsize()
			if qsz < 0x10:
				self.logger.warning('clearMerkleRoots running out! only %d left' % (qsz,))
			MRD = self.clearMerkleRoots.get()
			self.LowestClearMerkleRoots = min(self.clearMerkleRoots.qsize(), self.LowestClearMerkleRoots)
			rollPrevBlk = True
		(merkleRoot, merkleTree, cb) = MRD
		return (merkleRoot, merkleTree, cb, prevBlock, bits, rollPrevBlk)
<|endoftext|>