<commit_before>"""
Starting in CherryPy 3.1, cherrypy.server is implemented as an
:ref:`Engine Plugin<plugins>`. It's an instance of
:class:`cherrypy._cpserver.Server`, which is a subclass of
:class:`cherrypy.process.servers.ServerAdapter`. The ``ServerAdapter`` class
is designed to control other servers, as well.

Multiple servers/ports
======================

If you need to start more than one HTTP server (to serve on multiple ports, or
protocols, etc.), you can manually register each one and then start them all
with engine.start::

    s1 = ServerAdapter(cherrypy.engine, MyWSGIServer(host='0.0.0.0', port=80))
    s2 = ServerAdapter(cherrypy.engine, another.HTTPServer(host='127.0.0.1', SSL=True))
    s1.subscribe()
    s2.subscribe()
    cherrypy.engine.start()

.. index:: SCGI

FastCGI/SCGI
============

There are also Flup\ **F**\ CGIServer and Flup\ **S**\ CGIServer classes in
:mod:`cherrypy.process.servers`. To start an fcgi server, for example,
wrap an instance of it in a ServerAdapter::

    addr = ('0.0.0.0', 4000)
    f = servers.FlupFCGIServer(application=cherrypy.tree, bindAddress=addr)
    s = servers.ServerAdapter(cherrypy.engine, httpserver=f, bind_addr=addr)
    s.subscribe()

The :doc:`cherryd</deployguide/cherryd>` startup script will do the above for
you via its `-f` flag.
Note that you need to download and install `flup <http://trac.saddi.com/flup>`_
yourself, whether you use ``cherryd`` or not.

.. _fastcgi:
.. index:: FastCGI

FastCGI
-------

A very simple setup lets your cherry run with FastCGI.
You just need the flup library,
plus a running Apache server (with ``mod_fastcgi``) or lighttpd server.

CherryPy code
^^^^^^^^^^^^^

hello.py::

    #!/usr/bin/python
    import cherrypy

    class HelloWorld:
        \"""Sample request handler class.\"""
        def index(self):
            return "Hello world!"
        index.exposed = True

    cherrypy.tree.mount(HelloWorld())
    # CherryPy autoreload must be disabled for the flup server to work
    cherrypy.config.update({'engine.autoreload_on':False})

Then run :doc:`/deployguide/cherryd` with the '-f' arg::

    cherryd -c <myconfig> -d -f -i hello.py

Apache
^^^^^^

At the top level in httpd.conf::

    FastCgiIpcDir /tmp
    FastCgiServer /path/to/cherry.fcgi -idle-timeout 120 -processes 4

And inside the relevant VirtualHost section::

    # FastCGI config
    AddHandler fastcgi-script .fcgi
    ScriptAliasMatch (.*$) /path/to/cherry.fcgi$1

Lighttpd
^^^^^^^^

For `Lighttpd <http://www.lighttpd.net/>`_ you can follow these
instructions. Within ``lighttpd.conf`` make sure ``mod_fastcgi`` is
active within ``server.modules``. Then, within your ``$HTTP["host"]``
directive, configure your fastcgi script like the following::

    $HTTP["url"] =~ "" {
      fastcgi.server = (
        "/" => (
          "script.fcgi" => (
            "bin-path" => "/path/to/your/script.fcgi",
            "socket"          => "/tmp/script.sock",
            "check-local"     => "disable",
            "disable-time"    => 1,
            "min-procs"       => 1,
            "max-procs"       => 1, # adjust as needed
          ),
        ),
      )
    } # end of $HTTP["url"] =~ "^/"

Please see `Lighttpd FastCGI Docs
<http://redmine.lighttpd.net/wiki/lighttpd/Docs:ModFastCGI>`_ for an explanation
of the possible configuration options.
"""

import sys
import time
import warnings

class ServerAdapter(object):
    """Adapter for an HTTP server.

    If you need to start more than one HTTP server (to serve on multiple
    ports, or protocols, etc.), you can manually register each one and then
    start them all with bus.start:

        s1 = ServerAdapter(bus, MyWSGIServer(host='0.0.0.0', port=80))
        s2 = ServerAdapter(bus, another.HTTPServer(host='127.0.0.1', SSL=True))
        s1.subscribe()
        s2.subscribe()
        bus.start()
    """

    def __init__(self, bus, httpserver=None, bind_addr=None):
        self.bus = bus
        self.httpserver = httpserver
        self.bind_addr = bind_addr
        self.interrupt = None
        self.running = False

    def subscribe(self):
        self.bus.subscribe('start', self.start)
        self.bus.subscribe('stop', self.stop)

    def unsubscribe(self):
        self.bus.unsubscribe('start', self.start)
        self.bus.unsubscribe('stop', self.stop)

    def start(self):
        """Start the HTTP server."""
        if self.bind_addr is None:
            on_what = "unknown interface (dynamic?)"
        elif isinstance(self.bind_addr, tuple):
            host, port = self.bind_addr
            on_what = "%s:%s" % (host, port)
        else:
            on_what = "socket file: %s" % self.bind_addr

        if self.running:
            self.bus.log("Already serving on %s" % on_what)
            return

        self.interrupt = None
        if not self.httpserver:
            raise ValueError("No HTTP server has been created.")

        # Start the httpserver in a new thread.
        if isinstance(self.bind_addr, tuple):
            wait_for_free_port(*self.bind_addr)

        import threading
        t = threading.Thread(target=self._start_http_thread)
        t.setName("HTTPServer " + t.getName())
        t.start()

        self.wait()
        self.running = True
        self.bus.log("Serving on %s" % on_what)
    start.priority = 75

    def _start_http_thread(self):
        """HTTP servers MUST be running in new threads, so that the
        main thread persists to receive KeyboardInterrupt's. If an
        exception is raised in the httpserver's thread then it's
        trapped here, and the bus (and therefore our httpserver)
        are shut down.
        """
        try:
            self.httpserver.start()
        except KeyboardInterrupt:
            self.bus.log("<Ctrl-C> hit: shutting down HTTP server")
            self.interrupt = sys.exc_info()[1]
            self.bus.exit()
        except SystemExit:
            self.bus.log("SystemExit raised: shutting down HTTP server")
            self.interrupt = sys.exc_info()[1]
            self.bus.exit()
            raise
        except:
            self.interrupt = sys.exc_info()[1]
            self.bus.log("Error in HTTP server: shutting down",
                         traceback=True, level=40)
            self.bus.exit()
            raise

    def wait(self):
        """Wait until the HTTP server is ready to receive requests."""
        while not getattr(self.httpserver, "ready", False):
            if self.interrupt:
                raise self.interrupt
            time.sleep(.1)

        # Wait for port to be occupied
        if isinstance(self.bind_addr, tuple):
            host, port = self.bind_addr
            wait_for_occupied_port(host, port)

    def stop(self):
        """Stop the HTTP server."""
        if self.running:
            # stop() MUST block until the server is *truly* stopped.
            self.httpserver.stop()
            # Wait for the socket to be truly freed.
            if isinstance(self.bind_addr, tuple):
                wait_for_free_port(*self.bind_addr)
            self.running = False
            self.bus.log("HTTP Server %s shut down" % self.httpserver)
        else:
            self.bus.log("HTTP Server %s already shut down" % self.httpserver)
    stop.priority = 25

    def restart(self):
        """Restart the HTTP server."""
        self.stop()
        self.start()

class FlupCGIServer(object):
    """Adapter for a flup.server.cgi.WSGIServer."""

    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs
        self.ready = False

    def start(self):
        """Start the CGI server."""
        # We have to instantiate the server class here because its __init__
        # starts a threadpool. If we do it too early, daemonize won't work.
        from flup.server.cgi import WSGIServer

        self.cgiserver = WSGIServer(*self.args, **self.kwargs)
        self.ready = True
        self.cgiserver.run()

    def stop(self):
        """Stop the HTTP server."""
        self.ready = False

class FlupFCGIServer(object):
    """Adapter for a flup.server.fcgi.WSGIServer."""

    def __init__(self, *args, **kwargs):
        if kwargs.get('bindAddress', None) is None:
            import socket
            if not hasattr(socket, 'fromfd'):
                raise ValueError(
                    'Dynamic FCGI server not available on this platform. '
                    'You must use a static or external one by providing a '
                    'legal bindAddress.')
        self.args = args
        self.kwargs = kwargs
        self.ready = False

    def start(self):
        """Start the FCGI server."""
        # We have to instantiate the server class here because its __init__
        # starts a threadpool. If we do it too early, daemonize won't work.
        from flup.server.fcgi import WSGIServer
        self.fcgiserver = WSGIServer(*self.args, **self.kwargs)
        # TODO: report this bug upstream to flup.
        # If we don't set _oldSIGs on Windows, we get:
        #   File "C:\Python24\Lib\site-packages\flup\server\threadedserver.py",
        #   line 108, in run
        #     self._restoreSignalHandlers()
        #   File "C:\Python24\Lib\site-packages\flup\server\threadedserver.py",
        #   line 156, in _restoreSignalHandlers
        #     for signum,handler in self._oldSIGs:
        #   AttributeError: 'WSGIServer' object has no attribute '_oldSIGs'
        self.fcgiserver._installSignalHandlers = lambda: None
        self.fcgiserver._oldSIGs = []
        self.ready = True
        self.fcgiserver.run()

    def stop(self):
        """Stop the HTTP server."""
        # Forcibly stop the fcgi server main event loop.
        self.fcgiserver._keepGoing = False
        # Force all worker threads to die off.
        self.fcgiserver._threadPool.maxSpare = self.fcgiserver._threadPool._idleCount
        self.ready = False

class FlupSCGIServer(object):
    """Adapter for a flup.server.scgi.WSGIServer."""

    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs
        self.ready = False

    def start(self):
        """Start the SCGI server."""
        # We have to instantiate the server class here because its __init__
        # starts a threadpool. If we do it too early, daemonize won't work.
        from flup.server.scgi import WSGIServer
        self.scgiserver = WSGIServer(*self.args, **self.kwargs)
        # TODO: report this bug upstream to flup.
        # If we don't set _oldSIGs on Windows, we get:
        #   File "C:\Python24\Lib\site-packages\flup\server\threadedserver.py",
        #   line 108, in run
        #     self._restoreSignalHandlers()
        #   File "C:\Python24\Lib\site-packages\flup\server\threadedserver.py",
        #   line 156, in _restoreSignalHandlers
        #     for signum,handler in self._oldSIGs:
        #   AttributeError: 'WSGIServer' object has no attribute '_oldSIGs'
        self.scgiserver._installSignalHandlers = lambda: None
        self.scgiserver._oldSIGs = []
        self.ready = True
        self.scgiserver.run()

    def stop(self):
        """Stop the HTTP server."""
        self.ready = False
        # Forcibly stop the scgi server main event loop.
        self.scgiserver._keepGoing = False
        # Force all worker threads to die off.
        self.scgiserver._threadPool.maxSpare = 0

def client_host(server_host):
    """Return the host on which a client can connect to the given listener."""
    if server_host == '0.0.0.0':
        # 0.0.0.0 is INADDR_ANY, which should answer on localhost.
        return '127.0.0.1'
    if server_host in ('::', '::0', '::0.0.0.0'):
        # :: is IN6ADDR_ANY, which should answer on localhost.
        # ::0 and ::0.0.0.0 are non-canonical but common ways to write IN6ADDR_ANY.
        return '::1'
    return server_host

def check_port(host, port, timeout=1.0):
    """Raise an error if the given port is not free on the given host."""
    if not host:
        raise ValueError("Host values of '' or None are not allowed.")
    host = client_host(host)
    port = int(port)

    import socket

    # AF_INET or AF_INET6 socket
    # Get the correct address family for our host (allows IPv6 addresses)
    try:
        info = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
                                  socket.SOCK_STREAM)
    except socket.gaierror:
        if ':' in host:
            info = [(socket.AF_INET6, socket.SOCK_STREAM, 0, "", (host, port, 0, 0))]
        else:
            info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (host, port))]

    for res in info:
        af, socktype, proto, canonname, sa = res
        s = None
        try:
            s = socket.socket(af, socktype, proto)
            # See http://groups.google.com/group/cherrypy-users/
            #        browse_frm/thread/bbfe5eb39c904fe0
            s.settimeout(timeout)
            s.connect((host, port))
            s.close()
        except socket.error:
            if s:
                s.close()
        else:
            raise IOError("Port %s is in use on %s; perhaps the previous "
                          "httpserver did not shut down properly." %
                          (repr(port), repr(host)))

# Feel free to increase these defaults on slow systems:
free_port_timeout = 0.1
occupied_port_timeout = 1.0

def wait_for_free_port(host, port, timeout=None):
    """Wait for the specified port to become free (drop requests)."""
    if not host:
        raise ValueError("Host values of '' or None are not allowed.")
    if timeout is None:
        timeout = free_port_timeout

    for trial in range(50):
        try:
            # we are expecting a free port, so reduce the timeout
            check_port(host, port, timeout=timeout)
        except IOError:
            # Give the old server thread time to free the port.
            time.sleep(timeout)
        else:
            return

    raise IOError("Port %r not free on %r" % (port, host))

def wait_for_occupied_port(host, port, timeout=None):
    """Wait for the specified port to become active (receive requests)."""
    if not host:
        raise ValueError("Host values of '' or None are not allowed.")
    if timeout is None:
        timeout = occupied_port_timeout

    for trial in range(50):
        try:
            check_port(host, port, timeout=timeout)
        except IOError:
            # port is occupied
            return
        else:
            time.sleep(timeout)

    if host == client_host(host):
        raise IOError("Port %r not bound on %r" % (port, host))

    # On systems where a loopback interface is not available and the
    #  server is bound to all interfaces, it's difficult to determine
    #  whether the server is in fact occupying the port. In this case,
    #  just issue a warning and move on. See issue #1100.
    msg = "Unable to verify that the server is bound on %r" % port
    warnings.warn(msg)
<commit_msg>Update documentation to remove reference to old configuration alias.<commit_after>"""
Starting in CherryPy 3.1, cherrypy.server is implemented as an
:ref:`Engine Plugin<plugins>`. It's an instance of
:class:`cherrypy._cpserver.Server`, which is a subclass of
:class:`cherrypy.process.servers.ServerAdapter`. The ``ServerAdapter`` class
is designed to control other servers, as well.

Multiple servers/ports
======================

If you need to start more than one HTTP server (to serve on multiple ports, or
protocols, etc.), you can manually register each one and then start them all
with engine.start::

    s1 = ServerAdapter(cherrypy.engine, MyWSGIServer(host='0.0.0.0', port=80))
    s2 = ServerAdapter(cherrypy.engine, another.HTTPServer(host='127.0.0.1', SSL=True))
    s1.subscribe()
    s2.subscribe()
    cherrypy.engine.start()

.. index:: SCGI

FastCGI/SCGI
============

There are also Flup\ **F**\ CGIServer and Flup\ **S**\ CGIServer classes in
:mod:`cherrypy.process.servers`. To start an fcgi server, for example,
wrap an instance of it in a ServerAdapter::

    addr = ('0.0.0.0', 4000)
    f = servers.FlupFCGIServer(application=cherrypy.tree, bindAddress=addr)
    s = servers.ServerAdapter(cherrypy.engine, httpserver=f, bind_addr=addr)
    s.subscribe()

The :doc:`cherryd</deployguide/cherryd>` startup script will do the above for
you via its `-f` flag.
Note that you need to download and install `flup <http://trac.saddi.com/flup>`_
yourself, whether you use ``cherryd`` or not.

.. _fastcgi:
.. index:: FastCGI

FastCGI
-------

A very simple setup lets your cherry run with FastCGI.
You just need the flup library,
plus a running Apache server (with ``mod_fastcgi``) or lighttpd server.

CherryPy code
^^^^^^^^^^^^^

hello.py::

    #!/usr/bin/python
    import cherrypy

    class HelloWorld:
        \"""Sample request handler class.\"""
        def index(self):
            return "Hello world!"
        index.exposed = True

    cherrypy.tree.mount(HelloWorld())
    # CherryPy autoreload must be disabled for the flup server to work
    cherrypy.config.update({'engine.autoreload.on':False})

Then run :doc:`/deployguide/cherryd` with the '-f' arg::

    cherryd -c <myconfig> -d -f -i hello.py

Apache
^^^^^^

At the top level in httpd.conf::

    FastCgiIpcDir /tmp
    FastCgiServer /path/to/cherry.fcgi -idle-timeout 120 -processes 4

And inside the relevant VirtualHost section::

    # FastCGI config
    AddHandler fastcgi-script .fcgi
    ScriptAliasMatch (.*$) /path/to/cherry.fcgi$1

Lighttpd
^^^^^^^^

For `Lighttpd <http://www.lighttpd.net/>`_ you can follow these
instructions. Within ``lighttpd.conf`` make sure ``mod_fastcgi`` is
active within ``server.modules``. Then, within your ``$HTTP["host"]``
directive, configure your fastcgi script like the following::

    $HTTP["url"] =~ "" {
      fastcgi.server = (
        "/" => (
          "script.fcgi" => (
            "bin-path" => "/path/to/your/script.fcgi",
            "socket"          => "/tmp/script.sock",
            "check-local"     => "disable",
            "disable-time"    => 1,
            "min-procs"       => 1,
            "max-procs"       => 1, # adjust as needed
          ),
        ),
      )
    } # end of $HTTP["url"] =~ "^/"

Please see `Lighttpd FastCGI Docs
<http://redmine.lighttpd.net/wiki/lighttpd/Docs:ModFastCGI>`_ for an explanation
of the possible configuration options.
"""

import sys
import time
import warnings

class ServerAdapter(object):
    """Adapter for an HTTP server.

    If you need to start more than one HTTP server (to serve on multiple
    ports, or protocols, etc.), you can manually register each one and then
    start them all with bus.start:

        s1 = ServerAdapter(bus, MyWSGIServer(host='0.0.0.0', port=80))
        s2 = ServerAdapter(bus, another.HTTPServer(host='127.0.0.1', SSL=True))
        s1.subscribe()
        s2.subscribe()
        bus.start()
    """

    def __init__(self, bus, httpserver=None, bind_addr=None):
        self.bus = bus
        self.httpserver = httpserver
        self.bind_addr = bind_addr
        self.interrupt = None
        self.running = False

    def subscribe(self):
        self.bus.subscribe('start', self.start)
        self.bus.subscribe('stop', self.stop)

    def unsubscribe(self):
        self.bus.unsubscribe('start', self.start)
        self.bus.unsubscribe('stop', self.stop)

    def start(self):
        """Start the HTTP server."""
        if self.bind_addr is None:
            on_what = "unknown interface (dynamic?)"
        elif isinstance(self.bind_addr, tuple):
            host, port = self.bind_addr
            on_what = "%s:%s" % (host, port)
        else:
            on_what = "socket file: %s" % self.bind_addr

        if self.running:
            self.bus.log("Already serving on %s" % on_what)
            return

        self.interrupt = None
        if not self.httpserver:
            raise ValueError("No HTTP server has been created.")

        # Start the httpserver in a new thread.
        if isinstance(self.bind_addr, tuple):
            wait_for_free_port(*self.bind_addr)

        import threading
        t = threading.Thread(target=self._start_http_thread)
        t.setName("HTTPServer " + t.getName())
        t.start()

        self.wait()
        self.running = True
        self.bus.log("Serving on %s" % on_what)
    start.priority = 75

    def _start_http_thread(self):
        """HTTP servers MUST be running in new threads, so that the
        main thread persists to receive KeyboardInterrupt's. If an
        exception is raised in the httpserver's thread then it's
        trapped here, and the bus (and therefore our httpserver)
        are shut down.
        """
        try:
            self.httpserver.start()
        except KeyboardInterrupt:
            self.bus.log("<Ctrl-C> hit: shutting down HTTP server")
            self.interrupt = sys.exc_info()[1]
            self.bus.exit()
        except SystemExit:
            self.bus.log("SystemExit raised: shutting down HTTP server")
            self.interrupt = sys.exc_info()[1]
            self.bus.exit()
            raise
        except:
            self.interrupt = sys.exc_info()[1]
            self.bus.log("Error in HTTP server: shutting down",
                         traceback=True, level=40)
            self.bus.exit()
            raise

    def wait(self):
        """Wait until the HTTP server is ready to receive requests."""
        while not getattr(self.httpserver, "ready", False):
            if self.interrupt:
                raise self.interrupt
            time.sleep(.1)

        # Wait for port to be occupied
        if isinstance(self.bind_addr, tuple):
            host, port = self.bind_addr
            wait_for_occupied_port(host, port)

    def stop(self):
        """Stop the HTTP server."""
        if self.running:
            # stop() MUST block until the server is *truly* stopped.
            self.httpserver.stop()
            # Wait for the socket to be truly freed.
            if isinstance(self.bind_addr, tuple):
                wait_for_free_port(*self.bind_addr)
            self.running = False
            self.bus.log("HTTP Server %s shut down" % self.httpserver)
        else:
            self.bus.log("HTTP Server %s already shut down" % self.httpserver)
    stop.priority = 25

    def restart(self):
        """Restart the HTTP server."""
        self.stop()
        self.start()

class FlupCGIServer(object):
    """Adapter for a flup.server.cgi.WSGIServer."""

    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs
        self.ready = False

    def start(self):
        """Start the CGI server."""
        # We have to instantiate the server class here because its __init__
        # starts a threadpool. If we do it too early, daemonize won't work.
        from flup.server.cgi import WSGIServer

        self.cgiserver = WSGIServer(*self.args, **self.kwargs)
        self.ready = True
        self.cgiserver.run()

    def stop(self):
        """Stop the HTTP server."""
        self.ready = False

class FlupFCGIServer(object):
    """Adapter for a flup.server.fcgi.WSGIServer."""

    def __init__(self, *args, **kwargs):
        if kwargs.get('bindAddress', None) is None:
            import socket
            if not hasattr(socket, 'fromfd'):
                raise ValueError(
                    'Dynamic FCGI server not available on this platform. '
                    'You must use a static or external one by providing a '
                    'legal bindAddress.')
        self.args = args
        self.kwargs = kwargs
        self.ready = False

    def start(self):
        """Start the FCGI server."""
        # We have to instantiate the server class here because its __init__
        # starts a threadpool. If we do it too early, daemonize won't work.
        from flup.server.fcgi import WSGIServer
        self.fcgiserver = WSGIServer(*self.args, **self.kwargs)
        # TODO: report this bug upstream to flup.
        # If we don't set _oldSIGs on Windows, we get:
        #   File "C:\Python24\Lib\site-packages\flup\server\threadedserver.py",
        #   line 108, in run
        #     self._restoreSignalHandlers()
        #   File "C:\Python24\Lib\site-packages\flup\server\threadedserver.py",
        #   line 156, in _restoreSignalHandlers
        #     for signum,handler in self._oldSIGs:
        #   AttributeError: 'WSGIServer' object has no attribute '_oldSIGs'
        self.fcgiserver._installSignalHandlers = lambda: None
        self.fcgiserver._oldSIGs = []
        self.ready = True
        self.fcgiserver.run()

    def stop(self):
        """Stop the HTTP server."""
        # Forcibly stop the fcgi server main event loop.
        self.fcgiserver._keepGoing = False
        # Force all worker threads to die off.
        self.fcgiserver._threadPool.maxSpare = self.fcgiserver._threadPool._idleCount
        self.ready = False

class FlupSCGIServer(object):
    """Adapter for a flup.server.scgi.WSGIServer."""

    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs
        self.ready = False

    def start(self):
        """Start the SCGI server."""
        # We have to instantiate the server class here because its __init__
        # starts a threadpool. If we do it too early, daemonize won't work.
        from flup.server.scgi import WSGIServer
        self.scgiserver = WSGIServer(*self.args, **self.kwargs)
        # TODO: report this bug upstream to flup.
        # If we don't set _oldSIGs on Windows, we get:
        #   File "C:\Python24\Lib\site-packages\flup\server\threadedserver.py",
        #   line 108, in run
        #     self._restoreSignalHandlers()
        #   File "C:\Python24\Lib\site-packages\flup\server\threadedserver.py",
        #   line 156, in _restoreSignalHandlers
        #     for signum,handler in self._oldSIGs:
        #   AttributeError: 'WSGIServer' object has no attribute '_oldSIGs'
        self.scgiserver._installSignalHandlers = lambda: None
        self.scgiserver._oldSIGs = []
        self.ready = True
        self.scgiserver.run()

    def stop(self):
        """Stop the HTTP server."""
        self.ready = False
        # Forcibly stop the scgi server main event loop.
        self.scgiserver._keepGoing = False
        # Force all worker threads to die off.
        self.scgiserver._threadPool.maxSpare = 0

def client_host(server_host):
    """Return the host on which a client can connect to the given listener."""
    if server_host == '0.0.0.0':
        # 0.0.0.0 is INADDR_ANY, which should answer on localhost.
        return '127.0.0.1'
    if server_host in ('::', '::0', '::0.0.0.0'):
        # :: is IN6ADDR_ANY, which should answer on localhost.
        # ::0 and ::0.0.0.0 are non-canonical but common ways to write IN6ADDR_ANY.
        return '::1'
    return server_host

def check_port(host, port, timeout=1.0):
    """Raise an error if the given port is not free on the given host."""
    if not host:
        raise ValueError("Host values of '' or None are not allowed.")
    host = client_host(host)
    port = int(port)

    import socket

    # AF_INET or AF_INET6 socket
    # Get the correct address family for our host (allows IPv6 addresses)
    try:
        info = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
                                  socket.SOCK_STREAM)
    except socket.gaierror:
        if ':' in host:
            info = [(socket.AF_INET6, socket.SOCK_STREAM, 0, "", (host, port, 0, 0))]
        else:
            info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (host, port))]

    for res in info:
        af, socktype, proto, canonname, sa = res
        s = None
        try:
            s = socket.socket(af, socktype, proto)
            # See http://groups.google.com/group/cherrypy-users/
            #        browse_frm/thread/bbfe5eb39c904fe0
            s.settimeout(timeout)
            s.connect((host, port))
            s.close()
        except socket.error:
            if s:
                s.close()
        else:
            raise IOError("Port %s is in use on %s; perhaps the previous "
                          "httpserver did not shut down properly." %
                          (repr(port), repr(host)))

# Feel free to increase these defaults on slow systems:
free_port_timeout = 0.1
occupied_port_timeout = 1.0

def wait_for_free_port(host, port, timeout=None):
    """Wait for the specified port to become free (drop requests)."""
    if not host:
        raise ValueError("Host values of '' or None are not allowed.")
    if timeout is None:
        timeout = free_port_timeout

    for trial in range(50):
        try:
            # we are expecting a free port, so reduce the timeout
            check_port(host, port, timeout=timeout)
        except IOError:
            # Give the old server thread time to free the port.
            time.sleep(timeout)
        else:
            return

    raise IOError("Port %r not free on %r" % (port, host))

def wait_for_occupied_port(host, port, timeout=None):
    """Wait for the specified port to become active (receive requests)."""
    if not host:
        raise ValueError("Host values of '' or None are not allowed.")
    if timeout is None:
        timeout = occupied_port_timeout

    for trial in range(50):
        try:
            check_port(host, port, timeout=timeout)
        except IOError:
            # port is occupied
            return
        else:
            time.sleep(timeout)

    if host == client_host(host):
        raise IOError("Port %r not bound on %r" % (port, host))

    # On systems where a loopback interface is not available and the
    #  server is bound to all interfaces, it's difficult to determine
    #  whether the server is in fact occupying the port. In this case,
    #  just issue a warning and move on. See issue #1100.
    msg = "Unable to verify that the server is bound on %r" % port
    warnings.warn(msg)
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
from os.path import join, dirname
from nose.tools import raises

from indra.statements import *
from indra.sources.geneways.geneways_symbols_parser import \
        GenewaysSymbols
from indra.sources.geneways.geneways_action_parser import \
        GenewaysActionParser
from indra.sources.geneways.geneways_actionmention_parser import \
        GenewaysActionMentionParser
from indra.sources.geneways.geneways_api import process_geneways_files

# Path to the Geneways test/dummy data folder
path_this = os.path.dirname(os.path.abspath(__file__))
data_folder = os.path.join(path_this,
        '../../data/tests_data/geneways_tests_data')
symbols_file = os.path.join(data_folder, 'human_symbols.txt')
actionmention_file = os.path.join(data_folder,
        'human_actionmention.txt')

def test_geneways_symbols_parser():
    symbols = GenewaysSymbols(symbols_file)

    print(symbols.symbol_to_id('Akt') == ['1'])
    assert(symbols.symbol_to_id('Akt') == ['1'])
    assert(symbols.symbol_to_id('c-Src') == ['2'])

    assert(symbols.id_to_symbol('1') == 'Akt')
    assert(symbols.id_to_symbol('2') == 'c-Src')

    assert(len(symbols.symbols_with_multiple_ids()) == 0)

def test_geneways_actionmention_parser():
    parser = GenewaysActionMentionParser(actionmention_file)

    assert(len(parser.action_mentions) == 4)
    mention0 = parser.action_mentions[0]
    mention1 = parser.action_mentions[1]
    mention2 = parser.action_mentions[2]
    mention3 = parser.action_mentions[3]

    #Make sure that the parser reads in the TSV file into the correct fields

    #First action mention
    assert(mention0.hiid == '1')
    assert(mention0.actionmentionid == '1')
    assert(mention0.negative == '0')
    assert(mention0.upstream == 'c-Src')
    assert(mention0.actiontype == 'phosphorylate')
    assert(mention0.downstream == 'Akt')
    assert(mention0.pmid == '19262695')
    assert(mention0.isFullText == '1')
    assert(mention0.sentencenumber == '4')
    assert(mention0.score == '0.56')
    assert(mention0.prec == '0.78')

    #Second action mention
    assert(mention1.hiid == '1')
    assert(mention1.actionmentionid == '2')
    assert(mention1.negative == '0')
    assert(mention1.upstream == 'c-Src')
    assert(mention1.actiontype == 'phosphorylate')
    assert(mention1.downstream == 'Akt')
    assert(mention1.pmid == '2')
    assert(mention1.isFullText == '0')
    assert(mention1.sentencenumber == '7')
    assert(mention1.score == '0.23')
    assert(mention1.prec == '0.34')

    #Third action mention
    assert(mention2.hiid == '2')
    assert(mention2.actionmentionid == '3')
    assert(mention2.negative == '1')
    assert(mention2.upstream == 'A')
    assert(mention2.actiontype == 'bind')
    assert(mention2.downstream == 'B')
    assert(mention2.pmid == '0')
    assert(mention2.isFullText == '0')
    assert(mention2.sentencenumber == '0')
    assert(mention2.score == '0.12')
    assert(mention2.prec == '0.48')

    #Fourth action mention
    assert(mention3.hiid == '3')
    assert(mention3.actionmentionid == '4')
    assert(mention3.negative == '0')
    assert(mention3.upstream == 'C')
    assert(mention3.actiontype == 'bind')
    assert(mention3.downstream == 'D')
    assert(mention3.pmid == '0')
    assert(mention3.isFullText == '0')
    assert(mention3.sentencenumber == '0')
    assert(mention3.score == '0.22')
    assert(mention3.prec == '0.55')

def test_geneways_action_parser():
    parser = GenewaysActionParser(data_folder)

    actions = parser.actions
    assert(len(actions) == 3)

    action0 = actions[0]
    action1 = actions[1]
    action2 = actions[2]

    assert(action0.hiid == '1')
    assert(action0.up == '2')
    assert(action0.dn == '1')
    assert(action0.actiontype == 'phosphorylate')
    assert(action0.action_count == '1')
    assert(action0.actionmention_count == '2')
    assert(action0.plo == 'P')
    assert(action0.max_score == '0.77')
    assert(action0.max_prec == '0.88')
    assert(len(action0.action_mentions) == 2)

    assert(action1.hiid == '2')
    assert(action1.up == '3')
    assert(action1.dn == '4')
    assert(action1.actiontype == 'bind')
    assert(action1.action_count == '1')
    assert(action1.actionmention_count == '1')
    assert(action1.plo == 'P')
    assert(action1.max_score == '0.12')
    assert(action1.max_prec == '0.34')
    assert(len(action1.action_mentions) == 1)

    assert(action2.hiid == '3')
    assert(action2.up == '5')
    assert(action2.dn == '6')
    assert(action2.actiontype == 'bind')
    assert(action2.action_count == '1')
    assert(action2.actionmention_count == '1')
    assert(action2.plo == 'P')
    assert(action2.max_score == '0.16')
    assert(action2.max_prec == '0.17')
    assert(len(action2.action_mentions) == 1)

def test_geneways_processor():
    processor = process_geneways_files(data_folder)

    statements = processor.statements
    print('Number of statements: ', len(statements))
    assert(len(statements) == 3)
<commit_msg>Add initial test for Geneways processor<commit_after>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
from os.path import join, dirname
from nose.tools import raises

from indra.statements import *
from indra.sources.geneways.geneways_symbols_parser import \
        GenewaysSymbols
from indra.sources.geneways.geneways_action_parser import \
        GenewaysActionParser
from indra.sources.geneways.geneways_actionmention_parser import \
        GenewaysActionMentionParser
from indra.sources.geneways.geneways_api import process_geneways_files

# Path to the Geneways test/dummy data folder
path_this = os.path.dirname(os.path.abspath(__file__))
data_folder = os.path.join(path_this,
        '../../data/tests_data/geneways_tests_data')
symbols_file = os.path.join(data_folder, 'human_symbols.txt')
actionmention_file = os.path.join(data_folder,
        'human_actionmention.txt')

def test_geneways_symbols_parser():
    symbols = GenewaysSymbols(symbols_file)

    print(symbols.symbol_to_id('Akt') == ['1'])
    assert(symbols.symbol_to_id('Akt') == ['1'])
    assert(symbols.symbol_to_id('c-Src') == ['2'])

    assert(symbols.id_to_symbol('1') == 'Akt')
    assert(symbols.id_to_symbol('2') == 'c-Src')

    assert(len(symbols.symbols_with_multiple_ids()) == 0)

def test_geneways_actionmention_parser():
    parser = GenewaysActionMentionParser(actionmention_file)

    assert(len(parser.action_mentions) == 4)
    mention0 = parser.action_mentions[0]
    mention1 = parser.action_mentions[1]
    mention2 = parser.action_mentions[2]
    mention3 = parser.action_mentions[3]

    #Make sure that the parser reads in the TSV file into the correct fields

    #First action mention
    assert(mention0.hiid == '1')
    assert(mention0.actionmentionid == '1')
    assert(mention0.negative == '0')
    assert(mention0.upstream == 'c-Src')
    assert(mention0.actiontype == 'phosphorylate')
    assert(mention0.downstream == 'Akt')
    assert(mention0.pmid == '19262695')
    assert(mention0.isFullText == '1')
    assert(mention0.sentencenumber == '4')
    assert(mention0.score == '0.56')
    assert(mention0.prec == '0.78')

    #Second action mention
    assert(mention1.hiid == '1')
    assert(mention1.actionmentionid == '2')
    assert(mention1.negative == '0')
    assert(mention1.upstream == 'c-Src')
    assert(mention1.actiontype == 'phosphorylate')
    assert(mention1.downstream == 'Akt')
    assert(mention1.pmid == '2')
    assert(mention1.isFullText == '0')
    assert(mention1.sentencenumber == '7')
    assert(mention1.score == '0.23')
    assert(mention1.prec == '0.34')

    #Third action mention
    assert(mention2.hiid == '2')
    assert(mention2.actionmentionid == '3')
    assert(mention2.negative == '1')
    assert(mention2.upstream == 'A')
    assert(mention2.actiontype == 'bind')
    assert(mention2.downstream == 'B')
    assert(mention2.pmid == '0')
    assert(mention2.isFullText == '0')
    assert(mention2.sentencenumber == '0')
    assert(mention2.score == '0.12')
    assert(mention2.prec == '0.48')

    #Fourth action mention
    assert(mention3.hiid == '3')
    assert(mention3.actionmentionid == '4')
    assert(mention3.negative == '0')
    assert(mention3.upstream == 'C')
    assert(mention3.actiontype == 'bind')
    assert(mention3.downstream == 'D')
    assert(mention3.pmid == '0')
    assert(mention3.isFullText == '0')
    assert(mention3.sentencenumber == '0')
    assert(mention3.score == '0.22')
    assert(mention3.prec == '0.55')

def test_geneways_action_parser():
    parser = GenewaysActionParser(data_folder)

    actions = parser.actions
    assert(len(actions) == 3)

    action0 = actions[0]
    action1 = actions[1]
    action2 = actions[2]

    assert(action0.hiid == '1')
    assert(action0.up == '2')
    assert(action0.dn == '1')
    assert(action0.actiontype == 'phosphorylate')
    assert(action0.action_count == '1')
    assert(action0.actionmention_count == '2')
    assert(action0.plo == 'P')
    assert(action0.max_score == '0.77')
    assert(action0.max_prec == '0.88')
    assert(len(action0.action_mentions) == 2)

    assert(action1.hiid == '2')
    assert(action1.up == '3')
    assert(action1.dn == '4')
    assert(action1.actiontype == 'bind')
    assert(action1.action_count == '1')
    assert(action1.actionmention_count == '1')
    assert(action1.plo == 'P')
    assert(action1.max_score == '0.12')
    assert(action1.max_prec == '0.34')
    assert(len(action1.action_mentions) == 1)

    assert(action2.hiid == '3')
    assert(action2.up == '5')
    assert(action2.dn == '6')
    assert(action2.actiontype == 'bind')
    assert(action2.action_count == '1')
    assert(action2.actionmention_count == '1')
    assert(action2.plo == 'P')
    assert(action2.max_score == '0.16')
    assert(action2.max_prec == '0.17')
    assert(len(action2.action_mentions) == 1)

def test_geneways_processor():
    processor = process_geneways_files(data_folder)

    statements = processor.statements
    assert(len(statements) == 3)

    statement0 = statements[0]
    statement1 = statements[1]
    statement2 = statements[2]

    assert(isinstance(statement0, Phosphorylation))

    assert(isinstance(statement1, Complex))

    assert(isinstance(statement2, Complex))
<|endoftext|>


<commit_before>import uuid, json
import pandas as pd
from IPython.display import display_html, HTML
from mpcontribs.config import mp_level01_titles, mp_id_pattern
from mpcontribs.io.core.utils import nest_dict, clean_value
from mpcontribs.io.core.recdict import RecursiveDict
from urllib.parse import urlparse

class Table(pd.DataFrame):
    def __init__(self, data, columns=None, index=None, cid=None, name=None):
        super(Table, self).__init__(data=data, index=index, columns=columns)
        self.cid = cid
        self.name = name

    def to_dict(self):
        from pandas import MultiIndex
        for col in self.columns:
            self[col] = self[col].apply(lambda x: clean_value(x, max_dgts=6))
        rdct = super(Table, self).to_dict(orient='split', into=RecursiveDict)
        if not isinstance(self.index, MultiIndex):
            rdct.pop('index')
        rdct["@module"] = self.__class__.__module__
        rdct["@class"] = self.__class__.__name__
        return rdct

    @classmethod
    def from_dict(cls, d):
        index = None
        if 'index' in d:
            from pandas import MultiIndex
            index = MultiIndex.from_tuples(d['index'])
        obj = cls(
            d['data'], columns=d['columns'], index=index,
            cid=d['cid'], name=d['name']
        ) if 'cid' in d and 'name' in d else cls(
            d['data'], columns=d['columns'], index=index
        )
        return obj

    @classmethod
    def from_items(cls, rdct, **kwargs):
        return super(Table, cls).from_dict(RecursiveDict(rdct), **kwargs)

    def to_backgrid_dict(self):
        """Backgrid-conform dict from DataFrame"""
        # shorten global import times by importing django here
        import numpy as np
        from mpcontribs.io.core.utils import get_composition_from_string
        from pandas import MultiIndex
        import pymatgen.util as pmg_util
        from pymatgen.core.composition import CompositionError

        table = dict()
        nrows_max = 260
        nrows = self.shape[0]
        df = Table(self.head(n=nrows_max)) if nrows > nrows_max else self
        numeric_columns = df.select_dtypes(include=[np.number]).columns.tolist()

        if isinstance(df.index, MultiIndex):
            df.reset_index(inplace=True)

        table['columns'] = []
        table['rows'] = super(Table, df).to_dict(orient='records')

        for col_index, col in enumerate(list(df.columns)):
            cell_type = 'number'

            # avoid looping rows to minimize use of `df.iat` (time-consuming in 3d)
            if not col.startswith('level_') and col not in numeric_columns:
                is_url_column, prev_unit, old_col = True, None, col

                for row_index in range(df.shape[0]):
                    cell = str(df.iat[row_index, col_index])
                    cell_split = cell.split(' ', 1)

                    if not cell or len(cell_split) == 1: # empty cell or no space
                        is_url_column = bool(is_url_column and (not cell or mp_id_pattern.match(cell)))
                        if is_url_column:
                            if cell:
                                value = 'https://materialsproject.org/materials/{}'.format(cell)
                                table['rows'][row_index][col] = value
                        elif cell:
                            try:
                                composition = get_composition_from_string(cell)
                                composition = pmg_util.string.unicodeify(composition)
                                table['rows'][row_index][col] = composition
                            except (CompositionError, ValueError, OverflowError):
                                try:
                                    # https://stackoverflow.com/a/38020041
                                    result = urlparse(cell)
                                    if not all([result.scheme, result.netloc, result.path]):
                                        break
                                    is_url_column = True
                                except:
                                    break

                    else:
                        value, unit = cell_split # TODO convert cell_split[0] to float?
                        is_url_column = False
                        try:
                            float(value) # unit is only a unit if value is number
                        except ValueError:
                            continue
                        table['rows'][row_index].pop(old_col)
                        if prev_unit is None:
                            prev_unit = unit
                            col = '{} [{}]'.format(col, unit)
                        table['rows'][row_index][col] = cell if prev_unit != unit else value

                cell_type = 'uri' if is_url_column else 'string'

            col_split = col.split('.')
            nesting = [col_split[0]] if len(col_split) > 1 else []
            table['columns'].append({
                'name': col, 'cell': cell_type, 'nesting': nesting, 'editable': 0
            })
            if len(col_split) > 1:
                table['columns'][-1].update({'label': '.'.join(col_split[1:])})
            if len(table['columns']) > 12:
                table['columns'][-1]['renderable'] = 0

        header = RecursiveDict()
        for idx, col in enumerate(table['columns']):
            if 'label' in col:
                k, sk = col['name'].split('.')
                sk_split = sk.split()
                if len(sk_split) == 2:
                    d = {'name': sk_split[0], 'unit': sk_split[1], 'idx': idx}
                    if k not in header:
                        header[k] = [d]
                    else:
                        header[k].append(d)
                elif k in header:
                    header.pop(k)

        for k, skl in header.items():
            units = [sk['unit'] for sk in skl]
            if units.count(units[0]) == len(units):
                for sk in skl:
                    table['columns'][sk['idx']]['label'] = sk['name']
                    table['columns'][sk['idx']]['nesting'][0] = '{} {}'.format(k, sk['unit'])

        return table

    def render(self, project=None, total_records=None):
        """use BackGrid JS library to render Pandas DataFrame"""
        # if project given, this will result in an overview table of contributions
        # TODO check for index column in df other than the default numbering
        jtable = json.dumps(self.to_backgrid_dict())
        if total_records is None:
            total_records = self.shape[0]
        config = {"total_records": total_records}
        config['uuids'] = [str(uuid.uuid4()) for i in range(3)]
        if project is None:
            config['name'] = self.name
            config['cid'] = self.cid
        else:
            config['project'] = project
        jconfig = json.dumps(config)
        html = '

'.format(config['uuids'][0])
        html += '

'.format(config['uuids'][1])
        html += '

'.format(config['uuids'][2])
        html += f''
        return html

    def _ipython_display_(self):
        display(HTML(self.render()))

class Tables(RecursiveDict):
    """class to hold and display multiple data tables"""
    def __init__(self, content=RecursiveDict()):
        super(Tables, self).__init__(
            (key, value) for key, value in content.items()
            if isinstance(value, Table)
        )

    def __str__(self):
        return 'tables: {}'.format(' '.join(self.keys()))

    def _ipython_display_(self):
        for name, table in self.items():
            display_html('
{}
'.format(name), raw=True)
            display_html(table)

class TabularData(RecursiveDict):
    """class to hold and display all tabular data of a MPFile"""
    def __init__(self, document):
        super(TabularData, self).__init__()
        from pymatgen import Structure
        scope = []
        for key, value in document.iterate():
            if isinstance(value, Table):
                self[scope[0]].rec_update({'.'.join(scope[1:]): value})
            elif not isinstance(value, Structure):
                level, key = key
                level_reduction = bool(level < len(scope))
                if level_reduction:
                    del scope[level:]
                if value is None:
                    scope.append(key)
                    if scope[0] not in self:
                        self[scope[0]] = Tables()

    def __str__(self):
        return 'mp-ids: {}'.format(' '.join(self.keys()))

    def _ipython_display_(self):
        for identifier, tables in self.items():
            if isinstance(tables, dict) and tables:
                display_html('
Tabular Data for {}
'.format(identifier), raw=True)
                display_html(tables)
<commit_msg>backgrid table: unit handling not necessary anymore<commit_after>import uuid, json
import pandas as pd
from IPython.display import display_html, HTML
from mpcontribs.config import mp_level01_titles, mp_id_pattern
from mpcontribs.io.core.utils import nest_dict, clean_value
from mpcontribs.io.core.recdict import RecursiveDict
from urllib.parse import urlparse

class Table(pd.DataFrame):
    def __init__(self, data, columns=None, index=None, cid=None, name=None):
        super(Table, self).__init__(data=data, index=index, columns=columns)
        self.cid = cid
        self.name = name

    def to_dict(self):
        from pandas import MultiIndex
        for col in self.columns:
            self[col] = self[col].apply(lambda x: clean_value(x, max_dgts=6))
        rdct = super(Table, self).to_dict(orient='split', into=RecursiveDict)
        if not isinstance(self.index, MultiIndex):
            rdct.pop('index')
        rdct["@module"] = self.__class__.__module__
        rdct["@class"] = self.__class__.__name__
        return rdct

    @classmethod
    def from_dict(cls, d):
        index = None
        if 'index' in d:
            from pandas import MultiIndex
            index = MultiIndex.from_tuples(d['index'])
        obj = cls(
            d['data'], columns=d['columns'], index=index,
            cid=d['cid'], name=d['name']
        ) if 'cid' in d and 'name' in d else cls(
            d['data'], columns=d['columns'], index=index
        )
        return obj

    @classmethod
    def from_items(cls, rdct, **kwargs):
        return super(Table, cls).from_dict(RecursiveDict(rdct), **kwargs)

    def to_backgrid_dict(self):
        """Backgrid-conform dict from DataFrame"""
        # shorten global import times by importing django here
        import numpy as np
        from mpcontribs.io.core.utils import get_composition_from_string
        from pandas import MultiIndex
        import pymatgen.util as pmg_util
        from pymatgen.core.composition import CompositionError

        table = dict()
        nrows_max = 260
        nrows = self.shape[0]
        df = Table(self.head(n=nrows_max)) if nrows > nrows_max else self
        numeric_columns = df.select_dtypes(include=[np.number]).columns.tolist()

        if isinstance(df.index, MultiIndex):
            df.reset_index(inplace=True)

        table['columns'] = []
        table['rows'] = super(Table, df).to_dict(orient='records')

        for col_index, col in enumerate(list(df.columns)):
            cell_type = 'number'

            # avoid looping rows to minimize use of `df.iat` (time-consuming in 3d)
            if not col.startswith('level_') and col not in numeric_columns:
                is_url_column, prev_unit, old_col = True, None, col

                for row_index in range(df.shape[0]):
                    cell = str(df.iat[row_index, col_index])
                    is_url_column = bool(is_url_column and (not cell or mp_id_pattern.match(cell)))
                    if is_url_column:
                        if cell:
                            value = 'https://materialsproject.org/materials/{}'.format(cell)
                            table['rows'][row_index][col] = value
                    elif cell:
                        try:
                            composition = get_composition_from_string(cell)
                            composition = pmg_util.string.unicodeify(composition)
                            table['rows'][row_index][col] = composition
                        except (CompositionError, ValueError, OverflowError):
                            try:
                                # https://stackoverflow.com/a/38020041
                                result = urlparse(cell)
                                if not all([result.scheme, result.netloc, result.path]):
                                    break
                                is_url_column = True
                            except:
                                break

                cell_type = 'uri' if is_url_column else 'string'

            col_split = col.split('.')
            nesting = [col_split[0]] if len(col_split) > 1 else []
            table['columns'].append({
                'name': col, 'cell': cell_type, 'nesting': nesting, 'editable': 0
            })
            if len(col_split) > 1:
                table['columns'][-1].update({'label': '.'.join(col_split[1:])})
            if len(table['columns']) > 12:
                table['columns'][-1]['renderable'] = 0

        return table

    def render(self, project=None, total_records=None):
        """use BackGrid JS library to render Pandas DataFrame"""
        # if project given, this will result in an overview table of contributions
        # TODO check for index column in df other than the default numbering
        jtable = json.dumps(self.to_backgrid_dict())
        if total_records is None:
            total_records = self.shape[0]
        config = {"total_records": total_records}
        config['uuids'] = [str(uuid.uuid4()) for i in range(3)]
        if project is None:
            config['name'] = self.name
            config['cid'] = self.cid
        else:
            config['project'] = project
        jconfig = json.dumps(config)
        html = '

'.format(config['uuids'][0])
        html += '

'.format(config['uuids'][1])
        html += '

'.format(config['uuids'][2])
        html += f''
        return html

    def _ipython_display_(self):
        display(HTML(self.render()))

class Tables(RecursiveDict):
    """class to hold and display multiple data tables"""
    def __init__(self, content=RecursiveDict()):
        super(Tables, self).__init__(
            (key, value) for key, value in content.items()
            if isinstance(value, Table)
        )

    def __str__(self):
        return 'tables: {}'.format(' '.join(self.keys()))

    def _ipython_display_(self):
        for name, table in self.items():
            display_html('
{}
'.format(name), raw=True)
            display_html(table)

class TabularData(RecursiveDict):
    """class to hold and display all tabular data of a MPFile"""
    def __init__(self, document):
        super(TabularData, self).__init__()
        from pymatgen import Structure
        scope = []
        for key, value in document.iterate():
            if isinstance(value, Table):
                self[scope[0]].rec_update({'.'.join(scope[1:]): value})
            elif not isinstance(value, Structure):
                level, key = key
                level_reduction = bool(level < len(scope))
                if level_reduction:
                    del scope[level:]
                if value is None:
                    scope.append(key)
                    if scope[0] not in self:
                        self[scope[0]] = Tables()

    def __str__(self):
        return 'mp-ids: {}'.format(' '.join(self.keys()))

    def _ipython_display_(self):
        for identifier, tables in self.items():
            if isinstance(tables, dict) and tables:
                display_html('
Tabular Data for {}
'.format(identifier), raw=True)
                display_html(tables)
<|endoftext|>


<commit_before>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#

import types

class Hunk:

    # The new lines which result from this hunk are returned. This does
    # not check for conflicts; countConflicts() should be used for that
    def apply(self, src, srcLine):
	result = []
	fromLine = srcLine
	for line in self.lines:
	    if line[0] == " ":
		result.append(src[fromLine])
		fromLine = fromLine + 1
	    elif line[0] == "+":
		result.append(line[1:])
	    elif line[0] == "-":
		fromLine = fromLine + 1

	assert(fromLine == self.fromLen + srcLine)
	assert(len(result) == self.toLen)

	return result

    def countConflicts(self, src, srcLine):
	conflicts = 0
	for line in self.lines:
	    if line[0] == " " or line[0] == "-":
		if src[srcLine] != line[1:]:
		    conflicts += 1
		srcLine += 1

	return conflicts

    def write(self, f):
	f.write(self.asString())

    def asString(self):
	str = "@@ -%d,%s +%d,%d @@\n" % (self.fromStart, self.fromLen,
		self.toStart, self.toLen)
	str += "".join(self.lines)
	return str

    def __init__(self, fromStart, fromLen, toStart, toLen, lines, contextCount):
	self.fromStart = fromStart
	self.toStart = toStart
	self.fromLen = fromLen
	self.toLen = toLen
	self.lines = lines
	self.contextCount = contextCount

class FailedHunkList(list):

    def write(self, filename, oldName, newName):
	f = open(filename, "w")
	f.write("--- %s\n" % oldName)
	f.write("+++ %s\n" % newName)
	for hunk in self:
	    hunk.write(f)

# this just applies hunks, not files... in other words, the --- and +++
# lines should be omitted, and only a single file can be patched
def patch(oldLines, unifiedDiff):
    i = 0

    if type(unifiedDiff) == types.GeneratorType:
	list = []
	for l in unifiedDiff:
	    list.append(l)
	unifiedDiff = list

    last = len(unifiedDiff)
    hunks = []
    while i < last:
	(magic1, fromRange, toRange, magic2) = unifiedDiff[i].split()
	if magic1 != "@@" or magic2 != "@@" or fromRange[0] != "-" or \
	   toRange[0] != "+":
	    raise BadHunkHeader()

	(fromStart, fromLen) = fromRange.split(",")
	fromStart = int(fromStart[1:]) - 1
	fromLen = int(fromLen)

	(toStart, toLen) = toRange.split(",")
	toStart = int(toStart[1:]) - 1
	toLen = int(toLen)

	fromCount = 0
	toCount = 0
	contextCount = 0
	lines = []
	i += 1
	while i < last and unifiedDiff[i][0] != '@':
	    lines.append(unifiedDiff[i])
	    ch = unifiedDiff[i][0]
	    if ch == " ":
		fromCount += 1
		toCount += 1
		contextCount += 1
	    elif ch == "-":
		fromCount += 1
	    elif ch == "+":
		toCount += 1
	    else:
		raise BadHunk()

	    i += 1

	if toCount != toLen or fromCount != fromLen:
	    raise BadHunk()

	hunks.append(Hunk(fromStart, fromLen, toStart, toLen, lines,
			  contextCount))

    i = 0
    last = len(unifiedDiff)
    result = []
    failedHunks = FailedHunkList()

    fromLine = 0
    offset = 0

    for hunk in hunks:
	start = hunk.fromStart + offset
	conflicts = hunk.countConflicts(oldLines, start)
	best = (conflicts, 0)

	i = 0
	while best[0]:
	    i = i + 1
	    tried = 0
	    if (start - abs(i) >= 0):
		tried = 1
		conflicts = hunk.countConflicts(oldLines, start - i)
		if conflicts < best[0]:
		    best = (conflicts, -i)

		if not conflicts:
		    i = -i
		    break

	    if ((start + i) <= (len(oldLines) - hunk.fromLen)):
		tried = 1
		conflicts = hunk.countConflicts(oldLines, start + i)
		if conflicts < best[0]:
		    best = (conflicts, i)
		if not conflicts:
		    break

	    if not tried:
		break

	conflictCount = best[0]
	if conflictCount and (hunk.contextCount - conflictCount) < 2:
	    failedHunks.append(hunk)
	    continue

	offset = best[1]
	start += offset

	while (fromLine < start):
	    result.append(oldLines[fromLine])
	    fromLine = fromLine + 1

	result += hunk.apply(oldLines, start)
	fromLine += hunk.fromLen

    while (fromLine < len(oldLines)):
	result.append(oldLines[fromLine])
	fromLine = fromLine + 1

    return (result, failedHunks)

class BadHunkHeader(Exception):

    pass

class BadHunk(Exception):

    pass

def reverse(lines):
    result = []
    for line in lines:
	if line[0] == " ":
	    result.append(line)
	elif line[0] == "+":
	    result.append("-" + line[1:])
	elif line[0] == "-":
	    result.append("+" + line[1:])
	elif line[0] == "@":
	    fields = line.split()
	    new = [ fields[0], "-" + fields[2][1:],
		    "+" + fields[1][1:], fields[3] ]
	    result.append(" ".join(new) + "\n")

    return result

<commit_msg>use a generator for reverse<commit_after>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#

import types

class Hunk:

    # The new lines which result from this hunk are returned. This does
    # not check for conflicts; countConflicts() should be used for that
    def apply(self, src, srcLine):
	result = []
	fromLine = srcLine
	for line in self.lines:
	    if line[0] == " ":
		result.append(src[fromLine])
		fromLine = fromLine + 1
	    elif line[0] == "+":
		result.append(line[1:])
	    elif line[0] == "-":
		fromLine = fromLine + 1

	assert(fromLine == self.fromLen + srcLine)
	assert(len(result) == self.toLen)

	return result

    def countConflicts(self, src, srcLine):
	conflicts = 0
	for line in self.lines:
	    if line[0] == " " or line[0] == "-":
		if src[srcLine] != line[1:]:
		    conflicts += 1
		srcLine += 1

	return conflicts

    def write(self, f):
	f.write(self.asString())

    def asString(self):
	str = "@@ -%d,%s +%d,%d @@\n" % (self.fromStart, self.fromLen,
		self.toStart, self.toLen)
	str += "".join(self.lines)
	return str

    def __init__(self, fromStart, fromLen, toStart, toLen, lines, contextCount):
	self.fromStart = fromStart
	self.toStart = toStart
	self.fromLen = fromLen
	self.toLen = toLen
	self.lines = lines
	self.contextCount = contextCount

class FailedHunkList(list):

    def write(self, filename, oldName, newName):
	f = open(filename, "w")
	f.write("--- %s\n" % oldName)
	f.write("+++ %s\n" % newName)
	for hunk in self:
	    hunk.write(f)

# this just applies hunks, not files... in other words, the --- and +++
# lines should be omitted, and only a single file can be patched
def patch(oldLines, unifiedDiff):
    i = 0

    if type(unifiedDiff) == types.GeneratorType:
	list = []
	for l in unifiedDiff:
	    list.append(l)
	unifiedDiff = list

    last = len(unifiedDiff)
    hunks = []
    while i < last:
	(magic1, fromRange, toRange, magic2) = unifiedDiff[i].split()
	if magic1 != "@@" or magic2 != "@@" or fromRange[0] != "-" or \
	   toRange[0] != "+":
	    raise BadHunkHeader()

	(fromStart, fromLen) = fromRange.split(",")
	fromStart = int(fromStart[1:]) - 1
	fromLen = int(fromLen)

	(toStart, toLen) = toRange.split(",")
	toStart = int(toStart[1:]) - 1
	toLen = int(toLen)

	fromCount = 0
	toCount = 0
	contextCount = 0
	lines = []
	i += 1
	while i < last and unifiedDiff[i][0] != '@':
	    lines.append(unifiedDiff[i])
	    ch = unifiedDiff[i][0]
	    if ch == " ":
		fromCount += 1
		toCount += 1
		contextCount += 1
	    elif ch == "-":
		fromCount += 1
	    elif ch == "+":
		toCount += 1
	    else:
		raise BadHunk()

	    i += 1

	if toCount != toLen or fromCount != fromLen:
	    raise BadHunk()

	hunks.append(Hunk(fromStart, fromLen, toStart, toLen, lines,
			  contextCount))

    i = 0
    last = len(unifiedDiff)
    result = []
    failedHunks = FailedHunkList()

    fromLine = 0
    offset = 0

    for hunk in hunks:
	start = hunk.fromStart + offset
	conflicts = hunk.countConflicts(oldLines, start)
	best = (conflicts, 0)

	i = 0
	while best[0]:
	    i = i + 1
	    tried = 0
	    if (start - abs(i) >= 0):
		tried = 1
		conflicts = hunk.countConflicts(oldLines, start - i)
		if conflicts < best[0]:
		    best = (conflicts, -i)

		if not conflicts:
		    i = -i
		    break

	    if ((start + i) <= (len(oldLines) - hunk.fromLen)):
		tried = 1
		conflicts = hunk.countConflicts(oldLines, start + i)
		if conflicts < best[0]:
		    best = (conflicts, i)
		if not conflicts:
		    break

	    if not tried:
		break

	conflictCount = best[0]
	if conflictCount and (hunk.contextCount - conflictCount) < 2:
	    failedHunks.append(hunk)
	    continue

	offset = best[1]
	start += offset

	while (fromLine < start):
	    result.append(oldLines[fromLine])
	    fromLine = fromLine + 1

	result += hunk.apply(oldLines, start)
	fromLine += hunk.fromLen

    while (fromLine < len(oldLines)):
	result.append(oldLines[fromLine])
	fromLine = fromLine + 1

    return (result, failedHunks)

class BadHunkHeader(Exception):

    pass

class BadHunk(Exception):

    pass

def reverse(lines):
    for line in lines:
	if line[0] == " ":
	    yield line
	elif line[0] == "+":
	    yield "-" + line[1:]
	elif line[0] == "-":
	    yield "+" + line[1:]
	elif line[0] == "@":
	    fields = line.split()
	    new = [ fields[0], "-" + fields[2][1:],
		    "+" + fields[1][1:], fields[3] ]
	    yield " ".join(new) + "\n"

<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =============================================================================
# Lint as: python3
"""Executor for MetaLearner."""

import os
import sys
import json
import collections
from typing import Any, Dict, List, Tuple

from nitroml.components.metalearning import artifacts
from absl import logging
from tfx.components.base import base_executor
from tfx.types import artifact_utils
from tfx.types.artifact import Artifact
from tfx.utils import io_utils
import tensorflow_data_validation as tfdv
import kerastuner

_MAX_INPUTS = 10
_DEFAULT_FILE_NAME = 'meta_hyperparameters.txt'

OUTPUT_MODEL = 'metalearned_model'
OUTPUT_HYPERPARAMS = 'output_hyperparameters'
MAJORITY_VOTING = 'majority_voting'
METALEARNING_ALGORITHMS = [
    MAJORITY_VOTING,
]

class MetaLearnerExecutor(base_executor.BaseExecutor):
  """Executor for MetaLearnerExecutor."""

  def _create_search_space_using_voting(
      self, candidate_hparams: List[Dict[str,
                                         Any]]) -> kerastuner.HyperParameters:
    """Convert List of HParams to kerastuner.HyperaParameters.

      Args:
        candidate_hparams: List of Dict of HParams with same keys.

      Returns:
        discrete_search_space: A kerastuner.HyperParameters object representing a discrete search
        space created using voting. For example, when

        `candidate_hparams` is [{`learning_rate`: 0.01, `num_nodes`: 32},
                                {`learning_rate`: 0.001, `num_nodes`: 128},
                                {`learning_rate`: 0.01, `num_nodes`: 128},
                                {`learning_rate`: 0.001, `num_nodes`: 128}]

        then, `discrete_search_space` depicts the following discrete search space
        {`learning_rate`: [0.01, 0.001], `num_nodes`: [128]}

      Raises:
        ValueError: An error occured when `candidate_hparams` is empty or None.
    """

    if not candidate_hparams:
      raise ValueError(
          f'Expected a non-empty list of candidate_hparams. Got {candidate_hparams}'
      )

    hparams = candidate_hparams[0].keys()

    search_space = {}
    for key in hparams:
      search_space[key] = collections.Counter(
          [candidate[key] for candidate in candidate_hparams]).most_common()

    discrete_search_space = kerastuner.HyperParameters()

    for key, value_list in search_space.items():
      max_vote = -1
      candidate_values = []
      for value, count in value_list:
        if count >= max_vote:
          candidate_values.append(value)
        else:
          break

      discrete_search_space.Choice(
          key, candidate_values, default=candidate_values[0])

    return discrete_search_space

  def Do(self, input_dict: Dict[str, List[Artifact]],
         output_dict: Dict[str, List[Artifact]],
         exec_properties: Dict[str, Any]) -> None:
    """Recommends a tuner config.

    Args:
      input_dict: Input dict from input key to a list of artifacts, including:
        - meta_train_features_N: MetaFeatures for Nth train dataset.
        - hparams_train_N: HParms for Nth train dataset.
        The maximum value `N` being _MAX_INPUTS.
      output_dict: Output dict from key to a list of artifacts.
      exec_properties: A dict of execution properties.
    Raises:
    """

    algorithm = exec_properties['algorithm']
    custom_config = exec_properties['custom_config']

    train_stats = {}
    # This should be agnostic to meta-feature type.
    for ix in range(_MAX_INPUTS):
      metafeature_key = f'meta_train_features_{ix}'
      if (metafeature_key in input_dict):
        metafeature_uri = os.path.join(
            artifact_utils.get_single_uri(input_dict[metafeature_key]),
            artifacts.MetaFeatures.DEFAULT_FILE_NAME)
        logging.info('Found %s at %s.', metafeature_key, metafeature_uri)
        metafeatures = json.loads(io_utils.read_string_file(metafeature_uri))
        logging.info('metafeatures %s.', metafeatures['metafeature'])

    all_hparams = []
    for ix in range(_MAX_INPUTS):
      hparam_key = f'hparams_train_{ix}'
      if (hparam_key in input_dict):
        hyperparameters_file = io_utils.get_only_uri_in_dir(
            artifact_utils.get_single_uri(input_dict[hparam_key]))
        logging.info('Found %s at %s.', hparam_key, hyperparameters_file)
        hparams_json = json.loads(
            io_utils.read_string_file(hyperparameters_file))
        all_hparams.append(hparams_json['values'])
        logging.info('File %s.', hparams_json)

    if algorithm == 'majority_voting':

      discrete_search_space = self._create_search_space_using_voting(
          all_hparams)
      voted_hparams_config = discrete_search_space.get_config()
      logging.info('Discrete Search Space: %s', voted_hparams_config)

      meta_hparams_path = os.path.join(
          artifact_utils.get_single_uri(output_dict[OUTPUT_HYPERPARAMS]),
          _DEFAULT_FILE_NAME)

      io_utils.write_string_file(meta_hparams_path,
                                 json.dumps(voted_hparams_config))
      logging.info('Meta HParams saved at %s', meta_hparams_path)

    else:
      raise NotImplementedError(
          f'The algorithm "{algorithm}" is not supported.')<commit_msg>added comment for metafeature in executor<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =============================================================================
# Lint as: python3
"""Executor for MetaLearner."""

import os
import sys
import json
import collections
from typing import Any, Dict, List, Tuple

from nitroml.components.metalearning import artifacts
from absl import logging
from tfx.components.base import base_executor
from tfx.types import artifact_utils
from tfx.types.artifact import Artifact
from tfx.utils import io_utils
import tensorflow_data_validation as tfdv
import kerastuner

_MAX_INPUTS = 10
_DEFAULT_FILE_NAME = 'meta_hyperparameters.txt'

OUTPUT_MODEL = 'metalearned_model'
OUTPUT_HYPERPARAMS = 'output_hyperparameters'
MAJORITY_VOTING = 'majority_voting'
METALEARNING_ALGORITHMS = [
    MAJORITY_VOTING,
]

class MetaLearnerExecutor(base_executor.BaseExecutor):
  """Executor for MetaLearnerExecutor."""

  def _create_search_space_using_voting(
      self, candidate_hparams: List[Dict[str,
                                         Any]]) -> kerastuner.HyperParameters:
    """Convert List of HParams to kerastuner.HyperaParameters.

      Args:
        candidate_hparams: List of Dict of HParams with same keys.

      Returns:
        discrete_search_space: A kerastuner.HyperParameters object representing a discrete search
        space created using voting. For example, when

        `candidate_hparams` is [{`learning_rate`: 0.01, `num_nodes`: 32},
                                {`learning_rate`: 0.001, `num_nodes`: 128},
                                {`learning_rate`: 0.01, `num_nodes`: 128},
                                {`learning_rate`: 0.001, `num_nodes`: 128}]

        then, `discrete_search_space` depicts the following discrete search space
        {`learning_rate`: [0.01, 0.001], `num_nodes`: [128]}

      Raises:
        ValueError: An error occured when `candidate_hparams` is empty or None.
    """

    if not candidate_hparams:
      raise ValueError(
          f'Expected a non-empty list of candidate_hparams. Got {candidate_hparams}'
      )

    hparams = candidate_hparams[0].keys()

    search_space = {}
    for key in hparams:
      search_space[key] = collections.Counter(
          [candidate[key] for candidate in candidate_hparams]).most_common()

    discrete_search_space = kerastuner.HyperParameters()

    for key, value_list in search_space.items():
      max_vote = -1
      candidate_values = []
      for value, count in value_list:
        if count >= max_vote:
          candidate_values.append(value)
        else:
          break

      discrete_search_space.Choice(
          key, candidate_values, default=candidate_values[0])

    return discrete_search_space

  def Do(self, input_dict: Dict[str, List[Artifact]],
         output_dict: Dict[str, List[Artifact]],
         exec_properties: Dict[str, Any]) -> None:
    """Recommends a tuner config.

    Args:
      input_dict: Input dict from input key to a list of artifacts, including:
        - meta_train_features_N: MetaFeatures for Nth train dataset.
        - hparams_train_N: HParms for Nth train dataset.
        The maximum value `N` being _MAX_INPUTS.
      output_dict: Output dict from key to a list of artifacts.
      exec_properties: A dict of execution properties.
    Raises:
    """

    algorithm = exec_properties['algorithm']
    custom_config = exec_properties['custom_config']

    train_stats = {}
    # This should be agnostic to meta-feature type.
    for ix in range(_MAX_INPUTS):
      metafeature_key = f'meta_train_features_{ix}'
      if (metafeature_key in input_dict):
        metafeature_uri = os.path.join(
            artifact_utils.get_single_uri(input_dict[metafeature_key]),
            artifacts.MetaFeatures.DEFAULT_FILE_NAME)
        logging.info('Found %s at %s.', metafeature_key, metafeature_uri)
        metafeatures = json.loads(io_utils.read_string_file(metafeature_uri))
        # Only logging metafeatures for now. MetaFeature will be used in upcoming
        # algorithms.
        logging.info('metafeatures %s.', metafeatures['metafeature'])

    all_hparams = []
    for ix in range(_MAX_INPUTS):
      hparam_key = f'hparams_train_{ix}'
      if (hparam_key in input_dict):
        hyperparameters_file = io_utils.get_only_uri_in_dir(
            artifact_utils.get_single_uri(input_dict[hparam_key]))
        logging.info('Found %s at %s.', hparam_key, hyperparameters_file)
        hparams_json = json.loads(
            io_utils.read_string_file(hyperparameters_file))
        all_hparams.append(hparams_json['values'])
        logging.info('File %s.', hparams_json)

    if algorithm == 'majority_voting':

      discrete_search_space = self._create_search_space_using_voting(
          all_hparams)
      voted_hparams_config = discrete_search_space.get_config()
      logging.info('Discrete Search Space: %s', voted_hparams_config)

      meta_hparams_path = os.path.join(
          artifact_utils.get_single_uri(output_dict[OUTPUT_HYPERPARAMS]),
          _DEFAULT_FILE_NAME)

      io_utils.write_string_file(meta_hparams_path,
                                 json.dumps(voted_hparams_config))
      logging.info('Meta HParams saved at %s', meta_hparams_path)

    else:
      raise NotImplementedError(
          f'The algorithm "{algorithm}" is not supported.')<|endoftext|>