<commit_before># coding: utf-8
#
# Copyright 2010 Alexandre Fiori
# based on the original Tornado by Facebook
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""A non-blocking, single-threaded HTTP server.

Typical applications have little direct interaction with the `HTTPConnection`
class, which is the HTTP parser executed on incoming connections.

It is a protocol class that inherits Twisted's `LineReceiver
<http://twistedmatrix.com/documents/current/api/
twisted.protocols.basic.LineReceiver.html>`_, and is usually created by
`cyclone.web.Application`, our connection factory.

This module also defines the `HTTPRequest` class which is exposed via
`cyclone.web.RequestHandler.request`.
"""

from __future__ import absolute_import, division, with_statement

import Cookie
import socket
import time

from twisted.python import log
from twisted.protocols import basic
from twisted.internet import address
from twisted.internet import defer
from twisted.internet import interfaces

from cyclone.escape import utf8, native_str, parse_qs_bytes
from cyclone import httputil
from cyclone.util import bytes_type

class _BadRequestException(Exception):
    """Exception class for malformed HTTP requests."""
    pass

class HTTPConnection(basic.LineReceiver):
    """Handles a connection to an HTTP client, executing HTTP requests.

    We parse HTTP headers and bodies, and execute the request callback
    until the HTTP conection is closed.

    If ``xheaders`` is ``True``, we support the ``X-Real-Ip`` and ``X-Scheme``
    headers, which override the remote IP and HTTP scheme for all requests.
    These headers are useful when running Tornado behind a reverse proxy or
    load balancer.
    """
    delimiter = "\r\n"

    def connectionMade(self):
        self._headersbuffer = []
        self._contentbuffer = []
        self._finish_callback = None
        self.no_keep_alive = False
        self.content_length = None
        self.request_callback = self.factory
        self.xheaders = self.factory.settings.get('xheaders', False)
        self._request = None
        self._request_finished = False

    def connectionLost(self, reason):
        if self._finish_callback:
            self._finish_callback.callback(reason.getErrorMessage())
            self._finish_callback = None

    def notifyFinish(self):
        if self._finish_callback is None:
            self._finish_callback = defer.Deferred()
        return self._finish_callback

    def lineReceived(self, line):
        if line:
            self._headersbuffer.append(line + self.delimiter)
        else:
            buff = "".join(self._headersbuffer)
            self._headersbuffer = []
            self._on_headers(buff)

    def rawDataReceived(self, data):
        if self.content_length is not None:
            data, rest = data[:self.content_length], data[self.content_length:]
            self.content_length -= len(data)
        else:
            rest = ''

        self._contentbuffer.append(data)
        if self.content_length == 0:
            buff = "".join(self._contentbuffer)
            self._contentbuffer = []
            self.content_length = None
            self._on_request_body(buff)
            self.setLineMode(rest)

    def write(self, chunk):
        assert self._request, "Request closed"
        self.transport.write(chunk)

    def finish(self):
        assert self._request, "Request closed"
        self._request_finished = True
        self._finish_request()

    def _on_write_complete(self):
        if self._request_finished:
            self._finish_request()

    def _finish_request(self):
        if self.no_keep_alive:
            disconnect = True
        else:
            connection_header = self._request.headers.get("Connection")
            if self._request.supports_http_1_1():
                disconnect = connection_header == "close"
            elif ("Content-Length" in self._request.headers
                    or self._request.method in ("HEAD", "GET")):
                disconnect = connection_header != "Keep-Alive"
            else:
                disconnect = True

        if self._finish_callback:
            self._finish_callback.callback(None)
            self._finish_callback = None
        self._request = None
        self._request_finished = False
        if disconnect is True:
            self.transport.loseConnection()

    def _on_headers(self, data):
        try:
            data = native_str(data.decode("latin1"))
            eol = data.find("\r\n")
            start_line = data[:eol]
            try:
                method, uri, version = start_line.split(" ")
            except ValueError:
                raise _BadRequestException("Malformed HTTP request line")
            if not version.startswith("HTTP/"):
                raise _BadRequestException(
                        "Malformed HTTP version in HTTP Request-Line")
            headers = httputil.HTTPHeaders.parse(data[eol:])
            peer = self.transport.getPeer()
            if isinstance(peer, address.UNIXAddress):
                remote_ip = "unix:%s" % self.transport.getHost().name
            else:
                remote_ip = self.transport.getPeer().host

            self._request = HTTPRequest(
                connection=self, method=method, uri=uri, version=version,
                headers=headers, remote_ip=remote_ip)

            content_length = int(headers.get("Content-Length", 0))
            if content_length:
                if headers.get("Expect") == "100-continue":
                    self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")
                self.content_length = content_length
                self.setRawMode()
                return

            self.request_callback(self._request)
        except _BadRequestException, e:
            log.msg("Malformed HTTP request from %s: %s",
                    self.transport.getPeer().host, e)
            self.transport.loseConnection()

    def _on_request_body(self, data):
        self._request.body = data
        content_type = self._request.headers.get("Content-Type", "")
        if self._request.method in ("POST", "PATCH", "PUT"):
            if content_type.startswith("application/x-www-form-urlencoded"):
                arguments = parse_qs_bytes(native_str(self._request.body))
                for name, values in arguments.iteritems():
                    values = [v for v in values if v]
                    if values:
                        self._request.arguments.setdefault(name,
                                                           []).extend(values)
            elif content_type.startswith("multipart/form-data"):
                fields = content_type.split(";")
                for field in fields:
                    k, sep, v, = field.strip().partition("=")
                    if k == "boundary" and v:
                        httputil.parse_multipart_form_data(
                            utf8(v), data,
                            self._request.arguments,
                            self._request.files)
                        break
                else:
                    log.msg("Invalid multipart/form-data")
        self.request_callback(self._request)

class HTTPRequest(object):
    """A single HTTP request.

    All attributes are type `str` unless otherwise noted.

    .. attribute:: method

       HTTP request method, e.g. "GET" or "POST"

    .. attribute:: uri

       The requested uri.

    .. attribute:: path

       The path portion of `uri`

    .. attribute:: query

       The query portion of `uri`

    .. attribute:: version

       HTTP version specified in request, e.g. "HTTP/1.1"

    .. attribute:: headers

       `HTTPHeader` dictionary-like object for request headers.  Acts like
       a case-insensitive dictionary with additional methods for repeated
       headers.

    .. attribute:: body

       Request body, if present, as a byte string.

    .. attribute:: remote_ip

       Client's IP address as a string.  If `HTTPConnection.xheaders` is set,
       will pass along the real IP address provided by a load balancer
       in the ``X-Real-Ip`` header

    .. attribute:: protocol

       The protocol used, either "http" or "https".
       If `HTTPConnection.xheaders` is set, will pass along the protocol used
       by a load balancer if
       reported via an ``X-Scheme`` header.

    .. attribute:: host

       The requested hostname, usually taken from the ``Host`` header.

    .. attribute:: arguments

       GET/POST arguments are available in the arguments property, which
       maps arguments names to lists of values (to support multiple values
       for individual names). Names are of type `str`, while arguments
       are byte strings.  Note that this is different from
       `RequestHandler.get_argument`, which returns argument values as
       unicode strings.

    .. attribute:: files

       File uploads are available in the files property, which maps file
       names to lists of :class:`HTTPFile`.

    .. attribute:: connection

       An HTTP request is attached to a single HTTP connection, which can
       be accessed through the "connection" attribute. Since connections
       are typically kept open in HTTP/1.1, multiple requests can be handled
       sequentially on a single connection.
    """
    def __init__(self, method, uri, version="HTTP/1.0", headers=None,
                 body=None, remote_ip=None, protocol=None, host=None,
                 files=None, connection=None):
        self.method = method
        self.uri = uri
        self.version = version
        self.headers = headers or httputil.HTTPHeaders()
        self.body = body or ""
        if connection and connection.xheaders:
            # Squid uses X-Forwarded-For, others use X-Real-Ip
            self.remote_ip = self.headers.get(
                "X-Real-Ip", self.headers.get("X-Forwarded-For", remote_ip))
            if not self._valid_ip(self.remote_ip):
                self.remote_ip = remote_ip
            # AWS uses X-Forwarded-Proto
            self.protocol = self.headers.get("X-Scheme",
                            self.headers.get("X-Forwarded-Proto", protocol))
            if self.protocol not in ("http", "https"):
                self.protocol = "http"
        else:
            self.remote_ip = remote_ip
            if connection and interfaces.ISSLTransport.providedBy(
                                                        connection.transport):
                self.protocol = "https"
            else:
                self.protocol = "http"
        self.host = host or self.headers.get("Host") or "127.0.0.1"
        self.files = files or {}
        self.connection = connection
        self._start_time = time.time()
        self._finish_time = None

        self.path, sep, self.query = uri.partition("?")
        self.arguments = parse_qs_bytes(self.query, keep_blank_values=True)

    def supports_http_1_1(self):
        """Returns True if this request supports HTTP/1.1 semantics"""
        return self.version == "HTTP/1.1"

    @property
    def cookies(self):
        """A dictionary of Cookie.Morsel objects."""
        if not hasattr(self, "_cookies"):
            self._cookies = Cookie.SimpleCookie()
            if "Cookie" in self.headers:
                try:
                    self._cookies.load(
                        native_str(self.headers["Cookie"]))
                except Exception:
                    self._cookies = {}
        return self._cookies

    def write(self, chunk):
        """Writes the given chunk to the response stream."""
        assert isinstance(chunk, bytes_type)
        self.connection.write(chunk)

    def finish(self):
        """Finishes this HTTP request on the open connection."""
        self.connection.finish()
        self._finish_time = time.time()

    def full_url(self):
        """Reconstructs the full URL for this request."""
        return self.protocol + "://" + self.host + self.uri

    def request_time(self):
        """Returns the amount of time it took for this request to execute."""
        if self._finish_time is None:
            return time.time() - self._start_time
        else:
            return self._finish_time - self._start_time

    def notifyFinish(self):
        """Returns a Deferred object, which is fired when the request is
        finished and the connection is closed.
        """
        return self.connection.notifyFinish()

    def __repr__(self):
        attrs = ("protocol", "host", "method", "uri", "version", "remote_ip",
                 "body")
        args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs])
        return "%s(%s, headers=%s)" % (
            self.__class__.__name__, args, dict(self.headers))

    def _valid_ip(self, ip):
        try:
            res = socket.getaddrinfo(ip, 0, socket.AF_UNSPEC,
                                     socket.SOCK_STREAM,
                                     0, socket.AI_NUMERICHOST)
            return bool(res)
        except socket.gaierror, e:
            if e.args[0] == socket.EAI_NONAME:
                return False
            raise
        return True
<commit_msg>fix on getPeer<commit_after># coding: utf-8
#
# Copyright 2010 Alexandre Fiori
# based on the original Tornado by Facebook
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""A non-blocking, single-threaded HTTP server.

Typical applications have little direct interaction with the `HTTPConnection`
class, which is the HTTP parser executed on incoming connections.

It is a protocol class that inherits Twisted's `LineReceiver
<http://twistedmatrix.com/documents/current/api/
twisted.protocols.basic.LineReceiver.html>`_, and is usually created by
`cyclone.web.Application`, our connection factory.

This module also defines the `HTTPRequest` class which is exposed via
`cyclone.web.RequestHandler.request`.
"""

from __future__ import absolute_import, division, with_statement

import Cookie
import socket
import time

from twisted.python import log
from twisted.protocols import basic
from twisted.internet import address
from twisted.internet import defer
from twisted.internet import interfaces

from cyclone.escape import utf8, native_str, parse_qs_bytes
from cyclone import httputil
from cyclone.util import bytes_type

class _BadRequestException(Exception):
    """Exception class for malformed HTTP requests."""
    pass

class HTTPConnection(basic.LineReceiver):
    """Handles a connection to an HTTP client, executing HTTP requests.

    We parse HTTP headers and bodies, and execute the request callback
    until the HTTP conection is closed.

    If ``xheaders`` is ``True``, we support the ``X-Real-Ip`` and ``X-Scheme``
    headers, which override the remote IP and HTTP scheme for all requests.
    These headers are useful when running Tornado behind a reverse proxy or
    load balancer.
    """
    delimiter = "\r\n"

    def connectionMade(self):
        self._headersbuffer = []
        self._contentbuffer = []
        self._finish_callback = None
        self.no_keep_alive = False
        self.content_length = None
        self.request_callback = self.factory
        self.xheaders = self.factory.settings.get('xheaders', False)
        self._request = None
        self._request_finished = False

    def connectionLost(self, reason):
        if self._finish_callback:
            self._finish_callback.callback(reason.getErrorMessage())
            self._finish_callback = None

    def notifyFinish(self):
        if self._finish_callback is None:
            self._finish_callback = defer.Deferred()
        return self._finish_callback

    def lineReceived(self, line):
        if line:
            self._headersbuffer.append(line + self.delimiter)
        else:
            buff = "".join(self._headersbuffer)
            self._headersbuffer = []
            self._on_headers(buff)

    def rawDataReceived(self, data):
        if self.content_length is not None:
            data, rest = data[:self.content_length], data[self.content_length:]
            self.content_length -= len(data)
        else:
            rest = ''

        self._contentbuffer.append(data)
        if self.content_length == 0:
            buff = "".join(self._contentbuffer)
            self._contentbuffer = []
            self.content_length = None
            self._on_request_body(buff)
            self.setLineMode(rest)

    def write(self, chunk):
        assert self._request, "Request closed"
        self.transport.write(chunk)

    def finish(self):
        assert self._request, "Request closed"
        self._request_finished = True
        self._finish_request()

    def _on_write_complete(self):
        if self._request_finished:
            self._finish_request()

    def _finish_request(self):
        if self.no_keep_alive:
            disconnect = True
        else:
            connection_header = self._request.headers.get("Connection")
            if self._request.supports_http_1_1():
                disconnect = connection_header == "close"
            elif ("Content-Length" in self._request.headers
                    or self._request.method in ("HEAD", "GET")):
                disconnect = connection_header != "Keep-Alive"
            else:
                disconnect = True

        if self._finish_callback:
            self._finish_callback.callback(None)
            self._finish_callback = None
        self._request = None
        self._request_finished = False
        if disconnect is True:
            self.transport.loseConnection()

    def _on_headers(self, data):
        try:
            data = native_str(data.decode("latin1"))
            eol = data.find("\r\n")
            start_line = data[:eol]
            try:
                method, uri, version = start_line.split(" ")
            except ValueError:
                raise _BadRequestException("Malformed HTTP request line")
            if not version.startswith("HTTP/"):
                raise _BadRequestException(
                        "Malformed HTTP version in HTTP Request-Line")
            headers = httputil.HTTPHeaders.parse(data[eol:])
            peer = self.transport.getPeer()
            if isinstance(peer, address.UNIXAddress):
                remote_ip = "unix:%s" % self.transport.getHost().name
            else:
                remote_ip = self.transport.getPeer().host

            self._request = HTTPRequest(
                connection=self, method=method, uri=uri, version=version,
                headers=headers, remote_ip=remote_ip)

            content_length = int(headers.get("Content-Length", 0))
            if content_length:
                if headers.get("Expect") == "100-continue":
                    self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")
                self.content_length = content_length
                self.setRawMode()
                return

            self.request_callback(self._request)
        except _BadRequestException, e:
            log.msg("Malformed HTTP request from %s: %s", remote_ip, e)
            self.transport.loseConnection()

    def _on_request_body(self, data):
        self._request.body = data
        content_type = self._request.headers.get("Content-Type", "")
        if self._request.method in ("POST", "PATCH", "PUT"):
            if content_type.startswith("application/x-www-form-urlencoded"):
                arguments = parse_qs_bytes(native_str(self._request.body))
                for name, values in arguments.iteritems():
                    values = [v for v in values if v]
                    if values:
                        self._request.arguments.setdefault(name,
                                                           []).extend(values)
            elif content_type.startswith("multipart/form-data"):
                fields = content_type.split(";")
                for field in fields:
                    k, sep, v, = field.strip().partition("=")
                    if k == "boundary" and v:
                        httputil.parse_multipart_form_data(
                            utf8(v), data,
                            self._request.arguments,
                            self._request.files)
                        break
                else:
                    log.msg("Invalid multipart/form-data")
        self.request_callback(self._request)

class HTTPRequest(object):
    """A single HTTP request.

    All attributes are type `str` unless otherwise noted.

    .. attribute:: method

       HTTP request method, e.g. "GET" or "POST"

    .. attribute:: uri

       The requested uri.

    .. attribute:: path

       The path portion of `uri`

    .. attribute:: query

       The query portion of `uri`

    .. attribute:: version

       HTTP version specified in request, e.g. "HTTP/1.1"

    .. attribute:: headers

       `HTTPHeader` dictionary-like object for request headers.  Acts like
       a case-insensitive dictionary with additional methods for repeated
       headers.

    .. attribute:: body

       Request body, if present, as a byte string.

    .. attribute:: remote_ip

       Client's IP address as a string.  If `HTTPConnection.xheaders` is set,
       will pass along the real IP address provided by a load balancer
       in the ``X-Real-Ip`` header

    .. attribute:: protocol

       The protocol used, either "http" or "https".
       If `HTTPConnection.xheaders` is set, will pass along the protocol used
       by a load balancer if
       reported via an ``X-Scheme`` header.

    .. attribute:: host

       The requested hostname, usually taken from the ``Host`` header.

    .. attribute:: arguments

       GET/POST arguments are available in the arguments property, which
       maps arguments names to lists of values (to support multiple values
       for individual names). Names are of type `str`, while arguments
       are byte strings.  Note that this is different from
       `RequestHandler.get_argument`, which returns argument values as
       unicode strings.

    .. attribute:: files

       File uploads are available in the files property, which maps file
       names to lists of :class:`HTTPFile`.

    .. attribute:: connection

       An HTTP request is attached to a single HTTP connection, which can
       be accessed through the "connection" attribute. Since connections
       are typically kept open in HTTP/1.1, multiple requests can be handled
       sequentially on a single connection.
    """
    def __init__(self, method, uri, version="HTTP/1.0", headers=None,
                 body=None, remote_ip=None, protocol=None, host=None,
                 files=None, connection=None):
        self.method = method
        self.uri = uri
        self.version = version
        self.headers = headers or httputil.HTTPHeaders()
        self.body = body or ""
        if connection and connection.xheaders:
            # Squid uses X-Forwarded-For, others use X-Real-Ip
            self.remote_ip = self.headers.get(
                "X-Real-Ip", self.headers.get("X-Forwarded-For", remote_ip))
            if not self._valid_ip(self.remote_ip):
                self.remote_ip = remote_ip
            # AWS uses X-Forwarded-Proto
            self.protocol = self.headers.get("X-Scheme",
                            self.headers.get("X-Forwarded-Proto", protocol))
            if self.protocol not in ("http", "https"):
                self.protocol = "http"
        else:
            self.remote_ip = remote_ip
            if connection and interfaces.ISSLTransport.providedBy(
                                                        connection.transport):
                self.protocol = "https"
            else:
                self.protocol = "http"
        self.host = host or self.headers.get("Host") or "127.0.0.1"
        self.files = files or {}
        self.connection = connection
        self._start_time = time.time()
        self._finish_time = None

        self.path, sep, self.query = uri.partition("?")
        self.arguments = parse_qs_bytes(self.query, keep_blank_values=True)

    def supports_http_1_1(self):
        """Returns True if this request supports HTTP/1.1 semantics"""
        return self.version == "HTTP/1.1"

    @property
    def cookies(self):
        """A dictionary of Cookie.Morsel objects."""
        if not hasattr(self, "_cookies"):
            self._cookies = Cookie.SimpleCookie()
            if "Cookie" in self.headers:
                try:
                    self._cookies.load(
                        native_str(self.headers["Cookie"]))
                except Exception:
                    self._cookies = {}
        return self._cookies

    def write(self, chunk):
        """Writes the given chunk to the response stream."""
        assert isinstance(chunk, bytes_type)
        self.connection.write(chunk)

    def finish(self):
        """Finishes this HTTP request on the open connection."""
        self.connection.finish()
        self._finish_time = time.time()

    def full_url(self):
        """Reconstructs the full URL for this request."""
        return self.protocol + "://" + self.host + self.uri

    def request_time(self):
        """Returns the amount of time it took for this request to execute."""
        if self._finish_time is None:
            return time.time() - self._start_time
        else:
            return self._finish_time - self._start_time

    def notifyFinish(self):
        """Returns a Deferred object, which is fired when the request is
        finished and the connection is closed.
        """
        return self.connection.notifyFinish()

    def __repr__(self):
        attrs = ("protocol", "host", "method", "uri", "version", "remote_ip",
                 "body")
        args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs])
        return "%s(%s, headers=%s)" % (
            self.__class__.__name__, args, dict(self.headers))

    def _valid_ip(self, ip):
        try:
            res = socket.getaddrinfo(ip, 0, socket.AF_UNSPEC,
                                     socket.SOCK_STREAM,
                                     0, socket.AI_NUMERICHOST)
            return bool(res)
        except socket.gaierror, e:
            if e.args[0] == socket.EAI_NONAME:
                return False
            raise
        return True
<|endoftext|>


<commit_before>#! /usr/bin/env python

"""Creates run length distribution figures."""

from __future__ import absolute_import

import os
import numpy
import matplotlib.pyplot as plt
from pdb import set_trace

#__all__ = []

rldColors = ('k', 'c', 'm', 'r', 'k', 'c', 'm', 'r', 'k', 'c', 'm', 'r')
rldUnsuccColors = ('k', 'c', 'm', 'k', 'c', 'm', 'k', 'c', 'm', 'k', 'c', 'm')  # should not be too short
# Used as a global to store the largest xmax and align the FV ECD figures.
fmax = None
evalfmax = None

def beautifyRLD(figHandle, figureName, maxEvalsF, fileFormat=('png', 'eps'),
                text=None, verbose=True):
    """Format the figure of the run length distribution and save into files."""
    axisHandle = figHandle.gca()
    axisHandle.set_xscale('log')
    plt.xlim(1.0, maxEvalsF)
    plt.ylim(0.0, 1.0)
    axisHandle.set_xlabel('log10 of FEvals / DIM')
    axisHandle.set_ylabel('proportion of trials')
    # Grid options
    axisHandle.grid('True')
    xtic = axisHandle.get_xticks()
    newxtic = []
    for j in xtic:
        newxtic.append('%d' % round(numpy.log10(j)))
    axisHandle.set_xticklabels(newxtic)

    plt.text(0.5, 0.93, text, horizontalalignment="center",
             transform=axisHandle.transAxes)
             #bbox=dict(ec='k', fill=False),

    #set_trace()
    plt.legend(loc='best')
    #if legend:
        #axisHandle.legend(legend, locLegend)

    # Save figure
    for entry in fileFormat:
        plt.savefig(figureName + '.' + entry, dpi = 300,
                    format = entry)
        if verbose:
            print 'Wrote figure in %s.' %(figureName + '.' + entry)

def plotRLDistr(indexEntries, fvalueToReach, maxEvalsF, verbose=True):
    """Creates run length distributions from a sequence of indexEntries.

    Keyword arguments:
    indexEntries
    fvalueToReach
    verbose

    Outputs:
    res -- resulting plot.
    fsolved -- number of different functions solved.
    funcs -- number of different function considered.
    """

    x = []
    nn = 0
    fsolved = set()
    funcs = set()
    for i in indexEntries:
        funcs.add(i.funcId)
        for j in i.hData:
            if j[0] <= fvalueToReach:
                #This loop is needed because though some number of function
                #evaluations might be below maxEvals, the target function value
                #might not be reached yet. This is because the horizontal data
                #do not go to maxEvals.

                for k in range(1, i.nbRuns() + 1):
                    if j[i.nbRuns() + k] <= fvalueToReach:
                        x.append(j[k] / i.dim)
                        fsolved.add(i.funcId)
                break
        nn += i.nbRuns()

    label = ('%+d:%d/%d' %
             (numpy.log10(fvalueToReach), len(fsolved), len(funcs)))
    n = len(x)
    if n == 0:
        res = plt.plot([], [], label=label)
    else:
        x.sort()
        x2 = numpy.hstack([numpy.repeat(x, 2), maxEvalsF])
        #maxEvalsF: used for the limit of the plot.
        y2 = numpy.hstack([0.0,
                           numpy.repeat(numpy.arange(1, n+1) / float(nn), 2)])
        res = plt.plot(x2, y2, label=label)

    return res#, fsolved, funcs

def beautifyFVD(figHandle, figureName, fileFormat=('png','eps'),
                isStoringXMax=False, text=None, verbose=True):
    """Formats the figure of the run length distribution.

    Keyword arguments:
    isStoringMaxF -- if set to True, the first call BeautifyVD sets the global
                     fmax and all subsequent call will have the same maximum
                     xlim.
    """

    axisHandle = figHandle.gca()
    axisHandle.set_xscale('log')

    if isStoringXMax:
        global fmax
    else:
        fmax = None

    if not fmax:
        xmin, fmax = plt.xlim()
    plt.xlim(1., fmax)

    #axisHandle.invert_xaxis()
    plt.ylim(0.0, 1.0)
    axisHandle.set_xlabel('log10 of Df / Dftarget')
    # axisHandle.set_ylabel('proportion of successful trials')
    # Grid options
    axisHandle.grid('True')
    xtic = axisHandle.get_xticks()
    newxtic = []
    for j in xtic:
        newxtic.append('%d' % round(numpy.log10(j)))
    axisHandle.set_xticklabels(newxtic)
    axisHandle.set_yticklabels(())

    plt.text(0.98, 0.02, text, horizontalalignment="right",
             transform=axisHandle.transAxes)
             #bbox=dict(ec='k', fill=False),

    # Save figure
    for entry in fileFormat:
        plt.savefig(figureName + '.' + entry, dpi = 300,
                    format = entry)
        if verbose:
            print 'Wrote figure in %s.' %(figureName + '.' + entry)

def plotFVDistr(indexEntries, fvalueToReach, maxEvalsF, verbose=True):
    """Creates empirical cumulative distribution functions of final function
    values plot from a sequence of indexEntries.

    Keyword arguments:
    indexEntries -- sequence of IndexEntry to process.
    fvalueToReach -- float used for the lower limit of the plot
    maxEvalsF -- indicates which vertical data to display.
    verbose -- controls verbosity.

    Outputs: a plot of a run length distribution.
    """

    x = []
    nn = 0
    for i in indexEntries:
        for j in i.vData:
            if j[0] >= maxEvalsF * i.dim:
                break
        x.extend(j[i.nbRuns()+1:] / fvalueToReach)
        nn += i.nbRuns()

    x.sort()
    x2 = numpy.hstack([numpy.repeat(x, 2)])
    #not efficient if some vals are repeated a lot
    #y2 = numpy.hstack([0.0, numpy.repeat(numpy.arange(1, n)/float(nn), 2),
                       #float(n)/nn, float(n)/nn])
    y2 = numpy.hstack([0.0, numpy.repeat(numpy.arange(1, nn)/float(nn), 2),
                       1.0])
    #set_trace()
    res = plt.plot(x2, y2)

    return res

def main(indexEntriesAlg0, indexEntriesAlg1, valuesOfInterest,
         isStoringXMax=False, outputdir='', info='default', verbose=True):
    """Generate figures of empirical cumulative distribution functions.

    Keyword arguments:
    indexEntries -- list of IndexEntry instances to process.
    valuesOfInterest -- target function values to be displayed.
    isStoringXMax -- if set to True, the first call BeautifyVD sets the globals
                     fmax and maxEvals and all subsequent calls will use these
                     values as rightmost xlim in the generated figures.
     -- if set to True, the first call BeautifyVD sets the global
                     fmax and all subsequent call will have the same maximum
                     xlim.
    outputdir -- output directory (must exist)
    info --- string suffix for output file names.

    Outputs:
    Image files of the empirical cumulative distribution functions.
    """

    #sortedIndexEntries = sortIndexEntries(indexEntries)

    plt.rc("axes", labelsize=20, titlesize=24)
    plt.rc("xtick", labelsize=20)
    plt.rc("ytick", labelsize=20)
    plt.rc("font", size=20)
    plt.rc("legend", fontsize=20)

    maxEvalsFactor = max(i.mMaxEvals()/i.dim for i in indexEntriesAlg0+indexEntriesAlg1) ** 1.05

    set_trace()

    dictFunc0 = indexEntriesAlg0.dictByFunc()
    dictFunc1 = indexEntriesAlg1.dictByFunc()
    funcs = set.union(set(dictFunc0), set(dictFunc0))

    #maxEvalsFactorCeil = numpy.power(10,
                                     #numpy.ceil(numpy.log10(maxEvalsFactor)))

    if isStoringXMax:
        global evalfmax
    else:
        evalfmax = None

    if not evalfmax:
        evalfmax = maxEvalsFactor

    figureName = os.path.join(outputdir,'pprldistr%s' %('_' + info))
    fig = plt.figure()
    legend = []
    for j in range(len(valuesOfInterest)):
        tmp = plotRLDistr(indexEntries, valuesOfInterest[j], evalfmax,
                          verbose)
        #set_trace()
        if not tmp is None:
            plt.setp(tmp, 'color', rldColors[j])
            #set_trace()
            #legend.append('%+d:%d/%d' %
                          #(numpy.log10(valuesOfInterest[j]), len(fsolved),
                           #len(f)))
            if rldColors[j] == 'r':  # 1e-8 in bold
                plt.setp(tmp, 'linewidth', 3)

    funcs = list(i.funcId for i in indexEntries)
    if len(funcs) > 1:
        text = 'f%d-%d' %(min(funcs), max(funcs))
    else:
        text = 'f%d' %(funcs[0])

    beautifyRLD(fig, figureName, evalfmax, text=text, verbose=verbose)
    plt.close(fig)

    #figureName = os.path.join(outputdir,'ppfvdistr_%s' %(info))
    #fig = plt.figure()
    #for j in range(len(valuesOfInterest)):
        ##set_trace()
        #tmp = plotFVDistr(indexEntries, valuesOfInterest[j],
                          #maxEvalsFactor, verbose=verbose)
        ##if not tmp is None:
        #plt.setp(tmp, 'color', rldColors[j])
        #if rldColors [j] == 'r':  # 1e-8 in bold
            #plt.setp(tmp, 'linewidth', 3)

    #tmp = numpy.floor(numpy.log10(maxEvalsFactor))
    ## coloring left to right:
    ##maxEvalsF = numpy.power(10, numpy.arange(tmp, 0, -1) - 1)
    ## coloring right to left:
    #maxEvalsF = numpy.power(10, numpy.arange(0, tmp))

    ##The last index of valuesOfInterest is still used in this loop.
    ##set_trace()
    #for k in range(len(maxEvalsF)):
        #tmp = plotFVDistr(indexEntries, valuesOfInterest[j],
                          #maxEvalsF=maxEvalsF[k], verbose=verbose)
        #plt.setp(tmp, 'color', rldUnsuccColors[k])

    #beautifyFVD(fig, figureName, text=text, isStoringXMax=isStoringXMax,
                #verbose=verbose)

    #plt.close(fig)

    plt.rcdefaults()
<commit_msg>Implemented ECDF of log(ERT1/ERT0) for an absolute target function value.<commit_after>#! /usr/bin/env python

"""Creates run length distribution figures."""

from __future__ import absolute_import

import os
import numpy
import matplotlib.pyplot as plt
from bbob_pproc import bootstrap
from pdb import set_trace

#__all__ = []

rldColors = ('k', 'c', 'm', 'r', 'k', 'c', 'm', 'r', 'k', 'c', 'm', 'r')
rldUnsuccColors = ('k', 'c', 'm', 'k', 'c', 'm', 'k', 'c', 'm', 'k', 'c', 'm')  # should not be too short
# Used as a global to store the largest xmax and align the FV ECD figures.
fmax = None
evalfmax = None

def beautify(figHandle, figureName, fileFormat=('png', 'eps'),
             text=None, verbose=True):
    """Format the figure of the run length distribution and save into files."""
    axisHandle = figHandle.gca()
    #try:
    axisHandle.set_xscale('log')
    #plt.xlim(1.0, maxEvalsF)
    plt.ylim(0.0, 1.0)
    axisHandle.set_xlabel('log10 of ERT1/ERT0')
    axisHandle.set_ylabel('proportion of instances...')
    # Grid options
    axisHandle.grid('True')
    xtic = axisHandle.get_xticks()
    newxtic = []
    for j in xtic:
        newxtic.append('%d' % round(numpy.log10(j)))
    axisHandle.set_xticklabels(newxtic)

    #plt.text(0.5, 0.93, text, horizontalalignment="center",
             #transform=axisHandle.transAxes)
             #bbox=dict(ec='k', fill=False),

    #set_trace()
    #plt.legend(loc='best')
    #if legend:
        #axisHandle.legend(legend, locLegend)

    # Save figure
    for entry in fileFormat:
        plt.savefig(figureName + '.' + entry, dpi = 300,
                    format = entry)
        if verbose:
            print 'Wrote figure in %s.' %(figureName + '.' + entry)

def plotLogAbs(indexEntries0, indexEntries1, fvalueToReach, isByInstance=False,
               verbose=True):
    """Creates one run length distribution from a sequence of indexEntries.

    Keyword arguments:
    indexEntries0 -- reference
    indexEntries1
    fvalueToReach
    maxEvalsF
    isByInstance -- loop over the function instances instead of the functions
    verbose

    Outputs:
    res -- resulting plot.
    fsolved -- number of different functions solved.
    funcs -- number of different function considered.
    """

    x = []
    nn = 0
    fsolved = set()
    funcs = set()
    funIndexEntries0 = indexEntries0.dictByFunc()
    funIndexEntries1 = indexEntries1.dictByFunc()
    for func in set(funIndexEntries0.keys()).union(funIndexEntries1.keys()):
        try:
            i0 = funIndexEntries0[func][0]
            i1 = funIndexEntries1[func][0]
        except KeyError:
            continue

        if isByInstance:
            dictinstance = []

        ERT = []
        if not isByInstance:
            for i, entry in enumerate((i0, i1)):
                for j in entry.hData:
                    if j[0] <= fvalueToReach:
                        break
                success = (j[entry.nbRuns()+1:] <= fvalueToReach)
                fevals = j[1:entry.nbRuns()+1]
                for j, issuccess in enumerate(success):
                    if not issuccess:
                        fevals[j] = entry.vData[-1, 1+j]
                tmp = bootstrap.sp(fevals, issuccessful=success)
                #set_trace()
                if tmp[2]: #success probability is larger than 0
                    ERT.append(tmp[0])
                else:
                    set_trace()
                    ERT.append(numpy.inf)
            if not all(numpy.isinf(ERT)):
                x.append(ERT[1]/ERT[0])
                nn += 1
            #TODO check it is the same as ERT[1]
        else:
            for i, entry in enumerate((i0, i1)):
                dictinstance = {}
                for j in range(len(entry.itrials)):
                    dictinstance.setdefault(entry.itrials[j], []).append(j)
                ERT.append(dictinstance.copy())
                for k in dictinstance:
                    for j in entry.hData:
                        if j[0] <= fvalueToReach:
                            break
                    success = (j[list(entry.nbRuns()+i for i in dictinstance[k])] <= fvalueToReach)
                    fevals = j[list(1+i for i in dictinstance[k])]
                    for l, issuccess in enumerate(success):
                        if issuccess:
                            fevals[l] = entry.vData[-1, 1+dictinstance[k][l]]
                    tmp = bootstrap.sp(fevals, issuccessful=success)
                    if tmp[2]: #success probability is larger than 0
                        ERT[i][k] = tmp[0]
                    else:
                        ERT[i][k] = numpy.inf
                    ERT[i][k] = tmp[0]
            s0 = set(ERT[0])
            s1 = set(ERT[1])
            #Could be done simpler
            for j in s0 - s1:
                x.append(0)
            for j in s0 & s1:
                x.append(ERT[1][j]/ERT[0][j])
            for j in s1 - s0:
                x.append(inf)
            nn += len(s0 | s1)

    #set_trace()
    #label = ('%+d:%d/%d' %
             #(numpy.log10(fvalueToReach), len(fsolved), len(funcs)))
    n = len(x)
    if n == 0:
        res = plt.plot([], [], label=label)
    else:
        x.sort()
        x2 = numpy.hstack([numpy.repeat(x, 2)])
        #maxEvalsF: used for the limit of the plot.
        y2 = numpy.hstack([0.0,
                           numpy.repeat(numpy.arange(1, n) / float(nn), 2),
                           n/float(nn)])
        res = plt.plot(x2, y2) #, label=label)

    return res#, fsolved, funcs

def main(indexEntriesAlg0, indexEntriesAlg1, valuesOfInterest,
         isStoringXMax=False, outputdir='', info='default', verbose=True):
    """Generate figures of empirical cumulative distribution functions.

    Keyword arguments:
    indexEntries -- list of IndexEntry instances to process.
    valuesOfInterest -- target function values to be displayed.
    isStoringXMax -- if set to True, the first call BeautifyVD sets the globals
                     fmax and maxEvals and all subsequent calls will use these
                     values as rightmost xlim in the generated figures.
     -- if set to True, the first call BeautifyVD sets the global
                     fmax and all subsequent call will have the same maximum
                     xlim.
    outputdir -- output directory (must exist)
    info --- string suffix for output file names.

    Outputs:
    Image files of the empirical cumulative distribution functions.
    """

    #sortedIndexEntries = sortIndexEntries(indexEntries)

    plt.rc("axes", labelsize=20, titlesize=24)
    plt.rc("xtick", labelsize=20)
    plt.rc("ytick", labelsize=20)
    plt.rc("font", size=20)
    plt.rc("legend", fontsize=20)

    #maxEvalsFactor = max(i.mMaxEvals()/i.dim for i in indexEntriesAlg0+indexEntriesAlg1) ** 1.05

    #set_trace()

    #dictFunc0 = indexEntriesAlg0.dictByFunc()
    #dictFunc1 = indexEntriesAlg1.dictByFunc()
    #funcs = set.union(set(dictFunc0), set(dictFunc0))

    #maxEvalsFactorCeil = numpy.power(10,
                                     #numpy.ceil(numpy.log10(maxEvalsFactor)))

    #if isStoringXMax:
        #global evalfmax
    #else:
        #evalfmax = None

    #if not evalfmax:
        #evalfmax = maxEvalsFactor

    figureName = os.path.join(outputdir,'pplog%s' %('_' + info))
    fig = plt.figure()
    legend = []
    for j in range(len(valuesOfInterest)):
        tmp = plotLogAbs(indexEntriesAlg0, indexEntriesAlg1,
                         valuesOfInterest[j], verbose=verbose)
        #set_trace()
        if not tmp is None:
            plt.setp(tmp, 'color', rldColors[j])
            #set_trace()
            #legend.append('%+d:%d/%d' %
                          #(numpy.log10(valuesOfInterest[j]), len(fsolved),
                           #len(f)))
            if rldColors[j] == 'r':  # 1e-8 in bold
                plt.setp(tmp, 'linewidth', 3)

    #funcs = list(i.funcId for i in indexEntries)
    #if len(funcs) > 1:
        #text = 'f%d-%d' %(min(funcs), max(funcs))
    #else:
        #text = 'f%d' %(funcs[0])
    beautify(fig, figureName, verbose=verbose)
    plt.close(fig)

    #figureName = os.path.join(outputdir,'ppfvdistr_%s' %(info))
    #fig = plt.figure()
    #for j in range(len(valuesOfInterest)):
        ##set_trace()
        #tmp = plotFVDistr(indexEntries, valuesOfInterest[j],
                          #maxEvalsFactor, verbose=verbose)
        ##if not tmp is None:
        #plt.setp(tmp, 'color', rldColors[j])
        #if rldColors [j] == 'r':  # 1e-8 in bold
            #plt.setp(tmp, 'linewidth', 3)

    #tmp = numpy.floor(numpy.log10(maxEvalsFactor))
    ## coloring left to right:
    ##maxEvalsF = numpy.power(10, numpy.arange(tmp, 0, -1) - 1)
    ## coloring right to left:
    #maxEvalsF = numpy.power(10, numpy.arange(0, tmp))

    ##The last index of valuesOfInterest is still used in this loop.
    ##set_trace()
    #for k in range(len(maxEvalsF)):
        #tmp = plotFVDistr(indexEntries, valuesOfInterest[j],
                          #maxEvalsF=maxEvalsF[k], verbose=verbose)
        #plt.setp(tmp, 'color', rldUnsuccColors[k])

    #beautifyFVD(fig, figureName, text=text, isStoringXMax=isStoringXMax,
                #verbose=verbose)

    #plt.close(fig)

    plt.rcdefaults()
<|endoftext|>


<commit_before>"""praw constants."""

__version__ = '4.0.0b12'

API_PATH = {
    'accept_mod_invite':      'api/accept_moderator_invite',
    'access_token_url':       'api/v1/access_token/',
    'approve':                'api/approve/',
    'authorize':              'api/v1/authorize/',
    'banned':                 'r/{subreddit}/about/banned/',
    'block':                  'api/block',
    'blocked':                'prefs/blocked/',
    'by_id':                  'by_id/',
    'captcha':                'captcha/',
    'clearflairtemplates':    'api/clearflairtemplates/',
    'collapse_message':       'api/collapse_message/',
    'comment':                'api/comment/',
    'comment_replies':        'message/comments/',
    'comments':               'comments/',
    'compose':                'api/compose/',
    'contest_mode':           'api/set_contest_mode/',
    'contributor':            'r/{subreddit}/about/contributors/',
    'controversial':          'controversial/',
    'del':                    'api/del/',
    'deleteflair':            'api/deleteflair',
    'delete_redditor':        'api/delete_user',
    'delete_sr_header':       'r/{subreddit}/api/delete_sr_header',
    'delete_sr_image':        'r/{subreddit}/api/delete_sr_img',
    'distinguish':            'api/distinguish/',
    'domain':                 'domain/{domain}/',
    'duplicates':             'duplicates/{submission_id}/',
    'edit':                   'api/editusertext/',
    'edited':                 'r/{subreddit}/about/edited/',
    'flair':                  'r/{subreddit}/api/flair/',
    'flairconfig':            'api/flairconfig/',
    'flaircsv':               'r/{subreddit}/api/flaircsv/',
    'flairlist':              'r/{subreddit}/api/flairlist/',
    'flairselector':          'r/{subreddit}/api/flairselector/',
    'flairtemplate':          'api/flairtemplate/',
    'friend':                 'api/friend/',
    'friend_v1':              'api/v1/me/friends/{user}',
    'friends':                'api/v1/me/friends/',
    'gild_thing':             'api/v1/gold/gild/{fullname}/',
    'gild_user':              'api/v1/gold/give/{username}/',
    'help':                   'help/',
    'hide':                   'api/hide/',
    'ignore_reports':         'api/ignore_reports/',
    'inbox':                  'message/inbox/',
    'info':                   'api/info/',
    'leavecontributor':       'api/leavecontributor',
    'leavemoderator':         'api/leavemoderator',
    'liveabout':              'api/live/{id}/about/',
    'livecreate':             'api/live/create',
    'me':                     'api/v1/me',
    'mentions':               'message/mentions',
    'message':                'message/messages/{messageid}/',
    'messages':               'message/messages/',
    'moderator':              'r/{subreddit}/about/moderators/',
    'moderator_messages':     'r/{subreddit}/message/moderator/',
    'moderator_unread':       'r/{subreddit}/message/moderator/unread/',
    'modlog':                 'r/{subreddit}/about/log/',
    'modqueue':               'r/{subreddit}/about/modqueue/',
    'morechildren':           'api/morechildren/',
    'my_contributor':         'subreddits/mine/contributor/',
    'my_moderator':           'subreddits/mine/moderator/',
    'my_multireddits':        'api/multi/mine/',
    'my_subreddits':          'subreddits/mine/subscriber/',
    'new':                    'new/',
    'marknsfw':               'api/marknsfw/',
    'multireddit':            'user/{user}/m/{multi}/',
    'multireddit_api':        'api/multi/user/{user}/m/{multi}/',
    'multireddit_base':       'api/multi/',
    'multireddit_copy':       'api/multi/copy/',
    'multireddit_rename':     'api/multi/rename/',
    'multireddit_update':     'api/multi/user/{user}/m/{multi}/r/{subreddit}',
    'multireddit_user':       'api/multi/user/{user}/',
    'mute_sender':            'api/mute_message_author/',
    'muted':                  'r/{subreddit}/about/muted/',
    'read_message':           'api/read_message/',
    'reddit_url':             '/',
    'register':               'api/register/',
    'remove':                 'api/remove/',
    'report':                 'api/report/',
    'reports':                'r/{subreddit}/about/reports/',
    'rising':                 'rising/',
    'save':                   'api/save/',
    'saved':                  'saved/',
    'search':                 'r/{subreddit}/search/',
    'select_flair':           'r/{subreddit}/api/selectflair/',
    'sent':                   'message/sent/',
    'sticky':                 'r/{subreddit}/about/sticky/',
    'sticky_submission':      'api/set_subreddit_sticky/',
    'site_admin':             'api/site_admin/',
    'spam':                   'r/{subreddit}/about/spam/',
    'stylesheet':             'r/{subreddit}/about/stylesheet/',
    'sub_recommendations':    'api/recommend/sr/{subreddits}',
    'submission':             'comments/{id}/',
    'submission_replies':     'message/selfreply/',
    'submit':                 'api/submit/',
    'subreddit':              'r/{subreddit}/',
    'subreddit_about':        'r/{subreddit}/about/',
    'subreddit_comments':     'r/{subreddit}/comments/',
    'subreddit_css':          'api/subreddit_stylesheet/',
    'subreddit_random':       'r/{subreddit}/random/',
    'subreddit_settings':     'r/{subreddit}/about/edit/',
    'subreddit_traffic':      'r/{subreddit}/about/traffic/',
    'subreddits_default':     'subreddits/default/',
    'subreddits_gold':        'subreddits/gold/',
    'subreddits_new':         'subreddits/new/',
    'subreddits_popular':     'subreddits/popular/',
    'subreddits_name_search': 'api/search_reddit_names/',
    'subreddits_search':      'subreddits/search/',
    'subscribe':              'api/subscribe/',
    'suggested_sort':         'api/set_suggested_sort/',
    'top':                    'top/',
    'uncollapse_message':     'api/uncollapse_message/',
    'unfriend':               'api/unfriend/',
    'unhide':                 'api/unhide/',
    'unmarknsfw':             'api/unmarknsfw/',
    'unmoderated':            'r/{subreddit}/about/unmoderated/',
    'unmute_sender':          'api/unmute_message_author/',
    'unignore_reports':       'api/unignore_reports/',
    'unread':                 'message/unread/',
    'unread_message':         'api/unread_message/',
    'unsave':                 'api/unsave/',
    'upload_image':           'api/upload_sr_img',
    'user':                   'user/{user}/',
    'user_about':             'user/{user}/about/',
    'username_available':     'api/username_available/',
    'vote':                   'api/vote/',
    'wiki_edit':              'r/{subreddit}/api/wiki/edit/',
    'wiki_page':              'r/{subreddit}/wiki/{page}',
    'wiki_page_editor':       'r/{subreddit}/api/wiki/alloweditor/{method}',
    'wiki_page_settings':     'r/{subreddit}/wiki/settings/{page}',
    'wiki_pages':             'r/{subreddit}/wiki/pages/',
    'wikibanned':             'r/{subreddit}/about/wikibanned/',
    'wikicontributor':        'r/{subreddit}/about/wikicontributors/'}

JPEG_HEADER = b'\xff\xd8\xff'
MAX_IMAGE_SIZE = 512000
MIN_PNG_SIZE = 67
MIN_JPEG_SIZE = 128
PNG_HEADER = b'\x89\x50\x4e\x47\x0d\x0a\x1a\x0a'

USER_AGENT_FORMAT = '{{}} PRAW/{}'.format(__version__)
<commit_msg>Bump to 4.0.0b13.<commit_after>"""praw constants."""

__version__ = '4.0.0b13'

API_PATH = {
    'accept_mod_invite':      'api/accept_moderator_invite',
    'access_token_url':       'api/v1/access_token/',
    'approve':                'api/approve/',
    'authorize':              'api/v1/authorize/',
    'banned':                 'r/{subreddit}/about/banned/',
    'block':                  'api/block',
    'blocked':                'prefs/blocked/',
    'by_id':                  'by_id/',
    'captcha':                'captcha/',
    'clearflairtemplates':    'api/clearflairtemplates/',
    'collapse_message':       'api/collapse_message/',
    'comment':                'api/comment/',
    'comment_replies':        'message/comments/',
    'comments':               'comments/',
    'compose':                'api/compose/',
    'contest_mode':           'api/set_contest_mode/',
    'contributor':            'r/{subreddit}/about/contributors/',
    'controversial':          'controversial/',
    'del':                    'api/del/',
    'deleteflair':            'api/deleteflair',
    'delete_redditor':        'api/delete_user',
    'delete_sr_header':       'r/{subreddit}/api/delete_sr_header',
    'delete_sr_image':        'r/{subreddit}/api/delete_sr_img',
    'distinguish':            'api/distinguish/',
    'domain':                 'domain/{domain}/',
    'duplicates':             'duplicates/{submission_id}/',
    'edit':                   'api/editusertext/',
    'edited':                 'r/{subreddit}/about/edited/',
    'flair':                  'r/{subreddit}/api/flair/',
    'flairconfig':            'api/flairconfig/',
    'flaircsv':               'r/{subreddit}/api/flaircsv/',
    'flairlist':              'r/{subreddit}/api/flairlist/',
    'flairselector':          'r/{subreddit}/api/flairselector/',
    'flairtemplate':          'api/flairtemplate/',
    'friend':                 'api/friend/',
    'friend_v1':              'api/v1/me/friends/{user}',
    'friends':                'api/v1/me/friends/',
    'gild_thing':             'api/v1/gold/gild/{fullname}/',
    'gild_user':              'api/v1/gold/give/{username}/',
    'help':                   'help/',
    'hide':                   'api/hide/',
    'ignore_reports':         'api/ignore_reports/',
    'inbox':                  'message/inbox/',
    'info':                   'api/info/',
    'leavecontributor':       'api/leavecontributor',
    'leavemoderator':         'api/leavemoderator',
    'liveabout':              'api/live/{id}/about/',
    'livecreate':             'api/live/create',
    'me':                     'api/v1/me',
    'mentions':               'message/mentions',
    'message':                'message/messages/{messageid}/',
    'messages':               'message/messages/',
    'moderator':              'r/{subreddit}/about/moderators/',
    'moderator_messages':     'r/{subreddit}/message/moderator/',
    'moderator_unread':       'r/{subreddit}/message/moderator/unread/',
    'modlog':                 'r/{subreddit}/about/log/',
    'modqueue':               'r/{subreddit}/about/modqueue/',
    'morechildren':           'api/morechildren/',
    'my_contributor':         'subreddits/mine/contributor/',
    'my_moderator':           'subreddits/mine/moderator/',
    'my_multireddits':        'api/multi/mine/',
    'my_subreddits':          'subreddits/mine/subscriber/',
    'new':                    'new/',
    'marknsfw':               'api/marknsfw/',
    'multireddit':            'user/{user}/m/{multi}/',
    'multireddit_api':        'api/multi/user/{user}/m/{multi}/',
    'multireddit_base':       'api/multi/',
    'multireddit_copy':       'api/multi/copy/',
    'multireddit_rename':     'api/multi/rename/',
    'multireddit_update':     'api/multi/user/{user}/m/{multi}/r/{subreddit}',
    'multireddit_user':       'api/multi/user/{user}/',
    'mute_sender':            'api/mute_message_author/',
    'muted':                  'r/{subreddit}/about/muted/',
    'read_message':           'api/read_message/',
    'reddit_url':             '/',
    'register':               'api/register/',
    'remove':                 'api/remove/',
    'report':                 'api/report/',
    'reports':                'r/{subreddit}/about/reports/',
    'rising':                 'rising/',
    'save':                   'api/save/',
    'saved':                  'saved/',
    'search':                 'r/{subreddit}/search/',
    'select_flair':           'r/{subreddit}/api/selectflair/',
    'sent':                   'message/sent/',
    'sticky':                 'r/{subreddit}/about/sticky/',
    'sticky_submission':      'api/set_subreddit_sticky/',
    'site_admin':             'api/site_admin/',
    'spam':                   'r/{subreddit}/about/spam/',
    'stylesheet':             'r/{subreddit}/about/stylesheet/',
    'sub_recommendations':    'api/recommend/sr/{subreddits}',
    'submission':             'comments/{id}/',
    'submission_replies':     'message/selfreply/',
    'submit':                 'api/submit/',
    'subreddit':              'r/{subreddit}/',
    'subreddit_about':        'r/{subreddit}/about/',
    'subreddit_comments':     'r/{subreddit}/comments/',
    'subreddit_css':          'api/subreddit_stylesheet/',
    'subreddit_random':       'r/{subreddit}/random/',
    'subreddit_settings':     'r/{subreddit}/about/edit/',
    'subreddit_traffic':      'r/{subreddit}/about/traffic/',
    'subreddits_default':     'subreddits/default/',
    'subreddits_gold':        'subreddits/gold/',
    'subreddits_new':         'subreddits/new/',
    'subreddits_popular':     'subreddits/popular/',
    'subreddits_name_search': 'api/search_reddit_names/',
    'subreddits_search':      'subreddits/search/',
    'subscribe':              'api/subscribe/',
    'suggested_sort':         'api/set_suggested_sort/',
    'top':                    'top/',
    'uncollapse_message':     'api/uncollapse_message/',
    'unfriend':               'api/unfriend/',
    'unhide':                 'api/unhide/',
    'unmarknsfw':             'api/unmarknsfw/',
    'unmoderated':            'r/{subreddit}/about/unmoderated/',
    'unmute_sender':          'api/unmute_message_author/',
    'unignore_reports':       'api/unignore_reports/',
    'unread':                 'message/unread/',
    'unread_message':         'api/unread_message/',
    'unsave':                 'api/unsave/',
    'upload_image':           'api/upload_sr_img',
    'user':                   'user/{user}/',
    'user_about':             'user/{user}/about/',
    'username_available':     'api/username_available/',
    'vote':                   'api/vote/',
    'wiki_edit':              'r/{subreddit}/api/wiki/edit/',
    'wiki_page':              'r/{subreddit}/wiki/{page}',
    'wiki_page_editor':       'r/{subreddit}/api/wiki/alloweditor/{method}',
    'wiki_page_settings':     'r/{subreddit}/wiki/settings/{page}',
    'wiki_pages':             'r/{subreddit}/wiki/pages/',
    'wikibanned':             'r/{subreddit}/about/wikibanned/',
    'wikicontributor':        'r/{subreddit}/about/wikicontributors/'}

JPEG_HEADER = b'\xff\xd8\xff'
MAX_IMAGE_SIZE = 512000
MIN_PNG_SIZE = 67
MIN_JPEG_SIZE = 128
PNG_HEADER = b'\x89\x50\x4e\x47\x0d\x0a\x1a\x0a'

USER_AGENT_FORMAT = '{{}} PRAW/{}'.format(__version__)
<|endoftext|>


<commit_before>"""praw constants."""
import sys

__version__ = '6.1.1'

API_PATH = {
    'about_edited':           'r/{subreddit}/about/edited/',
    'about_log':              'r/{subreddit}/about/log/',
    'about_modqueue':         'r/{subreddit}/about/modqueue/',
    'about_reports':          'r/{subreddit}/about/reports/',
    'about_spam':             'r/{subreddit}/about/spam/',
    'about_sticky':           'r/{subreddit}/about/sticky/',
    'about_stylesheet':       'r/{subreddit}/about/stylesheet/',
    'about_traffic':          'r/{subreddit}/about/traffic/',
    'about_unmoderated':      'r/{subreddit}/about/unmoderated/',
    'accept_mod_invite':      'r/{subreddit}/api/accept_moderator_invite',
    'approve':                'api/approve/',
    'block':                  'api/block',
    'block_user':             '/api/block_user/',
    'blocked':                'prefs/blocked/',
    'collapse':               'api/collapse_message/',
    'comment':                'api/comment/',
    'comment_replies':        'message/comments/',
    'compose':                'api/compose/',
    'contest_mode':           'api/set_contest_mode/',
    'del':                    'api/del/',
    'delete_message':         'api/del_msg',
    'delete_sr_banner':       'r/{subreddit}/api/delete_sr_banner',
    'delete_sr_header':       'r/{subreddit}/api/delete_sr_header',
    'delete_sr_icon':         'r/{subreddit}/api/delete_sr_icon',
    'delete_sr_image':        'r/{subreddit}/api/delete_sr_img',
    'deleteflair':            'r/{subreddit}/api/deleteflair',
    'distinguish':            'api/distinguish/',
    'domain':                 'domain/{domain}/',
    'duplicates':             'duplicates/{submission_id}/',
    'edit':                   'api/editusertext/',
    'emoji_delete':           'api/v1/{subreddit}/emoji/{emoji_name}',
    'emoji_lease':            'api/v1/{subreddit}/emoji_asset_upload_s3.json',
    'emoji_list':             'api/v1/{subreddit}/emojis/all',
    'emoji_upload':           'api/v1/{subreddit}/emoji.json',
    'flair':                  'r/{subreddit}/api/flair/',
    'flairconfig':            'r/{subreddit}/api/flairconfig/',
    'flaircsv':               'r/{subreddit}/api/flaircsv/',
    'flairlist':              'r/{subreddit}/api/flairlist/',
    'flairselector':          'r/{subreddit}/api/flairselector/',
    'flairtemplate':          'r/{subreddit}/api/flairtemplate/',
    'flairtemplate_v2':       'r/{subreddit}/api/flairtemplate_v2',
    'flairtemplateclear':     'r/{subreddit}/api/clearflairtemplates/',
    'flairtemplatedelete':    'r/{subreddit}/api/deleteflairtemplate/',
    'friend':                 'r/{subreddit}/api/friend/',
    'friend_v1':              'api/v1/me/friends/{user}',
    'friends':                'api/v1/me/friends/',
    'gild_thing':             'api/v1/gold/gild/{fullname}/',
    'gild_user':              'api/v1/gold/give/{username}/',
    'hide':                   'api/hide/',
    'ignore_reports':         'api/ignore_reports/',
    'inbox':                  'message/inbox/',
    'info':                   'api/info/',
    'karma':                  'api/v1/me/karma',
    'leavecontributor':       'api/leavecontributor',
    'link_flair':             'r/{subreddit}/api/link_flair_v2',
    'list_banned':            'r/{subreddit}/about/banned/',
    'list_contributor':       'r/{subreddit}/about/contributors/',
    'list_moderator':         'r/{subreddit}/about/moderators/',
    'list_muted':             'r/{subreddit}/about/muted/',
    'list_wikibanned':        'r/{subreddit}/about/wikibanned/',
    'list_wikicontributor':   'r/{subreddit}/about/wikicontributors/',
    'live_accept_invite':     'api/live/{id}/accept_contributor_invite',
    'live_add_update':        'api/live/{id}/update',
    'live_close':             'api/live/{id}/close_thread',
    'live_contributors':      'live/{id}/contributors',
    'live_discussions':       'live/{id}/discussions',
    'live_focus':             'live/{thread_id}/updates/{update_id}',
    'live_info':              'api/live/by_id/{ids}',
    'live_invite':            'api/live/{id}/invite_contributor',
    'live_leave':             'api/live/{id}/leave_contributor',
    'live_now':               'api/live/happening_now',
    'live_remove_contrib':    'api/live/{id}/rm_contributor',
    'live_remove_invite':     'api/live/{id}/rm_contributor_invite',
    'live_remove_update':     'api/live/{id}/delete_update',
    'live_report':            'api/live/{id}/report',
    'live_strike':            'api/live/{id}/strike_update',
    'live_update_perms':      'api/live/{id}/set_contributor_permissions',
    'live_update_thread':     'api/live/{id}/edit',
    'live_updates':           'live/{id}',
    'liveabout':              'api/live/{id}/about/',
    'livecreate':             'api/live/create',
    'lock':                   'api/lock/',
    'marknsfw':               'api/marknsfw/',
    'me':                     'api/v1/me',
    'media_asset':            'api/media/asset.json',
    'mentions':               'message/mentions',
    'message':                'message/messages/{id}/',
    'messages':               'message/messages/',
    'moderator_messages':     'r/{subreddit}/message/moderator/',
    'moderator_unread':       'r/{subreddit}/message/moderator/unread/',
    'modmail_archive':        'api/mod/conversations/{id}/archive',
    'modmail_bulk_read':      'api/mod/conversations/bulk/read',
    'modmail_conversation':   'api/mod/conversations/{id}',
    'modmail_conversations':  'api/mod/conversations/',
    'modmail_highlight':      'api/mod/conversations/{id}/highlight',
    'modmail_mute':           'api/mod/conversations/{id}/mute',
    'modmail_read':           'api/mod/conversations/read',
    'modmail_subreddits':     'api/mod/conversations/subreddits',
    'modmail_unarchive':      'api/mod/conversations/{id}/unarchive',
    'modmail_unmute':         'api/mod/conversations/{id}/unmute',
    'modmail_unread':         'api/mod/conversations/unread',
    'modmail_unread_count':   'api/mod/conversations/unread/count',
    'morechildren':           'api/morechildren/',
    'multireddit':            'user/{user}/m/{multi}/',
    'multireddit_api':        'api/multi/user/{user}/m/{multi}/',
    'multireddit_base':       'api/multi/',
    'multireddit_copy':       'api/multi/copy/',
    'multireddit_rename':     'api/multi/rename/',
    'multireddit_update':     'api/multi/user/{user}/m/{multi}/r/{subreddit}',
    'multireddit_user':       'api/multi/user/{user}/',
    'mute_sender':            'api/mute_message_author/',
    'my_contributor':         'subreddits/mine/contributor/',
    'my_moderator':           'subreddits/mine/moderator/',
    'my_multireddits':        'api/multi/mine/',
    'my_subreddits':          'subreddits/mine/subscriber/',
    'preferences':            'api/v1/me/prefs',
    'quarantine_opt_in':      'api/quarantine_optin',
    'quarantine_opt_out':     'api/quarantine_optout',
    'read_message':           'api/read_message/',
    ('removal_comment_'
     'message'):              'api/v1/modactions/removal_comment_message',
    'removal_link_message':   'api/v1/modactions/removal_link_message',
    'remove':                 'api/remove/',
    'report':                 'api/report/',
    'rules':                  'r/{subreddit}/about/rules',
    'save':                   'api/save/',
    'search':                 'r/{subreddit}/search/',
    'select_flair':           'r/{subreddit}/api/selectflair/',
    'sendreplies':            'api/sendreplies',
    'sent':                   'message/sent/',
    'setpermissions':         'r/{subreddit}/api/setpermissions/',
    'site_admin':             'api/site_admin/',
    'spoiler':                'api/spoiler/',
    'sticky_submission':      'api/set_subreddit_sticky/',
    'store_visits':           'api/store_visits',
    'sub_recommended':        'api/recommend/sr/{subreddits}',
    'submission':             'comments/{id}/',
    'submission_replies':     'message/selfreply/',
    'submit':                 'api/submit/',
    'subreddit':              'r/{subreddit}/',
    'subreddit_about':        'r/{subreddit}/about/',
    'subreddit_filter':       ('api/filter/user/{user}/f/{special}/'
                               'r/{subreddit}'),
    'subreddit_filter_list':  'api/filter/user/{user}/f/{special}',
    'subreddit_random':       'r/{subreddit}/random/',
    'subreddit_settings':     'r/{subreddit}/about/edit/',
    'subreddit_stylesheet':   'r/{subreddit}/api/subreddit_stylesheet/',
    'subreddits_by_topic':    'api/subreddits_by_topic',
    'subreddits_default':     'subreddits/default/',
    'subreddits_gold':        'subreddits/gold/',
    'subreddits_name_search': 'api/search_reddit_names/',
    'subreddits_new':         'subreddits/new/',
    'subreddits_popular':     'subreddits/popular/',
    'subreddits_search':      'subreddits/search/',
    'subscribe':              'api/subscribe/',
    'suggested_sort':         'api/set_suggested_sort/',
    'trophies':               'api/v1/user/{user}/trophies',
    'uncollapse':             'api/uncollapse_message/',
    'unfriend':               'r/{subreddit}/api/unfriend/',
    'unhide':                 'api/unhide/',
    'unignore_reports':       'api/unignore_reports/',
    'unlock':                 'api/unlock/',
    'unmarknsfw':             'api/unmarknsfw/',
    'unmute_sender':          'api/unmute_message_author/',
    'unread':                 'message/unread/',
    'unread_message':         'api/unread_message/',
    'unsave':                 'api/unsave/',
    'unspoiler':              'api/unspoiler/',
    'upload_image':           'r/{subreddit}/api/upload_sr_img',
    'user':                   'user/{user}/',
    'user_about':             'user/{user}/about/',
    'user_flair':             'r/{subreddit}/api/user_flair_v2',
    'users_new':              'users/new',
    'users_popular':          'users/popular',
    'users_search':           'users/search',
    'vote':                   'api/vote/',
    'widget_create':          'r/{subreddit}/api/widget',
    'widget_lease':           'r/{subreddit}/api/widget_image_upload_s3',
    'widget_modify':          'r/{subreddit}/api/widget/{widget_id}',
    'widget_order':           'r/{subreddit}/api/widget_order/{section}',
    'widgets':                'r/{subreddit}/api/widgets',
    'wiki_edit':              'r/{subreddit}/api/wiki/edit/',
    'wiki_page':              'r/{subreddit}/wiki/{page}',
    'wiki_page_editor':       'r/{subreddit}/api/wiki/alloweditor/{method}',
    'wiki_page_revisions':    'r/{subreddit}/wiki/revisions/{page}',
    'wiki_page_settings':     'r/{subreddit}/wiki/settings/{page}',
    'wiki_pages':             'r/{subreddit}/wiki/pages/',
    'wiki_revisions':         'r/{subreddit}/wiki/revisions/'}

JPEG_HEADER = b'\xff\xd8\xff'
MAX_IMAGE_SIZE = 512000
MIN_PNG_SIZE = 67
MIN_JPEG_SIZE = 128
PNG_HEADER = b'\x89\x50\x4e\x47\x0d\x0a\x1a\x0a'

USER_AGENT_FORMAT = '{{}} PRAW/{}'.format(__version__)

# pylint: disable=import-error,no-name-in-module,unused-import
if sys.version_info.major == 2:
    import ConfigParser as configparser  # NOQA
    from urlparse import urljoin, urlparse  # NOQA
else:
    import configparser  # NOQA
    from urllib.parse import urljoin, urlparse  # NOQA
<commit_msg>Touch 6.1.2.dev0 version<commit_after>"""praw constants."""
import sys

__version__ = '6.1.2.dev0'

API_PATH = {
    'about_edited':           'r/{subreddit}/about/edited/',
    'about_log':              'r/{subreddit}/about/log/',
    'about_modqueue':         'r/{subreddit}/about/modqueue/',
    'about_reports':          'r/{subreddit}/about/reports/',
    'about_spam':             'r/{subreddit}/about/spam/',
    'about_sticky':           'r/{subreddit}/about/sticky/',
    'about_stylesheet':       'r/{subreddit}/about/stylesheet/',
    'about_traffic':          'r/{subreddit}/about/traffic/',
    'about_unmoderated':      'r/{subreddit}/about/unmoderated/',
    'accept_mod_invite':      'r/{subreddit}/api/accept_moderator_invite',
    'approve':                'api/approve/',
    'block':                  'api/block',
    'block_user':             '/api/block_user/',
    'blocked':                'prefs/blocked/',
    'collapse':               'api/collapse_message/',
    'comment':                'api/comment/',
    'comment_replies':        'message/comments/',
    'compose':                'api/compose/',
    'contest_mode':           'api/set_contest_mode/',
    'del':                    'api/del/',
    'delete_message':         'api/del_msg',
    'delete_sr_banner':       'r/{subreddit}/api/delete_sr_banner',
    'delete_sr_header':       'r/{subreddit}/api/delete_sr_header',
    'delete_sr_icon':         'r/{subreddit}/api/delete_sr_icon',
    'delete_sr_image':        'r/{subreddit}/api/delete_sr_img',
    'deleteflair':            'r/{subreddit}/api/deleteflair',
    'distinguish':            'api/distinguish/',
    'domain':                 'domain/{domain}/',
    'duplicates':             'duplicates/{submission_id}/',
    'edit':                   'api/editusertext/',
    'emoji_delete':           'api/v1/{subreddit}/emoji/{emoji_name}',
    'emoji_lease':            'api/v1/{subreddit}/emoji_asset_upload_s3.json',
    'emoji_list':             'api/v1/{subreddit}/emojis/all',
    'emoji_upload':           'api/v1/{subreddit}/emoji.json',
    'flair':                  'r/{subreddit}/api/flair/',
    'flairconfig':            'r/{subreddit}/api/flairconfig/',
    'flaircsv':               'r/{subreddit}/api/flaircsv/',
    'flairlist':              'r/{subreddit}/api/flairlist/',
    'flairselector':          'r/{subreddit}/api/flairselector/',
    'flairtemplate':          'r/{subreddit}/api/flairtemplate/',
    'flairtemplate_v2':       'r/{subreddit}/api/flairtemplate_v2',
    'flairtemplateclear':     'r/{subreddit}/api/clearflairtemplates/',
    'flairtemplatedelete':    'r/{subreddit}/api/deleteflairtemplate/',
    'friend':                 'r/{subreddit}/api/friend/',
    'friend_v1':              'api/v1/me/friends/{user}',
    'friends':                'api/v1/me/friends/',
    'gild_thing':             'api/v1/gold/gild/{fullname}/',
    'gild_user':              'api/v1/gold/give/{username}/',
    'hide':                   'api/hide/',
    'ignore_reports':         'api/ignore_reports/',
    'inbox':                  'message/inbox/',
    'info':                   'api/info/',
    'karma':                  'api/v1/me/karma',
    'leavecontributor':       'api/leavecontributor',
    'link_flair':             'r/{subreddit}/api/link_flair_v2',
    'list_banned':            'r/{subreddit}/about/banned/',
    'list_contributor':       'r/{subreddit}/about/contributors/',
    'list_moderator':         'r/{subreddit}/about/moderators/',
    'list_muted':             'r/{subreddit}/about/muted/',
    'list_wikibanned':        'r/{subreddit}/about/wikibanned/',
    'list_wikicontributor':   'r/{subreddit}/about/wikicontributors/',
    'live_accept_invite':     'api/live/{id}/accept_contributor_invite',
    'live_add_update':        'api/live/{id}/update',
    'live_close':             'api/live/{id}/close_thread',
    'live_contributors':      'live/{id}/contributors',
    'live_discussions':       'live/{id}/discussions',
    'live_focus':             'live/{thread_id}/updates/{update_id}',
    'live_info':              'api/live/by_id/{ids}',
    'live_invite':            'api/live/{id}/invite_contributor',
    'live_leave':             'api/live/{id}/leave_contributor',
    'live_now':               'api/live/happening_now',
    'live_remove_contrib':    'api/live/{id}/rm_contributor',
    'live_remove_invite':     'api/live/{id}/rm_contributor_invite',
    'live_remove_update':     'api/live/{id}/delete_update',
    'live_report':            'api/live/{id}/report',
    'live_strike':            'api/live/{id}/strike_update',
    'live_update_perms':      'api/live/{id}/set_contributor_permissions',
    'live_update_thread':     'api/live/{id}/edit',
    'live_updates':           'live/{id}',
    'liveabout':              'api/live/{id}/about/',
    'livecreate':             'api/live/create',
    'lock':                   'api/lock/',
    'marknsfw':               'api/marknsfw/',
    'me':                     'api/v1/me',
    'media_asset':            'api/media/asset.json',
    'mentions':               'message/mentions',
    'message':                'message/messages/{id}/',
    'messages':               'message/messages/',
    'moderator_messages':     'r/{subreddit}/message/moderator/',
    'moderator_unread':       'r/{subreddit}/message/moderator/unread/',
    'modmail_archive':        'api/mod/conversations/{id}/archive',
    'modmail_bulk_read':      'api/mod/conversations/bulk/read',
    'modmail_conversation':   'api/mod/conversations/{id}',
    'modmail_conversations':  'api/mod/conversations/',
    'modmail_highlight':      'api/mod/conversations/{id}/highlight',
    'modmail_mute':           'api/mod/conversations/{id}/mute',
    'modmail_read':           'api/mod/conversations/read',
    'modmail_subreddits':     'api/mod/conversations/subreddits',
    'modmail_unarchive':      'api/mod/conversations/{id}/unarchive',
    'modmail_unmute':         'api/mod/conversations/{id}/unmute',
    'modmail_unread':         'api/mod/conversations/unread',
    'modmail_unread_count':   'api/mod/conversations/unread/count',
    'morechildren':           'api/morechildren/',
    'multireddit':            'user/{user}/m/{multi}/',
    'multireddit_api':        'api/multi/user/{user}/m/{multi}/',
    'multireddit_base':       'api/multi/',
    'multireddit_copy':       'api/multi/copy/',
    'multireddit_rename':     'api/multi/rename/',
    'multireddit_update':     'api/multi/user/{user}/m/{multi}/r/{subreddit}',
    'multireddit_user':       'api/multi/user/{user}/',
    'mute_sender':            'api/mute_message_author/',
    'my_contributor':         'subreddits/mine/contributor/',
    'my_moderator':           'subreddits/mine/moderator/',
    'my_multireddits':        'api/multi/mine/',
    'my_subreddits':          'subreddits/mine/subscriber/',
    'preferences':            'api/v1/me/prefs',
    'quarantine_opt_in':      'api/quarantine_optin',
    'quarantine_opt_out':     'api/quarantine_optout',
    'read_message':           'api/read_message/',
    ('removal_comment_'
     'message'):              'api/v1/modactions/removal_comment_message',
    'removal_link_message':   'api/v1/modactions/removal_link_message',
    'remove':                 'api/remove/',
    'report':                 'api/report/',
    'rules':                  'r/{subreddit}/about/rules',
    'save':                   'api/save/',
    'search':                 'r/{subreddit}/search/',
    'select_flair':           'r/{subreddit}/api/selectflair/',
    'sendreplies':            'api/sendreplies',
    'sent':                   'message/sent/',
    'setpermissions':         'r/{subreddit}/api/setpermissions/',
    'site_admin':             'api/site_admin/',
    'spoiler':                'api/spoiler/',
    'sticky_submission':      'api/set_subreddit_sticky/',
    'store_visits':           'api/store_visits',
    'sub_recommended':        'api/recommend/sr/{subreddits}',
    'submission':             'comments/{id}/',
    'submission_replies':     'message/selfreply/',
    'submit':                 'api/submit/',
    'subreddit':              'r/{subreddit}/',
    'subreddit_about':        'r/{subreddit}/about/',
    'subreddit_filter':       ('api/filter/user/{user}/f/{special}/'
                               'r/{subreddit}'),
    'subreddit_filter_list':  'api/filter/user/{user}/f/{special}',
    'subreddit_random':       'r/{subreddit}/random/',
    'subreddit_settings':     'r/{subreddit}/about/edit/',
    'subreddit_stylesheet':   'r/{subreddit}/api/subreddit_stylesheet/',
    'subreddits_by_topic':    'api/subreddits_by_topic',
    'subreddits_default':     'subreddits/default/',
    'subreddits_gold':        'subreddits/gold/',
    'subreddits_name_search': 'api/search_reddit_names/',
    'subreddits_new':         'subreddits/new/',
    'subreddits_popular':     'subreddits/popular/',
    'subreddits_search':      'subreddits/search/',
    'subscribe':              'api/subscribe/',
    'suggested_sort':         'api/set_suggested_sort/',
    'trophies':               'api/v1/user/{user}/trophies',
    'uncollapse':             'api/uncollapse_message/',
    'unfriend':               'r/{subreddit}/api/unfriend/',
    'unhide':                 'api/unhide/',
    'unignore_reports':       'api/unignore_reports/',
    'unlock':                 'api/unlock/',
    'unmarknsfw':             'api/unmarknsfw/',
    'unmute_sender':          'api/unmute_message_author/',
    'unread':                 'message/unread/',
    'unread_message':         'api/unread_message/',
    'unsave':                 'api/unsave/',
    'unspoiler':              'api/unspoiler/',
    'upload_image':           'r/{subreddit}/api/upload_sr_img',
    'user':                   'user/{user}/',
    'user_about':             'user/{user}/about/',
    'user_flair':             'r/{subreddit}/api/user_flair_v2',
    'users_new':              'users/new',
    'users_popular':          'users/popular',
    'users_search':           'users/search',
    'vote':                   'api/vote/',
    'widget_create':          'r/{subreddit}/api/widget',
    'widget_lease':           'r/{subreddit}/api/widget_image_upload_s3',
    'widget_modify':          'r/{subreddit}/api/widget/{widget_id}',
    'widget_order':           'r/{subreddit}/api/widget_order/{section}',
    'widgets':                'r/{subreddit}/api/widgets',
    'wiki_edit':              'r/{subreddit}/api/wiki/edit/',
    'wiki_page':              'r/{subreddit}/wiki/{page}',
    'wiki_page_editor':       'r/{subreddit}/api/wiki/alloweditor/{method}',
    'wiki_page_revisions':    'r/{subreddit}/wiki/revisions/{page}',
    'wiki_page_settings':     'r/{subreddit}/wiki/settings/{page}',
    'wiki_pages':             'r/{subreddit}/wiki/pages/',
    'wiki_revisions':         'r/{subreddit}/wiki/revisions/'}

JPEG_HEADER = b'\xff\xd8\xff'
MAX_IMAGE_SIZE = 512000
MIN_PNG_SIZE = 67
MIN_JPEG_SIZE = 128
PNG_HEADER = b'\x89\x50\x4e\x47\x0d\x0a\x1a\x0a'

USER_AGENT_FORMAT = '{{}} PRAW/{}'.format(__version__)

# pylint: disable=import-error,no-name-in-module,unused-import
if sys.version_info.major == 2:
    import ConfigParser as configparser  # NOQA
    from urlparse import urljoin, urlparse  # NOQA
else:
    import configparser  # NOQA
    from urllib.parse import urljoin, urlparse  # NOQA
<|endoftext|>


<commit_before>import math
from itertools import chain

from sklearn import preprocessing, tree, cross_validation
import numpy as np

from base import gob
from base.utils import coord_in_miles

def logify(x):
    return int(math.ceil(math.log(x+1,2)))

@gob.mapper()
def edge_vect(user):

    dists = [ logify(coord_in_miles(user['mloc'],rel)) for rel in user['rels'] ]
    # including the dist in the median is broken.
    fol_dist = int(np.median(dists))

    for dist,rel in zip(dists,user['rels']):
        flags = [rel['kind'] >>i & 1 for i in range(4)]
        vals = [logify(rel[k]) for k in ('mdist','folc')]
        yield flags+vals+[fol_dist, dist]

def _transformed(vects):
    # convert vects to a scaled numpy array
    vects_ = np.fromiter( chain.from_iterable(vects), int )
    vects_.shape = (len(vects_)//8),8
    X = np.array(vects_[:,:-1],float)
    y = np.array(vects_[:,-1],int)
    scaler = preprocessing.Scaler().fit(X)

    # In final system, you will want to only fit the training data.
    return scaler.transform(X), y

@gob.mapper(all_items=True)
def fl_learn(vects):
    #./gb.py -i edge_vect.22 fl_learn
    X, y = _transformed(vects)
    clf = tree.DecisionTreeClassifier()
    scores = cross_validation.cross_val_score(clf,X,y,cv=5)
    print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() / 2)
    return []

<commit_msg>take the median of the other edges<commit_after>import math
from itertools import chain

from sklearn import preprocessing, tree, cross_validation
import numpy as np

from base import gob
from base.utils import coord_in_miles

def logify(x):
    return int(math.ceil(math.log(x+1,2)))

@gob.mapper()
def edge_vect(user):

    dists = [ logify(coord_in_miles(user['mloc'],rel)) for rel in user['rels'] ]

    for index,rel in enumerate(user['rels']):
        oth_dists = dists[:index] + dists[index+1:]
        fol_dist = int(np.median(dists)) if oth_dists else 8
        flags = [rel['kind'] >>i & 1 for i in range(4)]
        vals = [logify(rel[k]) for k in ('mdist','folc')]
        yield flags+vals+[fol_dist, dists[index]]

def _transformed(vects):
    # convert vects to a scaled numpy array
    vects_ = np.fromiter( chain.from_iterable(vects), int )
    vects_.shape = (len(vects_)//8),8
    X = np.array(vects_[:,:-1],float)
    y = np.array(vects_[:,-1],int)
    scaler = preprocessing.Scaler().fit(X)

    # In final system, you will want to only fit the training data.
    return scaler.transform(X), y

@gob.mapper(all_items=True)
def fl_learn(vects):
    #./gb.py -i edge_vect.22 fl_learn
    X, y = _transformed(vects)
    clf = tree.DecisionTreeClassifier()
    scores = cross_validation.cross_val_score(clf,X,y,cv=5)
    print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() / 2)
    return []

<|endoftext|>


<commit_before>import sys
import urllib2
from BeautifulSoup import BeautifulSoup
import os.path
import tarfile
import bson
import json
import re
import shelve

class Preprocessor(object):
    DOWNLOADS_URL = "http://ghtorrent.org/downloads/"
    BSON_FILE_DIR = "dump/github/"

    def __init__(self):
        self.dataset = ''
        self.bson_file = ''
        self.keep_fields = []

    def preprocess(self):
        self.get_bson()
        self.convert_bson()

    def get_bson(self):
        if not os.path.isfile(self.dataset + '.tar.gz'):
            self.download(self.dataset + '.tar.gz')
        if not os.path.isfile(self.bson_file + '.bson'):
            self.extract(self.bson_file + '.bson')

    def download(self, target):
        stream = urllib2.urlopen(self.DOWNLOADS_URL + self.dataset + '.tar.gz')
        file = open(target, 'wb')
        file_size = int(stream.info().getheaders('Content-Length')[0])
        downloaded_size = 0
        block_size = 8192

        while True:
            buffer = stream.read(block_size)
            if not buffer:
                break

            downloaded_size += len(buffer)
            file.write(buffer)
            status = 'Downloading "' + self.dataset + '" dataset [%3.0f%%]' % (downloaded_size * 100. / file_size)
            status += chr(8) * (len(status) + 1)
            print(status),

        print('Downloading "' + self.dataset + '" dataset [finished]')
        file.close()

    def extract(self, file):
        tar = tarfile.open(self.dataset + '.tar.gz')
        tar.extractall()
        tar.close()
        print('Untarring "' + self.dataset + '" dataset [finished]')

    def convert_bson(self):
        raise NotImplementedError("Cannot call convert_bson on the base class: a subclass must implement this method instead")

class Commit_Comments_Preprocessor(Preprocessor):
    def __init__(self, group):
        super(Commit_Comments_Preprocessor, self).__init__()
        self.dataset = 'commit_comments-dump.2015-01-29'
        self.bson_file = self.BSON_FILE_DIR + 'commit_comments.bson'
        self.group = group
        self.keep_fields = ['id', 'body']
        if group not in self.keep_fields:
            self.keep_fields.append(group)

    def is_latin(self, string):
        try:
            string.encode('ascii')
        except UnicodeEncodeError:
            return False

        return True

    def convert_bson(self):
        output = open(self.dataset + '.json', 'wb')
        bson_file = open(self.bson_file, 'rb')

        if os.path.isfile('languages.shelf'):
            languages = shelve.open('languages.shelf')
        else:
            languages = {}

        # Read every BSON object as an iterator to save memory.
        for raw_json in bson.decode_file_iter(bson_file):
            if not self.is_latin(raw_json['body']):
                continue

            preprocessed_json = {}
            repository = str(re.search(r"repos/([^/]+/[^/]+)(/|$)", raw_json['url']).group(1))
            raw_json['language'] = ''
            if repository in languages:
                raw_json['language'] = languages[repository]
            for item in self.keep_fields:
                preprocessed_json[item] = raw_json[item]

            json.dump(preprocessed_json, output)
            output.write('\n')

        output.close()
        bson_file.close()
        os.remove(self.bson_file)
        os.removedirs(self.BSON_FILE_DIR)
        print('Converting BSON and removing unused fields [finished]')

class Repos_Preprocessor(Preprocessor):
    def __init__(self, date):
        super(Repos_Preprocessor, self).__init__()
        self.dataset = 'repos-dump.' + date
        self.bson_file = self.BSON_FILE_DIR + 'repos.bson'

    def convert_bson(self):
        bson_file = open(self.bson_file, 'rb')

        # Read every BSON object as an iterator to save memory.
        languages = shelve.open('languages.shelf')
        for raw_json in bson.decode_file_iter(bson_file):
            repository = raw_json['full_name'].encode('utf-8')
            language = raw_json['language'].encode('utf-8') if raw_json['language'] is not None else ''
            if repository not in languages:
                languages[repository] = language

        languages.close()
        bson_file.close()
        os.remove(self.bson_file)
        os.removedirs(self.BSON_FILE_DIR)
        print('Converting BSON and removing unused fields [finished]')

def main(argv):
    group = argv[0] if len(argv) > 0 else "id"

    if group == "language" and not os.path.isfile('languages.shelf'):
        # First prepare the languages as the commit comments dataset depends on
        # that. Fetch all the repo dumps.
        preprocessors = []
        html_page = urllib2.urlopen(Preprocessor.DOWNLOADS_URL)
        soup = BeautifulSoup(html_page)
        for link in soup.findAll('a'):
            href = link.get('href')
            if href.startswith('repos-dump'):
                date = href[11:-7]
                preprocessors[:0] = [Repos_Preprocessor(date)]

        for preprocessor in preprocessors:
            preprocessor.preprocess()

    commit_comments = Commit_Comments_Preprocessor(group)
    commit_comments.preprocess()

if __name__ == "__main__":
    main(sys.argv[1:])
<commit_msg>Progress in preprocess steps, shelve writeback<commit_after>import sys
import urllib2
from BeautifulSoup import BeautifulSoup
import os.path
import tarfile
import bson
import json
import re
import shelve

class Progress(object):
    def __init__(self, message, total_size):
        self.message = message
        self.total_size = total_size
        self.previous_percent = -1

    def show_progress(self, current_size):
        percent = int(current_size * 100. / self.total_size)
        if percent != self.previous_percent:
            status = self.message + ' [%3d%%]' % percent
            clear = chr(8) * len(status)
            if self.previous_percent != -1:
                sys.stdout.write(clear)

            sys.stdout.write(status)
            sys.stdout.flush()
            self.previous_percent = percent
            if percent == 100:
                sys.stdout.write(clear)

class ProgressFile(file, Progress):
    def __init__(self, *a, **kw):
        message = kw.pop('message', None)

        file.__init__(self, *a, **kw)

        if message is None:
            message = 'Reading file ' + self.name

        Progress.__init__(self, message, os.path.getsize(self.name))

    def read(self, size):
        self.show_progress(self.tell())
        return file.read(self, size)

class Preprocessor(object):
    DOWNLOADS_URL = "http://ghtorrent.org/downloads/"
    BSON_FILE_DIR = "dump/github/"

    def __init__(self):
        self.dataset = ''
        self.bson_file = ''
        self.keep_fields = []

    def preprocess(self):
        self.get_bson()
        self.convert_bson()

    def get_bson(self):
        if not os.path.isfile(self.dataset + '.tar.gz'):
            self.download(self.dataset + '.tar.gz')
        if not os.path.isfile(self.bson_file + '.bson'):
            self.extract(self.bson_file + '.bson')

    def download(self, target):
        stream = urllib2.urlopen(self.DOWNLOADS_URL + self.dataset + '.tar.gz')
        file = open(target, 'wb')
        file_size = int(stream.info().getheaders('Content-Length')[0])
        downloaded_size = 0
        block_size = 8192
        message = 'Downloading "' + self.dataset + '" dataset'
        progress = Progress(message, file_size)

        while True:
            buffer = stream.read(block_size)
            if not buffer:
                break

            downloaded_size += len(buffer)
            file.write(buffer)
            progress.show_progress(downloaded_size)

        print(message + ' [finished]')
        file.close()

    def extract(self, file):
        message = 'Untarring "' + self.dataset + '" dataset'
        tar = tarfile.open(fileobj=ProgressFile(self.dataset + '.tar.gz', message=message))
        tar.extractall()
        tar.close()
        print(message + ' [finished]')

    def convert_bson(self):
        raise NotImplementedError("Cannot call convert_bson on the base class: a subclass must implement this method instead")

class Commit_Comments_Preprocessor(Preprocessor):
    def __init__(self, group):
        super(Commit_Comments_Preprocessor, self).__init__()
        self.dataset = 'commit_comments-dump.2015-01-29'
        self.bson_file = self.BSON_FILE_DIR + 'commit_comments.bson'
        self.group = group
        self.keep_fields = ['id', 'body']
        if group not in self.keep_fields:
            self.keep_fields.append(group)

    def is_latin(self, string):
        try:
            string.encode('ascii')
        except UnicodeEncodeError:
            return False

        return True

    def convert_bson(self):
        output = open(self.dataset + '.json', 'wb')
        message = 'Converting BSON and removing unused fields'
        bson_file = ProgressFile(self.bson_file, 'rb', message=message)

        if os.path.isfile('languages.shelf'):
            languages = shelve.open('languages.shelf')
        else:
            languages = {}

        # Read every BSON object as an iterator to save memory.
        for raw_json in bson.decode_file_iter(bson_file):
            if not self.is_latin(raw_json['body']):
                continue

            preprocessed_json = {}
            repository = str(re.search(r"repos/([^/]+/[^/]+)(/|$)", raw_json['url']).group(1))
            raw_json['language'] = ''
            if repository in languages:
                raw_json['language'] = languages[repository]
            for item in self.keep_fields:
                preprocessed_json[item] = raw_json[item]

            json.dump(preprocessed_json, output)
            output.write('\n')

        output.close()
        bson_file.close()
        os.remove(self.bson_file)
        os.removedirs(self.BSON_FILE_DIR)
        print(message + ' [finished]')

class Repos_Preprocessor(Preprocessor):
    def __init__(self, date):
        super(Repos_Preprocessor, self).__init__()
        self.dataset = 'repos-dump.' + date
        self.bson_file = self.BSON_FILE_DIR + 'repos.bson'

    def convert_bson(self):
        message = 'Converting BSON and removing unused fields'
        bson_file = ProgressFile(self.bson_file, 'rb', message=message)

        # Read every BSON object as an iterator to save memory.
        languages = shelve.open('languages.shelf', writeback=True)
        for raw_json in bson.decode_file_iter(bson_file):
            repository = raw_json['full_name'].encode('utf-8')
            language = raw_json['language'].encode('utf-8') if raw_json['language'] is not None else ''
            if repository not in languages:
                languages[repository] = language

        languages.close()
        bson_file.close()
        os.remove(self.bson_file)
        os.removedirs(self.BSON_FILE_DIR)
        print('Converting BSON and removing unused fields [finished]')

def main(argv):
    group = argv[0] if len(argv) > 0 else "id"

    if group == "language" and not os.path.isfile('languages.shelf'):
        # First prepare the languages as the commit comments dataset depends on
        # that. Fetch all the repo dumps.
        preprocessors = []
        html_page = urllib2.urlopen(Preprocessor.DOWNLOADS_URL)
        soup = BeautifulSoup(html_page)
        for link in soup.findAll('a'):
            href = link.get('href')
            if href.startswith('repos-dump'):
                date = href[11:-7]
                preprocessors[:0] = [Repos_Preprocessor(date)]

        for preprocessor in preprocessors:
            preprocessor.preprocess()

    commit_comments = Commit_Comments_Preprocessor(group)
    commit_comments.preprocess()

if __name__ == "__main__":
    main(sys.argv[1:])
<|endoftext|>