<commit_before>#! /usr/bin/env python
# throxy.py - HTTP proxy to simulate dial-up access
# Copyright (C) 2007 Johann C. Rocholl <johann@rocholl.net>
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""
Throxy (Throttling Proxy)

This is a simple HTTP proxy, with the following features:

* Simulates a slow connection (like dial-up).
* Optionally dumps HTTP headers and content for debugging.
* Supports multiple connections, without threading.
* Written in pure Python.

To use it, run this script on your local machine and adjust your
browser settings to use 127.0.0.1:8080 as HTTP proxy.
"""

import sys
import asyncore
import socket
import time
import re

__revision__ = '$Rev$'

KILO = 1000 # decimal or binary kilo
MIN_FRAGMENT_SIZE = 512 # bytes

request_match = re.compile(r'^([A-Z]+) (\S+) (HTTP/\S+)$').match

class Header:

    def __init__(self):
        self.data = ''
        self.lines = []
        self.complete = False

    def append(self, new_data):
        self.data += new_data
        while not self.complete:
            newline = self.data.find('\n')
            if newline < 0:
                break # No complete line found.
            line = self.data[:newline].rstrip('\r')
            if len(line):
                self.lines.append(line)
            else:
                self.complete = True
                self.content_type = self.extract('Content-Type', '')
            self.data = self.data[newline+1:]
        if self.complete:
            # Give remaining data back to caller. It may contain
            # content, or even the start of the next request.
            rest = self.data
            self.data = ''
            return rest
        else:
            return ''

    def extract(self, name, default=None):
        name = name.lower()
        for line in self.lines:
            if not line.count(':'):
                continue
            key, value = line.split(':', 1)
            if key.lower() == name:
                return value.strip()
        return default

    def extract_host(self):
        self.host = self.extract('Host')
        if self.host is None:
            return
        # print "client %s:%d wants to talk to" % self.client.addr, self.host
        if self.host.count(':'):
            self.host_name, self.host_port = self.host.split(':')
        else:
            self.host_name = self.host
            self.host_port = 80
        self.host_ip = socket.gethostbyname(self.host_name)
        self.host_addr = (self.host_ip, self.host_port)

    def extract_request(self):
        match = request_match(self.lines[0])
        if not match:
            raise ValueError("malformed request line " + self.lines[0])
        self.method, self.url, self.proto = match.groups()
        if self.method.upper() == 'CONNECT':
            raise ValueError("method CONNECT is not supported")
        prefix = 'http://' + self.host
        if not self.url.startswith(prefix):
            raise ValueError("URL doesn't start with " + prefix)
        self.path = self.url[len(prefix):]

    def dump_title(self, from_addr, to_addr, direction, what):
        print '==== %s %s (%s:%d => %s:%d) ====' % (
            direction, what,
            from_addr[0], from_addr[1],
            to_addr[0], to_addr[1])

    def dump(self, from_addr, to_addr, direction='sending'):
        self.dump_title(from_addr, to_addr, direction, 'headers')
        print '\n'.join(self.lines)
        print

    def dump_content(self, content, from_addr, to_addr, direction='sending'):
        self.dump_title(from_addr, to_addr, direction, 'content')
        if self.content_type.startswith('text'):
            print content
        elif len(content) < 400:
            print repr(content)
        else:
            print "(%d bytes of %s)" % (len(content), self.content_type)
        print

class ThrottleSender(asyncore.dispatcher):

    def __init__(self, kbps, channel=None):
        if channel is None:
            asyncore.dispatcher.__init__(self)
        else:
            asyncore.dispatcher.__init__(self, channel)
        self.interval = 1.0
        self.bytes_per_second = int(kbps * KILO) / 8
        self.transmit_log = []
        self.buffer = []

    def log_sent_bytes(self, bytes):
        self.transmit_log.append((time.time(), bytes))

    def trim_log(self, horizon):
        while len(self.transmit_log) and self.transmit_log[0][0] <= horizon:
            self.transmit_log.pop(0)

    def weighted_bytes(self):
        """Compute recent bandwidth usage, in bytes per second."""
        now = time.time()
        self.trim_log(now - self.interval)
        if len(self.transmit_log) == 0:
            return 0
        weighted = 0.0
        for timestamp, bytes in self.transmit_log:
            age = now - timestamp # Event's age in seconds.
            assert 0 <= age <= self.interval
            weight = 2.0 * (self.interval - age) / self.interval
            assert 0.0 <= weight <= 2.0
            weighted += weight * bytes # Newer entries count more.
        return int(weighted / self.interval)

    def weighted_kbps(self):
        """Compute recent bandwidth usage, in kbps."""
        return 8 * self.weighted_bytes() / float(KILO)

    def sendable(self):
        """How many bytes can we send without exceeding bandwidth?"""
        return max(0, self.bytes_per_second - self.weighted_bytes())

    def writable(self):
        return (len(self.buffer) and
                self.sendable() / 2 > MIN_FRAGMENT_SIZE)

    def handle_write(self):
        max_bytes = self.sendable() / 2
        if max_bytes < MIN_FRAGMENT_SIZE:
            return
        # print "sendable", max_bytes
        bytes = self.send(self.buffer[0][:max_bytes])
        self.log_sent_bytes(bytes)
        if bytes == len(self.buffer[0]):
            self.buffer.pop(0)
        else:
            self.buffer[0] = self.buffer[0][bytes:]

class ClientChannel(ThrottleSender):

    def __init__(self, channel, addr):
        ThrottleSender.__init__(self, options.download, channel)
        self.addr = addr
        self.header = Header()
        self.content_length = 0
        self.server = None
        self.handle_connect()

    def readable(self):
        return self.server is None or len(self.server.buffer) == 0

    def handle_read(self):
        data = self.recv(8192)
        while len(data):
            if self.content_length:
                bytes = min(self.content_length, len(data))
                self.server.buffer.append(data[:bytes])
                if options.dump_send_content:
                    self.header.dump_content(
                        data[:bytes], self.addr, self.header.host_addr)
                data = data[bytes:]
                self.content_length -= bytes
            if not len(data):
                break
            if self.header.complete and self.content_length == 0:
                print "*** starting new header for %s:%d ***" % self.addr
                self.header = Header()
                self.server = None
            data = self.header.append(data)
            if self.header.complete:
                self.content_length = int(
                    self.header.extract('Content-Length', 0))
                self.header.extract_host()
                if options.dump_send_headers:
                    self.header.dump(self.addr, self.header.host_addr)
                self.server = ServerChannel(self, self.header)

    def handle_connect(self):
        if not options.quiet:
            print >> sys.stderr, "client %s:%d connected" % self.addr

    def handle_close(self):
        self.close()
        if not options.quiet:
            print >> sys.stderr, "client %s:%d disconnected" % self.addr

class ServerChannel(ThrottleSender):

    def __init__(self, client, header):
        ThrottleSender.__init__(self, options.upload)
        self.client = client
        self.addr = header.host_addr
        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
        self.connect(self.addr)
        self.send_header(header)
        self.header = Header()

    def send_header(self, header):
        header.extract_request()
        self.send_line(' '.join(
            (header.method, header.path, header.proto)))
        self.send_line('Connection: close')
        for line in header.lines[1:]:
            if not (line.startswith('Keep-Alive: ') or
                    line.startswith('Connection: ') or
                    line.startswith('Proxy-')):
                self.send_line(line)
        self.send_line('')

    def send_line(self, line):
        self.buffer.append(line + '\r\n')

    def receive_header(self, header):
        for line in header.lines:
            if not (line.startswith('Keep-Alive: ') or
                    line.startswith('Connection: ') or
                    line.startswith('Proxy-')):
                self.receive_line(line)
        self.receive_line('')

    def receive_line(self, line):
        self.client.buffer.append(line + '\r\n')

    def readable(self):
        return len(self.client.buffer) == 0

    def handle_read(self):
        data = self.recv(8192)
        if not self.header.complete:
            data = self.header.append(data)
            if self.header.complete:
                if options.dump_recv_headers:
                    self.header.dump(self.addr, self.client.addr, 'receiving')
                self.receive_header(self.header)
        if self.header.complete and len(data):
            if options.dump_recv_content:
                self.header.dump_content(
                    data, self.addr, self.client.addr, 'receiving')
            self.client.buffer.append(data)

    def handle_connect(self):
        if not options.quiet:
            print >> sys.stderr, "server %s:%d connected" % self.addr

    def handle_close(self):
        self.client.should_close = True
        self.close()
        if not options.quiet:
            print >> sys.stderr, "server %s:%d disconnected" % self.addr

class ProxyServer(asyncore.dispatcher):

    def __init__(self):
        asyncore.dispatcher.__init__(self)
        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
        self.bind((options.interface, options.port))
        self.listen(5)
        print >> sys.stderr, "listening on port", options.port

    def handle_accept(self):
        channel, addr = self.accept()
        if addr[0] == '127.0.0.1' or options.allow_remote:
            ClientChannel(channel, addr)
        else:
            channel.close()
            print >> sys.stderr, "remote client %s:%d not allowed" % addr

if __name__ == '__main__':
    from optparse import OptionParser
    version = '%prog ' + __revision__.strip('$').replace('Rev: ', 'r')
    parser = OptionParser(version=version)
    parser.add_option('-i', dest='interface', action='store', type='string',
                      metavar='<ip>', default='',
                      help="listen on this interface only (default all)")
    parser.add_option('-p', dest='port', action='store', type='int',
                      metavar='<port>', default=8080,
                      help="listen on this port number (default 8080)")
    parser.add_option('-d', dest='download', action='store', type='float',
                      metavar='<kbps>', default=28.8,
                      help="download bandwidth in kbps (default 28.8)")
    parser.add_option('-u', dest='upload', action='store', type='float',
                      metavar='<kbps>', default=28.8,
                      help="upload bandwidth in kbps (default 28.8)")
    parser.add_option('-o', dest='allow_remote', action='store_true',
                      help="allow remote clients (WARNING: open proxy)")
    parser.add_option('-q', dest='quiet', action='store_true',
                      help="don't show connect and disconnect messages")
    parser.add_option('-s', dest='dump_send_headers', action='store_true',
                      help="dump headers sent to server")
    parser.add_option('-r', dest='dump_recv_headers', action='store_true',
                      help="dump headers received from server")
    parser.add_option('-S', dest='dump_send_content', action='store_true',
                      help="dump content sent to server")
    parser.add_option('-R', dest='dump_recv_content', action='store_true',
                      help="dump content received from server")
    options, args = parser.parse_args()
    proxy = ProxyServer()
    try:
        asyncore.loop(timeout=0.1)
    except:
        proxy.shutdown(2)
        proxy.close()
        raise
<commit_msg>Better human-readable content dumping, with output limits and gzip decompression.<commit_after>#! /usr/bin/env python
# throxy.py - HTTP proxy to simulate dial-up access
# Copyright (C) 2007 Johann C. Rocholl <johann@rocholl.net>
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""
Throxy (Throttling Proxy)

This is a simple HTTP proxy, with the following features:

* Simulates a slow connection (like dial-up).
* Optionally dumps HTTP headers and content for debugging.
* Supports multiple connections, without threading.
* Written in pure Python.

To use it, run this script on your local machine and adjust your
browser settings to use 127.0.0.1:8080 as HTTP proxy.
"""

import sys
import asyncore
import socket
import time
import gzip
import cStringIO
import re

__revision__ = '$Rev$'

KILO = 1000 # decimal or binary kilo
MIN_FRAGMENT_SIZE = 512 # bytes

request_match = re.compile(r'^([A-Z]+) (\S+) (HTTP/\S+)$').match

class Header:

    def __init__(self):
        self.data = ''
        self.lines = []
        self.complete = False

    def append(self, new_data):
        self.data += new_data
        while not self.complete:
            newline = self.data.find('\n')
            if newline < 0:
                break # No complete line found.
            line = self.data[:newline].rstrip('\r')
            if len(line):
                self.lines.append(line)
            else:
                self.complete = True
                self.content_type = self.extract('Content-Type')
                self.content_encoding = self.extract('Content-Encoding')
                if self.content_encoding == 'gzip':
                    self.gzip_data = cStringIO.StringIO()
            self.data = self.data[newline+1:]
        if self.complete:
            # Give remaining data back to caller. It may contain
            # content, or even the start of the next request.
            rest = self.data
            self.data = ''
            return rest
        else:
            return ''

    def extract(self, name, default=''):
        name = name.lower()
        for line in self.lines:
            if not line.count(':'):
                continue
            key, value = line.split(':', 1)
            if key.lower() == name:
                return value.strip()
        return default

    def extract_host(self):
        self.host = self.extract('Host')
        if self.host is None:
            return
        # print "client %s:%d wants to talk to" % self.client.addr, self.host
        if self.host.count(':'):
            self.host_name, self.host_port = self.host.split(':')
        else:
            self.host_name = self.host
            self.host_port = 80
        self.host_ip = socket.gethostbyname(self.host_name)
        self.host_addr = (self.host_ip, self.host_port)

    def extract_request(self):
        match = request_match(self.lines[0])
        if not match:
            raise ValueError("malformed request line " + self.lines[0])
        self.method, self.url, self.proto = match.groups()
        if self.method.upper() == 'CONNECT':
            raise ValueError("method CONNECT is not supported")
        prefix = 'http://' + self.host
        if not self.url.startswith(prefix):
            raise ValueError("URL doesn't start with " + prefix)
        self.path = self.url[len(prefix):]

    def dump_title(self, from_addr, to_addr, direction, what):
        print '==== %s %s (%s:%d => %s:%d) ====' % (
            direction, what,
            from_addr[0], from_addr[1],
            to_addr[0], to_addr[1])

    def dump(self, from_addr, to_addr, direction='sending'):
        self.dump_title(from_addr, to_addr, direction, 'headers')
        print '\n'.join(self.lines)
        print

    def dump_content(self, content, from_addr, to_addr, direction='sending'):
        self.dump_title(from_addr, to_addr, direction, 'content')
        if self.content_encoding:
            print "(%d bytes of %s with %s encoding)" % (len(content),
                repr(self.content_type), repr(self.content_encoding))
        else:
            print "(%d bytes of %s)" % (len(content), repr(self.content_type))
        if self.content_encoding == 'gzip':
            self.gzip_data.write(content)
            try:
                content = self.gunzip()
            except IOError, message:
                content = 'Could not gunzip: ' + message
        if self.content_type.startswith('text/'):
            limit = options.text_dump_limit
        elif self.content_type.startswith('application/') and \
                 self.content_type.count('xml'):
            limit = options.text_dump_limit
        else:
            limit = options.data_dump_limit
            content = repr(content)
        if len(content) < limit or limit == 0:
            print content
        else:
            print content[:limit] + ' ...'
        print

    def gunzip(self):
        gzip_file = gzip.GzipFile(
            fileobj=self.gzip_data, mode='rb')
        result = gzip_file.read()
        gzip_file.close()
        return result

class ThrottleSender(asyncore.dispatcher):

    def __init__(self, kbps, channel=None):
        if channel is None:
            asyncore.dispatcher.__init__(self)
        else:
            asyncore.dispatcher.__init__(self, channel)
        self.interval = 1.0
        self.bytes_per_second = int(kbps * KILO) / 8
        self.transmit_log = []
        self.buffer = []

    def log_sent_bytes(self, bytes):
        self.transmit_log.append((time.time(), bytes))

    def trim_log(self, horizon):
        while len(self.transmit_log) and self.transmit_log[0][0] <= horizon:
            self.transmit_log.pop(0)

    def weighted_bytes(self):
        """Compute recent bandwidth usage, in bytes per second."""
        now = time.time()
        self.trim_log(now - self.interval)
        if len(self.transmit_log) == 0:
            return 0
        weighted = 0.0
        for timestamp, bytes in self.transmit_log:
            age = now - timestamp # Event's age in seconds.
            assert 0 <= age <= self.interval
            weight = 2.0 * (self.interval - age) / self.interval
            assert 0.0 <= weight <= 2.0
            weighted += weight * bytes # Newer entries count more.
        return int(weighted / self.interval)

    def weighted_kbps(self):
        """Compute recent bandwidth usage, in kbps."""
        return 8 * self.weighted_bytes() / float(KILO)

    def sendable(self):
        """How many bytes can we send without exceeding bandwidth?"""
        return max(0, self.bytes_per_second - self.weighted_bytes())

    def writable(self):
        return (len(self.buffer) and
                self.sendable() / 2 > MIN_FRAGMENT_SIZE)

    def handle_write(self):
        max_bytes = self.sendable() / 2
        if max_bytes < MIN_FRAGMENT_SIZE:
            return
        # print "sendable", max_bytes
        bytes = self.send(self.buffer[0][:max_bytes])
        self.log_sent_bytes(bytes)
        if bytes == len(self.buffer[0]):
            self.buffer.pop(0)
        else:
            self.buffer[0] = self.buffer[0][bytes:]

class ClientChannel(ThrottleSender):

    def __init__(self, channel, addr):
        ThrottleSender.__init__(self, options.download, channel)
        self.addr = addr
        self.header = Header()
        self.content_length = 0
        self.server = None
        self.handle_connect()

    def readable(self):
        return self.server is None or len(self.server.buffer) == 0

    def handle_read(self):
        data = self.recv(8192)
        while len(data):
            if self.content_length:
                bytes = min(self.content_length, len(data))
                self.server.buffer.append(data[:bytes])
                if options.dump_send_content:
                    self.header.dump_content(
                        data[:bytes], self.addr, self.header.host_addr)
                data = data[bytes:]
                self.content_length -= bytes
            if not len(data):
                break
            if self.header.complete and self.content_length == 0:
                print >> sys.stderr, \
                      "**** new request from client %s:%d ****" % self.addr
                self.header = Header()
                self.server = None
            data = self.header.append(data)
            if self.header.complete:
                self.content_length = int(
                    self.header.extract('Content-Length', 0))
                self.header.extract_host()
                if options.dump_send_headers:
                    self.header.dump(self.addr, self.header.host_addr)
                self.server = ServerChannel(self, self.header)

    def handle_connect(self):
        if not options.quiet:
            print >> sys.stderr, "client %s:%d connected" % self.addr

    def handle_close(self):
        self.close()
        if not options.quiet:
            print >> sys.stderr, "client %s:%d disconnected" % self.addr

class ServerChannel(ThrottleSender):

    def __init__(self, client, header):
        ThrottleSender.__init__(self, options.upload)
        self.client = client
        self.addr = header.host_addr
        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
        self.connect(self.addr)
        self.send_header(header)
        self.header = Header()

    def send_header(self, header):
        header.extract_request()
        self.send_line(' '.join(
            (header.method, header.path, header.proto)))
        self.send_line('Connection: close')
        for line in header.lines[1:]:
            if not (line.startswith('Keep-Alive: ') or
                    line.startswith('Connection: ') or
                    line.startswith('Proxy-')):
                self.send_line(line)
        self.send_line('')

    def send_line(self, line):
        self.buffer.append(line + '\r\n')

    def receive_header(self, header):
        for line in header.lines:
            if not (line.startswith('Keep-Alive: ') or
                    line.startswith('Connection: ') or
                    line.startswith('Proxy-')):
                self.receive_line(line)
        self.receive_line('')

    def receive_line(self, line):
        self.client.buffer.append(line + '\r\n')

    def readable(self):
        return len(self.client.buffer) == 0

    def handle_read(self):
        data = self.recv(8192)
        if not self.header.complete:
            data = self.header.append(data)
            if self.header.complete:
                if options.dump_recv_headers:
                    self.header.dump(self.addr, self.client.addr, 'receiving')
                self.receive_header(self.header)
        if self.header.complete and len(data):
            if options.dump_recv_content:
                self.header.dump_content(
                    data, self.addr, self.client.addr, 'receiving')
            self.client.buffer.append(data)

    def handle_connect(self):
        if not options.quiet:
            print >> sys.stderr, "server %s:%d connected" % self.addr

    def handle_close(self):
        self.client.should_close = True
        self.close()
        if not options.quiet:
            print >> sys.stderr, "server %s:%d disconnected" % self.addr

class ProxyServer(asyncore.dispatcher):

    def __init__(self):
        asyncore.dispatcher.__init__(self)
        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
        self.bind((options.interface, options.port))
        self.listen(5)
        print >> sys.stderr, "listening on port", options.port

    def handle_accept(self):
        channel, addr = self.accept()
        if addr[0] == '127.0.0.1' or options.allow_remote:
            ClientChannel(channel, addr)
        else:
            channel.close()
            print >> sys.stderr, "remote client %s:%d not allowed" % addr

if __name__ == '__main__':
    from optparse import OptionParser
    version = '%prog ' + __revision__.strip('$').replace('Rev: ', 'r')
    parser = OptionParser(version=version)
    parser.add_option('-i', dest='interface', action='store', type='string',
                      metavar='<ip>', default='',
                      help="listen on this interface only (default all)")
    parser.add_option('-p', dest='port', action='store', type='int',
                      metavar='<port>', default=8080,
                      help="listen on this port number (default 8080)")
    parser.add_option('-d', dest='download', action='store', type='float',
                      metavar='<kbps>', default=28.8,
                      help="download bandwidth in kbps (default 28.8)")
    parser.add_option('-u', dest='upload', action='store', type='float',
                      metavar='<kbps>', default=28.8,
                      help="upload bandwidth in kbps (default 28.8)")
    parser.add_option('-o', dest='allow_remote', action='store_true',
                      help="allow remote clients (WARNING: open proxy)")
    parser.add_option('-q', dest='quiet', action='store_true',
                      help="don't show connect and disconnect messages")
    parser.add_option('-s', dest='dump_send_headers', action='store_true',
                      help="dump headers sent to server")
    parser.add_option('-r', dest='dump_recv_headers', action='store_true',
                      help="dump headers received from server")
    parser.add_option('-S', dest='dump_send_content', action='store_true',
                      help="dump content sent to server")
    parser.add_option('-R', dest='dump_recv_content', action='store_true',
                      help="dump content received from server")
    parser.add_option('-l', dest='text_dump_limit', action='store',
                      metavar='<bytes>', type='int', default=1024,
                      help="maximum length of dumped text (default 1024)")
    parser.add_option('-L', dest='data_dump_limit', action='store',
                      metavar='<bytes>', type='int', default=256,
                      help="maximum length of dumped data (default 256)")
    options, args = parser.parse_args()
    proxy = ProxyServer()
    try:
        asyncore.loop(timeout=0.1)
    except:
        proxy.shutdown(2)
        proxy.close()
        raise
<|endoftext|>


<commit_before>"""
Ahmed Abdulkareem
02/10/2015
Final Project
CPU Implementation
All rights reserved
"""

import argparse
import Memory
import ALU
import instruction_decoder
import config
from random import randrange

#instruction format
#0001 1000 0000 0000 0000 0010 0000 0011
#00110 000000001 000000000 000000101
# opc   src1       src2       dest
def CPU(args):

    #Memory.Mem('0x0', din = '0x20000102', we = True) # ADD R2, R1, R0
    #Memory.Mem('0x4', din = '0x20000203', we = True) # ADD R3, R2, R1
    #for i in range(16):
        #Memory.Mem(hex((i + 20) * 4), din = hex(randrange(1000)), we = True)
    Memory.Mem(hex(4*100), din = '0x1', we = True)
    Memory.Mem(hex(4*101), din = '0x3', we = True)
    Memory.Mem(hex(4*102), din = '0x5', we = True)
    Memory.Mem(hex(4*103), din = '0x9', we = True)
    config.REGISTER[0] = 4*100
    config.REGISTER[1] = 4*101
    config.REGISTER[2] = 4*102
    config.REGISTER[3] = 4*103
    config.REGISTER[9] = -8
    Memory.Mem('0x0', din = '0x30000004', we = True) # LOAD R4, [R0]
    Memory.Mem('0x4', din = '0x30040005', we = True) # LOAD R5, [R1]
    Memory.Mem('0x8', din = '0x281408063', we = True) # MULT R6, R5, R4
    Memory.Mem('0xC', din = '0x30080C04', we = True) # LOAD R4, [R2]
    Memory.Mem('0x10', din = '0x300C0805', we = True) # LOAD R5, [R3]
    Memory.Mem('0x14', din = '0x28100A07', we = True) # MULT R7, R4, R5
    Memory.Mem('0x18', din = '0xF8080C04', we = True) # MULT R7, R4, R5
    Memory.Mem('0x1C', din = '0x181C0C08', we = True) # ADD R8, R6, R7
    Memory.Mem('0x20', din = '0x48201208', we = True) # BEQ R8, R9
    Memory.Mem('0x24', din = '0xF8080C04', we = True) # LOOP
    Memory.Mem('0x28', din = '0x20040601', we = True) # SUB R1, R1, R3
    Memory.Mem('0x2C', din = '0x48041209', we = True) # BEQ R1, R9
    #Memory.Mem('0x8', din = '0x38180007', we = True) # STR R6, [R7]
    Memory.Mem('0x30', din = '0x68000000', we = True) # HALT
    print(config.REGISTER)
    print(Memory.MEM_SPACE)

    # loop until HALT
    decoded = {'OPCODE': 'None'}
    while 'OPCODE' in decoded and decoded['OPCODE'] != 'HALT':
        instruc = fetch(hex(config.PC))  # fetch
        decoded = decode(instruc['DATA']) # decode
        print('INSTRUCTION: %s, SRC1: %s, SRC2: %s, DEST: %s' % (decoded['OPCODE'], int(decoded['SRC1'], 16), int(decoded['SRC2'], 16), int(decoded['DEST'], 16)))
        execute(decoded) # execute
        ALU.ALU(0, 0, 'ADD') # add 4 to pc
        config.PC += 4

    print(Memory.MEM_SPACE)
    print(config.REGISTER)
    print("CLOCK: %d" % (config.GLOBAL_CLOCK/config.CLOCK_CYCLE))

def fetch(addr):
    """
    this function fetches an instruction
    from the address given
    addr --> input: address of instruction

    returns: The instruction fetched
    """

    return Memory.Mem(addr)

def decode(ins):
    """
    this function decodes instruction

    returns: instruction decoded
    """
    return instruction_decoder.decode(ins)

def execute(decoded_ins):
    """
    function to excute given instruction
    """

    opcode = decoded_ins['OPCODE']

    # if opcode is arithmetic operation
    if opcode == 'ADD' or opcode == 'SUB' or opcode == 'MULT' or opcode == 'XOR' or opcode == 'OR' or opcode == 'AND':
        config.REGISTER[int(decoded_ins['DEST'], 16)] = ALU.ALU(config.REGISTER[int(decoded_ins['SRC1'], 16)], config.REGISTER[int(decoded_ins['SRC2'], 16)], opcode)['RESULT']

    # if opcode is LOAD
    elif opcode == 'LOAD':
        reg = int(decoded_ins['SRC1'], 16)
        load(int(decoded_ins['DEST'], 16), hex(config.REGISTER[reg]))

    # if opcode is STR
    elif opcode == 'STR':
        register = int(decoded_ins['SRC1'], 16)
        value = hex(config.REGISTER[register])
        addr = hex(config.REGISTER[int(decoded_ins['DEST'], 16)])
        store(value, addr)

    # if opcode is BEQ
    elif opcode == 'BEQ':
        equal = ALU.ALU(config.REGISTER[int(decoded_ins['SRC1'], 16)], config.REGISTER[int(decoded_ins['SRC2'], 16)], 'SUB')['STATUS']
        if equal == 'ZERO':
            config.PC = config.JUMPS.pop()
        config.GLOBAL_CLOCK += 1 # compare latency (XOR)

    # if opcode is BP, BN, BZ
    elif opcode == 'BP':
        value = config.REGISTER[int(decoded_ins['SRC1'], 16)]
        if value > 0:
            config.PC = config.JUMPS.pop()
        config.GLOBAL_CLOCK += 1 # compare latency (XOR)

    # if opcode is BN
    elif opcode == 'BN':
        value = config.REGISTER[int(decoded_ins['SRC1'], 16)]
        if value < 0:
            config.PC = config.JUMPS.pop()
        config.GLOBAL_CLOCK += 1 # compare latency (XOR)

    # if opcode is BZ
    elif opcode == 'BZ':
        value = config.REGISTER[int(decoded_ins['SRC1'], 16)]
        if value == 0:
            config.PC = config.JUMPS.pop()
        config.GLOBAL_CLOCK += 1 # compare latency (XOR)

def load(dest, addr):
    """
    function to load value at specified address
    dest --> input: destination register
    addr --> input: address in memory
    """
    config.REGISTER[dest] = int(''.join(Memory.Mem(addr)['DATA']), 16)

def store(value, addr):
    """
    function to store given value
    at a specified address
    value --> input: register with value
    addr  --> input: address in memory
    """
    Memory.Mem(addr, din = value, we = True)

# -------------------------------------
# Parser
# -------------------------------------
parser = argparse.ArgumentParser(description = 'CPU Simulator')
parser.add_argument('-mem', metavar = 'Mem File', help = "path to memory image")
parser.set_defaults(func = CPU)

if __name__ == "__main__":
    args = parser.parse_args()
    args.func(args)
<commit_msg>branching works<commit_after>"""
Ahmed Abdulkareem
02/10/2015
Final Project
CPU Implementation
All rights reserved
"""

import argparse
import Memory
import ALU
import instruction_decoder
import config
from random import randrange

#instruction format
#0001 1000 0000 0000 0000 0010 0000 0011
#00110 000000001 000000000 000000101
# opc   src1       src2       dest
def CPU(args):

    #Memory.Mem('0x0', din = '0x20000102', we = True) # ADD R2, R1, R0
    #Memory.Mem('0x4', din = '0x20000203', we = True) # ADD R3, R2, R1
    #for i in range(16):
        #Memory.Mem(hex((i + 20) * 4), din = hex(randrange(1000)), we = True)
    Memory.Mem(hex(4*100), din = '0x1', we = True)
    Memory.Mem(hex(4*101), din = '0x3', we = True)
    Memory.Mem(hex(4*102), din = '0x5', we = True)
    Memory.Mem(hex(4*103), din = '0x9', we = True)
    config.REGISTER[0] = 4*100
    config.REGISTER[1] = 4*101
    config.REGISTER[2] = 4*102
    config.REGISTER[3] = 4*103
    config.REGISTER[9] = 1
    config.REGISTER[10] = -4
    """
    Memory.Mem('0x0', din = '0x30000004', we = True) # LOAD R4, [R0]
    Memory.Mem('0x4', din = '0x30040005', we = True) # LOAD R5, [R1]
    Memory.Mem('0x8', din = '0x281408063', we = True) # MULT R6, R5, R4
    Memory.Mem('0xC', din = '0x30080C04', we = True) # LOAD R4, [R2]
    Memory.Mem('0x10', din = '0x300C0805', we = True) # LOAD R5, [R3]
    Memory.Mem('0x14', din = '0x28100A07', we = True) # MULT R7, R4, R5
    Memory.Mem('0x18', din = '0xF8080C04', we = True) # MULT R7, R4, R5
    Memory.Mem('0x1C', din = '0x181C0C08', we = True) # ADD R8, R6, R7
    Memory.Mem('0x20', din = '0x48201208', we = True) # BEQ R8, R9
    Memory.Mem('0x24', din = '0xF8080C04', we = True) # LOOP
    Memory.Mem('0x28', din = '0x20040601', we = True) # SUB R1, R1, R3
    Memory.Mem('0x2C', din = '0x48041209', we = True) # BEQ R1, R9
    """
    Memory.Mem('0x0', din = '0xF8080C04', we = True) # LOOP
    Memory.Mem('0x4', din = '0x1828120A', we = True) # ADD R10, R10, R9
    Memory.Mem('0x8', din = '0x5828120A', we = True) # BEQ R10
    #Memory.Mem('0x8', din = '0x38180007', we = True) # STR R6, [R7]
    Memory.Mem('0xC', din = '0x68000000', we = True) # HALT
    print(config.REGISTER)
    print(Memory.MEM_SPACE)

    # loop until HALT
    decoded = {'OPCODE': 'None'}
    while 'OPCODE' in decoded and decoded['OPCODE'] != 'HALT':
        instruc = fetch(hex(config.PC))  # fetch
        decoded = decode(instruc['DATA']) # decode
        print('INSTRUCTION: %s, SRC1: %s, SRC2: %s, DEST: %s' % (decoded['OPCODE'], int(decoded['SRC1'], 16), int(decoded['SRC2'], 16), int(decoded['DEST'], 16)))
        execute(decoded) # execute
        ALU.ALU(0, 0, 'ADD') # add 4 to pc
        config.PC += 4

    print(Memory.MEM_SPACE)
    print(config.REGISTER)
    print("CLOCK: %d" % (config.GLOBAL_CLOCK/config.CLOCK_CYCLE))

def fetch(addr):
    """
    this function fetches an instruction
    from the address given
    addr --> input: address of instruction

    returns: The instruction fetched
    """

    return Memory.Mem(addr)

def decode(ins):
    """
    this function decodes instruction

    returns: instruction decoded
    """
    return instruction_decoder.decode(ins)

def execute(decoded_ins):
    """
    function to excute given instruction
    """

    opcode = decoded_ins['OPCODE']

    # if opcode is arithmetic operation
    if opcode == 'ADD' or opcode == 'SUB' or opcode == 'MULT' or opcode == 'XOR' or opcode == 'OR' or opcode == 'AND':
        config.REGISTER[int(decoded_ins['DEST'], 16)] = ALU.ALU(config.REGISTER[int(decoded_ins['SRC1'], 16)], config.REGISTER[int(decoded_ins['SRC2'], 16)], opcode)['RESULT']

    # if opcode is LOAD
    elif opcode == 'LOAD':
        reg = int(decoded_ins['SRC1'], 16)
        load(int(decoded_ins['DEST'], 16), hex(config.REGISTER[reg]))

    # if opcode is STR
    elif opcode == 'STR':
        register = int(decoded_ins['SRC1'], 16)
        value = hex(config.REGISTER[register])
        addr = hex(config.REGISTER[int(decoded_ins['DEST'], 16)])
        store(value, addr)

    # if opcode is BEQ
    elif opcode == 'BEQ':
        equal = ALU.ALU(config.REGISTER[int(decoded_ins['SRC1'], 16)], config.REGISTER[int(decoded_ins['SRC2'], 16)], 'SUB')['STATUS']
        if equal == 'ZERO':
            config.PC = config.JUMPS.pop()
        config.GLOBAL_CLOCK += 1 # compare latency (XOR)

    # if opcode is BP, BN, BZ
    elif opcode == 'BP':
        value = config.REGISTER[int(decoded_ins['SRC1'], 16)]
        if value > 0:
            config.PC = config.JUMPS.pop()
        config.GLOBAL_CLOCK += 1 # compare latency (XOR)

    # if opcode is BN
    elif opcode == 'BN':
        value = config.REGISTER[int(decoded_ins['SRC1'], 16)]
        if value < 0:
            config.PC = config.JUMPS.pop()
        config.GLOBAL_CLOCK += 1 # compare latency (XOR)

    # if opcode is BZ
    elif opcode == 'BZ':
        value = config.REGISTER[int(decoded_ins['SRC1'], 16)]
        if value == 0:
            config.PC = config.JUMPS.pop()
        config.GLOBAL_CLOCK += 1 # compare latency (XOR)

def load(dest, addr):
    """
    function to load value at specified address
    dest --> input: destination register
    addr --> input: address in memory
    """
    config.REGISTER[dest] = int(''.join(Memory.Mem(addr)['DATA']), 16)

def store(value, addr):
    """
    function to store given value
    at a specified address
    value --> input: register with value
    addr  --> input: address in memory
    """
    Memory.Mem(addr, din = value, we = True)

# -------------------------------------
# Parser
# -------------------------------------
parser = argparse.ArgumentParser(description = 'CPU Simulator')
parser.add_argument('-mem', metavar = 'Mem File', help = "path to memory image")
parser.set_defaults(func = CPU)

if __name__ == "__main__":
    args = parser.parse_args()
    args.func(args)
<|endoftext|>


<commit_before>'''
This script calculates the gross emissions in tonnes CO2e/ha for every loss pixel.
The properties of each pixel determine the appropriate emissions equation, the constants for the equation, and the
carbon pool values that go into the equation.
Unlike all other flux model components, this one uses C++ to quickly iterate through every pixel in each tile.
Before running the model, the C++ script must be compiled first.
Navigate to carbon-budget/emissions/cpp_util.
Compile the C++ file calc_emissions: c++ calc_emissions_v3.cpp -o calc_emissions_v3.exe -lgdal
calc_emissions_v3.exe should appear in the directory.
Return to carbon-budget/emissions using cd ..
Run mp_calculate_gross_emissions.py by typing python mp_calculate_gross_emissions.py. The Python script will call the
compiled C++ code as needed.
The other C++ scripts (equations.cpp and flu_val.cpp) do not need to be compiled.
Through the magic of C++, calc_emissions_v3.cpp reads them just fine without the user compiling them.
Emissions from each driver (including loss that had no driver assigned) gets its own tile, as does all emissions combined.
The other output shows which branch of the decision tree that determines the emissions equation applies to each pixel.
These codes are summarized in carbon-budget/emissions/node_codes.txt
'''

import multiprocessing
import calculate_gross_emissions
from functools import partial
from multiprocessing.pool import Pool
import utilities
import os
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

tile_list = uu.tile_list(cn.AGC_emis_year_dir)
# tile_list = ['50N_020E'] # test tiles
# tile_list = ['00N_110E', '80N_020E', '30N_080W', '00N_020E'] # test tiles: no mangrove or planted forest, mangrove only, planted forest only, mangrove and planted forest
print tile_list
print "There are {} tiles to process".format(str(len(tile_list)))

# For downloading all tiles in the folders.
# This takes about 40 minutes.
download_list = [
                 cn.AGC_emis_year_dir, cn.BGC_emis_year_dir, cn.deadwood_emis_year_2000_dir, cn.litter_emis_year_2000_dir, cn.soil_C_emis_year_2000_dir,
                 cn.peat_mask_dir, cn.ifl_dir, cn.planted_forest_type_unmasked_dir, cn.drivers_processed_dir, cn.climate_zone_processed_dir,
                 cn.bor_tem_trop_processed_dir, cn.burn_year_dir,
                 cn.plant_pre_2000_processed_dir,
                 cn.loss_dir
                ]

for input in download_list:
    uu.s3_folder_download(input, './cpp_util/')

# # For copying individual tiles to s3 for testing
# for tile in tile_list:
#
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.AGC_emis_year_dir, tile, cn.pattern_AGC_emis_year), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.BGC_emis_year_dir, tile, cn.pattern_BGC_emis_year), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.deadwood_emis_year_2000_dir, tile, cn.pattern_deadwood_emis_year_2000), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.litter_emis_year_2000_dir, tile, cn.pattern_litter_emis_year_2000), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.soil_C_emis_year_2000_dir, tile, cn.pattern_soil_C_emis_year_2000), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.peat_mask_dir, tile, cn.pattern_peat_mask), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.ifl_dir, tile, cn.pattern_ifl), './cpp_util/')
#     try:
#         uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.planted_forest_type_unmasked_dir, tile, cn.pattern_planted_forest_type_unmasked), './cpp_util/')
#     except:
#         print "No plantations in", tile
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.drivers_processed_dir, tile, cn.pattern_drivers), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.climate_zone_processed_dir, tile, cn.pattern_climate_zone), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.bor_tem_trop_processed_dir, tile, cn.pattern_bor_tem_trop_processed), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.burn_year_dir, tile, cn.pattern_burn_year), './cpp_util/')
#     try:
#         uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.plant_pre_2000_processed_dir, tile, cn.pattern_plant_pre_2000), './cpp_util/')
#     except:
#         print "No pre-2000 plantations in", tile
#     uu.s3_file_download('{0}{1}.tif'.format(cn.loss_dir, tile), './cpp_util/')

print "Removing loss pixels from plantations that existed in Indonesia and Malaysia before 2000..."
# Pixels that were in plantations that existed before 2000 should not be included in gross emissions.
# Pre-2000 plantations have not previously been masked, so that is done here.
count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(count/2)
pool.map(utilities.mask_loss_pre_2000_plantation, tile_list)

# # For single processor use
# for tile in tile_list:
#
#       utilities.mask_loss_pre_2000_plantation(tile)

# The C++ code expects a plantations tile for every input 10x10.
# However, not all Hansen tiles have plantations.
# This function creates "dummy" plantation tiles for all Hansen tiles that do not have plantations.
# That way, the C++ script gets all the necessary input files
pattern = cn.pattern_planted_forest_type_unmasked
folder = 'cpp_util/'

# All of the inputs that need to have dummy tiles made in order to match the tile list of the carbon pools
pattern_list = [cn.pattern_planted_forest_type_unmasked, cn.pattern_peat_mask, cn.pattern_ifl, cn.pattern_planted_forest_type_unmasked,
                cn.pattern_drivers, cn.pattern_bor_tem_trop_processed]

for pattern in pattern_list:
    count = multiprocessing.cpu_count()
    pool = multiprocessing.Pool(count-10)
    pool.map(partial(uu.make_blank_tile, pattern=pattern, folder=folder), tile_list)
    pool.close()
    pool.join()

# # For single processor use
# for pattern in pattern_list:
#     for tile in tile_list:
#
#         uu.make_blank_tile(tile, pattern, folder)

# Calculates gross emissions for each tile
count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(count/3)
pool.map(calculate_gross_emissions.calc_emissions, tile_list)

# # For single processor use
# for tile in tile_list:
#
#       calculate_gross_emissions.calc_emissions(tile)

uu.upload_final_set(cn.gross_emis_commod_dir, cn.pattern_gross_emis_commod)
uu.upload_final_set(cn.gross_emis_shifting_ag_dir, cn.pattern_gross_emis_shifting_ag)
uu.upload_final_set(cn.gross_emis_forestry_dir, cn.pattern_gross_emis_forestry)
uu.upload_final_set(cn.gross_emis_wildfire_dir, cn.pattern_gross_emis_wildfire)
uu.upload_final_set(cn.gross_emis_urban_dir, cn.pattern_gross_emis_urban)
uu.upload_final_set(cn.gross_emis_no_driver_dir, cn.pattern_gross_emis_no_driver)
uu.upload_final_set(cn.gross_emis_nodes_dir, cn.pattern_gross_emis_nodes)
uu.upload_final_set(cn.gross_emis_all_drivers_dir, cn.pattern_gross_emis_all_drivers)

<commit_msg>Got to the emissions calculation function but froze with count/2. Trying with count/4 at the same time as count/3 in case count/3 freezes as well..<commit_after>'''
This script calculates the gross emissions in tonnes CO2e/ha for every loss pixel.
The properties of each pixel determine the appropriate emissions equation, the constants for the equation, and the
carbon pool values that go into the equation.
Unlike all other flux model components, this one uses C++ to quickly iterate through every pixel in each tile.
Before running the model, the C++ script must be compiled first.
Navigate to carbon-budget/emissions/cpp_util.
Compile the C++ file calc_emissions: c++ calc_emissions_v3.cpp -o calc_emissions_v3.exe -lgdal
calc_emissions_v3.exe should appear in the directory.
Return to carbon-budget/emissions using cd ..
Run mp_calculate_gross_emissions.py by typing python mp_calculate_gross_emissions.py. The Python script will call the
compiled C++ code as needed.
The other C++ scripts (equations.cpp and flu_val.cpp) do not need to be compiled.
Through the magic of C++, calc_emissions_v3.cpp reads them just fine without the user compiling them.
Emissions from each driver (including loss that had no driver assigned) gets its own tile, as does all emissions combined.
The other output shows which branch of the decision tree that determines the emissions equation applies to each pixel.
These codes are summarized in carbon-budget/emissions/node_codes.txt
'''

import multiprocessing
import calculate_gross_emissions
from functools import partial
from multiprocessing.pool import Pool
import utilities
import os
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

tile_list = uu.tile_list(cn.AGC_emis_year_dir)
# tile_list = ['50N_020E'] # test tiles
# tile_list = ['00N_110E', '80N_020E', '30N_080W', '00N_020E'] # test tiles: no mangrove or planted forest, mangrove only, planted forest only, mangrove and planted forest
print tile_list
print "There are {} tiles to process".format(str(len(tile_list)))

# For downloading all tiles in the folders.
# This takes about 40 minutes.
download_list = [
                 cn.AGC_emis_year_dir, cn.BGC_emis_year_dir, cn.deadwood_emis_year_2000_dir, cn.litter_emis_year_2000_dir, cn.soil_C_emis_year_2000_dir,
                 cn.peat_mask_dir, cn.ifl_dir, cn.planted_forest_type_unmasked_dir, cn.drivers_processed_dir, cn.climate_zone_processed_dir,
                 cn.bor_tem_trop_processed_dir, cn.burn_year_dir,
                 cn.plant_pre_2000_processed_dir,
                 cn.loss_dir
                ]

for input in download_list:
    uu.s3_folder_download(input, './cpp_util/')

# # For copying individual tiles to s3 for testing
# for tile in tile_list:
#
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.AGC_emis_year_dir, tile, cn.pattern_AGC_emis_year), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.BGC_emis_year_dir, tile, cn.pattern_BGC_emis_year), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.deadwood_emis_year_2000_dir, tile, cn.pattern_deadwood_emis_year_2000), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.litter_emis_year_2000_dir, tile, cn.pattern_litter_emis_year_2000), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.soil_C_emis_year_2000_dir, tile, cn.pattern_soil_C_emis_year_2000), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.peat_mask_dir, tile, cn.pattern_peat_mask), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.ifl_dir, tile, cn.pattern_ifl), './cpp_util/')
#     try:
#         uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.planted_forest_type_unmasked_dir, tile, cn.pattern_planted_forest_type_unmasked), './cpp_util/')
#     except:
#         print "No plantations in", tile
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.drivers_processed_dir, tile, cn.pattern_drivers), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.climate_zone_processed_dir, tile, cn.pattern_climate_zone), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.bor_tem_trop_processed_dir, tile, cn.pattern_bor_tem_trop_processed), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.burn_year_dir, tile, cn.pattern_burn_year), './cpp_util/')
#     try:
#         uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.plant_pre_2000_processed_dir, tile, cn.pattern_plant_pre_2000), './cpp_util/')
#     except:
#         print "No pre-2000 plantations in", tile
#     uu.s3_file_download('{0}{1}.tif'.format(cn.loss_dir, tile), './cpp_util/')

print "Removing loss pixels from plantations that existed in Indonesia and Malaysia before 2000..."
# Pixels that were in plantations that existed before 2000 should not be included in gross emissions.
# Pre-2000 plantations have not previously been masked, so that is done here.
count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(count/2)
pool.map(utilities.mask_loss_pre_2000_plantation, tile_list)

# # For single processor use
# for tile in tile_list:
#
#       utilities.mask_loss_pre_2000_plantation(tile)

# The C++ code expects a plantations tile for every input 10x10.
# However, not all Hansen tiles have plantations.
# This function creates "dummy" plantation tiles for all Hansen tiles that do not have plantations.
# That way, the C++ script gets all the necessary input files
pattern = cn.pattern_planted_forest_type_unmasked
folder = 'cpp_util/'

# All of the inputs that need to have dummy tiles made in order to match the tile list of the carbon pools
pattern_list = [cn.pattern_planted_forest_type_unmasked, cn.pattern_peat_mask, cn.pattern_ifl, cn.pattern_planted_forest_type_unmasked,
                cn.pattern_drivers, cn.pattern_bor_tem_trop_processed]

for pattern in pattern_list:
    count = multiprocessing.cpu_count()
    pool = multiprocessing.Pool(count-10)
    pool.map(partial(uu.make_blank_tile, pattern=pattern, folder=folder), tile_list)
    pool.close()
    pool.join()

# # For single processor use
# for pattern in pattern_list:
#     for tile in tile_list:
#
#         uu.make_blank_tile(tile, pattern, folder)

# Calculates gross emissions for each tile
count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(count/4)
pool.map(calculate_gross_emissions.calc_emissions, tile_list)

# # For single processor use
# for tile in tile_list:
#
#       calculate_gross_emissions.calc_emissions(tile)

uu.upload_final_set(cn.gross_emis_commod_dir, cn.pattern_gross_emis_commod)
uu.upload_final_set(cn.gross_emis_shifting_ag_dir, cn.pattern_gross_emis_shifting_ag)
uu.upload_final_set(cn.gross_emis_forestry_dir, cn.pattern_gross_emis_forestry)
uu.upload_final_set(cn.gross_emis_wildfire_dir, cn.pattern_gross_emis_wildfire)
uu.upload_final_set(cn.gross_emis_urban_dir, cn.pattern_gross_emis_urban)
uu.upload_final_set(cn.gross_emis_no_driver_dir, cn.pattern_gross_emis_no_driver)
uu.upload_final_set(cn.gross_emis_nodes_dir, cn.pattern_gross_emis_nodes)
uu.upload_final_set(cn.gross_emis_all_drivers_dir, cn.pattern_gross_emis_all_drivers)

<|endoftext|>


<commit_before>__title__ = "SSLyze"
__version__ = "4.0.4"
__description__ = "Fast and powerful SSL/TLS scanning library."
__author__ = "Alban Diquet"
__author_email__ = "nabla.c0d3@gmail.com"
__url__ = "https://github.com/nabla-c0d3/sslyze"
__license__ = "AGPL"
__copyright__ = "Copyright 2020 Alban Diquet"
<commit_msg>Bump version number<commit_after>__title__ = "SSLyze"
__version__ = "4.1.0"
__description__ = "Fast and powerful SSL/TLS scanning library."
__author__ = "Alban Diquet"
__author_email__ = "nabla.c0d3@gmail.com"
__url__ = "https://github.com/nabla-c0d3/sslyze"
__license__ = "AGPL"
__copyright__ = "Copyright 2020 Alban Diquet"
<|endoftext|>


<commit_before># stackcore.py - JSONModel/Enumeration + other utility classes that don't really belong now that the API's multi-file
# This file is relatively safe to "import *"

import datetime
from math import floor
from six.moves import urllib

## JSONModel base class
class JSONModel(object):
    """The base class of all the objects which describe API objects directly - ie, those which take JSON objects as parameters to their constructor."""

    def __init__(self, json, site, skip_ext = False):
        self.json = json
        self.json_ob = DictObject(json)
        self.site = site

        # we have four ways of specifying a field:
        #   - ('a', 'b') in alias: .a = JSON.b
        #   - 'name' in transfer: .name = JSON.name
        #   - ('a', 'b', t) in alias: .a = t(JSON.b)
        #   - ('name', t) in transfer: .name = t(JSON.name)
        # we unify these all into one set of descriptions
        alias = self.alias if hasattr(self, 'alias') else ()
        transfer = self.transfer if hasattr(self, 'transfer') else ()
        fields = ([A + (None,) for A in alias if len(A) == 2] +
                  [(k, k, None) for k in transfer if isinstance(k, str)] +
                  [A for A in alias if len(A) == 3] +
                  [(k[0],) + k for k in transfer if not isinstance(k, str)])

        for dest, key, transform in fields:
            if hasattr(self.json_ob, key):
                value = getattr(self.json_ob, key)

                if isinstance(transform, ComplexTransform):
                    value = transform(key, value, self)
                elif transform is not None:
                    value = transform(value)

                setattr(self, dest, value)
            elif isinstance(transform, ComplexTransform):
                value = transform.no_value(key, self)
                if value is not None:
                    setattr(self, dest, value)

        if hasattr(self, '_extend') and not skip_ext:
            self._extend(self.json_ob, site)

    def fetch(self):
        """Fetches all the data that the model can describe, not just the attributes which were specified in the original response."""
        if hasattr(self, 'fetch_callback'):
            res = self.fetch_callback(self)

            if isinstance(res, dict):
                self.__init__(res, self.site)
            elif hasattr(res, 'json'):
                self.__init__(res.json, self.site)
            else:
                raise ValueError('Supplied fetch callback did not return a usable value.')
        else:
            return False

    # Allows the easy creation of updateable, partial classes
    @classmethod
    def partial(cls, fetch_callback, site, populate):
        """Creates a partial description of the API object, with the proviso that the full set of data can be fetched later."""

        model = cls({}, site, True)

        for k, v in populate.items():
            setattr(model, k, v)

        model.fetch_callback = fetch_callback
        return model

    # for use with Lazy classes that need a callback to actually set the model property
    def _up(self, a):
        """Returns a function which can be used with the LazySequence class to actually update the results properties on the model with the
new fetched data."""

        def inner(m):
            setattr(self, a, m)
        return inner

def LaterClassIn(name, module):
    def constructor(*a, **kw):
        cls = getattr(module, name)
        return cls(*a, **kw)
    constructor.__name__ = name
    return constructor

# a convenience 'type constructor' for producing datetime's from UNIX timestamps
UNIXTimestamp = lambda n: datetime.datetime.fromtimestamp(n)

# Some transforms are more complicated and need access to the model object
# itself, including Sites, etc. This is a base class to indicate this to the
# constructor.
class ComplexTransform(object):
    def no_value(self, key, model):
        pass

class ListOf(ComplexTransform):
    def __init__(self, transform):
        self.transform = transform

    def no_value(self, key, model):
        return []

    def __call__(self, key, value, model):
        if isinstance(self.transform, ComplexTransform):
            return [self.transform(key, v, model) for v in value]
        else:
            return [self.transform(v) for v in value]

class ModelRef(ComplexTransform):
    '''A convenience for foreign model references that take a JSON value and a reference to the underlying site object.'''
    def __init__(self, model_type):
        self.model_type = model_type

    def __call__(self, key, value, model):
        return self.model_type(value, model.site)

class LazySequenceField(ComplexTransform):
    def __init__(self, m_type, url_format, count = None, update_key = None, response_key = None, **kw):
        self.m_type = m_type
        self.url_format = url_format
        self.count = count
        self.update_key = update_key
        self.response_key = response_key
        self.kw = kw

    def no_value(self, key, model):
        model_id = getattr(model, 'id', getattr(model.json_ob, key + '_id', None))
        url = self.url_format.format(id = model_id)
        response_key = key if self.response_key is None else self.response_key
        update_key = key if self.update_key is None else self.update_key
        return StackExchangeLazySequence(self.m_type, self.count, model.site,
                                            url, model._up(update_key),
                                            response_key, **self.kw)

    def __call__(self, key, value, model):
        return self.no_value(key, model)

class Enumeration(object):
    """Provides a base class for enumeration classes. (Similar to 'enum' types in other languages.)"""

    @classmethod
    def from_string(cls, text, typ = None):
        'Returns the appropriate enumeration value for the given string, mapping underscored names to CamelCase, or the input string if a mapping could not be made.'
        if typ is not None:
            if hasattr(typ, '_map') and text in typ._map:
                return getattr(typ, typ._map[text])
            elif hasattr(typ, text[0].upper() + text[1:]):
                return getattr(typ, text[0].upper() + text[1:])
            elif '_' in text:
                real_name = ''.join(x.title() for x in text.split('_'))
                if hasattr(typ, real_name):
                    return getattr(typ, real_name)
                else:
                    return text
            else:
                return text
        else:
            return cls.from_string(text, cls)

class StackExchangeError(Exception):
    """A generic error thrown on a bad HTTP request during a StackExchange API request."""
    UNKNOWN = -1

    def __init__(self, code = UNKNOWN, name = None, message = None):
        self.code = code
        self.name = name
        self.message = message

    def __str__(self):
        if self.code == self.UNKNOWN:
            return 'unrecognised error'
        else:
            return '%d [%s]: %s' % (self.code, self.name, self.message)

class EmptyResultset(tuple):
    def __new__(cls, json):
        instance = tuple.__new__(cls, [])

        for key in ('page_size', 'page', 'has_more', 'total', 'quota_max', 'quota_remaining', 'type'):
            if key in json:
                setattr(instance, key, json[key])

        return instance

class StackExchangeResultset(tuple):
    """Defines an immutable, paginated resultset. This class can be used as a tuple, but provides extended metadata as well, including methods
to fetch the next page."""

    def __new__(cls, items, build_info, has_more = True, page = 1, pagesize = None):
        if pagesize is None:
            pagesize = len(items)

        instance = tuple.__new__(cls, items)
        instance.page, instance.pagesize, instance.build_info = page, pagesize, build_info
        instance.items = items
        instance.has_more = has_more

        return instance

    def reload(self):
        """Refreshes the data in the resultset with fresh API data. Note that this doesn't work with extended resultsets."""
        # kind of a cheat, but oh well
        return self.fetch_page(self.page)

    def fetch_page(self, page, **kw):
        """Returns a new resultset containing data from the specified page of the results. It re-uses all parameters that were passed in
to the initial function which created the resultset."""
        new_params = list(self.build_info)
        new_params[4] = new_params[4].copy()
        new_params[4].update(kw)
        new_params[4]['page'] = page

        new_set = new_params[0].build(*new_params[1:])
        new_set.page = page
        return new_set

    def fetch_extended(self, page):
        """Returns a new resultset containing data from this resultset AND from the specified page."""
        next = self.fetch_page(page)
        extended = self + next

        # max(0, ...) is so a non-zero, positive result for page is always found
        return StackExchangeResultset(extended, self.build_info, next.has_more, page)

    def fetch_next(self):
        """Returns the resultset of the data in the next page."""
        return self.fetch_page(self.page + 1)

    def extend_next(self):
        """Returns a new resultset containing data from this resultset AND from the next page."""
        return self.fetch_extended(self.page + 1)

    def fetch(self):
        # Do nothing, but allow multiple fetch calls
        return self

    def __iter__(self):
        return self.next()

    def next(self):
        for obj in self.items:
            yield obj

        current = self
        while current.has_more:
            for obj in current.items:
                yield obj

            try:
                current = current.fetch_next()
                if len(current) == 0:
                    return
            except urllib.error.HTTPError:
                return

class NeedsAwokenError(Exception):
    """An error raised when an attempt is made to access a property of a lazy collection that requires the data to have been fetched,
but whose data has not yet been requested."""

    def __init__(self, lazy):
        self.lazy = lazy
    def __str__(self):
        return 'Could not return requested data; the sequence of "%s" has not been fetched.' % self.lazy.m_lazy

class StackExchangeLazySequence(list):
    """Provides a sequence which *can* contain extra data available on an object. It is 'lazy' in the sense that data is only fetched when
required - not on object creation."""

    def __init__(self, m_type, count, site, url, fetch = None, collection = None, **kw):
        self.m_type = m_type
        self.count = count
        self.site = site
        self.url = url
        self.fetch_callback = fetch
        self.kw = kw
        self.collection = collection if collection != None else self._collection(url)

    def _collection(self, c):
        return c.split('/')[-1]

    def __len__(self):
        if self.count != None:
            return self.count
        else:
            raise NeedsAwokenError(self)

    def __repr__(self):
        if self.count is None:
            return '<unfetched sequence: %s>' % self.m_type.__name__
        else:
            return list.__repr__(self)

    def fetch(self, **direct_kw):
        """Fetch, from the API, the data this sequence is meant to hold."""
        # If we have any default parameters, include them, but overwrite any
        # passed in here directly.
        kw = dict(self.kw)
        kw.update(direct_kw)

        res = self.site.build(self.url, self.m_type, self.collection, kw)
        if self.fetch_callback != None:
            self.fetch_callback(res)
        return res

class StackExchangeLazyObject(list):
    """Provides a proxy to fetching a single item from a collection, lazily."""

    def __init__(self, m_type, site, url, fetch = None, collection = None):
        self.m_type = m_type
        self.site = site
        self.url = url
        self.fetch_callback = fetch
        self.collection = collection if collection != None else self._collection(url)

    def fetch(self, **kw):
        """Fetch, from the API, the data supposed to be held."""
        res = self.site.build(self.url, self.m_type, self.collection, kw)[0]
        if self.fetch_callback != None:
            self.fetch_callback(res)
        return res

    def __getattr__(self, key):
        raise NeedsAwokenError

#### Hack, because I can't be bothered to fix my mistaking JSON's output for an object not a dict
# (Si jeunesse savait, si vieillesse pouvait...)
# Attrib: Eli Bendersky, http://stackoverflow.com/questions/1305532/convert-python-dict-to-object/1305663#1305663
class DictObject:
    def __init__(self, entries):
        self.__dict__.update(entries)

class JSONMangler(object):
    """This class handles all sorts of random JSON-handling stuff"""

    @staticmethod
    def paginated_to_resultset(site, json, typ, collection, params):
        # N.B.: We ignore the 'collection' parameter for now, given that it is
        # no longer variable in v2.x, having been replaced by a generic field
        # 'items'. To perhaps be removed completely at some later point.
        items = []

        # create strongly-typed objects from the JSON items
        for json_item in json['items']:
            json_item['_params_'] = params[-1] # convenient access to the kw hash
            items.append(typ(json_item, site))

        rs = StackExchangeResultset(items, params, json['has_more'])
        if 'total' in json:
            rs.total = json['total']

        return rs

    @staticmethod
    def normal_to_resultset(site, json, typ, collection):
        # the parameter 'collection' may be need in future, and was needed pre-2.0
        return tuple([typ(x, site) for x in json['items']])

    @classmethod
    def json_to_resultset(cls, site, json, typ, collection, params = None):
        # this is somewhat of a special case, introduced by some filters in
        # post-2.0 allowing only 'metadata' to be returned
        if 'items' not in json:
            return EmptyResultset(json)
        elif 'has_more' in json:
            # we have a paginated resultset
            return cls.paginated_to_resultset(site, json, typ, collection, params)
        else:
            # this isn't paginated (unlikely but possible - eg badges)
            return cls.normal_to_resultset(site, json, typ, collection)

def format_relative_date(date):
    """Takes a datetime object and returns the date formatted as a string e.g. "3 minutes ago", like the real site.
    This is based roughly on George Edison's code from StackApps:
    http://stackapps.com/questions/1009/how-to-format-time-since-xxx-e-g-4-minutes-ago-similar-to-stack-exchange-site/1018#1018"""

    now = datetime.datetime.now()
    diff = (now - date).seconds

    # Anti-repetition! These simplify the code somewhat.
    plural = lambda d: 's' if d != 1 else ''
    frmt   = lambda d: (diff / float(d), plural(diff / float(d)))

    if diff < 60:
        return '%d second%s ago' % frmt(1)
    elif diff < 3600:
        return '%d minute%s ago' % frmt(60)
    elif diff < 86400:
        return '%d hour%s ago' % frmt(3600)
    elif diff < 172800:
        return 'yesterday'
    else:
        return date.strftime('M j / y - H:i')

class Sort(Enumeration):
    Activity = 'activity'
    Views = 'views'
    Creation = 'creation'
    Votes = 'votes'

ASC = 'asc'
DESC = 'desc'
<commit_msg>Add relative_to parameter to format_relative_date<commit_after># stackcore.py - JSONModel/Enumeration + other utility classes that don't really belong now that the API's multi-file
# This file is relatively safe to "import *"

import datetime
from math import floor
from six.moves import urllib

## JSONModel base class
class JSONModel(object):
    """The base class of all the objects which describe API objects directly - ie, those which take JSON objects as parameters to their constructor."""

    def __init__(self, json, site, skip_ext = False):
        self.json = json
        self.json_ob = DictObject(json)
        self.site = site

        # we have four ways of specifying a field:
        #   - ('a', 'b') in alias: .a = JSON.b
        #   - 'name' in transfer: .name = JSON.name
        #   - ('a', 'b', t) in alias: .a = t(JSON.b)
        #   - ('name', t) in transfer: .name = t(JSON.name)
        # we unify these all into one set of descriptions
        alias = self.alias if hasattr(self, 'alias') else ()
        transfer = self.transfer if hasattr(self, 'transfer') else ()
        fields = ([A + (None,) for A in alias if len(A) == 2] +
                  [(k, k, None) for k in transfer if isinstance(k, str)] +
                  [A for A in alias if len(A) == 3] +
                  [(k[0],) + k for k in transfer if not isinstance(k, str)])

        for dest, key, transform in fields:
            if hasattr(self.json_ob, key):
                value = getattr(self.json_ob, key)

                if isinstance(transform, ComplexTransform):
                    value = transform(key, value, self)
                elif transform is not None:
                    value = transform(value)

                setattr(self, dest, value)
            elif isinstance(transform, ComplexTransform):
                value = transform.no_value(key, self)
                if value is not None:
                    setattr(self, dest, value)

        if hasattr(self, '_extend') and not skip_ext:
            self._extend(self.json_ob, site)

    def fetch(self):
        """Fetches all the data that the model can describe, not just the attributes which were specified in the original response."""
        if hasattr(self, 'fetch_callback'):
            res = self.fetch_callback(self)

            if isinstance(res, dict):
                self.__init__(res, self.site)
            elif hasattr(res, 'json'):
                self.__init__(res.json, self.site)
            else:
                raise ValueError('Supplied fetch callback did not return a usable value.')
        else:
            return False

    # Allows the easy creation of updateable, partial classes
    @classmethod
    def partial(cls, fetch_callback, site, populate):
        """Creates a partial description of the API object, with the proviso that the full set of data can be fetched later."""

        model = cls({}, site, True)

        for k, v in populate.items():
            setattr(model, k, v)

        model.fetch_callback = fetch_callback
        return model

    # for use with Lazy classes that need a callback to actually set the model property
    def _up(self, a):
        """Returns a function which can be used with the LazySequence class to actually update the results properties on the model with the
new fetched data."""

        def inner(m):
            setattr(self, a, m)
        return inner

def LaterClassIn(name, module):
    def constructor(*a, **kw):
        cls = getattr(module, name)
        return cls(*a, **kw)
    constructor.__name__ = name
    return constructor

# a convenience 'type constructor' for producing datetime's from UNIX timestamps
UNIXTimestamp = lambda n: datetime.datetime.fromtimestamp(n)

# Some transforms are more complicated and need access to the model object
# itself, including Sites, etc. This is a base class to indicate this to the
# constructor.
class ComplexTransform(object):
    def no_value(self, key, model):
        pass

class ListOf(ComplexTransform):
    def __init__(self, transform):
        self.transform = transform

    def no_value(self, key, model):
        return []

    def __call__(self, key, value, model):
        if isinstance(self.transform, ComplexTransform):
            return [self.transform(key, v, model) for v in value]
        else:
            return [self.transform(v) for v in value]

class ModelRef(ComplexTransform):
    '''A convenience for foreign model references that take a JSON value and a reference to the underlying site object.'''
    def __init__(self, model_type):
        self.model_type = model_type

    def __call__(self, key, value, model):
        return self.model_type(value, model.site)

class LazySequenceField(ComplexTransform):
    def __init__(self, m_type, url_format, count = None, update_key = None, response_key = None, **kw):
        self.m_type = m_type
        self.url_format = url_format
        self.count = count
        self.update_key = update_key
        self.response_key = response_key
        self.kw = kw

    def no_value(self, key, model):
        model_id = getattr(model, 'id', getattr(model.json_ob, key + '_id', None))
        url = self.url_format.format(id = model_id)
        response_key = key if self.response_key is None else self.response_key
        update_key = key if self.update_key is None else self.update_key
        return StackExchangeLazySequence(self.m_type, self.count, model.site,
                                            url, model._up(update_key),
                                            response_key, **self.kw)

    def __call__(self, key, value, model):
        return self.no_value(key, model)

class Enumeration(object):
    """Provides a base class for enumeration classes. (Similar to 'enum' types in other languages.)"""

    @classmethod
    def from_string(cls, text, typ = None):
        'Returns the appropriate enumeration value for the given string, mapping underscored names to CamelCase, or the input string if a mapping could not be made.'
        if typ is not None:
            if hasattr(typ, '_map') and text in typ._map:
                return getattr(typ, typ._map[text])
            elif hasattr(typ, text[0].upper() + text[1:]):
                return getattr(typ, text[0].upper() + text[1:])
            elif '_' in text:
                real_name = ''.join(x.title() for x in text.split('_'))
                if hasattr(typ, real_name):
                    return getattr(typ, real_name)
                else:
                    return text
            else:
                return text
        else:
            return cls.from_string(text, cls)

class StackExchangeError(Exception):
    """A generic error thrown on a bad HTTP request during a StackExchange API request."""
    UNKNOWN = -1

    def __init__(self, code = UNKNOWN, name = None, message = None):
        self.code = code
        self.name = name
        self.message = message

    def __str__(self):
        if self.code == self.UNKNOWN:
            return 'unrecognised error'
        else:
            return '%d [%s]: %s' % (self.code, self.name, self.message)

class EmptyResultset(tuple):
    def __new__(cls, json):
        instance = tuple.__new__(cls, [])

        for key in ('page_size', 'page', 'has_more', 'total', 'quota_max', 'quota_remaining', 'type'):
            if key in json:
                setattr(instance, key, json[key])

        return instance

class StackExchangeResultset(tuple):
    """Defines an immutable, paginated resultset. This class can be used as a tuple, but provides extended metadata as well, including methods
to fetch the next page."""

    def __new__(cls, items, build_info, has_more = True, page = 1, pagesize = None):
        if pagesize is None:
            pagesize = len(items)

        instance = tuple.__new__(cls, items)
        instance.page, instance.pagesize, instance.build_info = page, pagesize, build_info
        instance.items = items
        instance.has_more = has_more

        return instance

    def reload(self):
        """Refreshes the data in the resultset with fresh API data. Note that this doesn't work with extended resultsets."""
        # kind of a cheat, but oh well
        return self.fetch_page(self.page)

    def fetch_page(self, page, **kw):
        """Returns a new resultset containing data from the specified page of the results. It re-uses all parameters that were passed in
to the initial function which created the resultset."""
        new_params = list(self.build_info)
        new_params[4] = new_params[4].copy()
        new_params[4].update(kw)
        new_params[4]['page'] = page

        new_set = new_params[0].build(*new_params[1:])
        new_set.page = page
        return new_set

    def fetch_extended(self, page):
        """Returns a new resultset containing data from this resultset AND from the specified page."""
        next = self.fetch_page(page)
        extended = self + next

        # max(0, ...) is so a non-zero, positive result for page is always found
        return StackExchangeResultset(extended, self.build_info, next.has_more, page)

    def fetch_next(self):
        """Returns the resultset of the data in the next page."""
        return self.fetch_page(self.page + 1)

    def extend_next(self):
        """Returns a new resultset containing data from this resultset AND from the next page."""
        return self.fetch_extended(self.page + 1)

    def fetch(self):
        # Do nothing, but allow multiple fetch calls
        return self

    def __iter__(self):
        return self.next()

    def next(self):
        for obj in self.items:
            yield obj

        current = self
        while current.has_more:
            for obj in current.items:
                yield obj

            try:
                current = current.fetch_next()
                if len(current) == 0:
                    return
            except urllib.error.HTTPError:
                return

class NeedsAwokenError(Exception):
    """An error raised when an attempt is made to access a property of a lazy collection that requires the data to have been fetched,
but whose data has not yet been requested."""

    def __init__(self, lazy):
        self.lazy = lazy
    def __str__(self):
        return 'Could not return requested data; the sequence of "%s" has not been fetched.' % self.lazy.m_lazy

class StackExchangeLazySequence(list):
    """Provides a sequence which *can* contain extra data available on an object. It is 'lazy' in the sense that data is only fetched when
required - not on object creation."""

    def __init__(self, m_type, count, site, url, fetch = None, collection = None, **kw):
        self.m_type = m_type
        self.count = count
        self.site = site
        self.url = url
        self.fetch_callback = fetch
        self.kw = kw
        self.collection = collection if collection != None else self._collection(url)

    def _collection(self, c):
        return c.split('/')[-1]

    def __len__(self):
        if self.count != None:
            return self.count
        else:
            raise NeedsAwokenError(self)

    def __repr__(self):
        if self.count is None:
            return '<unfetched sequence: %s>' % self.m_type.__name__
        else:
            return list.__repr__(self)

    def fetch(self, **direct_kw):
        """Fetch, from the API, the data this sequence is meant to hold."""
        # If we have any default parameters, include them, but overwrite any
        # passed in here directly.
        kw = dict(self.kw)
        kw.update(direct_kw)

        res = self.site.build(self.url, self.m_type, self.collection, kw)
        if self.fetch_callback != None:
            self.fetch_callback(res)
        return res

class StackExchangeLazyObject(list):
    """Provides a proxy to fetching a single item from a collection, lazily."""

    def __init__(self, m_type, site, url, fetch = None, collection = None):
        self.m_type = m_type
        self.site = site
        self.url = url
        self.fetch_callback = fetch
        self.collection = collection if collection != None else self._collection(url)

    def fetch(self, **kw):
        """Fetch, from the API, the data supposed to be held."""
        res = self.site.build(self.url, self.m_type, self.collection, kw)[0]
        if self.fetch_callback != None:
            self.fetch_callback(res)
        return res

    def __getattr__(self, key):
        raise NeedsAwokenError

#### Hack, because I can't be bothered to fix my mistaking JSON's output for an object not a dict
# (Si jeunesse savait, si vieillesse pouvait...)
# Attrib: Eli Bendersky, http://stackoverflow.com/questions/1305532/convert-python-dict-to-object/1305663#1305663
class DictObject:
    def __init__(self, entries):
        self.__dict__.update(entries)

class JSONMangler(object):
    """This class handles all sorts of random JSON-handling stuff"""

    @staticmethod
    def paginated_to_resultset(site, json, typ, collection, params):
        # N.B.: We ignore the 'collection' parameter for now, given that it is
        # no longer variable in v2.x, having been replaced by a generic field
        # 'items'. To perhaps be removed completely at some later point.
        items = []

        # create strongly-typed objects from the JSON items
        for json_item in json['items']:
            json_item['_params_'] = params[-1] # convenient access to the kw hash
            items.append(typ(json_item, site))

        rs = StackExchangeResultset(items, params, json['has_more'])
        if 'total' in json:
            rs.total = json['total']

        return rs

    @staticmethod
    def normal_to_resultset(site, json, typ, collection):
        # the parameter 'collection' may be need in future, and was needed pre-2.0
        return tuple([typ(x, site) for x in json['items']])

    @classmethod
    def json_to_resultset(cls, site, json, typ, collection, params = None):
        # this is somewhat of a special case, introduced by some filters in
        # post-2.0 allowing only 'metadata' to be returned
        if 'items' not in json:
            return EmptyResultset(json)
        elif 'has_more' in json:
            # we have a paginated resultset
            return cls.paginated_to_resultset(site, json, typ, collection, params)
        else:
            # this isn't paginated (unlikely but possible - eg badges)
            return cls.normal_to_resultset(site, json, typ, collection)

def format_relative_date(date, relative_to = None):
    """Takes a datetime object and returns the date formatted as a string e.g. "3 minutes ago", like the real site, relative to the given datetime. If no datetime is given, default to the current time."""
    # This is based roughly on George Edison's code from StackApps:
    #   http://stackapps.com/questions/1009/how-to-format-time-since-xxx-e-g-4-minutes-ago-similar-to-stack-exchange-site/1018#1018"""

    now = datetime.datetime.now() if relative_to is None else relative_to
    diff = (now - date).seconds

    # Anti-repetition! These simplify the code somewhat.
    plural = lambda d: 's' if d != 1 else ''
    frmt   = lambda d: (diff / float(d), plural(diff / float(d)))

    if diff < 60:
        return '%d second%s ago' % frmt(1)
    elif diff < 3600:
        return '%d minute%s ago' % frmt(60)
    elif diff < 86400:
        return '%d hour%s ago' % frmt(3600)
    elif diff < 172800:
        return 'yesterday'
    else:
        return date.strftime('M j / y - H:i')

class Sort(Enumeration):
    Activity = 'activity'
    Views = 'views'
    Creation = 'creation'
    Votes = 'votes'

ASC = 'asc'
DESC = 'desc'
<|endoftext|>