<commit_before>def ask(questionText, answerTypeAllowed = 0, allowedAnswers = None, acceptedAnswerText = "Answer accepted.", typeErrorText = "Your answer was of the wrong type.", badAnswerText = "Your answer was not one of the allowed answers.", genericErrorText = "An error occured."):
    """
    Ask a question on the command line.

    answerTypeAllowed : What type of answers to allow. Defaults to any type.
    questionText      : What question the user is asked. This is required.
    allowedAnswers    : What values are allowed. Defaults to any.
    typeErrorText     : The text printed if the input was of the wrong type.
    badAnswerText     : What is printed when the answer is not on eof the specified allowed ones.
    genericErrorText  : What happens when an error occurs.

    answerTypeAllowed Values:
    0 : Any type.
    1 : String
    2 : Integer
    3 : Float
    4 : Boolean
    """

    # TODO: Simplify more.

    def badAnswer():
        print(badAnswerText)
        input("Press enter to continue. ")

    if answerTypeAllowed not in list(range(5)):
        print("There was an error with internal coding :(")
        print("Please tell the maintainer this error:")
        print("answerTypeAllowed out of bounds when ask_question function called.")
        print("answerTypeAllowed was "+str(answerTypeAllowed)+" when allowed values are 0-4.")
        return

    while 1:
        userInput = input(questionText)
        if allowedAnswers is not None:
            if answerTypeAllowed == 0:
                if userInput in allowedAnswers:
                    print(acceptedAnswerText)
                    return userInput
                else:
                    badAnswer()
                    continue
            elif answerTypeAllowed == 1:
                if str(userInput) in allowedAnswers:
                    print(acceptedAnswerText)
                    return str(userInput)
                else:
                    badAnswer()
                    continue
            elif answerTypeAllowed == 2:
                try:
                    int(userInput)
                except ValueError:
                    print(typeErrorText)
                    input("Press enter to continue. ")
                    continue
                if int(userInput) in allowedAnswers:
                    print(acceptedAnswerText)
                    return int(userInput)
                else:
                    badAnswer()
                    continue
            elif answerTypeAllowed == 3:
                try:
                    float(userInput)
                except ValueError:
                    print(typeErrorText)
                    input("Press enter to continue. ")
                    continue
                if float(userInput) in allowedAnswers:
                    print(acceptedAnswerText)
                    return float(userInput)
                else:
                    badAnswer()
                    continue
            elif answerTypeAllowed == 4:
                if str(userInput).lower() in ['y', 'n', 'yes', 'no', '0', '1', 'true', 'false']:
                    if str(userInput).lower() in ['y', 'yes', '1', 'true']:
                        print(acceptedAnswerText)
                        return True
                    else:
                        print(acceptedAnswerText)
                        return False
                else:
                    print(typeErrorText)
                    input("Press enter to continue. ")
                    continue
            else:
                print(genericErrorText)
                input("Press enter to continue. ")
                continue
        elif allowedAnswers is None:
            if answerTypeAllowed == 0:
                print(acceptedAnswerText)
                return userInput
            elif answerTypeAllowed == 1:
                print(acceptedAnswerText)
                return str(userInput)
            elif answerTypeAllowed == 2:
                try:
                    int(userInput)
                except ValueError:
                    print(typeErrorText)
                    input("Press enter to continue. ")
                    continue
                print(acceptedAnswerText)
                return int(userInput)
            elif answerTypeAllowed == 3:
                try:
                    float(userInput)
                except ValueError:
                    print(typeErrorText)
                    input("Press enter to continue. ")
                    continue
                print(acceptedAnswerText)
                return float(userInput)
            elif answerTypeAllowed == 4:
                if str(userInput).lower() in ['y', 'n', 'yes', 'no', '0', '1', 'true', 'false']:
                    if str(userInput).lower() in ['y', 'yes', '1', 'true']:
                        print(acceptedAnswerText)
                        return True
                    else:
                        print(acceptedAnswerText)
                        return False
                else:
                    print(typeErrorText)
                    input("Press enter to continue. ")
                    continue
            else:
                print(genericErrorText)
                input("Press enter to continue. ")
                continue
        else:
            print(genericErrorText)
            input("Press enter to continue. ")
            continue

if __name__ == "__main__":
    print("Testing testing 123")
<commit_msg>Fixed up askquestion.py and also added a demo system.<commit_after>def ask(questionText, answerTypeAllowed = 0, allowedAnswers = None, acceptedAnswerText = "Answer accepted.", typeErrorText = "Your answer was of the wrong type.", badAnswerText = "Your answer was not one of the allowed answers.", genericErrorText = "An error occured.", defaultAnswer = None):
    """
    Ask a question on the command line.

    answerTypeAllowed : What type of answers to allow. Defaults to any type.
    questionText      : What question the user is asked. This is required.
    allowedAnswers    : What values are allowed. Defaults to any.
    acceptedAnswerText: What to say when the answer is accepted. `None` to supress printing.
    typeErrorText     : The text printed if the input was of the wrong type.
    badAnswerText     : What is printed when the answer is not one of the specified allowed ones.
    genericErrorText  : What happens when an error occurs.
    defaultAnswer     : If the user does not enter anything, what's the default answer. None to ignore.

    answerTypeAllowed Values:
    0 : Any type.
    1 : String
    2 : Integer
    3 : Float
    4 : Boolean
    """

    # TODO: Simplify more.

    def badAnswer():
        print(badAnswerText)
        input("Press enter to continue. ")

    if answerTypeAllowed not in list(range(5)):
        print("There was an error with internal coding :(")
        print("Please tell the maintainer this error:")
        print("answerTypeAllowed out of bounds when ask_question function called.")
        print("answerTypeAllowed was "+str(answerTypeAllowed)+" when allowed values are 0-4.")
        return

    while 1:
        userInput = input(questionText + " ")
        if (not userInput) and (defaultAnswer):
            if acceptedAnswerText: print(acceptedAnswerText)
            return defaultAnswer
        if allowedAnswers is not None:
            if answerTypeAllowed == 0:
                if userInput in allowedAnswers:
                    if acceptedAnswerText: print(acceptedAnswerText)
                    return userInput
                else:
                    badAnswer()
                    continue
            elif answerTypeAllowed == 1:
                if str(userInput) in allowedAnswers:
                    if acceptedAnswerText: print(acceptedAnswerText)
                    return str(userInput)
                else:
                    badAnswer()
                    continue
            elif answerTypeAllowed == 2:
                try:
                    int(userInput)
                except ValueError:
                    print(typeErrorText)
                    input("Press enter to continue. ")
                    continue
                if int(userInput) in allowedAnswers:
                    if acceptedAnswerText: print(acceptedAnswerText)
                    return int(userInput)
                else:
                    badAnswer()
                    continue
            elif answerTypeAllowed == 3:
                try:
                    float(userInput)
                except ValueError:
                    print(typeErrorText)
                    input("Press enter to continue. ")
                    continue
                if float(userInput) in allowedAnswers:
                    if acceptedAnswerText: print(acceptedAnswerText)
                    return float(userInput)
                else:
                    badAnswer()
                    continue
            elif answerTypeAllowed == 4:
                if str(userInput).lower() in ['y', 'n', 'yes', 'no', '0', '1', 'true', 'false']:
                    if str(userInput).lower() in ['y', 'yes', '1', 'true']:
                        if acceptedAnswerText: print(acceptedAnswerText)
                        return True
                    else:
                        if acceptedAnswerText: print(acceptedAnswerText)
                        return False
                else:
                    print(typeErrorText)
                    input("Press enter to continue. ")
                    continue
            else:
                print(genericErrorText)
                input("Press enter to continue. ")
                continue
        elif allowedAnswers is None:
            if answerTypeAllowed == 0:
                if acceptedAnswerText: print(acceptedAnswerText)
                return userInput
            elif answerTypeAllowed == 1:
                if acceptedAnswerText: print(acceptedAnswerText)
                return str(userInput)
            elif answerTypeAllowed == 2:
                try:
                    int(userInput)
                except ValueError:
                    print(typeErrorText)
                    input("Press enter to continue. ")
                    continue
                if acceptedAnswerText: print(acceptedAnswerText)
                return int(userInput)
            elif answerTypeAllowed == 3:
                try:
                    float(userInput)
                except ValueError:
                    print(typeErrorText)
                    input("Press enter to continue. ")
                    continue
                if acceptedAnswerText: print(acceptedAnswerText)
                return float(userInput)
            elif answerTypeAllowed == 4:
                if str(userInput).lower() in ['y', 'n', 'yes', 'no', '0', '1', 'true', 'false']:
                    if str(userInput).lower() in ['y', 'yes', '1', 'true']:
                        if acceptedAnswerText: print(acceptedAnswerText)
                        return True
                    else:
                        if acceptedAnswerText: print(acceptedAnswerText)
                        return False
                else:
                    print(typeErrorText)
                    input("Press enter to continue. ")
                    continue
            else:
                print(genericErrorText)
                input("Press enter to continue. ")
                continue
        else:
            print(genericErrorText)
            input("Press enter to continue. ")
            continue

if __name__ == "__main__":
    print("askquestion.py Version 1.0")
    result = ask("Do you wish to see some demos of askquestion.py (Y/n):", answerTypeAllowed = 4, allowedAnswers = None, acceptedAnswerText = None, defaultAnswer = True)
    if result:
        print("You just saw one! askquestion.py supports many different types of questions.")
        while 1:
            result = ask("""(1) Anything
(2) String (basically above, just always returns as string)
(3) Integer
(4) Float
(5) Boolean
Type the corresponding number to try one or zero to cancel (0-5):""", answerTypeAllowed = 2, allowedAnswers = list(range(0, 6)), acceptedAnswerText = None)
            if result == 0:
                break
            elif result == 1:
                result = ask("Enter anything:", answerTypeAllowed = 0, acceptedAnswerText = None)
                print("You entered "+str(result)+"!")
            elif result == 2:
                result = ask("Enter 'foo', 'bar' or 'derp':", answerTypeAllowed = 1, allowedAnswers = ["foo", "bar", "derp"], acceptedAnswerText = None)
                print("You entered "+result+"!")
            elif result == 3:
                result = ask("Enter a number from one to ten (1-10):", answerTypeAllowed = 2, allowedAnswers = list(range(10)),  acceptedAnswerText = None)
                print("You entered "+str(result)+"!")
            elif result == 4:
                result = ask("Please enter a float:", answerTypeAllowed = 3, acceptedAnswerText = None)
                print("You entered "+str(result)+"!")
            else:
                result = ask("Please enter a boolean (Y/n):", answerTypeAllowed = 4, acceptedAnswerText = None, defaultAnswer = True)
                print("You entered "+str(result)+"!")
    print("Thanks for using askquestion.py! Exiting.")
<|endoftext|>


<commit_before>"""
WSGI config for devent project.

It exposes the WSGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/1.7/howto/deployment/wsgi/
"""

import os
os.environ["DJANGO_SETTINGS_MODULE"] = "devent.settings"

from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()
<commit_msg>configure djstatic<commit_after>"""
WSGI config for devent project.

It exposes the WSGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/1.7/howto/deployment/wsgi/
"""

import os
os.environ["DJANGO_SETTINGS_MODULE"] = "devent.settings"

from django.core.wsgi import get_wsgi_application
from dj_static import Cling

application = Cling(get_wsgi_application())
<|endoftext|>


<commit_before>from metadata import *

# Returns the relevant portion of the pad
def read_decrypt_pad(sid, uid, decrypt_index, clen):
    metadataFile = get_metadatafile_name(uid, sid)
    metadata = read_metadata(metadataFile)
    assert decrypt_index >= 0 and decrypt_index < metadata["n_bytes"]

    storeFile = metadata["store_filename"]
    with open(storeFile, "rb") as store:
        pad = store.read()
        d = metadata["direction"]
        endIndex = decrypt_index - d * clen
        assert endIndex >= 0 and endIndex < metadata["n_bytes"]

        print("Decrypt index skipped: {}\nDecrypt index used: {}\n".
              format(decrypt_index_skipped(sid, uid, decrypt_index),
                     decrypt_index_used(sid, uid, decrypt_index)))
        with open(metadataFile, "w") as mfile:
            updates = {"decrypt_log":
                       "{}{}-{},".format(metadata["decrypt_log"],
                                         decrypt_index,
                                         endIndex),
                       "decrypt_max":
                       max(metadata["decrypt_max"], endIndex)
                       if metadata["direction"] == -1 else
                       min(metadata["decrypt_max"], endIndex)}
            metadata = update_metadata(metadata, updates)
            mfile.write(json.dumps(metadata))

        return pad[decrypt_index:endIndex:-d]

# Returns (pad, index)
def read_encrypt_pad(uid, rid, mlen):
    """ Read the next mlen bytes from the pad.

    Advances the pointer in metadata.
    Will return new pad data each call.

    Args:
        uid: User ID of the sender.
        rid: User ID of the recipient.
        mlen: Amount of pad to retrieve in BYTES.
    Returns:
        Returns a tuple of (pad data, index)
        pad data is a string.
        index is the start index of the portion of the pad used.
    """
    metadataFile = get_metadatafile_name(uid, rid)
    metadata = read_metadata(metadataFile)

    storeFile = metadata["store_filename"]
    with open(storeFile, "rb") as store:
        pad = store.read()
        e = metadata["encrypt_index"]
        d = metadata["direction"]
        endIndex = e + d * mlen
        assert endIndex >= 0 and endIndex < metadata["n_bytes"]

        with open(metadataFile, "w") as mfile:
            updates = {"encrypt_index": e+d*mlen}
            metadata = update_metadata(metadata, updates)
            mfile.write(json.dumps(metadata))
        return (pad[e:endIndex:d], e)

# True if the decrypt index requested has been used before
#  to decrypt
def decrypt_index_used(sid, uid, decrypt_index):
    metadataFile = get_metadatafile_name(uid, sid)
    metadata = read_metadata(metadataFile)

    # decrypt_log is of the form "1-10,10-15,15-22,"
    # indexes_used stores ["1-10","10-15","15-22"]
    indexes_used = metadata["decrypt_log"].split(",")[:-1]
    print "indexes_used in decrypt_index_used is:"
    print indexes_used

    # If decrypt_index is within the range of any of
    #  the elements in the log, then the index has
    #  been used before
    d = metadata["direction"]
    temp = [s.split("-") for s in indexes_used]
    inclusive_ranges = [(int(r[0]), int(r[1]) + d) for r in temp]
    if d == 1:
        return any(map(lambda rt: rt[0] >= decrypt_index
                       and decrypt_index >= rt[1], inclusive_ranges))
    if d == -1:
        return any(map(lambda rt: rt[0] <= decrypt_index
                       and decrypt_index <= rt[1], inclusive_ranges))

# True if this decrypt index requested has skipped over
#  a portion of the pad
def decrypt_index_skipped(sid, uid, decrypt_index, history=-1):
    metadataFile = get_metadatafile_name(uid, sid)
    metadata = read_metadata(metadataFile)

    decrypt_optimum = metadata["decrypt_max"]
    d = metadata["direction"]
    if d == 1:
        return decrypt_index < decrypt_optimum
    if d == -1:
        return decrypt_index > decrypt_optimum

# Returns UID of this device
def whoami(override_true_id=None):
    # Override just in case
    if override_true_id:
        return override_true_id

    files = os.listdir(".")
    metadata_file = filter(lambda f: f.find(METADATA_STEM) > -1, files)[0]
    uid = metadata_file[:metadata_file.index(".")]
    return uid
<commit_msg>Remove print lines in decrypt_index_used<commit_after>from metadata import *

# Returns the relevant portion of the pad
def read_decrypt_pad(sid, uid, decrypt_index, clen):
    metadataFile = get_metadatafile_name(uid, sid)
    metadata = read_metadata(metadataFile)
    assert decrypt_index >= 0 and decrypt_index < metadata["n_bytes"]

    storeFile = metadata["store_filename"]
    with open(storeFile, "rb") as store:
        pad = store.read()
        d = metadata["direction"]
        endIndex = decrypt_index - d * clen
        assert endIndex >= 0 and endIndex < metadata["n_bytes"]

        print("Decrypt index skipped: {}\nDecrypt index used: {}\n".
              format(decrypt_index_skipped(sid, uid, decrypt_index),
                     decrypt_index_used(sid, uid, decrypt_index)))
        with open(metadataFile, "w") as mfile:
            updates = {"decrypt_log":
                       "{}{}-{},".format(metadata["decrypt_log"],
                                         decrypt_index,
                                         endIndex),
                       "decrypt_max":
                       max(metadata["decrypt_max"], endIndex)
                       if metadata["direction"] == -1 else
                       min(metadata["decrypt_max"], endIndex)}
            metadata = update_metadata(metadata, updates)
            mfile.write(json.dumps(metadata))

        return pad[decrypt_index:endIndex:-d]

# Returns (pad, index)
def read_encrypt_pad(uid, rid, mlen):
    """ Read the next mlen bytes from the pad.

    Advances the pointer in metadata.
    Will return new pad data each call.

    Args:
        uid: User ID of the sender.
        rid: User ID of the recipient.
        mlen: Amount of pad to retrieve in BYTES.
    Returns:
        Returns a tuple of (pad data, index)
        pad data is a string.
        index is the start index of the portion of the pad used.
    """
    metadataFile = get_metadatafile_name(uid, rid)
    metadata = read_metadata(metadataFile)

    storeFile = metadata["store_filename"]
    with open(storeFile, "rb") as store:
        pad = store.read()
        e = metadata["encrypt_index"]
        d = metadata["direction"]
        endIndex = e + d * mlen
        assert endIndex >= 0 and endIndex < metadata["n_bytes"]

        with open(metadataFile, "w") as mfile:
            updates = {"encrypt_index": e+d*mlen}
            metadata = update_metadata(metadata, updates)
            mfile.write(json.dumps(metadata))
        return (pad[e:endIndex:d], e)

# True if the decrypt index requested has been used before
#  to decrypt
def decrypt_index_used(sid, uid, decrypt_index):
    metadataFile = get_metadatafile_name(uid, sid)
    metadata = read_metadata(metadataFile)

    # decrypt_log is of the form "1-10,10-15,15-22,"
    # indexes_used stores ["1-10","10-15","15-22"]
    indexes_used = metadata["decrypt_log"].split(",")[:-1]

    # If decrypt_index is within the range of any of
    #  the elements in the log, then the index has
    #  been used before
    d = metadata["direction"]
    temp = [s.split("-") for s in indexes_used]
    inclusive_ranges = [(int(r[0]), int(r[1]) + d) for r in temp]
    if d == 1:
        return any(map(lambda rt: rt[0] >= decrypt_index
                       and decrypt_index >= rt[1], inclusive_ranges))
    if d == -1:
        return any(map(lambda rt: rt[0] <= decrypt_index
                       and decrypt_index <= rt[1], inclusive_ranges))

# True if this decrypt index requested has skipped over
#  a portion of the pad
def decrypt_index_skipped(sid, uid, decrypt_index, history=-1):
    metadataFile = get_metadatafile_name(uid, sid)
    metadata = read_metadata(metadataFile)

    decrypt_optimum = metadata["decrypt_max"]
    d = metadata["direction"]
    if d == 1:
        return decrypt_index < decrypt_optimum
    if d == -1:
        return decrypt_index > decrypt_optimum

# Returns UID of this device
def whoami(override_true_id=None):
    # Override just in case
    if override_true_id:
        return override_true_id

    files = os.listdir(".")
    metadata_file = filter(lambda f: f.find(METADATA_STEM) > -1, files)[0]
    uid = metadata_file[:metadata_file.index(".")]
    return uid
<|endoftext|>


<commit_before>import json
import logging

from jwcrypto import jwk
from rest_framework import serializers
from rest_framework.exceptions import PermissionDenied
from rest_framework.mixins import CreateModelMixin, DestroyModelMixin
from rest_framework.permissions import IsAuthenticated
from rest_framework.viewsets import GenericViewSet

from tunnistamo.api_common import OidcTokenAuthentication

from .models import UserDevice

logger = logging.getLogger(__name__)

class DeviceScopeAuthentication(OidcTokenAuthentication):
    scopes_needed = ['devices']

def generate_secret_key():
    key = jwk.JWK()
    # Generate 256-bit AES key for encryption
    key = key.generate(kty='oct', alg='A256KW', use='enc')
    return json.loads(key.export())

class PublicKeySerializer(serializers.Serializer):
    kty = serializers.ChoiceField(['EC'])
    use = serializers.ChoiceField(['sig'])
    crv = serializers.ChoiceField(['P-256'])
    x = serializers.CharField()
    y = serializers.CharField()
    alg = serializers.ChoiceField(['ES256'])

    def to_representation(self, instance):
        return instance

    def validate(self, data):
        # TODO
        return data

class UserDeviceSerializer(serializers.ModelSerializer):
    user = serializers.ReadOnlyField(source='user.uuid')
    public_key = PublicKeySerializer()

    class Meta:
        model = UserDevice
        fields = (
            'id', 'user', 'public_key', 'secret_key', 'app_version',
            'os', 'os_version', 'device_model', 'auth_counter'
        )
        read_only_fields = ('user', 'secret_key', 'auth_counter')

    def create(self, validated_data):
        validated_data['secret_key'] = generate_secret_key()
        return UserDevice.objects.create(**validated_data)

class UserDeviceViewSet(CreateModelMixin, DestroyModelMixin, GenericViewSet):
    queryset = UserDevice.objects.all()
    serializer_class = UserDeviceSerializer
    authentication_classes = (DeviceScopeAuthentication,)
    permission_classes = (IsAuthenticated,)

    def perform_create(self, serializer):
        serializer.save(user=self.request.user)

    def perform_destroy(self, instance):
        if instance.user != self.request.user:
            raise PermissionDenied()
        instance.delete()
<commit_msg>Use ScopePermission in UserDeviceViewSet<commit_after>import json
import logging

from jwcrypto import jwk
from rest_framework import serializers
from rest_framework.exceptions import PermissionDenied
from rest_framework.mixins import CreateModelMixin, DestroyModelMixin
from rest_framework.permissions import IsAuthenticated
from rest_framework.viewsets import GenericViewSet

from tunnistamo.api_common import OidcTokenAuthentication, ScopePermission

from .models import UserDevice

logger = logging.getLogger(__name__)

def generate_secret_key():
    key = jwk.JWK()
    # Generate 256-bit AES key for encryption
    key = key.generate(kty='oct', alg='A256KW', use='enc')
    return json.loads(key.export())

class PublicKeySerializer(serializers.Serializer):
    kty = serializers.ChoiceField(['EC'])
    use = serializers.ChoiceField(['sig'])
    crv = serializers.ChoiceField(['P-256'])
    x = serializers.CharField()
    y = serializers.CharField()
    alg = serializers.ChoiceField(['ES256'])

    def to_representation(self, instance):
        return instance

class UserDeviceSerializer(serializers.ModelSerializer):
    user = serializers.ReadOnlyField(source='user.uuid')
    public_key = PublicKeySerializer()

    class Meta:
        model = UserDevice
        fields = (
            'id', 'user', 'public_key', 'secret_key', 'app_version',
            'os', 'os_version', 'device_model', 'auth_counter'
        )
        read_only_fields = ('user', 'secret_key', 'auth_counter')

    def create(self, validated_data):
        validated_data['secret_key'] = generate_secret_key()
        return UserDevice.objects.create(**validated_data)

class UserDeviceViewSet(CreateModelMixin, DestroyModelMixin, GenericViewSet):
    queryset = UserDevice.objects.all()
    serializer_class = UserDeviceSerializer
    authentication_classes = (OidcTokenAuthentication,)
    permission_classes = (IsAuthenticated, ScopePermission)
    required_scopes = ('devices',)

    def perform_create(self, serializer):
        serializer.save(user=self.request.user)

    def perform_destroy(self, instance):
        if instance.user != self.request.user:
            raise PermissionDenied()
        instance.delete()
<|endoftext|>


<commit_before>import time
import errno
import gevent
import operator
from collections import OrderedDict
from .protocol import BaseProtocol
from .p2p_protocol import P2PProtocol
from .service import WiredService
from .multiplexer import MultiplexerError, Packet
from .muxsession import MultiplexedSession
from .crypto import ECIESDecryptionError
from devp2p import slogging
import gevent.socket
from devp2p import rlpxcipher
from rlp.utils import decode_hex

log = slogging.get_logger('p2p.peer')

class UnknownCommandError(Exception):

    "raised if we recive an unknown command for a known protocol"
    pass

class Peer(gevent.Greenlet):

    remote_client_version = ''
    offset_based_dispatch = False
    wait_read_timeout = 0.001
    dumb_remote_timeout = 10.0

    def __init__(self, peermanager, connection, remote_pubkey=None):
        super(Peer, self).__init__()
        self.is_stopped = False
        self.hello_received = False
        self.peermanager = peermanager
        self.connection = connection
        self.config = peermanager.config
        self.protocols = OrderedDict()
        log.debug('peer init', peer=self)

        # create multiplexed encrypted session
        privkey = decode_hex(self.config['node']['privkey_hex'])
        hello_packet = P2PProtocol.get_hello_packet(self)
        self.mux = MultiplexedSession(privkey, hello_packet, remote_pubkey=remote_pubkey)
        self.remote_pubkey = remote_pubkey
        self.remote_capabilities = None

        # register p2p protocol
        assert issubclass(self.peermanager.wire_protocol, P2PProtocol)
        self.connect_service(self.peermanager)

        # assure, we don't get messages while replies are not read
        self.safe_to_read = gevent.event.Event()
        self.safe_to_read.set()

        self.greenlets = dict()

        # Stop peer if hello not received in self.dumb_remote_timeout seconds
        self.greenlets['dumb_checker'] = gevent.spawn_later(self.dumb_remote_timeout, self.check_if_dumb_remote)

    @property
    def remote_pubkey(self):
        "if peer is responder, then the remote_pubkey will not be available"
        "before the first packet is received"
        return self.mux.remote_pubkey

    @remote_pubkey.setter
    def remote_pubkey(self, value):
        self.remote_pubkey_available = True if value else False
        self.mux.remote_pubkey = value

    def __repr__(self):
        try:
            pn = self.connection.getpeername()
        except gevent.socket.error:
            pn = ('not ready',)
        try:
            cv = '/'.join(self.remote_client_version.split('/')[:2])
        except:
            cv = self.remote_client_version
        return '<Peer%r %s>' % (pn, cv)
        # return '<Peer%r>' % repr(pn)

    def report_error(self, reason):
        try:
            ip_port = self.ip_port
        except:
            ip_port = 'ip_port not available fixme'
        self.peermanager.errors.add(ip_port, reason, self.remote_client_version)

    @property
    def ip_port(self):
        try:
            return self.connection.getpeername()
        except Exception as e:
            log.debug('ip_port failed', e=e)
            raise e

    def connect_service(self, service):
        assert isinstance(service, WiredService)
        protocol_class = service.wire_protocol
        assert issubclass(protocol_class, BaseProtocol)
        # create protcol instance which connects peer with serivce
        protocol = protocol_class(self, service)
        # register protocol
        assert protocol_class not in self.protocols
        log.debug('registering protocol', protocol=protocol.name, peer=self)
        self.protocols[protocol_class] = protocol
        self.mux.add_protocol(protocol.protocol_id)
        protocol.start()

    def has_protocol(self, protocol):
        assert issubclass(protocol, BaseProtocol)
        return protocol in self.protocols

    def receive_hello(self, proto, version, client_version_string, capabilities,
                      listen_port, remote_pubkey):
        log.debug('received hello', proto=proto, version=version,
                  client_version=client_version_string, capabilities=capabilities)
        assert isinstance(remote_pubkey, bytes)
        assert len(remote_pubkey) == 64
        if self.remote_pubkey_available:
            assert self.remote_pubkey == remote_pubkey
        self.hello_received = True

        # enable backwards compatibility for legacy peers
        if version < 5:
            self.offset_based_dispatch = True
            max_window_size = 2**32  # disable chunked transfers

        # call peermanager
        agree = self.peermanager.on_hello_received(
            proto, version, client_version_string, capabilities, listen_port, remote_pubkey)
        if not agree:
            return

        self.remote_client_version = client_version_string
        self.remote_pubkey = remote_pubkey
        self.remote_capabilities = capabilities

        # register in common protocols
        log.debug('connecting services', services=self.peermanager.wired_services)
        remote_services = dict()
        for name, version in capabilities:
            if not name in remote_services:
                remote_services[name] = []
            remote_services[name].append(version)
        for service in sorted(self.peermanager.wired_services, key=operator.attrgetter('name')):
            proto = service.wire_protocol
            assert isinstance(service, WiredService)
            assert isinstance(proto.name, bytes)
            if proto.name in remote_services:
                if proto.version in remote_services[proto.name]:
                    if service != self.peermanager:  # p2p protocol already registered
                        self.connect_service(service)
                else:
                    log.debug('wrong version', service=proto.name, local_version=proto.version,
                              remote_version=remote_services[proto.name])
                    self.report_error('wrong version')

    @property
    def capabilities(self):
        return [(s.wire_protocol.name, s.wire_protocol.version)
                for s in self.peermanager.wired_services]

    # sending p2p messages

    def send_packet(self, packet):
        for i, protocol in enumerate(self.protocols.values()):
            if packet.protocol_id == protocol.protocol_id:
                break

        assert packet.protocol_id == protocol.protocol_id, 'no protocol found'
        log.debug('send packet', cmd=protocol.cmd_by_id[packet.cmd_id], protcol=protocol.name,
                  peer=self)
        # rewrite cmd_id (backwards compatibility)
        if self.offset_based_dispatch:
            for i, protocol in enumerate(self.protocols.values()):
                if packet.protocol_id == protocol.protocol_id:
                    break
                packet.cmd_id += (0 if protocol.max_cmd_id == 0 else protocol.max_cmd_id + 1)
            packet.protocol_id = 0
        self.mux.add_packet(packet)

    # receiving p2p messages

    def protocol_cmd_id_from_packet(self, packet):
        # offset-based dispatch (backwards compatibility)
        if self.offset_based_dispatch:
            max_id = 0
            for protocol in self.protocols.values():
                if packet.cmd_id < max_id + protocol.max_cmd_id + 1:
                    return protocol, packet.cmd_id - max_id
                max_id += protocol.max_cmd_id + 1
            raise UnknownCommandError('no protocol for id %s' % packet.cmd_id)
        # new-style dispatch based on protocol_id
        for i, protocol in enumerate(self.protocols.values()):
            if packet.protocol_id == protocol.protocol_id:
                return protocol, packet.cmd_id
        raise UnknownCommandError('no protocol for protocol id %s' % packet.protocol_id)

    def _handle_packet(self, packet):
        assert isinstance(packet, Packet)
        try:
            protocol, cmd_id = self.protocol_cmd_id_from_packet(packet)
            log.debug('recv packet', peer=self, cmd=protocol.cmd_by_id[cmd_id], protocol=protocol.name, orig_cmd_id=packet.cmd_id)
            packet.cmd_id = cmd_id  # rewrite
            protocol.receive_packet(packet)
        except UnknownCommandError as e:
            log.debug('received unknown cmd', peer=self, error=e, packet=packet)
        except Exception as e:
            log.debug('failed to handle packet', peer=self, error=e)
            self.stop()

    def send(self, data):
        if not data:
            return
        self.safe_to_read.clear()  # make sure we don't accept any data until message is sent
        try:
            self.connection.sendall(data)  # check if gevent chunkes and switches contexts
            log.debug('wrote data', size=len(data), ts=time.time())
        except gevent.socket.error as e:
            log.debug('write error', errno=e.errno, reason=e.strerror)
            self.report_error('write error %r' % e.strerror)
            self.stop()
        except gevent.socket.timeout:
            log.debug('write timeout')
            self.report_error('write timeout')
            self.stop()
        self.safe_to_read.set()

    def _run_egress_message(self):
        while not self.is_stopped:
            self.send(self.mux.message_queue.get())

    def _run_decoded_packets(self):
        # handle decoded packets
        while not self.is_stopped:
            self._handle_packet(self.mux.packet_queue.get())  # get_packet blocks

    def _run_ingress_message(self):
        log.debug('peer starting main loop')
        assert not self.connection.closed, "connection is closed"
        self.greenlets['decoder'] = gevent.spawn(self._run_decoded_packets)
        self.greenlets['sender'] = gevent.spawn(self._run_egress_message)

        while not self.is_stopped:
            self.safe_to_read.wait()
            try:
                gevent.socket.wait_read(self.connection.fileno())
            except gevent.socket.error as e:
                log.debug('read error', errno=e.errno, reason=e.strerror, peer=self)
                self.report_error('network error %s' % e.strerror)
                if e.errno in(errno.EBADF,):
                    # ('Bad file descriptor')
                    self.stop()
                else:
                    raise e
                    break
            try:
                imsg = self.connection.recv(4096)
            except gevent.socket.error as e:
                log.debug('read error', errno=e.errno, reason=e.strerror, peer=self)
                self.report_error('network error %s' % e.strerror)
                if e.errno in(errno.ENETDOWN, errno.ECONNRESET, errno.ETIMEDOUT,
                              errno.EHOSTUNREACH, errno.ECONNABORTED):
                    # (Network down, Connection reset by peer, timeout, no route to host,
                    # Software caused connection abort (windows weirdness))
                    self.stop()
                else:
                    raise e
                    break
            if imsg:
                log.debug('read data', ts=time.time(), size=len(imsg))
                try:
                    self.mux.add_message(imsg)
                except (rlpxcipher.RLPxSessionError, ECIESDecryptionError) as e:
                    log.debug('rlpx session error', peer=self, error=e)
                    self.report_error('rlpx session error')
                    self.stop()
                except MultiplexerError as e:
                    log.debug('multiplexer error', peer=self, error=e)
                    self.report_error('multiplexer error')
                    self.stop()

    _run = _run_ingress_message

    def stop(self):
        if not self.is_stopped:
            try:
                self.is_stopped = True
                log.debug('peer stopped', peer=self)
                for g in self.greenlets.values():
                    try:
                        g.kill()
                    except gevent.GreenletExit:
                        pass
                self.greenlets = None
                for p in self.protocols.values():
                    p.stop()
                self.protocols = None  # break circular dependency with BaseProtocol
            except Exception as e:
                log.debug("failed to gracefully shutdown peer", error=e)
            finally:
                self.peermanager.peers.remove(self)
                self.kill()

    def check_if_dumb_remote(self):
        "Stop peer if hello not received"
        if not self.hello_received:
            self.report_error('No hello in {} seconds'.format(self.dumb_remote_timeout))
            self.stop()
<commit_msg>fixed bytes/str problem<commit_after>import time
import errno
import gevent
import operator
from collections import OrderedDict
from .protocol import BaseProtocol
from .p2p_protocol import P2PProtocol
from .service import WiredService
from .multiplexer import MultiplexerError, Packet
from .muxsession import MultiplexedSession
from .crypto import ECIESDecryptionError
from devp2p import slogging
import gevent.socket
from devp2p import rlpxcipher
from rlp.utils import decode_hex

log = slogging.get_logger('p2p.peer')

class UnknownCommandError(Exception):

    "raised if we recive an unknown command for a known protocol"
    pass

class Peer(gevent.Greenlet):

    remote_client_version = ''
    offset_based_dispatch = False
    wait_read_timeout = 0.001
    dumb_remote_timeout = 10.0

    def __init__(self, peermanager, connection, remote_pubkey=None):
        super(Peer, self).__init__()
        self.is_stopped = False
        self.hello_received = False
        self.peermanager = peermanager
        self.connection = connection
        self.config = peermanager.config
        self.protocols = OrderedDict()
        log.debug('peer init', peer=self)

        # create multiplexed encrypted session
        privkey = decode_hex(self.config['node']['privkey_hex'])
        hello_packet = P2PProtocol.get_hello_packet(self)
        self.mux = MultiplexedSession(privkey, hello_packet, remote_pubkey=remote_pubkey)
        self.remote_pubkey = remote_pubkey
        self.remote_capabilities = None

        # register p2p protocol
        assert issubclass(self.peermanager.wire_protocol, P2PProtocol)
        self.connect_service(self.peermanager)

        # assure, we don't get messages while replies are not read
        self.safe_to_read = gevent.event.Event()
        self.safe_to_read.set()

        self.greenlets = dict()

        # Stop peer if hello not received in self.dumb_remote_timeout seconds
        self.greenlets['dumb_checker'] = gevent.spawn_later(self.dumb_remote_timeout, self.check_if_dumb_remote)

    @property
    def remote_pubkey(self):
        "if peer is responder, then the remote_pubkey will not be available"
        "before the first packet is received"
        return self.mux.remote_pubkey

    @remote_pubkey.setter
    def remote_pubkey(self, value):
        self.remote_pubkey_available = True if value else False
        self.mux.remote_pubkey = value

    def __repr__(self):
        try:
            pn = self.connection.getpeername()
        except gevent.socket.error:
            pn = ('not ready',)
        try:
            cv = '/'.join(self.remote_client_version.split('/')[:2])
        except:
            cv = self.remote_client_version
        return '<Peer%r %s>' % (pn, cv)
        # return '<Peer%r>' % repr(pn)

    def report_error(self, reason):
        try:
            ip_port = self.ip_port
        except:
            ip_port = 'ip_port not available fixme'
        self.peermanager.errors.add(ip_port, reason, self.remote_client_version)

    @property
    def ip_port(self):
        try:
            return self.connection.getpeername()
        except Exception as e:
            log.debug('ip_port failed', e=e)
            raise e

    def connect_service(self, service):
        assert isinstance(service, WiredService)
        protocol_class = service.wire_protocol
        assert issubclass(protocol_class, BaseProtocol)
        # create protcol instance which connects peer with serivce
        protocol = protocol_class(self, service)
        # register protocol
        assert protocol_class not in self.protocols
        log.debug('registering protocol', protocol=protocol.name, peer=self)
        self.protocols[protocol_class] = protocol
        self.mux.add_protocol(protocol.protocol_id)
        protocol.start()

    def has_protocol(self, protocol):
        assert issubclass(protocol, BaseProtocol)
        return protocol in self.protocols

    def receive_hello(self, proto, version, client_version_string, capabilities,
                      listen_port, remote_pubkey):
        log.debug('received hello', proto=proto, version=version,
                  client_version=client_version_string, capabilities=capabilities)
        assert isinstance(remote_pubkey, bytes)
        assert len(remote_pubkey) == 64
        if self.remote_pubkey_available:
            assert self.remote_pubkey == remote_pubkey
        self.hello_received = True

        # enable backwards compatibility for legacy peers
        if version < 5:
            self.offset_based_dispatch = True
            max_window_size = 2**32  # disable chunked transfers

        # call peermanager
        agree = self.peermanager.on_hello_received(
            proto, version, client_version_string, capabilities, listen_port, remote_pubkey)
        if not agree:
            return

        self.remote_client_version = client_version_string
        self.remote_pubkey = remote_pubkey
        self.remote_capabilities = capabilities

        # register in common protocols
        log.debug('connecting services', services=self.peermanager.wired_services)
        remote_services = dict()
        for name, version in capabilities:
            if isinstance(name, bytes) and not isinstance(name, str):
                name = name.decode('utf-8')
            if not name in remote_services:
                remote_services[name] = []
            remote_services[name].append(version)
        for service in sorted(self.peermanager.wired_services, key=operator.attrgetter('name')):
            proto = service.wire_protocol
            assert isinstance(service, WiredService)
            assert isinstance(proto.name, (bytes, str))
            if proto.name in remote_services:
                if proto.version in remote_services[proto.name]:
                    if service != self.peermanager:  # p2p protocol already registered
                        self.connect_service(service)
                else:
                    log.debug('wrong version', service=proto.name, local_version=proto.version,
                              remote_version=remote_services[proto.name])
                    self.report_error('wrong version')

    @property
    def capabilities(self):
        return [(s.wire_protocol.name, s.wire_protocol.version)
                for s in self.peermanager.wired_services]

    # sending p2p messages

    def send_packet(self, packet):
        for i, protocol in enumerate(self.protocols.values()):
            if packet.protocol_id == protocol.protocol_id:
                break

        assert packet.protocol_id == protocol.protocol_id, 'no protocol found'
        log.debug('send packet', cmd=protocol.cmd_by_id[packet.cmd_id], protcol=protocol.name,
                  peer=self)
        # rewrite cmd_id (backwards compatibility)
        if self.offset_based_dispatch:
            for i, protocol in enumerate(self.protocols.values()):
                if packet.protocol_id == protocol.protocol_id:
                    break
                packet.cmd_id += (0 if protocol.max_cmd_id == 0 else protocol.max_cmd_id + 1)
            packet.protocol_id = 0
        self.mux.add_packet(packet)

    # receiving p2p messages

    def protocol_cmd_id_from_packet(self, packet):
        # offset-based dispatch (backwards compatibility)
        if self.offset_based_dispatch:
            max_id = 0
            for protocol in self.protocols.values():
                if packet.cmd_id < max_id + protocol.max_cmd_id + 1:
                    return protocol, packet.cmd_id - max_id
                max_id += protocol.max_cmd_id + 1
            raise UnknownCommandError('no protocol for id %s' % packet.cmd_id)
        # new-style dispatch based on protocol_id
        for i, protocol in enumerate(self.protocols.values()):
            if packet.protocol_id == protocol.protocol_id:
                return protocol, packet.cmd_id
        raise UnknownCommandError('no protocol for protocol id %s' % packet.protocol_id)

    def _handle_packet(self, packet):
        assert isinstance(packet, Packet)
        try:
            protocol, cmd_id = self.protocol_cmd_id_from_packet(packet)
            log.debug('recv packet', peer=self, cmd=protocol.cmd_by_id[cmd_id], protocol=protocol.name, orig_cmd_id=packet.cmd_id)
            packet.cmd_id = cmd_id  # rewrite
            protocol.receive_packet(packet)
        except UnknownCommandError as e:
            log.debug('received unknown cmd', peer=self, error=e, packet=packet)
        except Exception as e:
            log.debug('failed to handle packet', peer=self, error=e)
            self.stop()

    def send(self, data):
        if not data:
            return
        self.safe_to_read.clear()  # make sure we don't accept any data until message is sent
        try:
            self.connection.sendall(data)  # check if gevent chunkes and switches contexts
            log.debug('wrote data', size=len(data), ts=time.time())
        except gevent.socket.error as e:
            log.debug('write error', errno=e.errno, reason=e.strerror)
            self.report_error('write error %r' % e.strerror)
            self.stop()
        except gevent.socket.timeout:
            log.debug('write timeout')
            self.report_error('write timeout')
            self.stop()
        self.safe_to_read.set()

    def _run_egress_message(self):
        while not self.is_stopped:
            self.send(self.mux.message_queue.get())

    def _run_decoded_packets(self):
        # handle decoded packets
        while not self.is_stopped:
            self._handle_packet(self.mux.packet_queue.get())  # get_packet blocks

    def _run_ingress_message(self):
        log.debug('peer starting main loop')
        assert not self.connection.closed, "connection is closed"
        self.greenlets['decoder'] = gevent.spawn(self._run_decoded_packets)
        self.greenlets['sender'] = gevent.spawn(self._run_egress_message)

        while not self.is_stopped:
            self.safe_to_read.wait()
            try:
                gevent.socket.wait_read(self.connection.fileno())
            except gevent.socket.error as e:
                log.debug('read error', errno=e.errno, reason=e.strerror, peer=self)
                self.report_error('network error %s' % e.strerror)
                if e.errno in(errno.EBADF,):
                    # ('Bad file descriptor')
                    self.stop()
                else:
                    raise e
                    break
            try:
                imsg = self.connection.recv(4096)
            except gevent.socket.error as e:
                log.debug('read error', errno=e.errno, reason=e.strerror, peer=self)
                self.report_error('network error %s' % e.strerror)
                if e.errno in(errno.ENETDOWN, errno.ECONNRESET, errno.ETIMEDOUT,
                              errno.EHOSTUNREACH, errno.ECONNABORTED):
                    # (Network down, Connection reset by peer, timeout, no route to host,
                    # Software caused connection abort (windows weirdness))
                    self.stop()
                else:
                    raise e
                    break
            if imsg:
                log.debug('read data', ts=time.time(), size=len(imsg))
                try:
                    self.mux.add_message(imsg)
                except (rlpxcipher.RLPxSessionError, ECIESDecryptionError) as e:
                    log.debug('rlpx session error', peer=self, error=e)
                    self.report_error('rlpx session error')
                    self.stop()
                except MultiplexerError as e:
                    log.debug('multiplexer error', peer=self, error=e)
                    self.report_error('multiplexer error')
                    self.stop()

    _run = _run_ingress_message

    def stop(self):
        if not self.is_stopped:
            try:
                self.is_stopped = True
                log.debug('peer stopped', peer=self)
                for g in self.greenlets.values():
                    try:
                        g.kill()
                    except gevent.GreenletExit:
                        pass
                self.greenlets = None
                for p in self.protocols.values():
                    p.stop()
                self.protocols = None  # break circular dependency with BaseProtocol
            except Exception as e:
                log.debug("failed to gracefully shutdown peer", error=e)
            finally:
                self.peermanager.peers.remove(self)
                self.kill()

    def check_if_dumb_remote(self):
        "Stop peer if hello not received"
        if not self.hello_received:
            self.report_error('No hello in {} seconds'.format(self.dumb_remote_timeout))
            self.stop()
<|endoftext|>


<commit_before>"""
Proxies for the libgeos_c shared lib, GEOS-specific exceptions, and utilities
"""

import atexit
import logging
import os
import sys
import threading
import ctypes
from ctypes import cdll, CDLL, CFUNCTYPE, c_char_p, c_void_p, string_at
from ctypes.util import find_library

import ftools
from ctypes_declarations import prototype, EXCEPTION_HANDLER_FUNCTYPE

# Begin by creating a do-nothing handler and adding to this module's logger.
class NullHandler(logging.Handler):
    def emit(self, record):
        pass
LOG = logging.getLogger(__name__)
LOG.addHandler(NullHandler())

# Find and load the GEOS and C libraries
# If this ever gets any longer, we'll break it into separate modules

def load_dll(libname, fallbacks=None):
    lib = find_library(libname)
    if lib is not None:
        return CDLL(lib)
    else:
        if fallbacks is not None:
            for name in fallbacks:
                try:
                    return CDLL(name)
                except OSError:
                    # move on to the next fallback
                    pass
        # the end
        raise OSError(
            "Could not find library %s or load any of its variants %s" % (
                libname, fallbacks or []))

if sys.platform.startswith('linux'):
    _lgeos = load_dll('geos_c', fallbacks=['libgeos_c.so.1', 'libgeos_c.so'])
    free = load_dll('c').free
    free.argtypes = [c_void_p]
    free.restype = None

elif sys.platform == 'darwin':
    alt_paths = [
            # The Framework build from Kyng Chaos:
            "/Library/Frameworks/GEOS.framework/Versions/Current/GEOS",
            # macports
            '/opt/local/lib/libgeos_c.dylib',
    ]
    _lgeos = load_dll('geos_c', fallbacks=alt_paths)
    free = load_dll('c').free
    free.argtypes = [c_void_p]
    free.restype = None

elif sys.platform == 'win32':
    try:
        egg_dlls = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                     r"..\DLLs"))
        wininst_dlls =  os.path.abspath(os.__file__ + "../../../DLLs")
        original_path = os.environ['PATH']
        os.environ['PATH'] = "%s;%s;%s" % (egg_dlls, wininst_dlls, original_path)
        _lgeos = CDLL("geos.dll")
    except (ImportError, WindowsError, OSError):
        raise
    def free(m):
        try:
            cdll.msvcrt.free(m)
        except WindowsError:
            # XXX: See http://trac.gispython.org/projects/PCL/ticket/149
            pass

elif sys.platform == 'sunos5':
    _lgeos = load_dll('geos_c', fallbacks=['libgeos_c.so.1', 'libgeos_c.so'])
    free = CDLL('libc.so.1').free
    free.argtypes = [c_void_p]
    free.restype = None

else: # other *nix systems
    _lgeos = load_dll('geos_c', fallbacks=['libgeos_c.so.1', 'libgeos_c.so'])
    free = load_dll('c', fallbacks=['libc.so.6']).free
    free.argtypes = [c_void_p]
    free.restype = None

def _geos_c_version():
    func = _lgeos.GEOSversion
    func.argtypes = []
    func.restype = c_char_p
    v = func()
    c_ver = v.split('-')[2]
    # Ditch any SVN revision numbering that may have crept in without (ftm)
    # importing `re`.
    c_ver = c_ver.split()[0]
    return tuple(int(n) for n in c_ver.split('.')), v

geos_capi_version, geos_version_string = _geos_c_version()
geos_c_version = geos_capi_version

# If we have the new interface, then record a baseline so that we know what
# additional functions are declared in ctypes_declarations.
if geos_c_version >= (1,5,0):
    start_set = set(_lgeos.__dict__)

# Apply prototypes for the libgeos_c functions
prototype(_lgeos, geos_c_version)

# If we have the new interface, automatically detect all function
# declarations, and declare their re-entrant counterpart.
if geos_c_version >= (1,5,0):
    end_set = set(_lgeos.__dict__)
    new_func_names = end_set - start_set

    for func_name in new_func_names:
        new_func_name = "%s_r" % func_name
        if hasattr(_lgeos, new_func_name):
            new_func = getattr(_lgeos, new_func_name)
            old_func = getattr(_lgeos, func_name)
            new_func.restype = old_func.restype
            if old_func.argtypes is None:
                # Handle functions that didn't take an argument before,
                # finishGEOS.
                new_func.argtypes = [c_void_p]
            else:
                new_func.argtypes = [c_void_p] + old_func.argtypes
            if old_func.errcheck is not None:
                new_func.errcheck = old_func.errcheck

    # Handle special case.
    _lgeos.initGEOS_r.restype = c_void_p
    _lgeos.initGEOS_r.argtypes = [EXCEPTION_HANDLER_FUNCTYPE, EXCEPTION_HANDLER_FUNCTYPE]
    _lgeos.finishGEOS_r.argtypes = [c_void_p]

# Exceptions

class ReadingError(Exception):
    pass

class DimensionError(Exception):
    pass

class TopologicalError(Exception):
    pass

class PredicateError(Exception):
    pass

def error_handler(fmt, list):
    LOG.error("%s", list)
error_h = EXCEPTION_HANDLER_FUNCTYPE(error_handler)

def notice_handler(fmt, list):
    LOG.warning("%s", list)
notice_h = EXCEPTION_HANDLER_FUNCTYPE(notice_handler)

def cleanup():
    if _lgeos is not None :
        _lgeos.finishGEOS()

atexit.register(cleanup)

# Errcheck functions

def errcheck_wkb(result, func, argtuple):
    if not result:
        return None
    size_ref = argtuple[-1]
    size = size_ref.contents
    retval = ctypes.string_at(result, size.value)[:]
    lgeos.GEOSFree(result)
    return retval

def errcheck_just_free(result, func, argtuple):
    retval = string_at(result)
    lgeos.GEOSFree(result)
    return retval

def errcheck_predicate(result, func, argtuple):
    if result == 2:
        raise PredicateError("Failed to evaluate %s" % repr(func))
    return result

class LGEOSBase(threading.local):
    """Proxy for the GEOS_C DLL/SO

    This is a base class. Do not instantiate.
    """
    methods = {}
    def __init__(self, dll):
        self._lgeos = dll
        self.geos_handle = None

class LGEOS14(LGEOSBase):
    """Proxy for the GEOS_C DLL/SO API version 1.4
    """
    geos_capi_version = (1, 4, 0)
    def __init__(self, dll):
        super(LGEOS14, self).__init__(dll)
        self.geos_handle = self._lgeos.initGEOS(notice_h, error_h)
        keys = self._lgeos.__dict__.keys()
        for key in keys:
            setattr(self, key, getattr(self._lgeos, key))
        self.GEOSFree = self._lgeos.free
        self.GEOSGeomToWKB_buf.errcheck = errcheck_wkb
        self.GEOSGeomToWKT.errcheck = errcheck_just_free
        self.GEOSRelate.errcheck = errcheck_just_free
        for pred in ( self.GEOSDisjoint,
              self.GEOSTouches,
              self.GEOSIntersects,
              self.GEOSCrosses,
              self.GEOSWithin,
              self.GEOSContains,
              self.GEOSOverlaps,
              self.GEOSEquals,
              self.GEOSEqualsExact,
              self.GEOSisEmpty,
              self.GEOSisValid,
              self.GEOSisSimple,
              self.GEOSisRing,
              self.GEOSHasZ
              ):
            pred.errcheck = errcheck_predicate

        self.methods['area'] = self.GEOSArea
        self.methods['boundary'] = self.GEOSBoundary
        self.methods['buffer'] = self.GEOSBuffer
        self.methods['centroid'] = self.GEOSGetCentroid
        self.methods['representative_point'] = self.GEOSPointOnSurface
        self.methods['convex_hull'] = self.GEOSConvexHull
        self.methods['distance'] = self.GEOSDistance
        self.methods['envelope'] = self.GEOSEnvelope
        self.methods['length'] = self.GEOSLength
        self.methods['has_z'] = self.GEOSHasZ
        self.methods['is_empty'] = self.GEOSisEmpty
        self.methods['is_ring'] = self.GEOSisRing
        self.methods['is_simple'] = self.GEOSisSimple
        self.methods['is_valid'] = self.GEOSisValid
        self.methods['disjoint'] = self.GEOSDisjoint
        self.methods['touches'] = self.GEOSTouches
        self.methods['intersects'] = self.GEOSIntersects
        self.methods['crosses'] = self.GEOSCrosses
        self.methods['within'] = self.GEOSWithin
        self.methods['contains'] = self.GEOSContains
        self.methods['overlaps'] = self.GEOSOverlaps
        self.methods['equals'] = self.GEOSEquals
        self.methods['equals_exact'] = self.GEOSEqualsExact
        self.methods['relate'] = self.GEOSRelate
        self.methods['difference'] = self.GEOSDifference
        self.methods['symmetric_difference'] = self.GEOSSymDifference
        self.methods['union'] = self.GEOSUnion
        self.methods['intersection'] = self.GEOSIntersection
        self.methods['simplify'] = self.GEOSSimplify
        self.methods['topology_preserve_simplify'] = \
            self.GEOSTopologyPreserveSimplify

        self.methods['cascaded_union'] = self.GEOSUnionCascaded

class LGEOS15(LGEOSBase):
    """Proxy for the reentrant GEOS_C DLL/SO API version 1.5
    """
    geos_capi_version = (1, 5, 0)
    def __init__(self, dll):
        super(LGEOS15, self).__init__(dll)
        self.geos_handle = self._lgeos.initGEOS_r(notice_h, error_h)
        keys = self._lgeos.__dict__.keys()
        for key in filter(lambda x: not x.endswith('_r'), keys):
            if key + '_r' in keys:
                reentr_func = getattr(self._lgeos, key + '_r')
                attr = ftools.partial(reentr_func, self.geos_handle)
                attr.__name__ = reentr_func.__name__
                setattr(self, key, attr)
            else:
                setattr(self, key, getattr(self._lgeos, key))
        if not hasattr(self, 'GEOSFree'):
            self.GEOSFree = self._lgeos.free
        self.GEOSGeomToWKB_buf.func.errcheck = errcheck_wkb
        self.GEOSGeomToWKT.func.errcheck = errcheck_just_free
        self.GEOSRelate.func.errcheck = errcheck_just_free
        for pred in ( self.GEOSDisjoint,
              self.GEOSTouches,
              self.GEOSIntersects,
              self.GEOSCrosses,
              self.GEOSWithin,
              self.GEOSContains,
              self.GEOSOverlaps,
              self.GEOSEquals,
              self.GEOSEqualsExact,
              self.GEOSisEmpty,
              self.GEOSisValid,
              self.GEOSisSimple,
              self.GEOSisRing,
              self.GEOSHasZ
              ):
            pred.func.errcheck = errcheck_predicate

        self.GEOSisValidReason.func.errcheck = errcheck_just_free

        self.methods['area'] = self.GEOSArea
        self.methods['boundary'] = self.GEOSBoundary
        self.methods['buffer'] = self.GEOSBuffer
        self.methods['centroid'] = self.GEOSGetCentroid
        self.methods['representative_point'] = self.GEOSPointOnSurface
        self.methods['convex_hull'] = self.GEOSConvexHull
        self.methods['distance'] = self.GEOSDistance
        self.methods['envelope'] = self.GEOSEnvelope
        self.methods['length'] = self.GEOSLength
        self.methods['has_z'] = self.GEOSHasZ
        self.methods['is_empty'] = self.GEOSisEmpty
        self.methods['is_ring'] = self.GEOSisRing
        self.methods['is_simple'] = self.GEOSisSimple
        self.methods['is_valid'] = self.GEOSisValid
        self.methods['disjoint'] = self.GEOSDisjoint
        self.methods['touches'] = self.GEOSTouches
        self.methods['intersects'] = self.GEOSIntersects
        self.methods['crosses'] = self.GEOSCrosses
        self.methods['within'] = self.GEOSWithin
        self.methods['contains'] = self.GEOSContains
        self.methods['overlaps'] = self.GEOSOverlaps
        self.methods['equals'] = self.GEOSEquals
        self.methods['equals_exact'] = self.GEOSEqualsExact
        self.methods['relate'] = self.GEOSRelate
        self.methods['difference'] = self.GEOSDifference
        self.methods['symmetric_difference'] = self.GEOSSymDifference
        self.methods['union'] = self.GEOSUnion
        self.methods['intersection'] = self.GEOSIntersection
        self.methods['prepared_intersects'] = self.GEOSPreparedIntersects
        self.methods['prepared_contains'] = self.GEOSPreparedContains
        self.methods['prepared_contains_properly'] = \
            self.GEOSPreparedContainsProperly
        self.methods['prepared_covers'] = self.GEOSPreparedCovers
        self.methods['simplify'] = self.GEOSSimplify
        self.methods['topology_preserve_simplify'] = \
            self.GEOSTopologyPreserveSimplify

class LGEOS16(LGEOS15):
    """Proxy for the reentrant GEOS_C DLL/SO API version 1.6
    """
    geos_capi_version = (1, 6, 0)
    def __init__(self, dll):
        super(LGEOS16, self).__init__(dll)

class LGEOS16LR(LGEOS16):
    """Proxy for the reentrant GEOS_C DLL/SO API version 1.6 with linear
    referencing
    """
    geos_capi_version = geos_c_version
    def __init__(self, dll):
        super(LGEOS16LR, self).__init__(dll)

        self.methods['parallel_offset'] = self.GEOSSingleSidedBuffer
        self.methods['project'] = self.GEOSProject
        self.methods['project_normalized'] = self.GEOSProjectNormalized
        self.methods['interpolate'] = self.GEOSInterpolate
        self.methods['interpolate_normalized'] = \
            self.GEOSInterpolateNormalized

class LGEOS17(LGEOS16LR):
    """Proxy for the reentrant GEOS_C DLL/SO API version 1.7
    """
    geos_capi_version = (1, 7, 0)
    def __init__(self, dll):
        super(LGEOS17, self).__init__(dll)

        self.methods['unary_union'] = self.GEOSUnaryUnion
        self.methods['cascaded_union'] = self.methods['unary_union']

if geos_c_version >= (1, 7, 0):
    L = LGEOS17
elif geos_c_version >= (1, 6, 0):
    if hasattr(_lgeos, 'GEOSProject'):
        L = LGEOS16LR
    else:
        L = LGEOS16
elif geos_c_version >= (1, 5, 0):
        L = LGEOS15
else:
        L = LGEOS14

lgeos = L(_lgeos)

<commit_msg>fix cascaded union for CAPI 1.6.x<commit_after>"""
Proxies for the libgeos_c shared lib, GEOS-specific exceptions, and utilities
"""

import atexit
import logging
import os
import sys
import threading
import ctypes
from ctypes import cdll, CDLL, CFUNCTYPE, c_char_p, c_void_p, string_at
from ctypes.util import find_library

import ftools
from ctypes_declarations import prototype, EXCEPTION_HANDLER_FUNCTYPE

# Begin by creating a do-nothing handler and adding to this module's logger.
class NullHandler(logging.Handler):
    def emit(self, record):
        pass
LOG = logging.getLogger(__name__)
LOG.addHandler(NullHandler())

# Find and load the GEOS and C libraries
# If this ever gets any longer, we'll break it into separate modules

def load_dll(libname, fallbacks=None):
    lib = find_library(libname)
    if lib is not None:
        return CDLL(lib)
    else:
        if fallbacks is not None:
            for name in fallbacks:
                try:
                    return CDLL(name)
                except OSError:
                    # move on to the next fallback
                    pass
        # the end
        raise OSError(
            "Could not find library %s or load any of its variants %s" % (
                libname, fallbacks or []))

if sys.platform.startswith('linux'):
    _lgeos = load_dll('geos_c', fallbacks=['libgeos_c.so.1', 'libgeos_c.so'])
    free = load_dll('c').free
    free.argtypes = [c_void_p]
    free.restype = None

elif sys.platform == 'darwin':
    alt_paths = [
            # The Framework build from Kyng Chaos:
            "/Library/Frameworks/GEOS.framework/Versions/Current/GEOS",
            # macports
            '/opt/local/lib/libgeos_c.dylib',
    ]
    _lgeos = load_dll('geos_c', fallbacks=alt_paths)
    free = load_dll('c').free
    free.argtypes = [c_void_p]
    free.restype = None

elif sys.platform == 'win32':
    try:
        egg_dlls = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                     r"..\DLLs"))
        wininst_dlls =  os.path.abspath(os.__file__ + "../../../DLLs")
        original_path = os.environ['PATH']
        os.environ['PATH'] = "%s;%s;%s" % (egg_dlls, wininst_dlls, original_path)
        _lgeos = CDLL("geos.dll")
    except (ImportError, WindowsError, OSError):
        raise
    def free(m):
        try:
            cdll.msvcrt.free(m)
        except WindowsError:
            # XXX: See http://trac.gispython.org/projects/PCL/ticket/149
            pass

elif sys.platform == 'sunos5':
    _lgeos = load_dll('geos_c', fallbacks=['libgeos_c.so.1', 'libgeos_c.so'])
    free = CDLL('libc.so.1').free
    free.argtypes = [c_void_p]
    free.restype = None

else: # other *nix systems
    _lgeos = load_dll('geos_c', fallbacks=['libgeos_c.so.1', 'libgeos_c.so'])
    free = load_dll('c', fallbacks=['libc.so.6']).free
    free.argtypes = [c_void_p]
    free.restype = None

def _geos_c_version():
    func = _lgeos.GEOSversion
    func.argtypes = []
    func.restype = c_char_p
    v = func()
    c_ver = v.split('-')[2]
    # Ditch any SVN revision numbering that may have crept in without (ftm)
    # importing `re`.
    c_ver = c_ver.split()[0]
    return tuple(int(n) for n in c_ver.split('.')), v

geos_capi_version, geos_version_string = _geos_c_version()
geos_c_version = geos_capi_version

# If we have the new interface, then record a baseline so that we know what
# additional functions are declared in ctypes_declarations.
if geos_c_version >= (1,5,0):
    start_set = set(_lgeos.__dict__)

# Apply prototypes for the libgeos_c functions
prototype(_lgeos, geos_c_version)

# If we have the new interface, automatically detect all function
# declarations, and declare their re-entrant counterpart.
if geos_c_version >= (1,5,0):
    end_set = set(_lgeos.__dict__)
    new_func_names = end_set - start_set

    for func_name in new_func_names:
        new_func_name = "%s_r" % func_name
        if hasattr(_lgeos, new_func_name):
            new_func = getattr(_lgeos, new_func_name)
            old_func = getattr(_lgeos, func_name)
            new_func.restype = old_func.restype
            if old_func.argtypes is None:
                # Handle functions that didn't take an argument before,
                # finishGEOS.
                new_func.argtypes = [c_void_p]
            else:
                new_func.argtypes = [c_void_p] + old_func.argtypes
            if old_func.errcheck is not None:
                new_func.errcheck = old_func.errcheck

    # Handle special case.
    _lgeos.initGEOS_r.restype = c_void_p
    _lgeos.initGEOS_r.argtypes = [EXCEPTION_HANDLER_FUNCTYPE, EXCEPTION_HANDLER_FUNCTYPE]
    _lgeos.finishGEOS_r.argtypes = [c_void_p]

# Exceptions

class ReadingError(Exception):
    pass

class DimensionError(Exception):
    pass

class TopologicalError(Exception):
    pass

class PredicateError(Exception):
    pass

def error_handler(fmt, list):
    LOG.error("%s", list)
error_h = EXCEPTION_HANDLER_FUNCTYPE(error_handler)

def notice_handler(fmt, list):
    LOG.warning("%s", list)
notice_h = EXCEPTION_HANDLER_FUNCTYPE(notice_handler)

def cleanup():
    if _lgeos is not None :
        _lgeos.finishGEOS()

atexit.register(cleanup)

# Errcheck functions

def errcheck_wkb(result, func, argtuple):
    if not result:
        return None
    size_ref = argtuple[-1]
    size = size_ref.contents
    retval = ctypes.string_at(result, size.value)[:]
    lgeos.GEOSFree(result)
    return retval

def errcheck_just_free(result, func, argtuple):
    retval = string_at(result)
    lgeos.GEOSFree(result)
    return retval

def errcheck_predicate(result, func, argtuple):
    if result == 2:
        raise PredicateError("Failed to evaluate %s" % repr(func))
    return result

class LGEOSBase(threading.local):
    """Proxy for the GEOS_C DLL/SO

    This is a base class. Do not instantiate.
    """
    methods = {}
    def __init__(self, dll):
        self._lgeos = dll
        self.geos_handle = None

class LGEOS14(LGEOSBase):
    """Proxy for the GEOS_C DLL/SO API version 1.4
    """
    geos_capi_version = (1, 4, 0)
    def __init__(self, dll):
        super(LGEOS14, self).__init__(dll)
        self.geos_handle = self._lgeos.initGEOS(notice_h, error_h)
        keys = self._lgeos.__dict__.keys()
        for key in keys:
            setattr(self, key, getattr(self._lgeos, key))
        self.GEOSFree = self._lgeos.free
        self.GEOSGeomToWKB_buf.errcheck = errcheck_wkb
        self.GEOSGeomToWKT.errcheck = errcheck_just_free
        self.GEOSRelate.errcheck = errcheck_just_free
        for pred in ( self.GEOSDisjoint,
              self.GEOSTouches,
              self.GEOSIntersects,
              self.GEOSCrosses,
              self.GEOSWithin,
              self.GEOSContains,
              self.GEOSOverlaps,
              self.GEOSEquals,
              self.GEOSEqualsExact,
              self.GEOSisEmpty,
              self.GEOSisValid,
              self.GEOSisSimple,
              self.GEOSisRing,
              self.GEOSHasZ
              ):
            pred.errcheck = errcheck_predicate

        self.methods['area'] = self.GEOSArea
        self.methods['boundary'] = self.GEOSBoundary
        self.methods['buffer'] = self.GEOSBuffer
        self.methods['centroid'] = self.GEOSGetCentroid
        self.methods['representative_point'] = self.GEOSPointOnSurface
        self.methods['convex_hull'] = self.GEOSConvexHull
        self.methods['distance'] = self.GEOSDistance
        self.methods['envelope'] = self.GEOSEnvelope
        self.methods['length'] = self.GEOSLength
        self.methods['has_z'] = self.GEOSHasZ
        self.methods['is_empty'] = self.GEOSisEmpty
        self.methods['is_ring'] = self.GEOSisRing
        self.methods['is_simple'] = self.GEOSisSimple
        self.methods['is_valid'] = self.GEOSisValid
        self.methods['disjoint'] = self.GEOSDisjoint
        self.methods['touches'] = self.GEOSTouches
        self.methods['intersects'] = self.GEOSIntersects
        self.methods['crosses'] = self.GEOSCrosses
        self.methods['within'] = self.GEOSWithin
        self.methods['contains'] = self.GEOSContains
        self.methods['overlaps'] = self.GEOSOverlaps
        self.methods['equals'] = self.GEOSEquals
        self.methods['equals_exact'] = self.GEOSEqualsExact
        self.methods['relate'] = self.GEOSRelate
        self.methods['difference'] = self.GEOSDifference
        self.methods['symmetric_difference'] = self.GEOSSymDifference
        self.methods['union'] = self.GEOSUnion
        self.methods['intersection'] = self.GEOSIntersection
        self.methods['simplify'] = self.GEOSSimplify
        self.methods['topology_preserve_simplify'] = \
            self.GEOSTopologyPreserveSimplify
        self.methods['cascaded_union'] = self.GEOSUnionCascaded

class LGEOS15(LGEOSBase):
    """Proxy for the reentrant GEOS_C DLL/SO API version 1.5
    """
    geos_capi_version = (1, 5, 0)
    def __init__(self, dll):
        super(LGEOS15, self).__init__(dll)
        self.geos_handle = self._lgeos.initGEOS_r(notice_h, error_h)
        keys = self._lgeos.__dict__.keys()
        for key in filter(lambda x: not x.endswith('_r'), keys):
            if key + '_r' in keys:
                reentr_func = getattr(self._lgeos, key + '_r')
                attr = ftools.partial(reentr_func, self.geos_handle)
                attr.__name__ = reentr_func.__name__
                setattr(self, key, attr)
            else:
                setattr(self, key, getattr(self._lgeos, key))
        if not hasattr(self, 'GEOSFree'):
            self.GEOSFree = self._lgeos.free
        self.GEOSGeomToWKB_buf.func.errcheck = errcheck_wkb
        self.GEOSGeomToWKT.func.errcheck = errcheck_just_free
        self.GEOSRelate.func.errcheck = errcheck_just_free
        for pred in ( self.GEOSDisjoint,
              self.GEOSTouches,
              self.GEOSIntersects,
              self.GEOSCrosses,
              self.GEOSWithin,
              self.GEOSContains,
              self.GEOSOverlaps,
              self.GEOSEquals,
              self.GEOSEqualsExact,
              self.GEOSisEmpty,
              self.GEOSisValid,
              self.GEOSisSimple,
              self.GEOSisRing,
              self.GEOSHasZ
              ):
            pred.func.errcheck = errcheck_predicate

        self.GEOSisValidReason.func.errcheck = errcheck_just_free

        self.methods['area'] = self.GEOSArea
        self.methods['boundary'] = self.GEOSBoundary
        self.methods['buffer'] = self.GEOSBuffer
        self.methods['centroid'] = self.GEOSGetCentroid
        self.methods['representative_point'] = self.GEOSPointOnSurface
        self.methods['convex_hull'] = self.GEOSConvexHull
        self.methods['distance'] = self.GEOSDistance
        self.methods['envelope'] = self.GEOSEnvelope
        self.methods['length'] = self.GEOSLength
        self.methods['has_z'] = self.GEOSHasZ
        self.methods['is_empty'] = self.GEOSisEmpty
        self.methods['is_ring'] = self.GEOSisRing
        self.methods['is_simple'] = self.GEOSisSimple
        self.methods['is_valid'] = self.GEOSisValid
        self.methods['disjoint'] = self.GEOSDisjoint
        self.methods['touches'] = self.GEOSTouches
        self.methods['intersects'] = self.GEOSIntersects
        self.methods['crosses'] = self.GEOSCrosses
        self.methods['within'] = self.GEOSWithin
        self.methods['contains'] = self.GEOSContains
        self.methods['overlaps'] = self.GEOSOverlaps
        self.methods['equals'] = self.GEOSEquals
        self.methods['equals_exact'] = self.GEOSEqualsExact
        self.methods['relate'] = self.GEOSRelate
        self.methods['difference'] = self.GEOSDifference
        self.methods['symmetric_difference'] = self.GEOSSymDifference
        self.methods['union'] = self.GEOSUnion
        self.methods['intersection'] = self.GEOSIntersection
        self.methods['prepared_intersects'] = self.GEOSPreparedIntersects
        self.methods['prepared_contains'] = self.GEOSPreparedContains
        self.methods['prepared_contains_properly'] = \
            self.GEOSPreparedContainsProperly
        self.methods['prepared_covers'] = self.GEOSPreparedCovers
        self.methods['simplify'] = self.GEOSSimplify
        self.methods['topology_preserve_simplify'] = \
            self.GEOSTopologyPreserveSimplify
        self.methods['cascaded_union'] = self.GEOSUnionCascaded

class LGEOS16(LGEOS15):
    """Proxy for the reentrant GEOS_C DLL/SO API version 1.6
    """
    geos_capi_version = (1, 6, 0)
    def __init__(self, dll):
        super(LGEOS16, self).__init__(dll)

class LGEOS16LR(LGEOS16):
    """Proxy for the reentrant GEOS_C DLL/SO API version 1.6 with linear
    referencing
    """
    geos_capi_version = geos_c_version
    def __init__(self, dll):
        super(LGEOS16LR, self).__init__(dll)

        self.methods['parallel_offset'] = self.GEOSSingleSidedBuffer
        self.methods['project'] = self.GEOSProject
        self.methods['project_normalized'] = self.GEOSProjectNormalized
        self.methods['interpolate'] = self.GEOSInterpolate
        self.methods['interpolate_normalized'] = \
            self.GEOSInterpolateNormalized

class LGEOS17(LGEOS16LR):
    """Proxy for the reentrant GEOS_C DLL/SO API version 1.7
    """
    geos_capi_version = (1, 7, 0)
    def __init__(self, dll):
        super(LGEOS17, self).__init__(dll)

        self.methods['unary_union'] = self.GEOSUnaryUnion
        self.methods['cascaded_union'] = self.methods['unary_union']

if geos_c_version >= (1, 7, 0):
    L = LGEOS17
elif geos_c_version >= (1, 6, 0):
    if hasattr(_lgeos, 'GEOSProject'):
        L = LGEOS16LR
    else:
        L = LGEOS16
elif geos_c_version >= (1, 5, 0):
        L = LGEOS15
else:
        L = LGEOS14

lgeos = L(_lgeos)

<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# shredcompare -- find common cliques in shred lists
#
import getopt, sys, bsddb

class SHIF:
    "SHIF-A file metadata container."
    def __init__(self, name):
        self.name = name
        self.fp = open(name)
        self.comments = []
        id = self.fp.readline()
        if not id.startswith("#SHIF-A "):
            sys.stderr.write("shredcompare: %s is not a SHIF-A file."%self.fp.name)
            sys.exit(1)
        while True:
            line = self.fp.readline()
            if line == '%%\n':
                break
            (tag, value) = line.split(":")
            value = value.strip()
            if tag == "Normalization":
                self.normalization = value.split(", ")
                self.normalization.sort()
            elif tag == "Shred-Size":
                self.shredsize = int(value)
            elif tag == "Hash-Method":
                self.hash = value
            elif tag == "Generator-Program":
                self.generator = value
            elif tag == "Comment":
                self.comments.append(value)
    def __check_match(self, other, attr):
        if getattr(self, attr) != getattr(other, attr):
            sys.stderr.write("shredcompare: %s of %s and %s doesn't match." \
                             % (attr, self.name, other.name))
            sys.exit(1)
    def compatible(self, other):
        self.__check_match(other, 'normalization')
        self.__check_match(other, 'shredsize')
        self.__check_match(other, 'hash')

class Shred:
    "Represent a range of lines."
    def __init__(self, file, start, end):
        self.file = file
        self.start = start
        self.end = end
    def dump(self):
        rfp = open(self.file)
        for i in range(self.start):
            rfp.readline()
        text = ""
        for i in range(self.start, self.end):
            text += rfp.readline()
        rfp.close()
        return text
    def __range(self):
        return range(self.start, self.end)
    def intersects(self, other):
        return self.file == other.file and \
           self.start in other.__range() or other.start in self.__range()
    def merge(self, other):
        self.start = min(self.start, other.start)
        self.end   = max(self.end,   other.end)
    # Make these sortable
    def __cmp__(self, other):
        return cmp((self.file, self.start, self.end),
                   (other.file, other.start, other.end))
    def __hash_(self):
        return hash((self.file, self.start, self.end))
    # External representation
    def __str__(self):
        # 0-origin line numbers internally, 1-origin externally.
        return "%s:%d-%d" % (self.file, self.start, self.end)
    __repr__ = __str__

def merge_hashes(fp, dict):
    "Read and merge hashes corresponding to one file."
    file = fp.readline().rstrip()
    while True:
        line = fp.readline()
        if not line:
            return False
        if not line.strip():
            return True
        (start, end, hashval) = line.split()
        if dict.has_key(hashval):
            oldval = dict[hashval]
        else:
            oldval = ""
        dict[hashval] = oldval + file + "\t" + start + "\t" + end + "\n"

def item_factory(db):
    "Hide the ugliness that is sequential DB access in a generator."
    yield db.first()
    while True:
        try:
            yield db.next()
        except:
            yield None

if __name__ == '__main__':
    try:
        (optlist, args) = getopt.getopt(sys.argv[1:], 'd')
    except getopt.GetoptError:
        sys.stderr.write("usage: shredcompare [-h] [-d] file\n")
        sys.exit(2)
    local_duplicates = True
    for (opt, val) in optlist:
        if opt == '-d':
            local_duplicates = False
        elif opt == '-h':
	    sys.stderr.write("usage: shredcompare [-d] file\n");
	    sys.stderr.write(" -d      = discard local duplicates.\n");
	    sys.stderr.write(" -h      = help (display this message).\n");
	    sys.exit(0);

    shiflist = []
    # Read input metadata
    for file in args:
        shiflist.append(SHIF(file))
    # Metadata sanity check
    for i in range(len(args)-1):
        shiflist[i].compatible(shiflist[i+1])
    # Read in all hashes
    hashdict = bsddb.hashopen(None, 'c')
    for shif in shiflist:
        while merge_hashes(shif.fp, hashdict):
            continue
    # Nuke all unique hashes
    db_cursor = item_factory(hashdict)
    while True:
        item = db_cursor.next()
        if not item:
            break
        (key, match) = item
        if match.count("\n") == 1:
            del hashdict[key]
    # Maybe nuke all match sets that don't cross a tree boundary
    if not local_duplicates:
        pass			# FIXME
    # Turn the remaining matches into match objects, because we're
    # going to want to do logic on them.  We deferred it this long
    # to lower memory usage.
    matches = []
    db_cursor = item_factory(hashdict)
    while True:
        item = db_cursor.next()
        if not item:
            break
        (key, match) = item
        # Decode the multiline strings we had to create to get around
        # the bsddb key and value constraint.
        match = map(lambda x: x.split("\t"), match.rstrip().split("\n"))
        match = map(lambda x: Shred(x[0], int(x[1]), int(x[2])), match)
        matches.append(match)
    # We're done with the database file.
    hashdict.close()
    # Merge overlapping ranges if possible.
    # This is O(n**2) in the number of matching chunks.
    # Should performance ever become a problem, break this up into
    # separate passes over the chunks in each file. (Would correspond
    # to eliminating a bunch of off-diagonal entries.)
    # FIXME: code currently only does two-way merge.
    retry = True
    while retry:
        retry = False
        #print "Matches at start:", matches
        for i in range(len(matches)):
            for j in range(len(matches)):
                if i >= j:
                    continue
                # Neither shred must have been deleted
                if not matches[j] or not matches[i]:
                    continue
                #print "Checking:", matches[j], matches[i]
                # Toss them out if they don't intersect
                if not (matches[j][0].intersects(matches[i][0]) and \
                	matches[j][1].intersects(matches[i][1])):
                    #print "alpha and beta don't intersect"
                    continue
                # Merge lower chunk into upper; delete lower
                #print "Found a merge:", matches[j], matches[i]
                matches[j][0].merge(matches[i][0])
                matches[j][1].merge(matches[i][1])
                matches[i] = None
                retry = True
    matches = filter(lambda x: x, matches)
    matches.sort(lambda x, y: cmp(x[0], y[0]))	# by source chunk
    # OK, dump all matches.
    print "#SHIF-B 1.0"
    print "Merge-Program: shredcompare.py 1.0"
    print "Hash-Method: MD5"
    print "Shred-Size: %d" % shiflist[0].shredsize
    print "Normalization:", ",".join(shiflist[0].normalization)
    print "%%"
    # FIXME: do something appropriate with comments
    for match in matches:
        for range in match:
            print `range`
        print "-"

<commit_msg>About to change the data format.<commit_after>#!/usr/bin/env python
#
# shredcompare -- find common cliques in shred lists
#
import os, sys, time, stat, getopt, bsddb

class SHIF:
    "SHIF-A file metadata container."
    def __init__(self, name):
        self.name = name
        self.fp = open(name)
        self.comments = []
        id = self.fp.readline()
        if not id.startswith("#SHIF-A "):
            sys.stderr.write("shredcompare: %s is not a SHIF-A file."%self.fp.name)
            sys.exit(1)
        while True:
            line = self.fp.readline()
            if line == '%%\n':
                break
            (tag, value) = line.split(":")
            value = value.strip()
            if tag == "Normalization":
                self.normalization = value.split(", ")
                self.normalization.sort()
            elif tag == "Shred-Size":
                self.shredsize = int(value)
            elif tag == "Hash-Method":
                self.hash = value
            elif tag == "Generator-Program":
                self.generator = value
            elif tag == "Comment":
                self.comments.append(value)
    def __check_match(self, other, attr):
        if getattr(self, attr) != getattr(other, attr):
            sys.stderr.write("shredcompare: %s of %s and %s doesn't match." \
                             % (attr, self.name, other.name))
            sys.exit(1)
    def compatible(self, other):
        self.__check_match(other, 'normalization')
        self.__check_match(other, 'shredsize')
        self.__check_match(other, 'hash')

class Shred:
    "Represent a range of lines."
    def __init__(self, file, start, end):
        self.file = file
        self.start = start
        self.end = end
    def dump(self):
        rfp = open(self.file)
        for i in range(self.start):
            rfp.readline()
        text = ""
        for i in range(self.start, self.end):
            text += rfp.readline()
        rfp.close()
        return text
    def __range(self):
        return range(self.start, self.end)
    def intersects(self, other):
        return self.file == other.file and \
           self.start in other.__range() or other.start in self.__range()
    def merge(self, other):
        self.start = min(self.start, other.start)
        self.end   = max(self.end,   other.end)
    # Make these sortable
    def __cmp__(self, other):
        return cmp((self.file, self.start, self.end),
                   (other.file, other.start, other.end))
    def __hash_(self):
        return hash((self.file, self.start, self.end))
    # External representation
    def __str__(self):
        # 0-origin line numbers internally, 1-origin externally.
        return "%s:%d-%d" % (self.file, self.start, self.end)
    __repr__ = __str__

hashcount = total = 0

def merge_hashes(fp, dict):
    "Read and merge hashes corresponding to one file."
    global hashcount
    file = fp.readline().rstrip()
    while True:
        line = fp.readline()
        if not line:
            return False
        if not line.strip():
            return True
        (start, end, hashval) = line.split()
        if dict.has_key(hashval):
            oldval = dict[hashval]
        else:
            oldval = ""
        dict[hashval] = oldval + file + "\t" + start + "\t" + end + "\n"
        hashcount += 1
        if hashcount % 10000 == 0:
            sys.stderr.write("%02.2f%% read.\n" % (fp.tell() / (total * 0.01)))

def item_factory(db):
    "Hide the ugliness that is sequential DB access in a generator."
    yield db.first()
    while True:
        try:
            yield db.next()
        except:
            yield None

mark_time = None

def report_time(legend=None):
    "Report time since last call."
    global mark_time
    endtime = time.time()
    if mark_time:
        elapsed = endtime - mark_time
        hours = elapsed/3600; elapsed %= 3600
        minutes = elapsed/60; elapsed %= 60
        seconds = elapsed
        sys.stderr.write("%% %s: %dh, %dm, %ds\n" % \
            (legend, hours, minutes, seconds))
    mark_time = endtime

def filesize(file):
    return os.stat(file)[stat.ST_SIZE]

if __name__ == '__main__':
    try:
        (optlist, args) = getopt.getopt(sys.argv[1:], 'd')
    except getopt.GetoptError:
        sys.stderr.write("usage: shredcompare [-h] [-d] file\n")
        sys.exit(2)
    local_duplicates = True
    for (opt, val) in optlist:
        if opt == '-d':
            local_duplicates = False
        elif opt == '-h':
	    sys.stderr.write("usage: shredcompare [-d] file\n");
	    sys.stderr.write(" -d      = discard local duplicates.\n");
	    sys.stderr.write(" -h      = help (display this message).\n");
	    sys.exit(0);

    report_time()
    shiflist = []
    # Read input metadata
    for file in args:
        shiflist.append(SHIF(file))
    # Metadata sanity check
    for i in range(len(args)-1):
        shiflist[i].compatible(shiflist[i+1])
    # Compute amount of data to be read
    total = sum(map(filesize, args))
    sys.stderr.write("%d bytes of data to be read\n" % total)
    # Read in all hashes
    hashdict = bsddb.hashopen(None, 'c')
    for shif in shiflist:
        while merge_hashes(shif.fp, hashdict):
            continue
    report_time("Hash merge done, %d hashes" % hashcount)
    # Nuke all unique hashes
    db_cursor = item_factory(hashdict)
    while True:
        item = db_cursor.next()
        if not item:
            break
        (key, match) = item
        if match.count("\n") == 1:
            del hashdict[key]
    report_time("Global duplicates removed")
    # Maybe nuke all match sets that don't cross a tree boundary
    if not local_duplicates:
        pass
    report_time("Local duplicates removed")
    # Turn the remaining matches into match objects, because we're
    # going to want to do logic on them.  We deferred it this long
    # to lower memory usage.
    matches = []
    db_cursor = item_factory(hashdict)
    while True:
        item = db_cursor.next()
        if not item:
            break
        (key, match) = item
        # Decode the multiline strings we had to create to get around
        # the bsddb key and value constraint.
        match = map(lambda x: x.split("\t"), match.rstrip().split("\n"))
        match = map(lambda x: Shred(x[0], int(x[1]), int(x[2])), match)
        matches.append(match)
    # We're done with the database file.
    hashdict.close()
    report_time("Match objects converted")
    # Merge overlapping ranges if possible.
    # This is O(n**2) in the number of matching chunks.
    # Should performance ever become a problem, break this up into
    # separate passes over the chunks in each file. (Would correspond
    # to eliminating a bunch of off-diagonal entries.)
    # FIXME: code currently only does two-way merge.
    retry = True
    while retry:
        retry = False
        #print "Matches at start:", matches
        for i in range(len(matches)):
            for j in range(len(matches)):
                if i >= j:
                    continue
                # Neither shred must have been deleted
                if not matches[j] or not matches[i]:
                    continue
                #print "Checking:", matches[j], matches[i]
                # Toss them out if they don't intersect
                if not (matches[j][0].intersects(matches[i][0]) and \
                	matches[j][1].intersects(matches[i][1])):
                    #print "alpha and beta don't intersect"
                    continue
                # Merge lower chunk into upper; delete lower
                #print "Found a merge:", matches[j], matches[i]
                matches[j][0].merge(matches[i][0])
                matches[j][1].merge(matches[i][1])
                matches[i] = None
                retry = True
    matches = filter(lambda x: x, matches)
    matches.sort(lambda x, y: cmp(x[0], y[0]))	# by source chunk
    report_time("Merge complete")
    # OK, dump all matches.
    print "#SHIF-B 1.0"
    print "Merge-Program: shredcompare.py 1.0"
    print "Hash-Method: MD5"
    print "Shred-Size: %d" % shiflist[0].shredsize
    print "Normalization:", ",".join(shiflist[0].normalization)
    print "%%"
    # FIXME: do something appropriate with comments
    for match in matches:
        for range in match:
            print `range`
        print "-"

<|endoftext|>