<commit_before>"""
This module is for formatting and writing unit-tests in python. The general format is as follows
1. Use start to start a test and give it, as an argument, the name of the test
2. Use whatever check functions are relevant to test the run
3. Use finish to signal the end of the test
4. Repeat stages 1-3 as the number of tests you want to run in the file
5. Use print_results_and_exit to print the number of tests and assertions that passed/failed in the correct format
   before exiting with 0 if all tests passed or with 1 if there was a failed test

In addition you may want to use the 'info' functions in this module to add more detailed
messages in case of a failed check
"""

import os, sys, subprocess, traceback, platform
import pyrealsense2 as rs

n_assertions = 0
n_failed_assertions = 0
n_tests = 0
n_failed_tests = 0
test_failed = False
test_in_progress = False
test_info = {} # Dictionary for holding additional information to print in case of a failed check.

def set_env_vars(env_vars):
    """
    If this is the first time running this script we set the wanted environment, however it is impossible to change the
    current running environment so we rerun the script in a child process that inherits the environment we set
    :param env_vars: A dictionary where the keys are the name of the environment variable and the values are the
        wanted values in string form (environment variables must be strings)
    """
    if len(sys.argv) < 2:
        for env_var, val in env_vars.items():
            os.environ[env_var] = val
        sys.argv.append("rerun")
        if platform.system() == 'Linux' and "microsoft" not in platform.uname()[3].lower():
            cmd = ["python3"]
        else:
            cmd = ["py", "-3"]
        if sys.flags.verbose:
            cmd += ["-v"]
        cmd += sys.argv
        p = subprocess.run( cmd, stderr=subprocess.PIPE, universal_newlines=True )
        exit(p.returncode)

def find_first_device_or_exit():
    """
    :return: the first device that was found, if no device is found the test is skipped. That way we can still run
        the unit-tests when no device is connected and not fail the tests that check a connected device
    """
    c = rs.context()
    if not c.devices.size():  # if no device is connected we skip the test
        print("No device found, skipping test")
        exit(0)
    return c.devices[0]

def find_devices_by_product_line_or_exit(product_line):
    """
    :param product_line: The product line of the wanted devices
    :return: A list of devices of specific product line that was found, if no device is found the test is skipped.
        That way we can still run the unit-tests when no device is connected
        and not fail the tests that check a connected device
    """
    c = rs.context()
    devices_list = c.query_devices(product_line)
    if devices_list.size() == 0:
        print("No device of the" , product_line ,"product line was found; skipping test")
        exit(0)
    return devices_list

def print_stack():
    """
    Function for printing the current call stack. Used when an assertion fails
    """
    for line in traceback.format_stack():
        print(line)

"""
The following functions are for asserting test cases:
The check family of functions tests an expression and continues the test whether the assertion succeeded or failed.
The require family are equivalent but execution is aborted if the assertion fails. In this module, the require family
is used by sending abort=True to check functions
"""

def check_failed():
    """
    Function for when a check fails
    """
    global n_failed_assertions, test_failed
    n_failed_assertions += 1
    test_failed = True
    print_info()
    info_reset()

def abort():
    print("Abort was specified in a failed check. Aborting test")
    exit(1)

def check(exp, abort_if_failed = False):
    """
    Basic function for asserting expressions.
    :param exp: An expression to be asserted, if false the assertion failed
    :param abort_if_failed: If True and assertion failed the test will be aborted
    :return: True if assertion passed, False otherwise
    """
    global n_assertions
    n_assertions += 1
    if not exp:
        print("Check failed, received", exp)
        check_failed()
        print_stack()
        if abort_if_failed:
            abort()
        return False
    info_reset()
    return True

def check_equal(result, expected, abort_if_failed = False):
    """
    Used for asserting a variable has the expected value
    :param result: The actual value of a variable
    :param expected: The expected value of the variable
    :param abort_if_failed:  If True and assertion failed the test will be aborted
    :return: True if assertion passed, False otherwise
    """
    if type(expected) == list:
        print("check_equal should not be used for lists. Use check_equal_lists instead")
        if abort_if_failed:
            abort()
        return False
    global n_assertions
    n_assertions += 1
    if result != expected:
        print("Result was:" + result + "\nBut we expected: " + expected)
        check_failed()
        print_stack()
        if abort_if_failed:
            abort()
        return False
    info_reset()
    return True

def unreachable( abort_if_failed = False ):
    """
    Used to assert that a certain section of code (exp: an if block) is not reached
    :param abort_if_failed: If True and this function is reached the test will be aborted
    """
    check(False, abort_if_failed)

def unexpected_exception( abort_if_failed = False ):
    """
    Used to assert that an except block is not reached. It's different from unreachable because it expects
    to be in an except block and prints the stack of the error and not the call-stack for this function
    :param abort_if_failed: If True and this function is reached the test will be aborted
    """
    global n_assertions
    n_assertions += 1
    traceback.print_exc( file = sys.stdout )
    check_failed()
    if abort_if_failed:
        abort()
    info_reset()

def check_equal_lists(result, expected, abort_if_failed = False):
    """
    Used to assert that 2 lists are identical. python "equality" (using ==) requires same length & elements
    but not necessarily same ordering. Here we require exactly the same, including ordering.
    :param result: The actual list
    :param expected: The expected list
    :param abort_if_failed:  If True and assertion failed the test will be aborted
    :return: True if assertion passed, False otherwise
    """
    global n_assertions
    n_assertions += 1
    failed = False
    if len(result) != len(expected):
        failed = True
        print("Check equal lists failed due to lists of different sizes:")
        print("The resulted list has", len(result), "elements, but the expected list has", len(expected), "elements")
    i = 0
    for res, exp in zip(result, expected):
        if res != exp:
            failed = True
            print("Check equal lists failed due to unequal elements:")
            print("The element of index", i, "in both lists was not equal")
        i += 1
    if failed:
        print("Result list:", result)
        print("Expected list:", expected)
        check_failed()
        print_stack()
        if abort_if_failed:
            abort()
        return False
    info_reset()
    return True

def check_exception(exception, expected_type, expected_msg = None, abort_if_failed = False):
    """
    Used to assert a certain type of exception was raised, placed in the except block
    :param exception: The exception that was raised
    :param expected_type: The expected type of exception
    :param expected_msg: The expected message in the exception
    :param abort_if_failed:  If True and assertion failed the test will be aborted
    :return: True if assertion passed, False otherwise
    """
    failed = False
    if type(exception) != expected_type:
        print("Raised exception was of type", type(exception), "and not of type", expected_type, "as expected")
        failed = True
    if expected_msg and str(exception) != expected_msg:
        print("Exception had message:", str(exception), "\nBut we expected:", expected_msg)
        failed = True
    if failed:
        check_failed()
        print_stack()
        if abort_if_failed:
            abort()
        return False
    info_reset()
    return True

def check_frame_drops(frame, previous_frame_number, allowed_drops = 0):
    """
    Used for checking frame drops while streaming
    :param frame: Current frame being checked
    :param previous_frame_number: Number of the previous frame
    :param allowed_drops: Maximum number of frame drops we accept
    :return: False if dropped too many frames or frames were out of order, True otherwise
    """
    frame_number = frame.get_frame_number()
    failed = False
    if previous_frame_number > 0:
        dropped_frames = frame_number - (previous_frame_number + 1)  # should be 0 in windows, less than 5 in linux
        if dropped_frames > allowed_drops:
            print(dropped_frames, "frame(s) starting from frame", previous_frame_number + 1, "were dropped")
            failed = True
        if dropped_frames < 0:
            print("Frames repeated or out of order. Got frame", frame_number, "after frame",
                  previous_frame_number)
            failed = True
    if failed:
        check_failed()
        return False
    info_reset()
    return True

"""
The following functions are for adding additional information to the printed messages in case of a failed check.
"""

def info(name, value, persistent = False):
    """
    This function is used to store additional information to print in case of a failed test. This information is
    erased after the next check. The information is stored in the dictionary test_info, Keys are names (strings)
    and the items are tuples of value, persistent
    :param name: The name of the variable
    :param value: The value this variable stores
    :param persistent: If this parameter is True, the information stored will be kept after the following check
        and will only be erased at the end of the test ( or when info_reset is called with True)
    """
    global test_info
    test_info[name] = (value, persistent)

def info_reset(persistent = False):
    """
    erases the stored information
    :param persistent: If this parameter is True, even the persistent information will be erased
    """
    global test_info
    if persistent:
        test_info.clear()
    else:
        for name, value_persistent in test_info:
            persistent = value_persistent[1]
            if persistent:
                test_info.pop(name)

def print_info():
    global test_info
    if not test_info: # No information is stored
        return
    print("Printing information")
    for name, value in test_info:
        print("Name:", name, "        value:", value[0])

"""
The following functions are for formatting tests in a file
"""

def fail():
    """
    Function for manually failing a test in case you want a specific test that does not fit any check function
    """
    global test_in_progress, n_failed_tests, test_failed
    if not test_in_progress:
        raise RuntimeError("Tried to fail a test with no test running")
    if not test_failed:
        n_failed_tests += 1
        test_failed = True

def start(*test_name):
    """
    Used at the beginning of each test to reset the global variables
    :param test_name: Any number of arguments that combined give the name of this test
    :return:
    """
    global n_tests, test_failed, test_in_progress
    if test_in_progress:
        raise RuntimeError("Tried to start test before previous test finished. Aborting test")
    n_tests += 1
    test_failed = False
    test_in_progress = True
    info_reset(persistent=True)
    print(*test_name)

def finish():
    """
    Used at the end of each test to check if it passed and print the answer
    """
    global test_failed, n_failed_tests, test_in_progress
    if not test_in_progress:
        raise RuntimeError("Tried to finish a test without starting one")
    if test_failed:
        n_failed_tests += 1
        print("Test failed")
    else:
        print("Test passed")
    print()
    test_in_progress = False

def print_results_and_exit():
    """
    Used to print the results of the tests in the file. The format has to agree with the expected format in check_log()
    in run-unit-tests and with the C++ format using Catch
    """
    global n_assertions, n_tests, n_failed_assertions, n_failed_tests
    if n_failed_tests:
        passed = n_assertions - n_failed_assertions
        print("test cases:", n_tests, "|" , n_failed_tests,  "failed")
        print("assertions:", n_assertions, "|", passed, "passed |", n_failed_assertions, "failed")
        exit(1)
    print("All tests passed (" + str(n_assertions) + " assertions in " + str(n_tests) + " test cases)")
    exit(0)
<commit_msg>created information class<commit_after>"""
This module is for formatting and writing unit-tests in python. The general format is as follows
1. Use start to start a test and give it, as an argument, the name of the test
2. Use whatever check functions are relevant to test the run
3. Use finish to signal the end of the test
4. Repeat stages 1-3 as the number of tests you want to run in the file
5. Use print_results_and_exit to print the number of tests and assertions that passed/failed in the correct format
   before exiting with 0 if all tests passed or with 1 if there was a failed test

In addition you may want to use the 'info' functions in this module to add more detailed
messages in case of a failed check
"""

import os, sys, subprocess, traceback, platform
import pyrealsense2 as rs

n_assertions = 0
n_failed_assertions = 0
n_tests = 0
n_failed_tests = 0
test_failed = False
test_in_progress = False
test_info = {} # Dictionary for holding additional information to print in case of a failed check.

def set_env_vars(env_vars):
    """
    If this is the first time running this script we set the wanted environment, however it is impossible to change the
    current running environment so we rerun the script in a child process that inherits the environment we set
    :param env_vars: A dictionary where the keys are the name of the environment variable and the values are the
        wanted values in string form (environment variables must be strings)
    """
    if len(sys.argv) < 2:
        for env_var, val in env_vars.items():
            os.environ[env_var] = val
        sys.argv.append("rerun")
        if platform.system() == 'Linux' and "microsoft" not in platform.uname()[3].lower():
            cmd = ["python3"]
        else:
            cmd = ["py", "-3"]
        if sys.flags.verbose:
            cmd += ["-v"]
        cmd += sys.argv
        p = subprocess.run( cmd, stderr=subprocess.PIPE, universal_newlines=True )
        exit(p.returncode)

def find_first_device_or_exit():
    """
    :return: the first device that was found, if no device is found the test is skipped. That way we can still run
        the unit-tests when no device is connected and not fail the tests that check a connected device
    """
    c = rs.context()
    if not c.devices.size():  # if no device is connected we skip the test
        print("No device found, skipping test")
        exit(0)
    return c.devices[0]

def find_devices_by_product_line_or_exit(product_line):
    """
    :param product_line: The product line of the wanted devices
    :return: A list of devices of specific product line that was found, if no device is found the test is skipped.
        That way we can still run the unit-tests when no device is connected
        and not fail the tests that check a connected device
    """
    c = rs.context()
    devices_list = c.query_devices(product_line)
    if devices_list.size() == 0:
        print("No device of the" , product_line ,"product line was found; skipping test")
        exit(0)
    return devices_list

def print_stack():
    """
    Function for printing the current call stack. Used when an assertion fails
    """
    for line in traceback.format_stack():
        print(line)

"""
The following functions are for asserting test cases:
The check family of functions tests an expression and continues the test whether the assertion succeeded or failed.
The require family are equivalent but execution is aborted if the assertion fails. In this module, the require family
is used by sending abort=True to check functions
"""

def check_failed():
    """
    Function for when a check fails
    """
    global n_failed_assertions, test_failed
    n_failed_assertions += 1
    test_failed = True
    print_info()

def abort():
    print("Abort was specified in a failed check. Aborting test")
    exit(1)

def check(exp, abort_if_failed = False):
    """
    Basic function for asserting expressions.
    :param exp: An expression to be asserted, if false the assertion failed
    :param abort_if_failed: If True and assertion failed the test will be aborted
    :return: True if assertion passed, False otherwise
    """
    global n_assertions
    n_assertions += 1
    if not exp:
        print("Check failed, received", exp)
        check_failed()
        print_stack()
        if abort_if_failed:
            abort()
        return False
    reset_info()
    return True

def check_equal(result, expected, abort_if_failed = False):
    """
    Used for asserting a variable has the expected value
    :param result: The actual value of a variable
    :param expected: The expected value of the variable
    :param abort_if_failed:  If True and assertion failed the test will be aborted
    :return: True if assertion passed, False otherwise
    """
    if type(expected) == list:
        print("check_equal should not be used for lists. Use check_equal_lists instead")
        if abort_if_failed:
            abort()
        return False
    global n_assertions
    n_assertions += 1
    if result != expected:
        print("Result was:" + result + "\nBut we expected: " + expected)
        check_failed()
        print_stack()
        if abort_if_failed:
            abort()
        return False
    reset_info()
    return True

def unreachable( abort_if_failed = False ):
    """
    Used to assert that a certain section of code (exp: an if block) is not reached
    :param abort_if_failed: If True and this function is reached the test will be aborted
    """
    check(False, abort_if_failed)

def unexpected_exception( abort_if_failed = False ):
    """
    Used to assert that an except block is not reached. It's different from unreachable because it expects
    to be in an except block and prints the stack of the error and not the call-stack for this function
    :param abort_if_failed: If True and this function is reached the test will be aborted
    """
    global n_assertions
    n_assertions += 1
    traceback.print_exc( file = sys.stdout )
    check_failed()
    if abort_if_failed:
        abort()
    reset_info()

def check_equal_lists(result, expected, abort_if_failed = False):
    """
    Used to assert that 2 lists are identical. python "equality" (using ==) requires same length & elements
    but not necessarily same ordering. Here we require exactly the same, including ordering.
    :param result: The actual list
    :param expected: The expected list
    :param abort_if_failed:  If True and assertion failed the test will be aborted
    :return: True if assertion passed, False otherwise
    """
    global n_assertions
    n_assertions += 1
    failed = False
    if len(result) != len(expected):
        failed = True
        print("Check equal lists failed due to lists of different sizes:")
        print("The resulted list has", len(result), "elements, but the expected list has", len(expected), "elements")
    i = 0
    for res, exp in zip(result, expected):
        if res != exp:
            failed = True
            print("Check equal lists failed due to unequal elements:")
            print("The element of index", i, "in both lists was not equal")
        i += 1
    if failed:
        print("Result list:", result)
        print("Expected list:", expected)
        check_failed()
        print_stack()
        if abort_if_failed:
            abort()
        return False
    reset_info()
    return True

def check_exception(exception, expected_type, expected_msg = None, abort_if_failed = False):
    """
    Used to assert a certain type of exception was raised, placed in the except block
    :param exception: The exception that was raised
    :param expected_type: The expected type of exception
    :param expected_msg: The expected message in the exception
    :param abort_if_failed:  If True and assertion failed the test will be aborted
    :return: True if assertion passed, False otherwise
    """
    failed = False
    if type(exception) != expected_type:
        print("Raised exception was of type", type(exception), "and not of type", expected_type, "as expected")
        failed = True
    if expected_msg and str(exception) != expected_msg:
        print("Exception had message:", str(exception), "\nBut we expected:", expected_msg)
        failed = True
    if failed:
        check_failed()
        print_stack()
        if abort_if_failed:
            abort()
        return False
    reset_info()
    return True

def check_frame_drops(frame, previous_frame_number, allowed_drops = 0):
    """
    Used for checking frame drops while streaming
    :param frame: Current frame being checked
    :param previous_frame_number: Number of the previous frame
    :param allowed_drops: Maximum number of frame drops we accept
    :return: False if dropped too many frames or frames were out of order, True otherwise
    """
    frame_number = frame.get_frame_number()
    failed = False
    if previous_frame_number > 0:
        dropped_frames = frame_number - (previous_frame_number + 1)  # should be 0 in windows, less than 5 in linux
        if dropped_frames > allowed_drops:
            print(dropped_frames, "frame(s) starting from frame", previous_frame_number + 1, "were dropped")
            failed = True
        if dropped_frames < 0:
            print("Frames repeated or out of order. Got frame", frame_number, "after frame",
                  previous_frame_number)
            failed = True
    if failed:
        check_failed()
        return False
    reset_info()
    return True

"""
The following functions are for adding additional information to the printed messages in case of a failed check.
"""

class Information:
    """
    Class representing the information stored in test_info dictionary
    """
    def __init__(self, value, persistent = False):
        self.value = value
        self.persistent = persistent

def info(name, value, persistent = False):
    """
    This function is used to store additional information to print in case of a failed test. This information is
    erased after the next check. The information is stored in the dictionary test_info, Keys are names (strings)
    and the items are tuples of value, persistent
    :param name: The name of the variable
    :param value: The value this variable stores
    :param persistent: If this parameter is True, the information stored will be kept after the following check
        and will only be erased at the end of the test ( or when reset_info is called with True)
    """
    global test_info
    test_info[name] = Information(value, persistent)

def reset_info(persistent = False):
    """
    erases the stored information
    :param persistent: If this parameter is True, even the persistent information will be erased
    """
    global test_info
    if persistent:
        test_info.clear()
    else:
        for name, information in test_info:
            if information.persistent:
                test_info.pop(name)

def print_info():
    global test_info
    if not test_info: # No information is stored
        return
    print("Printing information")
    for name, information in test_info:
        print("Name:", name, "        value:", information.value)
    reset_info()

"""
The following functions are for formatting tests in a file
"""

def fail():
    """
    Function for manually failing a test in case you want a specific test that does not fit any check function
    """
    global test_in_progress, n_failed_tests, test_failed
    if not test_in_progress:
        raise RuntimeError("Tried to fail a test with no test running")
    if not test_failed:
        n_failed_tests += 1
        test_failed = True

def start(*test_name):
    """
    Used at the beginning of each test to reset the global variables
    :param test_name: Any number of arguments that combined give the name of this test
    :return:
    """
    global n_tests, test_failed, test_in_progress
    if test_in_progress:
        raise RuntimeError("Tried to start test before previous test finished. Aborting test")
    n_tests += 1
    test_failed = False
    test_in_progress = True
    reset_info(persistent=True)
    print(*test_name)

def finish():
    """
    Used at the end of each test to check if it passed and print the answer
    """
    global test_failed, n_failed_tests, test_in_progress
    if not test_in_progress:
        raise RuntimeError("Tried to finish a test without starting one")
    if test_failed:
        n_failed_tests += 1
        print("Test failed")
    else:
        print("Test passed")
    print()
    test_in_progress = False

def print_results_and_exit():
    """
    Used to print the results of the tests in the file. The format has to agree with the expected format in check_log()
    in run-unit-tests and with the C++ format using Catch
    """
    global n_assertions, n_tests, n_failed_assertions, n_failed_tests
    if n_failed_tests:
        passed = n_assertions - n_failed_assertions
        print("test cases:", n_tests, "|" , n_failed_tests,  "failed")
        print("assertions:", n_assertions, "|", passed, "passed |", n_failed_assertions, "failed")
        exit(1)
    print("All tests passed (" + str(n_assertions) + " assertions in " + str(n_tests) + " test cases)")
    exit(0)
<|endoftext|>


<commit_before>class Vehiculo:
    def __init__(self,marca, modelo, color, transmision, sku):
        self.marca = marca
        self.modelo = modelo
        self.color = color
        self.transmision = transmision
        self.sku = sku

    def getMarca(self):
        return self.marca
    def setMarca(self,marca):
        self.marca = marca

    def getModelo(self):
        return self.modelo
    def setMarca(self,modelo):
        self.modelo = modelo

    def getColor(self):
        return self.color
    def setColor(self,color):
        self.color = color

    def getTransmision(self):
        return self.transmision
    def setTransmision(self,transmision):
        self.transmision = transmision

    def getSku(self):
        return self.sku
    def setSku(self,color):
        self.sku = sku

    def atribVehiculo(self):
        return "Marca: {}\nModelo: {}\nColor: {}\nTransmisión: {}\nSKU: {}\n".format(self.marca,
        self.modelo, self.color, self.transmision, self.sku)
<commit_msg>Delete Vehiculo.py<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""

ZUGBRUECKE
Calling routines in Windows DLLs from Python scripts running on unixlike systems
https://github.com/pleiszenburg/zugbruecke

	src/zugbruecke/core/lib.py: General purpose routines

	Required to run on platform / side: [UNIX, WINE]

	Copyright (C) 2017-2021 Sebastian M. Ernst <ernst@pleiszenburg.de>

<LICENSE_BLOCK>
The contents of this file are subject to the GNU Lesser General Public License
Version 2.1 ("LGPL" or "License"). You may not use this file except in
compliance with the License. You may obtain a copy of the License at
https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt
https://github.com/pleiszenburg/zugbruecke/blob/master/LICENSE

Software distributed under the License is distributed on an "AS IS" basis,
WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
specific language governing rights and limitations under the License.
</LICENSE_BLOCK>

"""

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# IMPORT
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

import hashlib
import os
import random
import socket

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# LIBRARY ROUTINES
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

def get_free_port():

    s = socket.socket()
    s.bind(("", 0))
    port = s.getsockname()[1]
    s.close()

    return port

def get_hash_of_string(str_in):

    return hashlib.sha256(str_in.encode("utf-8")).hexdigest()

def get_location_of_file(filename=""):

    if filename == "":
        filename = __file__

    return os.path.split(os.path.realpath(filename))[0]

def get_randhashstr(dig):

    # Return hash string with dig digits
    return ("%0" + str(dig) + "x") % random.randrange(16 ** dig)

def generate_session_id():

    # A session id by default is an 8 digit hash string
    return get_randhashstr(8)
<commit_msg>cleanup, types<commit_after># -*- coding: utf-8 -*-

"""

ZUGBRUECKE
Calling routines in Windows DLLs from Python scripts running on unixlike systems
https://github.com/pleiszenburg/zugbruecke

    src/zugbruecke/core/lib.py: General purpose routines

    Required to run on platform / side: [UNIX, WINE]

    Copyright (C) 2017-2021 Sebastian M. Ernst <ernst@pleiszenburg.de>

<LICENSE_BLOCK>
The contents of this file are subject to the GNU Lesser General Public License
Version 2.1 ("LGPL" or "License"). You may not use this file except in
compliance with the License. You may obtain a copy of the License at
https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt
https://github.com/pleiszenburg/zugbruecke/blob/master/LICENSE

Software distributed under the License is distributed on an "AS IS" basis,
WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
specific language governing rights and limitations under the License.
</LICENSE_BLOCK>

"""

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# IMPORT
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

import hashlib
import random
import socket

from .typeguard import typechecked

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# LIBRARY ROUTINES
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

@typechecked
def get_free_port() -> int:

    s = socket.socket()
    s.bind(("", 0))
    port = s.getsockname()[1]
    s.close()

    return port

@typechecked
def get_hash_of_string(in_str: str) -> str:

    return hashlib.sha256(in_str.encode("utf-8")).hexdigest()

@typechecked
def get_randhashstr(digits: int) -> str:

    return "{{NUMBER:0{DIGITS:d}x}}".format(DIGITS = digits).format(NUMBER = random.randrange(16 ** digits))

@typechecked
def generate_session_id() -> str:

    return get_randhashstr(8) # 8 digit hash string by default
<|endoftext|>


<commit_before>from django.shortcuts import render

# Create your views here.
<commit_msg>add JWT authentication mechanism; right now, JWT is hard-coded into the request handling. There is probably a better way, using the same mechanism that handles token authentication.<commit_after>from django.utils.decorators import method_decorator
from django.views.decorators.csrf import ensure_csrf_cookie

from graphene_django.views import GraphQLView
from rest_framework_jwt.authentication import JSONWebTokenAuthentication

class JWTGraphQLView(GraphQLView):
    @method_decorator(ensure_csrf_cookie)
    def dispatch(self, request, *args, **kwargs):
        auth = JSONWebTokenAuthentication()
        user_auth_tuple = auth.authenticate(request)
        if user_auth_tuple is not None:
            request.user, request.auth = user_auth_tuple
        return super(JWTGraphQLView, self).dispatch(request, *args, **kwargs)
<|endoftext|>


<commit_before>"""
Equivalencies between different kinds of units

"""

# -----------------------------------------------------------------------------
# Copyright (c) 2018, yt Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the LICENSE file, distributed with this software.
# -----------------------------------------------------------------------------

from collections import OrderedDict

from unyt.dimensions import (
    temperature,
    mass,
    energy,
    length,
    rate,
    velocity,
    dimensionless,
    density,
    number_density,
    flux,
    current_cgs,
    current_mks,
    charge_cgs,
    charge_mks,
    magnetic_field_cgs,
    magnetic_field_mks,
    electric_potential_cgs,
    electric_potential_mks,
    electric_field_cgs,
    electric_field_mks,
    resistance_cgs,
    resistance_mks
)

from unyt._physical_ratios import speed_of_light_cm_per_s
from six import add_metaclass
import numpy as np

equivalence_registry = OrderedDict()

class RegisteredEquivalence(type):
    def __init__(cls, name, b, d):
        type.__init__(cls, name, b, d)
        if hasattr(cls, "_type_name") and not cls._skip_add:
            equivalence_registry[cls._type_name] = cls
        if hasattr(cls, "_alternate_names") and not cls._skip_add:
            for name in cls._alternate_names:
                equivalence_registry[name] = cls

@add_metaclass(RegisteredEquivalence)
class Equivalence(object):
    _skip_add = False
    _one_way = False

class NumberDensityEquivalence(Equivalence):
    _type_name = "number_density"
    dims = (density, number_density,)

    def convert(self, x, new_dims, mu=0.6):
        from unyt import physical_constants as pc
        if new_dims == number_density:
            return x/(mu*pc.mh)
        elif new_dims == density:
            return x*mu*pc.mh

    def __str__(self):
        return "number density: density <-> number density"

class ThermalEquivalence(Equivalence):
    _type_name = "thermal"
    dims = (temperature, energy,)

    def convert(self, x, new_dims):
        from unyt import physical_constants as pc
        if new_dims == energy:
            return pc.kboltz*x
        elif new_dims == temperature:
            return x/pc.kboltz

    def __str__(self):
        return "thermal: temperature <-> energy"

class MassEnergyEquivalence(Equivalence):
    _type_name = "mass_energy"
    dims = (mass, energy,)

    def convert(self, x, new_dims):
        from unyt import physical_constants as pc
        if new_dims == energy:
            return x*pc.clight*pc.clight
        elif new_dims == mass:
            return x/(pc.clight*pc.clight)

    def __str__(self):
        return "mass_energy: mass <-> energy"

class SpectralEquivalence(Equivalence):
    _type_name = "spectral"
    dims = (length, rate, energy,)

    def convert(self, x, new_dims):
        from unyt import physical_constants as pc
        if new_dims == energy:
            if x.units.dimensions == length:
                nu = pc.clight/x
            elif x.units.dimensions == rate:
                nu = x
            return pc.hcgs*nu
        elif new_dims == length:
            if x.units.dimensions == rate:
                return pc.clight/x
            elif x.units.dimensions == energy:
                return pc.hcgs*pc.clight/x
        elif new_dims == rate:
            if x.units.dimensions == length:
                return pc.clight/x
            elif x.units.dimensions == energy:
                return x/pc.hcgs

    def __str__(self):
        return "spectral: length <-> frequency <-> energy"

class SoundSpeedEquivalence(Equivalence):
    _type_name = "sound_speed"
    dims = (velocity, temperature, energy,)

    def convert(self, x, new_dims, mu=0.6, gamma=5./3.):
        from unyt import physical_constants as pc
        if new_dims == velocity:
            if x.units.dimensions == temperature:
                kT = pc.kboltz*x
            elif x.units.dimensions == energy:
                kT = x
            return np.sqrt(gamma*kT/(mu*pc.mh))
        else:
            kT = x*x*mu*pc.mh/gamma
            if new_dims == temperature:
                return kT/pc.kboltz
            else:
                return kT

    def __str__(self):
        return "sound_speed (ideal gas): velocity <-> temperature <-> energy"

class LorentzEquivalence(Equivalence):
    _type_name = "lorentz"
    dims = (dimensionless, velocity,)

    def convert(self, x, new_dims):
        from unyt import physical_constants as pc
        if new_dims == dimensionless:
            beta = x.in_cgs()/pc.clight
            return 1./np.sqrt(1.-beta**2)
        elif new_dims == velocity:
            return pc.clight*np.sqrt(1.-1./(x*x))

    def __str__(self):
        return "lorentz: velocity <-> dimensionless"

class SchwarzschildEquivalence(Equivalence):
    _type_name = "schwarzschild"
    dims = (mass, length,)

    def convert(self, x, new_dims):
        from unyt import physical_constants as pc
        if new_dims == length:
            return 2.*pc.G*x/(pc.clight*pc.clight)
        elif new_dims == mass:
            return 0.5*x*pc.clight*pc.clight/pc.G

    def __str__(self):
        return "schwarzschild: mass <-> length"

class ComptonEquivalence(Equivalence):
    _type_name = "compton"
    dims = (mass, length,)

    def convert(self, x, new_dims):
        from unyt import physical_constants as pc
        return pc.hcgs/(x*pc.clight)

    def __str__(self):
        return "compton: mass <-> length"

class EffectiveTemperature(Equivalence):
    _type_name = "effective_temperature"
    dims = (flux, temperature,)

    def convert(self, x, new_dims):
        from unyt import physical_constants as pc
        if new_dims == flux:
            return pc.stefan_boltzmann_constant_cgs*x**4
        elif new_dims == temperature:
            return (x/pc.stefan_boltzmann_constant_cgs)**0.25

    def __str__(self):
        return "effective_temperature: flux <-> temperature"

em_conversions = {
    charge_mks: ("esu", 0.1*speed_of_light_cm_per_s),
    magnetic_field_mks: ("gauss", 1.0e4),
    current_mks: ("statA", 0.1*speed_of_light_cm_per_s),
    electric_potential_mks: ("statV", 1.0e-8*speed_of_light_cm_per_s),
    resistance_mks: ("statohm", 1.0e9/(speed_of_light_cm_per_s**2)),
    charge_cgs: ("C", 10.0/speed_of_light_cm_per_s),
    magnetic_field_cgs: ("T", 1.0e-4),
    current_cgs: ("A", 10.0/speed_of_light_cm_per_s),
    electric_potential_cgs: ("V", 1.0e8/speed_of_light_cm_per_s),
    resistance_cgs: ("ohm", speed_of_light_cm_per_s**2*1.0e-9),
}

class ElectromagneticSI(Equivalence):
    _type_name = "SI"
    _alternate_names = ["si", "MKS", "mks"]
    _one_way = True
    dims = (current_cgs, charge_cgs, magnetic_field_cgs,
            electric_field_cgs, electric_potential_cgs,
            resistance_cgs)

    def convert(self, x, new_dims):
        old_dims = x.units.dimensions
        new_units, convert_factor = em_conversions[old_dims]
        return x.in_cgs().v*convert_factor, new_units

    def __str__(self):
        return "SI: EM CGS unit -> EM SI unit"

class ElectromagneticCGS(Equivalence):
    _type_name = "CGS"
    _alternate_names = ["cgs"]
    _one_way = True
    dims = (current_mks, charge_mks, magnetic_field_mks,
            electric_field_mks, electric_potential_mks,
            resistance_mks)

    def convert(self, x, new_dims):
        old_dims = x.units.dimensions
        new_units, convert_factor = em_conversions[old_dims]
        return x.in_mks().v*convert_factor, new_units

    def __str__(self):
        return "CGS: EM SI unit -> EM CGS unit"
<commit_msg>make RegisteredEquivalence metaclass private<commit_after>"""
Equivalencies between different kinds of units

"""

# -----------------------------------------------------------------------------
# Copyright (c) 2018, yt Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the LICENSE file, distributed with this software.
# -----------------------------------------------------------------------------

from collections import OrderedDict

from unyt.dimensions import (
    temperature,
    mass,
    energy,
    length,
    rate,
    velocity,
    dimensionless,
    density,
    number_density,
    flux,
    current_cgs,
    current_mks,
    charge_cgs,
    charge_mks,
    magnetic_field_cgs,
    magnetic_field_mks,
    electric_potential_cgs,
    electric_potential_mks,
    electric_field_cgs,
    electric_field_mks,
    resistance_cgs,
    resistance_mks
)

from unyt._physical_ratios import speed_of_light_cm_per_s
from six import add_metaclass
import numpy as np

equivalence_registry = OrderedDict()

class _RegisteredEquivalence(type):
    def __init__(cls, name, b, d):
        type.__init__(cls, name, b, d)
        if hasattr(cls, "_type_name") and not cls._skip_add:
            equivalence_registry[cls._type_name] = cls
        if hasattr(cls, "_alternate_names") and not cls._skip_add:
            for name in cls._alternate_names:
                equivalence_registry[name] = cls

@add_metaclass(_RegisteredEquivalence)
class Equivalence(object):
    _skip_add = False
    _one_way = False

class NumberDensityEquivalence(Equivalence):
    _type_name = "number_density"
    dims = (density, number_density,)

    def convert(self, x, new_dims, mu=0.6):
        from unyt import physical_constants as pc
        if new_dims == number_density:
            return x/(mu*pc.mh)
        elif new_dims == density:
            return x*mu*pc.mh

    def __str__(self):
        return "number density: density <-> number density"

class ThermalEquivalence(Equivalence):
    _type_name = "thermal"
    dims = (temperature, energy,)

    def convert(self, x, new_dims):
        from unyt import physical_constants as pc
        if new_dims == energy:
            return pc.kboltz*x
        elif new_dims == temperature:
            return x/pc.kboltz

    def __str__(self):
        return "thermal: temperature <-> energy"

class MassEnergyEquivalence(Equivalence):
    _type_name = "mass_energy"
    dims = (mass, energy,)

    def convert(self, x, new_dims):
        from unyt import physical_constants as pc
        if new_dims == energy:
            return x*pc.clight*pc.clight
        elif new_dims == mass:
            return x/(pc.clight*pc.clight)

    def __str__(self):
        return "mass_energy: mass <-> energy"

class SpectralEquivalence(Equivalence):
    _type_name = "spectral"
    dims = (length, rate, energy,)

    def convert(self, x, new_dims):
        from unyt import physical_constants as pc
        if new_dims == energy:
            if x.units.dimensions == length:
                nu = pc.clight/x
            elif x.units.dimensions == rate:
                nu = x
            return pc.hcgs*nu
        elif new_dims == length:
            if x.units.dimensions == rate:
                return pc.clight/x
            elif x.units.dimensions == energy:
                return pc.hcgs*pc.clight/x
        elif new_dims == rate:
            if x.units.dimensions == length:
                return pc.clight/x
            elif x.units.dimensions == energy:
                return x/pc.hcgs

    def __str__(self):
        return "spectral: length <-> frequency <-> energy"

class SoundSpeedEquivalence(Equivalence):
    _type_name = "sound_speed"
    dims = (velocity, temperature, energy,)

    def convert(self, x, new_dims, mu=0.6, gamma=5./3.):
        from unyt import physical_constants as pc
        if new_dims == velocity:
            if x.units.dimensions == temperature:
                kT = pc.kboltz*x
            elif x.units.dimensions == energy:
                kT = x
            return np.sqrt(gamma*kT/(mu*pc.mh))
        else:
            kT = x*x*mu*pc.mh/gamma
            if new_dims == temperature:
                return kT/pc.kboltz
            else:
                return kT

    def __str__(self):
        return "sound_speed (ideal gas): velocity <-> temperature <-> energy"

class LorentzEquivalence(Equivalence):
    _type_name = "lorentz"
    dims = (dimensionless, velocity,)

    def convert(self, x, new_dims):
        from unyt import physical_constants as pc
        if new_dims == dimensionless:
            beta = x.in_cgs()/pc.clight
            return 1./np.sqrt(1.-beta**2)
        elif new_dims == velocity:
            return pc.clight*np.sqrt(1.-1./(x*x))

    def __str__(self):
        return "lorentz: velocity <-> dimensionless"

class SchwarzschildEquivalence(Equivalence):
    _type_name = "schwarzschild"
    dims = (mass, length,)

    def convert(self, x, new_dims):
        from unyt import physical_constants as pc
        if new_dims == length:
            return 2.*pc.G*x/(pc.clight*pc.clight)
        elif new_dims == mass:
            return 0.5*x*pc.clight*pc.clight/pc.G

    def __str__(self):
        return "schwarzschild: mass <-> length"

class ComptonEquivalence(Equivalence):
    _type_name = "compton"
    dims = (mass, length,)

    def convert(self, x, new_dims):
        from unyt import physical_constants as pc
        return pc.hcgs/(x*pc.clight)

    def __str__(self):
        return "compton: mass <-> length"

class EffectiveTemperature(Equivalence):
    _type_name = "effective_temperature"
    dims = (flux, temperature,)

    def convert(self, x, new_dims):
        from unyt import physical_constants as pc
        if new_dims == flux:
            return pc.stefan_boltzmann_constant_cgs*x**4
        elif new_dims == temperature:
            return (x/pc.stefan_boltzmann_constant_cgs)**0.25

    def __str__(self):
        return "effective_temperature: flux <-> temperature"

em_conversions = {
    charge_mks: ("esu", 0.1*speed_of_light_cm_per_s),
    magnetic_field_mks: ("gauss", 1.0e4),
    current_mks: ("statA", 0.1*speed_of_light_cm_per_s),
    electric_potential_mks: ("statV", 1.0e-8*speed_of_light_cm_per_s),
    resistance_mks: ("statohm", 1.0e9/(speed_of_light_cm_per_s**2)),
    charge_cgs: ("C", 10.0/speed_of_light_cm_per_s),
    magnetic_field_cgs: ("T", 1.0e-4),
    current_cgs: ("A", 10.0/speed_of_light_cm_per_s),
    electric_potential_cgs: ("V", 1.0e8/speed_of_light_cm_per_s),
    resistance_cgs: ("ohm", speed_of_light_cm_per_s**2*1.0e-9),
}

class ElectromagneticSI(Equivalence):
    _type_name = "SI"
    _alternate_names = ["si", "MKS", "mks"]
    _one_way = True
    dims = (current_cgs, charge_cgs, magnetic_field_cgs,
            electric_field_cgs, electric_potential_cgs,
            resistance_cgs)

    def convert(self, x, new_dims):
        old_dims = x.units.dimensions
        new_units, convert_factor = em_conversions[old_dims]
        return x.in_cgs().v*convert_factor, new_units

    def __str__(self):
        return "SI: EM CGS unit -> EM SI unit"

class ElectromagneticCGS(Equivalence):
    _type_name = "CGS"
    _alternate_names = ["cgs"]
    _one_way = True
    dims = (current_mks, charge_mks, magnetic_field_mks,
            electric_field_mks, electric_potential_mks,
            resistance_mks)

    def convert(self, x, new_dims):
        old_dims = x.units.dimensions
        new_units, convert_factor = em_conversions[old_dims]
        return x.in_mks().v*convert_factor, new_units

    def __str__(self):
        return "CGS: EM SI unit -> EM CGS unit"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import math

# square root of 2 for diagonal distance
SQRT2 = math.sqrt(2)

def backtrace(node):
    """
    Backtrace according to the parent records and return the path.
    (including both start and end nodes)
    """
    path = [(node.x, node.y)]
    while node.parent:
        node = node.parent
        path.append((node.x, node.y))
    path.reverse()
    return path

def bi_backtrace(node_a, node_b):
    """
    Backtrace from start and end node, returns the path for bi-directional A*
    (including both start and end nodes)
    """
    path_a = backtrace(node_a)
    path_b = backtrace(node_b)
    path_b.reverse()
    return path_a + path_b

def raytrace(coords_a, coords_b):
    line = []
    x0, y0 = coords_a
    x1, y1 = coords_b

    dx = x1 - x0
    dy = y1 - y0

    t = 0
    grid_pos = [x0, y0]
    t_for_one = \
        abs(1.0 / dx) if dx > 0 else 10000, \
        abs(1.0 / dy) if dy > 0 else 10000

    frac_start_pos = (x0 + .5) - x0, (y0 + .5) - y0
    t_for_next_border = [
      (1 - frac_start_pos[0] if dx < 0 else frac_start_pos[0]) * t_for_one[0],
      (1 - frac_start_pos[1] if dx < 0 else frac_start_pos[1]) * t_for_one[1]
    ]

    step = \
        1 if dx >= 0 else -1, \
        1 if dy >= 0 else -1

    i = 0
    while t <= 1:
        i+=1
        print (grid_pos)
        line.append(grid_pos.copy())
        index = 0 if t_for_next_border[0] <= t_for_next_border[1] else 1
        t = t_for_next_border[index]
        t_for_next_border[index] += t_for_one[index]
        grid_pos[index] += step[index]
    return line

def bresenham(coords_a, coords_b):
    '''
    Given the start and end coordinates, return all the coordinates lying
    on the line formed by these coordinates, based on Bresenham's algorithm.
    http://en.wikipedia.org/wiki/Bresenham's_line_algorithm#Simplification
    '''
    line = []
    x0, y0 = coords_a
    x1, y1 = coords_b
    dx = abs(x1 - x0)
    dy = abs(y1 - y0)
    sx = 1 if x0 < x1 else -1
    sy = 1 if y0 < y1 else -1
    err = dx - dy

    while True:
        line += [[x0, y0]]
        if x0 == x1 and y0 == y1:
            break
        e2 = err * 2
        if e2 > -dy:
            err = err - dy
            x0 = x0 + sx
        if e2 < dx:
            err = err + dx
            y0 = y0 + sy

    return line

def expand_path(path):
    '''
    Given a compressed path, return a new path that has all the segments
    in it interpolated.
    '''
    expanded = []
    if len(path) < 2:
        return expanded
    for i in range(len(path)-1):
        expanded += bresenham(path[i], path[i + 1])
    expanded += [path[:-1]]
    return expanded
<commit_msg>remove debug code<commit_after># -*- coding: utf-8 -*-
import math

# square root of 2 for diagonal distance
SQRT2 = math.sqrt(2)

def backtrace(node):
    """
    Backtrace according to the parent records and return the path.
    (including both start and end nodes)
    """
    path = [(node.x, node.y)]
    while node.parent:
        node = node.parent
        path.append((node.x, node.y))
    path.reverse()
    return path

def bi_backtrace(node_a, node_b):
    """
    Backtrace from start and end node, returns the path for bi-directional A*
    (including both start and end nodes)
    """
    path_a = backtrace(node_a)
    path_b = backtrace(node_b)
    path_b.reverse()
    return path_a + path_b

def raytrace(coords_a, coords_b):
    line = []
    x0, y0 = coords_a
    x1, y1 = coords_b

    dx = x1 - x0
    dy = y1 - y0

    t = 0
    grid_pos = [x0, y0]
    t_for_one = \
        abs(1.0 / dx) if dx > 0 else 10000, \
        abs(1.0 / dy) if dy > 0 else 10000

    frac_start_pos = (x0 + .5) - x0, (y0 + .5) - y0
    t_for_next_border = [
      (1 - frac_start_pos[0] if dx < 0 else frac_start_pos[0]) * t_for_one[0],
      (1 - frac_start_pos[1] if dx < 0 else frac_start_pos[1]) * t_for_one[1]
    ]

    step = \
        1 if dx >= 0 else -1, \
        1 if dy >= 0 else -1

    while t <= 1:
        line.append(grid_pos.copy())
        index = 0 if t_for_next_border[0] <= t_for_next_border[1] else 1
        t = t_for_next_border[index]
        t_for_next_border[index] += t_for_one[index]
        grid_pos[index] += step[index]
    return line

def bresenham(coords_a, coords_b):
    '''
    Given the start and end coordinates, return all the coordinates lying
    on the line formed by these coordinates, based on Bresenham's algorithm.
    http://en.wikipedia.org/wiki/Bresenham's_line_algorithm#Simplification
    '''
    line = []
    x0, y0 = coords_a
    x1, y1 = coords_b
    dx = abs(x1 - x0)
    dy = abs(y1 - y0)
    sx = 1 if x0 < x1 else -1
    sy = 1 if y0 < y1 else -1
    err = dx - dy

    while True:
        line += [[x0, y0]]
        if x0 == x1 and y0 == y1:
            break
        e2 = err * 2
        if e2 > -dy:
            err = err - dy
            x0 = x0 + sx
        if e2 < dx:
            err = err + dx
            y0 = y0 + sy

    return line

def expand_path(path):
    '''
    Given a compressed path, return a new path that has all the segments
    in it interpolated.
    '''
    expanded = []
    if len(path) < 2:
        return expanded
    for i in range(len(path)-1):
        expanded += bresenham(path[i], path[i + 1])
    expanded += [path[:-1]]
    return expanded
<|endoftext|>


<commit_before>#!/usr/bin/python

from Crypto.Cipher import AES
import scrypt
import hashlib
from binascii import hexlify, unhexlify
import unittest
import sys
from .account import PrivateKey, PublicKey, Address
from .base58 import ripemd160,Base58,base58encode,base58decode

"""
This class and the methods require python3
"""
assert sys.version_info[0] == 3, "graphenelib requires python3"

def _encrypt_xor(a, b, aes):
    'Returns encrypt(a ^ b).'
    return aes.encrypt(unhexlify('%0.32x' % (int((a), 16) ^ int(hexlify(b), 16))))

def encrypt(privkey,passphrase):
    '''BIP0038 non-ec-multiply encryption. Returns BIP0038 encrypted privkey.'''
    privkeyraw    = bytes(privkey) # bytes
    privkeyhex    = repr(privkey)  # hex
    addr          = format(privkey.uncompressed.address,"BTC")
    salt          = hashlib.sha256(hashlib.sha256(bytes(addr,'ascii')).digest()).digest()[0:4]
    key           = scrypt.hash(passphrase, salt, 16384, 8, 8)
    (derived_half1, derived_half2) = (key[:32], key[32:])
    aes             = AES.new(derived_half2)
    encrypted_half1 = _encrypt_xor(privkeyhex[:32], derived_half1[:16], aes)
    encrypted_half2 = _encrypt_xor(privkeyhex[32:], derived_half1[16:], aes)
    # flag byte is forced 0xc0 because BTS only uses compressed keys
    payload    = b'\x01' + b'\x42' + b'\xc0' + salt + encrypted_half1 + encrypted_half2
    checksum   = hashlib.sha256(hashlib.sha256(payload).digest()).digest()[:4] # b58check for encrypted privkey
    privatkey  = hexlify(payload + checksum).decode('ascii')
    return Base58(privatkey)

def decrypt(encrypted_privkey,passphrase):
    '''BIP0038 non-ec-multiply decryption. Returns WIF privkey.'''
    d        = unhexlify(base58decode(encrypted_privkey))
    d        = d[2:]  # remove trailing 0x01 and 0x42
    flagbyte = d[0:1] # get flag byte
    d        = d[1:]  # get payload
    assert flagbyte == b'\xc0', "Flagbyte has to be 0xc0"
    salt     = d[0:4]
    d        = d[4:-4]
    key      = scrypt.hash(passphrase,salt, 16384, 8, 8)
    derivedhalf1   = key[0:32]
    derivedhalf2   = key[32:64]
    encryptedhalf1 = d[0:16]
    encryptedhalf2 = d[16:32]
    aes            = AES.new(derivedhalf2)
    decryptedhalf2 = aes.decrypt(encryptedhalf2)
    decryptedhalf1 = aes.decrypt(encryptedhalf1)
    privraw        = decryptedhalf1 + decryptedhalf2
    privraw        = ('%064x' % (int(hexlify(privraw), 16) ^ int(hexlify(derivedhalf1), 16)))
    wif            = Base58(privraw)
    privkey        = PrivateKey(format(wif,"wif"))
    addr           = format(privkey.uncompressed.address,"BTC")
    saltverify     = hashlib.sha256(hashlib.sha256(bytes(addr,'ascii')).digest()).digest()[0:4]
    if saltverify != salt :
        raise Exception('salt verification failed! Password is likely incorrect.')
    return wif

class Testcases(unittest.TestCase) :
    def test_encrypt(self):
        self.assertEqual([
                    format(encrypt(PrivateKey("5HqUkGuo62BfcJU5vNhTXKJRXuUi9QSE6jp8C3uBJ2BVHtB8WSd"),"TestingOneTwoThree"), "encwif"),
                    format(encrypt(PrivateKey("5KN7MzqK5wt2TP1fQCYyHBtDrXdJuXbUzm4A9rKAteGu3Qi5CVR"),"TestingOneTwoThree"), "encwif"),
                    format(encrypt(PrivateKey("5HtasZ6ofTHP6HCwTqTkLDuLQisYPah7aUnSKfC7h4hMUVw2gi5"),"Satoshi"), "encwif"),
                ],[
                    "6PRN5mjUTtud6fUXbJXezfn6oABoSr6GSLjMbrGXRZxSUcxThxsUW8epQi",
                    "6PRVWUbkzzsbcVac2qwfssoUJAN1Xhrg6bNk8J7Nzm5H7kxEbn2Nh2ZoGg",
                    "6PRNFFkZc2NZ6dJqFfhRoFNMR9Lnyj7dYGrzdgXXVMXcxoKTePPX1dWByq",
                ])

    def test_deencrypt(self):
        self.assertEqual([
                    format(decrypt("6PRN5mjUTtud6fUXbJXezfn6oABoSr6GSLjMbrGXRZxSUcxThxsUW8epQi","TestingOneTwoThree"),"wif"),
                    format(decrypt("6PRVWUbkzzsbcVac2qwfssoUJAN1Xhrg6bNk8J7Nzm5H7kxEbn2Nh2ZoGg","TestingOneTwoThree"),"wif"),
                    format(decrypt("6PRNFFkZc2NZ6dJqFfhRoFNMR9Lnyj7dYGrzdgXXVMXcxoKTePPX1dWByq","Satoshi"),"wif")
                ],[
                    "5HqUkGuo62BfcJU5vNhTXKJRXuUi9QSE6jp8C3uBJ2BVHtB8WSd",
                    "5KN7MzqK5wt2TP1fQCYyHBtDrXdJuXbUzm4A9rKAteGu3Qi5CVR",
                    "5HtasZ6ofTHP6HCwTqTkLDuLQisYPah7aUnSKfC7h4hMUVw2gi5",
                ])

if __name__ == '__main__':
    unittest.main()
<commit_msg>[Bip38] import cleanup<commit_after>#!/usr/bin/python

from Crypto.Cipher import AES
import scrypt
import hashlib
from binascii import hexlify, unhexlify
import unittest
import sys
from graphenebase.account import PrivateKey, PublicKey, Address
from graphenebase.base58 import ripemd160,Base58,base58encode,base58decode

"""
This class and the methods require python3
"""
assert sys.version_info[0] == 3, "graphenelib requires python3"

def _encrypt_xor(a, b, aes):
    'Returns encrypt(a ^ b).'
    return aes.encrypt(unhexlify('%0.32x' % (int((a), 16) ^ int(hexlify(b), 16))))

def encrypt(privkey,passphrase):
    '''BIP0038 non-ec-multiply encryption. Returns BIP0038 encrypted privkey.'''
    privkeyraw    = bytes(privkey) # bytes
    privkeyhex    = repr(privkey)  # hex
    addr          = format(privkey.uncompressed.address,"BTC")
    salt          = hashlib.sha256(hashlib.sha256(bytes(addr,'ascii')).digest()).digest()[0:4]
    key           = scrypt.hash(passphrase, salt, 16384, 8, 8)
    (derived_half1, derived_half2) = (key[:32], key[32:])
    aes             = AES.new(derived_half2)
    encrypted_half1 = _encrypt_xor(privkeyhex[:32], derived_half1[:16], aes)
    encrypted_half2 = _encrypt_xor(privkeyhex[32:], derived_half1[16:], aes)
    # flag byte is forced 0xc0 because BTS only uses compressed keys
    payload    = b'\x01' + b'\x42' + b'\xc0' + salt + encrypted_half1 + encrypted_half2
    checksum   = hashlib.sha256(hashlib.sha256(payload).digest()).digest()[:4] # b58check for encrypted privkey
    privatkey  = hexlify(payload + checksum).decode('ascii')
    return Base58(privatkey)

def decrypt(encrypted_privkey,passphrase):
    '''BIP0038 non-ec-multiply decryption. Returns WIF privkey.'''
    d        = unhexlify(base58decode(encrypted_privkey))
    d        = d[2:]  # remove trailing 0x01 and 0x42
    flagbyte = d[0:1] # get flag byte
    d        = d[1:]  # get payload
    assert flagbyte == b'\xc0', "Flagbyte has to be 0xc0"
    salt     = d[0:4]
    d        = d[4:-4]
    key      = scrypt.hash(passphrase,salt, 16384, 8, 8)
    derivedhalf1   = key[0:32]
    derivedhalf2   = key[32:64]
    encryptedhalf1 = d[0:16]
    encryptedhalf2 = d[16:32]
    aes            = AES.new(derivedhalf2)
    decryptedhalf2 = aes.decrypt(encryptedhalf2)
    decryptedhalf1 = aes.decrypt(encryptedhalf1)
    privraw        = decryptedhalf1 + decryptedhalf2
    privraw        = ('%064x' % (int(hexlify(privraw), 16) ^ int(hexlify(derivedhalf1), 16)))
    wif            = Base58(privraw)
    privkey        = PrivateKey(format(wif,"wif"))
    addr           = format(privkey.uncompressed.address,"BTC")
    saltverify     = hashlib.sha256(hashlib.sha256(bytes(addr,'ascii')).digest()).digest()[0:4]
    if saltverify != salt :
        raise Exception('salt verification failed! Password is likely incorrect.')
    return wif

class Testcases(unittest.TestCase) :
    def test_encrypt(self):
        self.assertEqual([
                    format(encrypt(PrivateKey("5HqUkGuo62BfcJU5vNhTXKJRXuUi9QSE6jp8C3uBJ2BVHtB8WSd"),"TestingOneTwoThree"), "encwif"),
                    format(encrypt(PrivateKey("5KN7MzqK5wt2TP1fQCYyHBtDrXdJuXbUzm4A9rKAteGu3Qi5CVR"),"TestingOneTwoThree"), "encwif"),
                    format(encrypt(PrivateKey("5HtasZ6ofTHP6HCwTqTkLDuLQisYPah7aUnSKfC7h4hMUVw2gi5"),"Satoshi"), "encwif"),
                ],[
                    "6PRN5mjUTtud6fUXbJXezfn6oABoSr6GSLjMbrGXRZxSUcxThxsUW8epQi",
                    "6PRVWUbkzzsbcVac2qwfssoUJAN1Xhrg6bNk8J7Nzm5H7kxEbn2Nh2ZoGg",
                    "6PRNFFkZc2NZ6dJqFfhRoFNMR9Lnyj7dYGrzdgXXVMXcxoKTePPX1dWByq",
                ])

    def test_deencrypt(self):
        self.assertEqual([
                    format(decrypt("6PRN5mjUTtud6fUXbJXezfn6oABoSr6GSLjMbrGXRZxSUcxThxsUW8epQi","TestingOneTwoThree"),"wif"),
                    format(decrypt("6PRVWUbkzzsbcVac2qwfssoUJAN1Xhrg6bNk8J7Nzm5H7kxEbn2Nh2ZoGg","TestingOneTwoThree"),"wif"),
                    format(decrypt("6PRNFFkZc2NZ6dJqFfhRoFNMR9Lnyj7dYGrzdgXXVMXcxoKTePPX1dWByq","Satoshi"),"wif")
                ],[
                    "5HqUkGuo62BfcJU5vNhTXKJRXuUi9QSE6jp8C3uBJ2BVHtB8WSd",
                    "5KN7MzqK5wt2TP1fQCYyHBtDrXdJuXbUzm4A9rKAteGu3Qi5CVR",
                    "5HtasZ6ofTHP6HCwTqTkLDuLQisYPah7aUnSKfC7h4hMUVw2gi5",
                ])

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>"""
This script combines training_OnlineContrastiveLoss.py with training_MultipleNegativesRankingLoss.py

Online constrative loss works well for classification (are question1 and question2 duplicates?), but it
performs less well for duplicate questions mining. MultipleNegativesRankingLoss works well for duplicate
questions mining, but it has some issues with classification as it does not push dissimilar pairs away.

This script combines both losses to get the best of both worlds.

Multi task learning is achieved quite easily by calling the model.fit method like this:
model.fit(train_objectives=[(train_dataloader_MultipleNegativesRankingLoss, train_loss_MultipleNegativesRankingLoss), (train_dataloader_constrative_loss, train_loss_constrative_loss)] ...)
"""

from torch.utils.data import DataLoader
from sentence_transformers import losses, util
from sentence_transformers import LoggingHandler, SentenceTransformer, evaluation
from sentence_transformers.readers import InputExample
import logging
from datetime import datetime
import csv
import os
from zipfile import ZipFile
import random

#### Just some code to print debug information to stdout
logging.basicConfig(format='%(asctime)s - %(message)s',
                    datefmt='%Y-%m-%d %H:%M:%S',
                    level=logging.INFO,
                    handlers=[LoggingHandler()])
logger = logging.getLogger(__name__)
#### /print debug information to stdout

#As base model, we use DistilBERT-base that was pre-trained on NLI and STSb data
model = SentenceTransformer('stsb-distilbert-base')

#Training for multiple epochs can be beneficial, as in each epoch a mini-batch is sampled differently
#hence, we get different negatives for each positive
num_epochs = 10

#Increasing the batch size improves the performance for MultipleNegativesRankingLoss. Choose it as large as possible
#I achieved the good results with a batch size of 300-350 (requires about 30 GB of GPU memory)
train_batch_size = 64

#As distance metric, we use cosine distance (cosine_distance = 1-cosine_similarity)
distance_metric = losses.SiameseDistanceMetric.COSINE_DISTANCE

#Negative pairs should have a distance of at least 0.5
margin = 0.5

dataset_path = 'quora-IR-dataset'
model_save_path = 'output/training_multi-task-learning'+datetime.now().strftime("%Y-%m-%d_%H-%M-%S")

os.makedirs(model_save_path, exist_ok=True)

# Check if the dataset exists. If not, download and extract
if not os.path.exists(dataset_path):
    logger.info("Dataset not found. Download")
    zip_save_path = 'quora-IR-dataset.zip'
    util.http_get(url='https://sbert.net/datasets/quora-IR-dataset.zip', path=zip_save_path)
    with ZipFile(zip_save_path, 'r') as zip:
        zip.extractall(dataset_path)

######### Read train data  ##########
train_samples_MultipleNegativesRankingLoss = []
train_samples_ConstrativeLoss = []

with open(os.path.join(dataset_path, "classification/train_pairs.tsv"), encoding='utf8') as fIn:
    reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
    for row in reader:
        train_samples_ConstrativeLoss.append(InputExample(texts=[row['question1'], row['question2']], label=int(row['is_duplicate'])))
        if row['is_duplicate'] == '1':
            train_samples_MultipleNegativesRankingLoss.append(InputExample(texts=[row['question1'], row['question2']], label=1))
            train_samples_MultipleNegativesRankingLoss.append(InputExample(texts=[row['question2'], row['question1']], label=1))  # if A is a duplicate of B, then B is a duplicate of A

# Create data loader and loss for MultipleNegativesRankingLoss
train_dataloader_MultipleNegativesRankingLoss = DataLoader(train_samples_MultipleNegativesRankingLoss, shuffle=True, batch_size=train_batch_size)
train_loss_MultipleNegativesRankingLoss = losses.MultipleNegativesRankingLoss(model)

# Create data loader and loss for OnlineContrastiveLoss
train_dataloader_ConstrativeLoss = DataLoader(train_samples_ConstrativeLoss, shuffle=True, batch_size=train_batch_size)
train_loss_ConstrativeLoss = losses.OnlineContrastiveLoss(model=model, distance_metric=distance_metric, margin=margin)

################### Development  Evaluators ##################
# We add 3 evaluators, that evaluate the model on Duplicate Questions pair classification,
# Duplicate Questions Mining, and Duplicate Questions Information Retrieval
evaluators = []

###### Classification ######
# Given (quesiton1, question2), is this a duplicate or not?
# The evaluator will compute the embeddings for both questions and then compute
# a cosine similarity. If the similarity is above a threshold, we have a duplicate.
dev_sentences1 = []
dev_sentences2 = []
dev_labels = []
with open(os.path.join(dataset_path, "classification/dev_pairs.tsv"), encoding='utf8') as fIn:
    reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
    for row in reader:
        dev_sentences1.append(row['question1'])
        dev_sentences2.append(row['question2'])
        dev_labels.append(int(row['is_duplicate']))

binary_acc_evaluator = evaluation.BinaryClassificationEvaluator(dev_sentences1, dev_sentences2, dev_labels)
evaluators.append(binary_acc_evaluator)

###### Duplicate Questions Mining ######
# Given a large corpus of questions, identify all duplicates in that corpus.

# For faster processing, we limit the development corpus to only 10,000 sentences.
max_dev_samples = 10000
dev_sentences = {}
dev_duplicates = []
with open(os.path.join(dataset_path, "duplicate-mining/dev_corpus.tsv"), encoding='utf8') as fIn:
    reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
    for row in reader:
        dev_sentences[row['qid']] = row['question']

        if len(dev_sentences) >= max_dev_samples:
            break

with open(os.path.join(dataset_path, "duplicate-mining/dev_duplicates.tsv"), encoding='utf8') as fIn:
    reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
    for row in reader:
        if row['qid1'] in dev_sentences and row['qid2'] in dev_sentences:
            dev_duplicates.append([row['qid1'], row['qid2']])

# The ParaphraseMiningEvaluator computes the cosine similarity between all sentences and
# extracts a list with the pairs that have the highest similarity. Given the duplicate
# information in dev_duplicates, it then computes and F1 score how well our duplicate mining worked
paraphrase_mining_evaluator = evaluation.ParaphraseMiningEvaluator(dev_sentences, dev_duplicates, name='dev')
evaluators.append(paraphrase_mining_evaluator)

###### Duplicate Questions Information Retrieval ######
# Given a question and a large corpus of thousands questions, find the most relevant (i.e. duplicate) question
# in that corpus.

# For faster processing, we limit the development corpus to only 10,000 sentences.
max_corpus_size = 100000

ir_queries = {}             #Our queries (qid => question)
ir_needed_qids = set()      #QIDs we need in the corpus
ir_corpus = {}              #Our corpus (qid => question)
ir_relevant_docs = {}       #Mapping of relevant documents for a given query (qid => set([relevant_question_ids])

with open(os.path.join(dataset_path, 'information-retrieval/dev-queries.tsv'), encoding='utf8') as fIn:
    next(fIn) #Skip header
    for line in fIn:
        qid, query, duplicate_ids = line.strip().split('\t')
        duplicate_ids = duplicate_ids.split(',')
        ir_queries[qid] = query
        ir_relevant_docs[qid] = set(duplicate_ids)

        for qid in duplicate_ids:
            ir_needed_qids.add(qid)

# First get all needed relevant documents (i.e., we must ensure, that the relevant questions are actually in the corpus
distraction_questions = {}
with open(os.path.join(dataset_path, 'information-retrieval/corpus.tsv'), encoding='utf8') as fIn:
    next(fIn) #Skip header
    for line in fIn:
        qid, question = line.strip().split('\t')

        if qid in ir_needed_qids:
            ir_corpus[qid] = question
        else:
            distraction_questions[qid] = question

# Now, also add some irrelevant questions to fill our corpus
other_qid_list = list(distraction_questions.keys())
random.shuffle(other_qid_list)

for qid in other_qid_list[0:max(0, max_corpus_size-len(ir_corpus))]:
    ir_corpus[qid] = distraction_questions[qid]

#Given queries, a corpus and a mapping with relevant documents, the InformationRetrievalEvaluator computes different IR
# metrices. For our use case MRR@k and Accuracy@k are relevant.
ir_evaluator = evaluation.InformationRetrievalEvaluator(ir_queries, ir_corpus, ir_relevant_docs)

evaluators.append(ir_evaluator)

# Create a SequentialEvaluator. This SequentialEvaluator runs all three evaluators in a sequential order.
# We optimize the model with respect to the score from the last evaluator (scores[-1])
seq_evaluator = evaluation.SequentialEvaluator(evaluators, main_score_function=lambda scores: scores[-1])

logger.info("Evaluate model without training")
seq_evaluator(model, epoch=0, steps=0, output_path=model_save_path)

# Train the model
model.fit(train_objectives=[(train_dataloader_MultipleNegativesRankingLoss, train_loss_MultipleNegativesRankingLoss), (train_dataloader_ConstrativeLoss, train_loss_ConstrativeLoss)],
          evaluator=seq_evaluator,
          epochs=num_epochs,
          warmup_steps=1000,
          output_path=model_save_path,
          output_path_ignore_not_empty=True
          )
<commit_msg>remove unsupported arg of fit from example<commit_after>"""
This script combines training_OnlineContrastiveLoss.py with training_MultipleNegativesRankingLoss.py

Online constrative loss works well for classification (are question1 and question2 duplicates?), but it
performs less well for duplicate questions mining. MultipleNegativesRankingLoss works well for duplicate
questions mining, but it has some issues with classification as it does not push dissimilar pairs away.

This script combines both losses to get the best of both worlds.

Multi task learning is achieved quite easily by calling the model.fit method like this:
model.fit(train_objectives=[(train_dataloader_MultipleNegativesRankingLoss, train_loss_MultipleNegativesRankingLoss), (train_dataloader_constrative_loss, train_loss_constrative_loss)] ...)
"""

from torch.utils.data import DataLoader
from sentence_transformers import losses, util
from sentence_transformers import LoggingHandler, SentenceTransformer, evaluation
from sentence_transformers.readers import InputExample
import logging
from datetime import datetime
import csv
import os
from zipfile import ZipFile
import random

#### Just some code to print debug information to stdout
logging.basicConfig(format='%(asctime)s - %(message)s',
                    datefmt='%Y-%m-%d %H:%M:%S',
                    level=logging.INFO,
                    handlers=[LoggingHandler()])
logger = logging.getLogger(__name__)
#### /print debug information to stdout

#As base model, we use DistilBERT-base that was pre-trained on NLI and STSb data
model = SentenceTransformer('stsb-distilbert-base')

#Training for multiple epochs can be beneficial, as in each epoch a mini-batch is sampled differently
#hence, we get different negatives for each positive
num_epochs = 10

#Increasing the batch size improves the performance for MultipleNegativesRankingLoss. Choose it as large as possible
#I achieved the good results with a batch size of 300-350 (requires about 30 GB of GPU memory)
train_batch_size = 64

#As distance metric, we use cosine distance (cosine_distance = 1-cosine_similarity)
distance_metric = losses.SiameseDistanceMetric.COSINE_DISTANCE

#Negative pairs should have a distance of at least 0.5
margin = 0.5

dataset_path = 'quora-IR-dataset'
model_save_path = 'output/training_multi-task-learning'+datetime.now().strftime("%Y-%m-%d_%H-%M-%S")

os.makedirs(model_save_path, exist_ok=True)

# Check if the dataset exists. If not, download and extract
if not os.path.exists(dataset_path):
    logger.info("Dataset not found. Download")
    zip_save_path = 'quora-IR-dataset.zip'
    util.http_get(url='https://sbert.net/datasets/quora-IR-dataset.zip', path=zip_save_path)
    with ZipFile(zip_save_path, 'r') as zip:
        zip.extractall(dataset_path)

######### Read train data  ##########
train_samples_MultipleNegativesRankingLoss = []
train_samples_ConstrativeLoss = []

with open(os.path.join(dataset_path, "classification/train_pairs.tsv"), encoding='utf8') as fIn:
    reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
    for row in reader:
        train_samples_ConstrativeLoss.append(InputExample(texts=[row['question1'], row['question2']], label=int(row['is_duplicate'])))
        if row['is_duplicate'] == '1':
            train_samples_MultipleNegativesRankingLoss.append(InputExample(texts=[row['question1'], row['question2']], label=1))
            train_samples_MultipleNegativesRankingLoss.append(InputExample(texts=[row['question2'], row['question1']], label=1))  # if A is a duplicate of B, then B is a duplicate of A

# Create data loader and loss for MultipleNegativesRankingLoss
train_dataloader_MultipleNegativesRankingLoss = DataLoader(train_samples_MultipleNegativesRankingLoss, shuffle=True, batch_size=train_batch_size)
train_loss_MultipleNegativesRankingLoss = losses.MultipleNegativesRankingLoss(model)

# Create data loader and loss for OnlineContrastiveLoss
train_dataloader_ConstrativeLoss = DataLoader(train_samples_ConstrativeLoss, shuffle=True, batch_size=train_batch_size)
train_loss_ConstrativeLoss = losses.OnlineContrastiveLoss(model=model, distance_metric=distance_metric, margin=margin)

################### Development  Evaluators ##################
# We add 3 evaluators, that evaluate the model on Duplicate Questions pair classification,
# Duplicate Questions Mining, and Duplicate Questions Information Retrieval
evaluators = []

###### Classification ######
# Given (quesiton1, question2), is this a duplicate or not?
# The evaluator will compute the embeddings for both questions and then compute
# a cosine similarity. If the similarity is above a threshold, we have a duplicate.
dev_sentences1 = []
dev_sentences2 = []
dev_labels = []
with open(os.path.join(dataset_path, "classification/dev_pairs.tsv"), encoding='utf8') as fIn:
    reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
    for row in reader:
        dev_sentences1.append(row['question1'])
        dev_sentences2.append(row['question2'])
        dev_labels.append(int(row['is_duplicate']))

binary_acc_evaluator = evaluation.BinaryClassificationEvaluator(dev_sentences1, dev_sentences2, dev_labels)
evaluators.append(binary_acc_evaluator)

###### Duplicate Questions Mining ######
# Given a large corpus of questions, identify all duplicates in that corpus.

# For faster processing, we limit the development corpus to only 10,000 sentences.
max_dev_samples = 10000
dev_sentences = {}
dev_duplicates = []
with open(os.path.join(dataset_path, "duplicate-mining/dev_corpus.tsv"), encoding='utf8') as fIn:
    reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
    for row in reader:
        dev_sentences[row['qid']] = row['question']

        if len(dev_sentences) >= max_dev_samples:
            break

with open(os.path.join(dataset_path, "duplicate-mining/dev_duplicates.tsv"), encoding='utf8') as fIn:
    reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
    for row in reader:
        if row['qid1'] in dev_sentences and row['qid2'] in dev_sentences:
            dev_duplicates.append([row['qid1'], row['qid2']])

# The ParaphraseMiningEvaluator computes the cosine similarity between all sentences and
# extracts a list with the pairs that have the highest similarity. Given the duplicate
# information in dev_duplicates, it then computes and F1 score how well our duplicate mining worked
paraphrase_mining_evaluator = evaluation.ParaphraseMiningEvaluator(dev_sentences, dev_duplicates, name='dev')
evaluators.append(paraphrase_mining_evaluator)

###### Duplicate Questions Information Retrieval ######
# Given a question and a large corpus of thousands questions, find the most relevant (i.e. duplicate) question
# in that corpus.

# For faster processing, we limit the development corpus to only 10,000 sentences.
max_corpus_size = 100000

ir_queries = {}             #Our queries (qid => question)
ir_needed_qids = set()      #QIDs we need in the corpus
ir_corpus = {}              #Our corpus (qid => question)
ir_relevant_docs = {}       #Mapping of relevant documents for a given query (qid => set([relevant_question_ids])

with open(os.path.join(dataset_path, 'information-retrieval/dev-queries.tsv'), encoding='utf8') as fIn:
    next(fIn) #Skip header
    for line in fIn:
        qid, query, duplicate_ids = line.strip().split('\t')
        duplicate_ids = duplicate_ids.split(',')
        ir_queries[qid] = query
        ir_relevant_docs[qid] = set(duplicate_ids)

        for qid in duplicate_ids:
            ir_needed_qids.add(qid)

# First get all needed relevant documents (i.e., we must ensure, that the relevant questions are actually in the corpus
distraction_questions = {}
with open(os.path.join(dataset_path, 'information-retrieval/corpus.tsv'), encoding='utf8') as fIn:
    next(fIn) #Skip header
    for line in fIn:
        qid, question = line.strip().split('\t')

        if qid in ir_needed_qids:
            ir_corpus[qid] = question
        else:
            distraction_questions[qid] = question

# Now, also add some irrelevant questions to fill our corpus
other_qid_list = list(distraction_questions.keys())
random.shuffle(other_qid_list)

for qid in other_qid_list[0:max(0, max_corpus_size-len(ir_corpus))]:
    ir_corpus[qid] = distraction_questions[qid]

#Given queries, a corpus and a mapping with relevant documents, the InformationRetrievalEvaluator computes different IR
# metrices. For our use case MRR@k and Accuracy@k are relevant.
ir_evaluator = evaluation.InformationRetrievalEvaluator(ir_queries, ir_corpus, ir_relevant_docs)

evaluators.append(ir_evaluator)

# Create a SequentialEvaluator. This SequentialEvaluator runs all three evaluators in a sequential order.
# We optimize the model with respect to the score from the last evaluator (scores[-1])
seq_evaluator = evaluation.SequentialEvaluator(evaluators, main_score_function=lambda scores: scores[-1])

logger.info("Evaluate model without training")
seq_evaluator(model, epoch=0, steps=0, output_path=model_save_path)

# Train the model
model.fit(train_objectives=[(train_dataloader_MultipleNegativesRankingLoss, train_loss_MultipleNegativesRankingLoss), (train_dataloader_ConstrativeLoss, train_loss_ConstrativeLoss)],
          evaluator=seq_evaluator,
          epochs=num_epochs,
          warmup_steps=1000,
          output_path=model_save_path
          )
<|endoftext|>


<commit_before>from osgeo import ogr
try:
    import simplejson as json
except ImportError:
    import json

from greenwich.base import Comparable
from greenwich.srs import SpatialReference

class Envelope(Comparable):
    """Rectangular bounding extent.

    This class closely resembles OGREnvelope which is not included in the SWIG
    bindings.
    """

    def __init__(self, *args):
        """Creates an envelope from lower-left and upper-right coordinates.

        Arguments:
        args -- min_x, min_y, max_x, max_y or a four-tuple
        """
        if len(args) == 1:
            args = args[0]
        try:
            extent = map(float, args)
        except (TypeError, ValueError) as exc:
            exc.args = ('Cannot create Envelope from "%s"' % args,)
            raise
        try:
            self.min_x, self.max_x = sorted(extent[::2])
            self.min_y, self.max_y = sorted(extent[1::2])
        except ValueError as exc:
            exc.args = ('Sequence length should be "4", not "%d"' % len(args),)
            raise

    def __add__(self, envp):
        combined = Envelope(tuple(self))
        combined.expand(envp)
        return combined

    def __contains__(self, envp):
        return self.contains(envp)

    def __getitem__(self, index):
        return self.tuple[index]

    def __iter__(self):
        for val in self.tuple:
            yield val

    def __len__(self):
        return len(self.__dict__)

    def __repr__(self):
        return '<%s: %r>' % (self.__class__.__name__, self.tuple)

    def __sub__(self, envp):
        return self.intersect(envp)

    def contains(self, envp):
        """Returns true if this envelope contains another.

        Arguments:
        envp -- Envelope or tuple of (minX, minY, maxX, maxY)
        """
        try:
            return self.ll <= envp.ll and self.ur >= envp.ur
        except AttributeError:
            # Perhaps we have a tuple, try again with an Envelope.
            return self.contains(Envelope(envp))

    def clip(self, envp):
        mid = len(self) / 2
        self.ll = map(max, self.ll, envp[:mid])
        self.ur = map(min, self.ur, envp[mid:])

    def expand(self, envp):
        """Expands this envelope by the given Envelope or tuple.

        Arguments:
        envp -- Envelope, two-tuple, or four-tuple
        """
        if len(envp) == 2:
            envp += envp
        mid = len(self) / 2
        self.ll = map(min, self.ll, envp[:mid])
        self.ur = map(max, self.ur, envp[mid:])

    @staticmethod
    def from_geom(geom):
        """Returns an Envelope from an OGR Geometry."""
        extent = geom.GetEnvelope()
        return Envelope(map(extent.__getitem__, (0, 2, 1, 3)))

    @property
    def height(self):
        return self.max_y - self.min_y

    def intersect(self, envp):
        """Returns the intersection of this and another Envelope."""
        intersection = Envelope(tuple(self))
        intersection.clip(envp)
        return intersection

    def intersects(self, envp):
        """Returns true if this envelope intersects another.

        Arguments:
        envp -- Envelope or tuple of (minX, minY, maxX, maxY)
        """
        try:
            return self.ll <= envp.ur and self.ur >= envp.ll
        except AttributeError:
            return self.intersects(Envelope(envp))

    @property
    def ll(self):
        """Returns the lower left coordinate."""
        return self.min_x, self.min_y

    @ll.setter
    def ll(self, coord):
        """Set lower-left from (x, y) tuple."""
        self.min_x, self.min_y = coord

    @property
    def lr(self):
        """Returns the lower right coordinate."""
        return self.max_x, self.min_y

    @lr.setter
    def lr(self, coord):
        self.max_x, self.min_y = coord

    def scale(self, factor_x, factor_y=None):
        """Returns a new envelope rescaled by the given factor(s)."""
        factor_y = factor_x if factor_y is None else factor_y
        w = self.width * factor_x / 2.0
        h = self.height * factor_y / 2.0
        return Envelope(self.min_x + w, self.min_y + h,
                        self.max_x - w, self.max_y - h)

    @property
    def polygon(self):
        """Returns an OGR Geometry for this envelope."""
        ring = ogr.Geometry(ogr.wkbLinearRing)
        for coord in self.ll, self.lr, self.ur, self.ul, self.ll:
            ring.AddPoint_2D(*coord)
        polyg = ogr.Geometry(ogr.wkbPolygon)
        polyg.AddGeometryDirectly(ring)
        return polyg

    @property
    def tuple(self):
        """Returns the maximum extent as a tuple."""
        return self.ll + self.ur

    @property
    def ul(self):
        """Returns the upper left coordinate."""
        return self.min_x, self.max_y

    @ul.setter
    def ul(self, coord):
        self.min_x, self.max_y = coord

    @property
    def ur(self):
        """Returns the upper right coordinate."""
        return self.max_x, self.max_y

    @ur.setter
    def ur(self, coord):
        """Returns the upper right coordinate."""
        self.max_x, self.max_y = coord

    @property
    def width(self):
        return self.max_x - self.min_x

@property
def __geo_interface__(self):
    return json.loads(self.ExportToJson())

# Monkey-patch ogr.Geometry to provide geo-interface support.
ogr.Geometry.__geo_interface__ = __geo_interface__

def Geometry(*args, **kwargs):
    """Returns an ogr.Geometry instance optionally created from a geojson str
    or dict. The spatial reference may also be provided.
    """
    # Look for geojson as a positional or keyword arg.
    arg = kwargs.pop('geojson', None) or len(args) and args[0]
    try:
        srs = kwargs.pop('srs', None) or arg.srs.wkt
    except AttributeError:
        srs = None
    if hasattr(arg, 'keys'):
        geom = ogr.CreateGeometryFromJson(json.dumps(arg))
    elif hasattr(arg, 'startswith') and arg.startswith('{'):
        geom = ogr.CreateGeometryFromJson(arg)
    elif hasattr(arg, 'wkb'):
        geom = ogr.CreateGeometryFromWkb(bytes(arg.wkb))
    else:
        geom = ogr.Geometry(*args, **kwargs)
    if srs and geom:
        if not isinstance(srs, SpatialReference):
            srs = SpatialReference(srs)
        geom.AssignSpatialReference(srs)
    return geom
<commit_msg>Handle wkb hex strings, default to epsg 4326<commit_after>from osgeo import ogr
try:
    import simplejson as json
except ImportError:
    import json

from greenwich.base import Comparable
from greenwich.srs import SpatialReference

class Envelope(Comparable):
    """Rectangular bounding extent.

    This class closely resembles OGREnvelope which is not included in the SWIG
    bindings.
    """

    def __init__(self, *args):
        """Creates an envelope from lower-left and upper-right coordinates.

        Arguments:
        args -- min_x, min_y, max_x, max_y or a four-tuple
        """
        if len(args) == 1:
            args = args[0]
        try:
            extent = map(float, args)
        except (TypeError, ValueError) as exc:
            exc.args = ('Cannot create Envelope from "%s"' % args,)
            raise
        try:
            self.min_x, self.max_x = sorted(extent[::2])
            self.min_y, self.max_y = sorted(extent[1::2])
        except ValueError as exc:
            exc.args = ('Sequence length should be "4", not "%d"' % len(args),)
            raise

    def __add__(self, envp):
        combined = Envelope(tuple(self))
        combined.expand(envp)
        return combined

    def __contains__(self, envp):
        return self.contains(envp)

    def __getitem__(self, index):
        return self.tuple[index]

    def __iter__(self):
        for val in self.tuple:
            yield val

    def __len__(self):
        return len(self.__dict__)

    def __repr__(self):
        return '<%s: %r>' % (self.__class__.__name__, self.tuple)

    def __sub__(self, envp):
        return self.intersect(envp)

    def contains(self, envp):
        """Returns true if this envelope contains another.

        Arguments:
        envp -- Envelope or tuple of (minX, minY, maxX, maxY)
        """
        try:
            return self.ll <= envp.ll and self.ur >= envp.ur
        except AttributeError:
            # Perhaps we have a tuple, try again with an Envelope.
            return self.contains(Envelope(envp))

    def clip(self, envp):
        mid = len(self) / 2
        self.ll = map(max, self.ll, envp[:mid])
        self.ur = map(min, self.ur, envp[mid:])

    def expand(self, envp):
        """Expands this envelope by the given Envelope or tuple.

        Arguments:
        envp -- Envelope, two-tuple, or four-tuple
        """
        if len(envp) == 2:
            envp += envp
        mid = len(self) / 2
        self.ll = map(min, self.ll, envp[:mid])
        self.ur = map(max, self.ur, envp[mid:])

    @staticmethod
    def from_geom(geom):
        """Returns an Envelope from an OGR Geometry."""
        extent = geom.GetEnvelope()
        return Envelope(map(extent.__getitem__, (0, 2, 1, 3)))

    @property
    def height(self):
        return self.max_y - self.min_y

    def intersect(self, envp):
        """Returns the intersection of this and another Envelope."""
        intersection = Envelope(tuple(self))
        intersection.clip(envp)
        return intersection

    def intersects(self, envp):
        """Returns true if this envelope intersects another.

        Arguments:
        envp -- Envelope or tuple of (minX, minY, maxX, maxY)
        """
        try:
            return self.ll <= envp.ur and self.ur >= envp.ll
        except AttributeError:
            return self.intersects(Envelope(envp))

    @property
    def ll(self):
        """Returns the lower left coordinate."""
        return self.min_x, self.min_y

    @ll.setter
    def ll(self, coord):
        """Set lower-left from (x, y) tuple."""
        self.min_x, self.min_y = coord

    @property
    def lr(self):
        """Returns the lower right coordinate."""
        return self.max_x, self.min_y

    @lr.setter
    def lr(self, coord):
        self.max_x, self.min_y = coord

    def scale(self, factor_x, factor_y=None):
        """Returns a new envelope rescaled by the given factor(s)."""
        factor_y = factor_x if factor_y is None else factor_y
        w = self.width * factor_x / 2.0
        h = self.height * factor_y / 2.0
        return Envelope(self.min_x + w, self.min_y + h,
                        self.max_x - w, self.max_y - h)

    @property
    def polygon(self):
        """Returns an OGR Geometry for this envelope."""
        ring = ogr.Geometry(ogr.wkbLinearRing)
        for coord in self.ll, self.lr, self.ur, self.ul, self.ll:
            ring.AddPoint_2D(*coord)
        polyg = ogr.Geometry(ogr.wkbPolygon)
        polyg.AddGeometryDirectly(ring)
        return polyg

    @property
    def tuple(self):
        """Returns the maximum extent as a tuple."""
        return self.ll + self.ur

    @property
    def ul(self):
        """Returns the upper left coordinate."""
        return self.min_x, self.max_y

    @ul.setter
    def ul(self, coord):
        self.min_x, self.max_y = coord

    @property
    def ur(self):
        """Returns the upper right coordinate."""
        return self.max_x, self.max_y

    @ur.setter
    def ur(self, coord):
        """Returns the upper right coordinate."""
        self.max_x, self.max_y = coord

    @property
    def width(self):
        return self.max_x - self.min_x

@property
def __geo_interface__(self):
    return json.loads(self.ExportToJson())

# Monkey-patch ogr.Geometry to provide geo-interface support.
ogr.Geometry.__geo_interface__ = __geo_interface__

def Geometry(*args, **kwargs):
    """Returns an ogr.Geometry instance optionally created from a geojson str
    or dict. The spatial reference may also be provided.
    """
    # Look for geojson as a positional or keyword arg.
    arg = kwargs.pop('geojson', None) or len(args) and args[0]
    try:
        srs = kwargs.pop('srs', None) or arg.srs.wkt
    except AttributeError:
        srs = None
    if hasattr(arg, 'keys'):
        geom = ogr.CreateGeometryFromJson(json.dumps(arg))
    elif hasattr(arg, 'startswith'):
        if arg.startswith('{'):
            geom = ogr.CreateGeometryFromJson(arg)
        # WKB as hexadecimal string.
        elif ord(arg[0]) in [0, 1]:
            geom = ogr.CreateGeometryFromWkb(arg)
        elif arg.startswith('<gml'):
            geom = ogr.CreateGeometryFromGML(arg)
    elif hasattr(arg, 'wkb'):
        geom = ogr.CreateGeometryFromWkb(bytes(arg.wkb))
    else:
        geom = ogr.Geometry(*args, **kwargs)
    if geom:
        if not isinstance(srs, SpatialReference):
            srs = SpatialReference(srs)
        srs = srs or SpatialReference(4326)
        geom.AssignSpatialReference(srs)
    return geom
<|endoftext|>