<commit_before># Copyright 2015 ARM Limited, All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
This module contains classes to represent Test Specification interface that defines the data to be generated by/from
a build system to give enough information to Greentea.
"""

class TestBinary:
    """
    Class representing a Test Binary.
    """
    KW_BIN_TYPE = "binary_type"
    KW_BIN_PATH = "path"

    BIN_TYPE_BOOTABLE = "bootable"
    BIN_TYPE_DEFAULT = BIN_TYPE_BOOTABLE
    SUPPORTED_BIN_TYPES = [BIN_TYPE_BOOTABLE]

    def __init__(self, path, binary_type):
        """
        ctor.

        :param path:
        :param binary_type:
        :return:
        """
        assert binary_type in TestBinary.SUPPORTED_BIN_TYPES, \
            "Binary type %s not supported. Supported types [%s]" % \
            (binary_type, ", ".join(TestBinary.SUPPORTED_BIN_TYPES))
        self.__path = path
        self.__flash_method = binary_type

    def get_path(self):
        """
        Gives binary path.
        :return:
        """
        return self.__path

class Test:
    """
    class representing a Test artifact that may contain more than one test binaries.
    """
    KW_TEST_NAME = "name"
    KW_TEST_BINS = "binaries"

    def __init__(self, name, default_flash_method=None):
        """
        ctor.

        :param name:
        :param default_flash_method:
        :return:
        """
        self.__name = name
        self.__default_flash_method = default_flash_method
        self.__binaries_by_flash_method = {}

    def get_name(self):
        """
        Gives test name.

        :return:
        """
        return self.__name

    def get_binary(self, binary_type=TestBinary.BIN_TYPE_DEFAULT):
        """
        Gives a test binary of specific flash type.

        :param binary_type:
        :return:
        """
        return self.__binaries_by_flash_method.get(binary_type, None)

    def parse(self, test_json):
        """
        Parse json contents into object.

        :param test_json:
        :return:
        """
        assert Test.KW_TEST_BINS in test_json, "Test spec should contain key `binaries`"
        for binary in test_json[Test.KW_TEST_BINS]:
            mandatory_keys = [TestBinary.KW_BIN_PATH]
            assert set(mandatory_keys).issubset(set(binary.keys())), \
                "Binary spec should contain key [%s]" % ",".join(mandatory_keys)
            fm = binary.get(TestBinary.KW_BIN_TYPE, self.__default_flash_method)
            assert fm is not None, "Binary type not specified in build and binary spec."
            tb = TestBinary(binary[TestBinary.KW_BIN_PATH], fm)
            self.__binaries_by_flash_method[fm] = tb

    def add_binary(self, path, binary_type):
        """
        Add binary to the test.

        :param path:
        :param binary_type:
        :return:
        """
        self.__binaries_by_flash_method[binary_type] = TestBinary(path, binary_type)

class TestBuild:
    """
    class for Test build.
    """
    KW_TEST_BUILD_NAME = "name"
    KW_PLATFORM = "platform"
    KW_TOOLCHAIN = "toolchain"
    KW_BAUD_RATE = "baud_rate"
    KW_BUILD_BASE_PATH = "base_path"
    KW_TESTS = "tests"
    KW_BIN_TYPE = "binary_type"

    def __init__(self, name, platform, toolchain, baud_rate, base_path, default_flash_method=None):
        """
        ctor.

        :param name:
        :param platform:
        :param toolchain:
        :param baud_rate:
        :param base_path:
        :param default_flash_method:
        :return:
        """
        self.__name = name
        self.__platform = platform
        self.__toolchain = toolchain
        self.__baud_rate = baud_rate
        self.__base_path = base_path
        self.__default_flash_method = default_flash_method
        self.__tests = {}

    def get_name(self):
        """
        Gives build name.

        :return:
        """
        return self.__name

    def get_platform(self):
        """
        Gives mbed classic platform name.

        :return:
        """
        return self.__platform

    def get_toolchain(self):
        """
        Gives toolchain

        :return:
        """
        return self.__toolchain

    def get_baudrate(self):
        """
        Gives baud rate.

        :return:
        """
        return self.__baud_rate

    def get_path(self):
        """
        Gives path.

        :return:
        """
        return self.__base_path

    def get_tests(self):
        """
        Gives tests dict keyed by test name.

        :return:
        """
        return self.__tests

    def parse(self, build_spec):
        """
        Parse Test build json.

        :param build_spec:
        :return:
        """
        assert TestBuild.KW_TESTS in build_spec, "Build spec should contain key '%s'" % TestBuild.KW_TESTS
        for name, test_json in build_spec[TestBuild.KW_TESTS].iteritems():
            test = Test(name, default_flash_method=self.__default_flash_method)
            test.parse(test_json)
            self.__tests[name] = test

    def add_test(self, name, test):
        """
        Add test.

        :param name:
        :param test:
        :return:
        """
        self.__tests[name] = test

class TestSpec:
    """
    Test specification. Contains Builds.
    """
    KW_BUILDS = "builds"

    def __init__(self):
        """
        ctor.

        :return:
        """
        self.__target_test_spec = {}

    def parse(self, spec):
        """
        Parse test spec json.

        :param spec:
        :return:
        """
        assert TestSpec.KW_BUILDS, "Test spec should contain key '%s'" % TestSpec.KW_BUILDS
        for build_name, build in spec[TestSpec.KW_BUILDS].iteritems():
            mandatory_keys = [TestBuild.KW_PLATFORM, TestBuild.KW_TOOLCHAIN,
                              TestBuild.KW_BAUD_RATE,
                              TestBuild.KW_BUILD_BASE_PATH]
            print set(mandatory_keys)
            print set(build.keys())
            print set(mandatory_keys).issubset(set(build.keys()))
            assert set(mandatory_keys).issubset(set(build.keys())), \
                "Build spec should contain keys [%s]. It has [%s]" % (",".join(mandatory_keys), ",".join(build.keys()))
            platform = build[TestBuild.KW_PLATFORM]
            toolchain = build[TestBuild.KW_TOOLCHAIN]

            # If there is no 'name' property in build, we will use build key as build name
            name = build.get(TestBuild.KW_TEST_BUILD_NAME, build_name)

            tb = TestBuild(name, platform, toolchain,
                           build[TestBuild.KW_BAUD_RATE],
                           build[TestBuild.KW_BUILD_BASE_PATH],
                           build.get(TestBuild.KW_BIN_TYPE, None))
            tb.parse(build)
            self.__target_test_spec[name] = tb

    def get_test_builds(self):
        """
        Gives test builds.
        :return:
        """
        return self.__target_test_spec.values()

    def get_test_build(self, build_name):
        """
        Gives test build with given name.

        :param build_name:
        :return:
        """
        return self.__target_test_spec.get(build_name, None)

    def add_test_builds(self, name, test_build):
        """
        Add test build.

        :param name:
        :param test_build:
        :return:
        """
        self.__target_test_spec[name] = test_build
<commit_msg>Remove debug prints from test_spec<commit_after># Copyright 2015 ARM Limited, All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
This module contains classes to represent Test Specification interface that defines the data to be generated by/from
a build system to give enough information to Greentea.
"""

class TestBinary:
    """
    Class representing a Test Binary.
    """
    KW_BIN_TYPE = "binary_type"
    KW_BIN_PATH = "path"

    BIN_TYPE_BOOTABLE = "bootable"
    BIN_TYPE_DEFAULT = BIN_TYPE_BOOTABLE
    SUPPORTED_BIN_TYPES = [BIN_TYPE_BOOTABLE]

    def __init__(self, path, binary_type):
        """
        ctor.

        :param path:
        :param binary_type:
        :return:
        """
        assert binary_type in TestBinary.SUPPORTED_BIN_TYPES, \
            "Binary type %s not supported. Supported types [%s]" % \
            (binary_type, ", ".join(TestBinary.SUPPORTED_BIN_TYPES))
        self.__path = path
        self.__flash_method = binary_type

    def get_path(self):
        """
        Gives binary path.
        :return:
        """
        return self.__path

class Test:
    """
    class representing a Test artifact that may contain more than one test binaries.
    """
    KW_TEST_NAME = "name"
    KW_TEST_BINS = "binaries"

    def __init__(self, name, default_flash_method=None):
        """
        ctor.

        :param name:
        :param default_flash_method:
        :return:
        """
        self.__name = name
        self.__default_flash_method = default_flash_method
        self.__binaries_by_flash_method = {}

    def get_name(self):
        """
        Gives test name.

        :return:
        """
        return self.__name

    def get_binary(self, binary_type=TestBinary.BIN_TYPE_DEFAULT):
        """
        Gives a test binary of specific flash type.

        :param binary_type:
        :return:
        """
        return self.__binaries_by_flash_method.get(binary_type, None)

    def parse(self, test_json):
        """
        Parse json contents into object.

        :param test_json:
        :return:
        """
        assert Test.KW_TEST_BINS in test_json, "Test spec should contain key `binaries`"
        for binary in test_json[Test.KW_TEST_BINS]:
            mandatory_keys = [TestBinary.KW_BIN_PATH]
            assert set(mandatory_keys).issubset(set(binary.keys())), \
                "Binary spec should contain key [%s]" % ",".join(mandatory_keys)
            fm = binary.get(TestBinary.KW_BIN_TYPE, self.__default_flash_method)
            assert fm is not None, "Binary type not specified in build and binary spec."
            tb = TestBinary(binary[TestBinary.KW_BIN_PATH], fm)
            self.__binaries_by_flash_method[fm] = tb

    def add_binary(self, path, binary_type):
        """
        Add binary to the test.

        :param path:
        :param binary_type:
        :return:
        """
        self.__binaries_by_flash_method[binary_type] = TestBinary(path, binary_type)

class TestBuild:
    """
    class for Test build.
    """
    KW_TEST_BUILD_NAME = "name"
    KW_PLATFORM = "platform"
    KW_TOOLCHAIN = "toolchain"
    KW_BAUD_RATE = "baud_rate"
    KW_BUILD_BASE_PATH = "base_path"
    KW_TESTS = "tests"
    KW_BIN_TYPE = "binary_type"

    def __init__(self, name, platform, toolchain, baud_rate, base_path, default_flash_method=None):
        """
        ctor.

        :param name:
        :param platform:
        :param toolchain:
        :param baud_rate:
        :param base_path:
        :param default_flash_method:
        :return:
        """
        self.__name = name
        self.__platform = platform
        self.__toolchain = toolchain
        self.__baud_rate = baud_rate
        self.__base_path = base_path
        self.__default_flash_method = default_flash_method
        self.__tests = {}

    def get_name(self):
        """
        Gives build name.

        :return:
        """
        return self.__name

    def get_platform(self):
        """
        Gives mbed classic platform name.

        :return:
        """
        return self.__platform

    def get_toolchain(self):
        """
        Gives toolchain

        :return:
        """
        return self.__toolchain

    def get_baudrate(self):
        """
        Gives baud rate.

        :return:
        """
        return self.__baud_rate

    def get_path(self):
        """
        Gives path.

        :return:
        """
        return self.__base_path

    def get_tests(self):
        """
        Gives tests dict keyed by test name.

        :return:
        """
        return self.__tests

    def parse(self, build_spec):
        """
        Parse Test build json.

        :param build_spec:
        :return:
        """
        assert TestBuild.KW_TESTS in build_spec, "Build spec should contain key '%s'" % TestBuild.KW_TESTS
        for name, test_json in build_spec[TestBuild.KW_TESTS].iteritems():
            test = Test(name, default_flash_method=self.__default_flash_method)
            test.parse(test_json)
            self.__tests[name] = test

    def add_test(self, name, test):
        """
        Add test.

        :param name:
        :param test:
        :return:
        """
        self.__tests[name] = test

class TestSpec:
    """
    Test specification. Contains Builds.
    """
    KW_BUILDS = "builds"

    def __init__(self):
        """
        ctor.

        :return:
        """
        self.__target_test_spec = {}

    def parse(self, spec):
        """
        Parse test spec json.

        :param spec:
        :return:
        """
        assert TestSpec.KW_BUILDS, "Test spec should contain key '%s'" % TestSpec.KW_BUILDS
        for build_name, build in spec[TestSpec.KW_BUILDS].iteritems():
            mandatory_keys = [TestBuild.KW_PLATFORM, TestBuild.KW_TOOLCHAIN,
                              TestBuild.KW_BAUD_RATE,
                              TestBuild.KW_BUILD_BASE_PATH]
            #print set(mandatory_keys)
            #print set(build.keys())
            #print set(mandatory_keys).issubset(set(build.keys()))
            assert set(mandatory_keys).issubset(set(build.keys())), \
                "Build spec should contain keys [%s]. It has [%s]" % (",".join(mandatory_keys), ",".join(build.keys()))
            platform = build[TestBuild.KW_PLATFORM]
            toolchain = build[TestBuild.KW_TOOLCHAIN]

            # If there is no 'name' property in build, we will use build key as build name
            name = build.get(TestBuild.KW_TEST_BUILD_NAME, build_name)

            tb = TestBuild(name, platform, toolchain,
                           build[TestBuild.KW_BAUD_RATE],
                           build[TestBuild.KW_BUILD_BASE_PATH],
                           build.get(TestBuild.KW_BIN_TYPE, None))
            tb.parse(build)
            self.__target_test_spec[name] = tb

    def get_test_builds(self):
        """
        Gives test builds.
        :return:
        """
        return self.__target_test_spec.values()

    def get_test_build(self, build_name):
        """
        Gives test build with given name.

        :param build_name:
        :return:
        """
        return self.__target_test_spec.get(build_name, None)

    def add_test_builds(self, name, test_build):
        """
        Add test build.

        :param name:
        :param test_build:
        :return:
        """
        self.__target_test_spec[name] = test_build
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  utilities/__init__.py
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above
#    copyright notice, this list of conditions and the following disclaimer
#    in the documentation and/or other materials provided with the
#    distribution.
#  * Neither the name of the  nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

import collections
import functools
import os
import re
import time

__version__ = '0.1'

class Cache(object):
	"""
	This class provides a simple to use cache object which can be applied
	as a decorator.
	"""
	def __init__(self, timeout):
		"""
		@type timeout: string, integer
		@param timeout: The amount of time in seconds that a cached
		result will be considered valid for.
		"""
		if isinstance(timeout, (str, unicode)):
			timeout = timedef_to_seconds(timeout)
		self.cache_timeout = timeout
		self.__cache = {}

	def __call__(self, *args):
		if not hasattr(self, '_target_function'):
			self._target_function = args[0]
			return self
		self.cache_clean()
		if not isinstance(args, collections.Hashable):
			return self._target_function(*args)
		result, expiration = self.__cache.get(args, (None, 0))
		if expiration > time.time():
			return result
		result = self._target_function(*args)
		self.__cache[args] = (result, time.time() + self.cache_timeout)
		return result

	def __repr__(self):
		return "<cached function {0}>".format(self._target_function.__name__)

	def cache_clean(self):
		"""
		Remove expired items from the cache.
		"""
		now = time.time()
		keys_for_removal = []
		for key, (value, expiration) in self.__cache.items():
			if expiration < now:
				keys_for_removal.append(key)
		for key in keys_for_removal:
			del self.__cache[key]

	def cache_clear(self):
		"""
		Remove all items from the cache.
		"""
		self.__cache = {}

class FileWalker:
	"""
	This class is used to easily iterate over files in a directory.
	"""
	def __init__(self, filespath, absolute_path = False, skip_files = False, skip_dirs = False, filter_func = None):
		"""
		@type filespath: string
		@param filespath: A path to either a file or a directory.  If a
		file is passed then that will be the only file returned during the
		iteration.  If a directory is passed, all files will be recursively
		returned during the iteration.

		@type absolute_path: boolean
		@param absolute_path: Whether or not the absolute path or a relative
		path should be returned.

		@type skip_files: boolean
		@param skip_files: Whether or not to skip files.

		@type skip_dirs: boolean
		@param skip_dirs: Whether or not to skip directories.

		@type filter_func: function
		@param filter_func: If defined, the filter_func function will be called
		for each file and if the function returns false the file will be
		skipped.
		"""
		if not (os.path.isfile(filespath) or os.path.isdir(filespath)):
			raise Exception(filespath + ' is neither a file or directory')
		if absolute_path:
			self.filespath = os.path.abspath(filespath)
		else:
			self.filespath = os.path.relpath(filespath)
		self.skip_files = skip_files
		self.skip_dirs = skip_dirs
		self.filter_func = filter_func
		if os.path.isdir(self.filespath):
			self.__iter__= self._next_dir
		elif os.path.isfile(self.filespath):
			self.__iter__ = self._next_file

	def _skip(self, cur_file):
		if self.skip_files and os.path.isfile(cur_file):
			return True
		if self.skip_dirs and os.path.isdir(cur_file):
			return True
		if self.filter_func != None:
			if not self.filter_func(cur_file):
				return True
		return False

	def _next_dir(self):
		for root, dirs, files in os.walk(self.filespath):
			for cur_file in files:
				cur_file = os.path.join(root, cur_file)
				if not self._skip(cur_file):
					yield cur_file
			for cur_dir in dirs:
				cur_dir = os.path.join(root, cur_dir)
				if not self._skip(cur_dir):
					yield cur_dir
		raise StopIteration

	def _next_file(self):
		if not self._skip(self.filespath):
			yield self.filespath
		raise StopIteration

class SectionConfigParser(object):
	"""
	Proxy access to a section of a ConfigParser object.
	"""
	__version__ = '0.1'
	def __init__(self, section_name, config_parser):
		"""
		@type section_name: string
		@param section_name: Name of the section to proxy access for.

		@type config_parser: ConfigParser.ConfigParser instance
		@param config_parser: ConfigParser object to proxy access for.
		"""
		self.section_name = section_name
		self.config_parser = config_parser

	def get_raw(self, option, opt_type, default = None):
		get_func = getattr(self.config_parser, 'get' + opt_type)
		if default == None:
			return get_func(self.section_name, option)
		elif self.config_parser.has_option(self.section_name, option):
			return get_func(self.section_name, option)
		else:
			return default

	def get(self, option, default = None):
		return self.get_raw(option, '', default)

	def getint(self, option, default = None):
		return self.get_raw(option, 'int', default)

	def getfloat(self, option, default = None):
		return self.get_raw(option, 'float', default)

	def getboolean(self, option, default = None):
		return self.get_raw(option, 'boolean', default)

	def has_option(self, option):
		return self.config_parser.has_option(self.section_name, option)

	def options(self):
		return self.config_parser.options(self.section_name)

	def items(self):
		return self.config_parser.items(self.section_name)

def parse_case_camel_to_snake(string):
	return re.sub('((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))', r'_\1', string).lower()

def parse_case_snake_to_camel(string, upper_first = True):
	string = string.split('_')
	first_part = string[0]
	if upper_first:
		first_part = first_part.title()
	second_part = ''.join(map(lambda word: word.title(), string[1:]))
	return first_part + second_part

def parse_server(server, default_port):
	"""
	Convert a server string to a tuple suitable for passing to connect, for
	example converting 'www.google.com:443' to ('www.google.com', 443).

	@type server: string
	@param server: The server string to convert.

	@type default_port: integer
	@param default_port: The port to use in case one is not specified in
	the server string.
	"""
	server = server.split(':')
	host = server[0]
	if len(server) == 1:
		return (host, default_port)
	else:
		port = server[1]
		if not port:
			port = default_port
		else:
			port = int(port)
		return (host, port)

def parse_timespan(timedef):
	"""
	Convert a string timespan definition to seconds, for example converting
	'1m30s' to 90.

	@type timedef: string
	@param timedef: The timespan definition to convert to seconds.
	"""
	converter_order = ['w', 'd', 'h', 'm', 's']
	converters = {
		'w': 604800,
		'd': 86400,
		'h': 3600,
		'm': 60,
		's': 1
	}
	timedef = timedef.lower()
	if timedef.isdigit():
		return int(timedef)
	elif len(timedef) == 0:
		return 0
	seconds = -1
	for spec in converter_order:
		timedef = timedef.split(spec)
		if len(timedef) == 1:
			timedef = timedef[0]
			continue
		elif len(timedef) > 2 or not timedef[0].isdigit():
			seconds = -1
			break
		adjustment = converters[spec]
		seconds = max(seconds, 0)
		seconds += (int(timedef[0]) * adjustment)
		timedef = timedef[1]
		if not len(timedef):
			break
	if seconds < 0:
		raise ValueError('invalid time format')
	return seconds

def unique(seq, key = None):
	"""
	Unique a list or tuple and preserve the order

	@type seq: list, tuple
	@param seq: The list or tuple to preserve unique items from.

	@type key: Function or None
	@param key: If key is provided it will be called during the
	comparison process.
	"""
	if key is None:
		key = lambda x: x
	preserved_type = type(seq)
	seen = {}
	result = []
	for item in seq:
		marker = key(item)
		if marker in seen:
			continue
		seen[marker] = 1
		result.append(item)
	return preserved_type(result)

def which(program):
	"""
	Locate an executable binary's full path by its name.

	@type program: string
	@param program: The executables name.
	"""
	is_exe = lambda fpath: (os.path.isfile(fpath) and os.access(fpath, os.X_OK))
	if is_exe(program):
		return program
	for path in os.environ["PATH"].split(os.pathsep):
		path = path.strip('"')
		exe_file = os.path.join(path, program)
		if is_exe(exe_file):
			return exe_file
	return None

def xfrange(start, stop = None, step = 1):
	"""
	Iterate through an arithmetic progression.

	@type start: int, long, float
	@param start: Starting number.

	@type stop: int, long, float
	@param stop: Stopping number.

	@type step: int, long, float
	@param step: Stepping size.
	"""
	if stop == None:
		stop = start
		start = 0
	start = float(start)
	while start < stop:
		yield start
		start += step
<commit_msg>Support set in the section config parser<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  utilities/__init__.py
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above
#    copyright notice, this list of conditions and the following disclaimer
#    in the documentation and/or other materials provided with the
#    distribution.
#  * Neither the name of the  nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

import collections
import functools
import os
import re
import time

__version__ = '0.1'

class Cache(object):
	"""
	This class provides a simple to use cache object which can be applied
	as a decorator.
	"""
	def __init__(self, timeout):
		"""
		@type timeout: string, integer
		@param timeout: The amount of time in seconds that a cached
		result will be considered valid for.
		"""
		if isinstance(timeout, (str, unicode)):
			timeout = timedef_to_seconds(timeout)
		self.cache_timeout = timeout
		self.__cache = {}

	def __call__(self, *args):
		if not hasattr(self, '_target_function'):
			self._target_function = args[0]
			return self
		self.cache_clean()
		if not isinstance(args, collections.Hashable):
			return self._target_function(*args)
		result, expiration = self.__cache.get(args, (None, 0))
		if expiration > time.time():
			return result
		result = self._target_function(*args)
		self.__cache[args] = (result, time.time() + self.cache_timeout)
		return result

	def __repr__(self):
		return "<cached function {0}>".format(self._target_function.__name__)

	def cache_clean(self):
		"""
		Remove expired items from the cache.
		"""
		now = time.time()
		keys_for_removal = []
		for key, (value, expiration) in self.__cache.items():
			if expiration < now:
				keys_for_removal.append(key)
		for key in keys_for_removal:
			del self.__cache[key]

	def cache_clear(self):
		"""
		Remove all items from the cache.
		"""
		self.__cache = {}

class FileWalker:
	"""
	This class is used to easily iterate over files in a directory.
	"""
	def __init__(self, filespath, absolute_path = False, skip_files = False, skip_dirs = False, filter_func = None):
		"""
		@type filespath: string
		@param filespath: A path to either a file or a directory.  If a
		file is passed then that will be the only file returned during the
		iteration.  If a directory is passed, all files will be recursively
		returned during the iteration.

		@type absolute_path: boolean
		@param absolute_path: Whether or not the absolute path or a relative
		path should be returned.

		@type skip_files: boolean
		@param skip_files: Whether or not to skip files.

		@type skip_dirs: boolean
		@param skip_dirs: Whether or not to skip directories.

		@type filter_func: function
		@param filter_func: If defined, the filter_func function will be called
		for each file and if the function returns false the file will be
		skipped.
		"""
		if not (os.path.isfile(filespath) or os.path.isdir(filespath)):
			raise Exception(filespath + ' is neither a file or directory')
		if absolute_path:
			self.filespath = os.path.abspath(filespath)
		else:
			self.filespath = os.path.relpath(filespath)
		self.skip_files = skip_files
		self.skip_dirs = skip_dirs
		self.filter_func = filter_func
		if os.path.isdir(self.filespath):
			self.__iter__= self._next_dir
		elif os.path.isfile(self.filespath):
			self.__iter__ = self._next_file

	def _skip(self, cur_file):
		if self.skip_files and os.path.isfile(cur_file):
			return True
		if self.skip_dirs and os.path.isdir(cur_file):
			return True
		if self.filter_func != None:
			if not self.filter_func(cur_file):
				return True
		return False

	def _next_dir(self):
		for root, dirs, files in os.walk(self.filespath):
			for cur_file in files:
				cur_file = os.path.join(root, cur_file)
				if not self._skip(cur_file):
					yield cur_file
			for cur_dir in dirs:
				cur_dir = os.path.join(root, cur_dir)
				if not self._skip(cur_dir):
					yield cur_dir
		raise StopIteration

	def _next_file(self):
		if not self._skip(self.filespath):
			yield self.filespath
		raise StopIteration

class SectionConfigParser(object):
	"""
	Proxy access to a section of a ConfigParser object.
	"""
	__version__ = '0.2'
	def __init__(self, section_name, config_parser):
		"""
		@type section_name: string
		@param section_name: Name of the section to proxy access for.

		@type config_parser: ConfigParser.ConfigParser instance
		@param config_parser: ConfigParser object to proxy access for.
		"""
		self.section_name = section_name
		self.config_parser = config_parser

	def get_raw(self, option, opt_type, default = None):
		get_func = getattr(self.config_parser, 'get' + opt_type)
		if default == None:
			return get_func(self.section_name, option)
		elif self.config_parser.has_option(self.section_name, option):
			return get_func(self.section_name, option)
		else:
			return default

	def get(self, option, default = None):
		return self.get_raw(option, '', default)

	def getint(self, option, default = None):
		return self.get_raw(option, 'int', default)

	def getfloat(self, option, default = None):
		return self.get_raw(option, 'float', default)

	def getboolean(self, option, default = None):
		return self.get_raw(option, 'boolean', default)

	def has_option(self, option):
		return self.config_parser.has_option(self.section_name, option)

	def options(self):
		return self.config_parser.options(self.section_name)

	def items(self):
		return self.config_parser.items(self.section_name)

	def set(self, option, value):
		self.config_parser.set(self.section_name, option, value)

def parse_case_camel_to_snake(string):
	return re.sub('((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))', r'_\1', string).lower()

def parse_case_snake_to_camel(string, upper_first = True):
	string = string.split('_')
	first_part = string[0]
	if upper_first:
		first_part = first_part.title()
	second_part = ''.join(map(lambda word: word.title(), string[1:]))
	return first_part + second_part

def parse_server(server, default_port):
	"""
	Convert a server string to a tuple suitable for passing to connect, for
	example converting 'www.google.com:443' to ('www.google.com', 443).

	@type server: string
	@param server: The server string to convert.

	@type default_port: integer
	@param default_port: The port to use in case one is not specified in
	the server string.
	"""
	server = server.split(':')
	host = server[0]
	if len(server) == 1:
		return (host, default_port)
	else:
		port = server[1]
		if not port:
			port = default_port
		else:
			port = int(port)
		return (host, port)

def parse_timespan(timedef):
	"""
	Convert a string timespan definition to seconds, for example converting
	'1m30s' to 90.

	@type timedef: string
	@param timedef: The timespan definition to convert to seconds.
	"""
	converter_order = ['w', 'd', 'h', 'm', 's']
	converters = {
		'w': 604800,
		'd': 86400,
		'h': 3600,
		'm': 60,
		's': 1
	}
	timedef = timedef.lower()
	if timedef.isdigit():
		return int(timedef)
	elif len(timedef) == 0:
		return 0
	seconds = -1
	for spec in converter_order:
		timedef = timedef.split(spec)
		if len(timedef) == 1:
			timedef = timedef[0]
			continue
		elif len(timedef) > 2 or not timedef[0].isdigit():
			seconds = -1
			break
		adjustment = converters[spec]
		seconds = max(seconds, 0)
		seconds += (int(timedef[0]) * adjustment)
		timedef = timedef[1]
		if not len(timedef):
			break
	if seconds < 0:
		raise ValueError('invalid time format')
	return seconds

def unique(seq, key = None):
	"""
	Unique a list or tuple and preserve the order

	@type seq: list, tuple
	@param seq: The list or tuple to preserve unique items from.

	@type key: Function or None
	@param key: If key is provided it will be called during the
	comparison process.
	"""
	if key is None:
		key = lambda x: x
	preserved_type = type(seq)
	seen = {}
	result = []
	for item in seq:
		marker = key(item)
		if marker in seen:
			continue
		seen[marker] = 1
		result.append(item)
	return preserved_type(result)

def which(program):
	"""
	Locate an executable binary's full path by its name.

	@type program: string
	@param program: The executables name.
	"""
	is_exe = lambda fpath: (os.path.isfile(fpath) and os.access(fpath, os.X_OK))
	if is_exe(program):
		return program
	for path in os.environ["PATH"].split(os.pathsep):
		path = path.strip('"')
		exe_file = os.path.join(path, program)
		if is_exe(exe_file):
			return exe_file
	return None

def xfrange(start, stop = None, step = 1):
	"""
	Iterate through an arithmetic progression.

	@type start: int, long, float
	@param start: Starting number.

	@type stop: int, long, float
	@param stop: Stopping number.

	@type step: int, long, float
	@param step: Stepping size.
	"""
	if stop == None:
		stop = start
		start = 0
	start = float(start)
	while start < stop:
		yield start
		start += step
<|endoftext|>


<commit_before>from django.contrib import admin
from sleep.models import *

class AllnighterAdmin(admin.ModelAdmin):
    list_display = ('user', 'date')
    search_fields = ['user__username']

class PartialSleepAdmin(admin.ModelAdmin):
    list_display = ('user','start_local_time')
    search_fields = ('user__username',)

class SleepAdmin(admin.ModelAdmin):
    fieldsets = [
        (None, {'fields' : ['user']}),
        ('Date/Time info', {'fields' : ['date', 'start_time', 'end_time','timezone']}),
        ('Other', {'fields' : ['comments','quality','sleepcycles']})
         ]
    list_display = ('user', 'start_local_time', 'end_local_time','comments')
    ordering = ('user','end_time',)
    search_fields = ['user__username','comments']

class SleeperProfileAdmin(admin.ModelAdmin):
    fieldsets = [
        (None, {'fields' : ['user','timezone','metrics']}),
        ('Friends', {'fields' : ['friends','follows']}),
        ('Privacy Settings', {'fields' : [ 'privacy', 'privacyLoggedIn', 'privacyFriends', 'autoAcceptGroups']}),
        ('Customizations', {'fields' : ['punchInDelay','useGravatar','moreMetrics', 'use12HourTime', 'mobile']}),
        ('Ideal Settings', {'fields' : ['idealSleep','idealSleepTimeWeekday', 'idealSleepTimeWeekend','idealWakeupWeekday','idealWakeupWeekend']}),
        ('Email Settings', {'fields' : ['emailreminders','emailTime','emailActivated']})
        ]
    list_display = ['user']
    ordering = ('user',)
    filter_horizontal = ('friends','follows','metrics')
    search_fields = ['user__username']

class FriendRequestAdmin(admin.ModelAdmin):
    list_display = ('requestor', 'requestee', 'accepted')
    search_fields = ['requestor__user__username','requestee__username']

class MembershipInline(admin.TabularInline):
    model = Membership

class SleeperGroupAdmin(admin.ModelAdmin):
    fields = ['name', 'description', 'privacy', 'defunctMembers']
    list_display = ('name', 'description','privacy')
    inlines = [MembershipInline]
    search_fields = ['name','description']
    ordering = ('name',)

class GroupInviteAdmin(admin.ModelAdmin):
    list_display = ('group', 'user', 'accepted')
    ordering = ('accepted','group','user')
    search_fields = ['user__username','group__name']

class MembershipAdmin(admin.ModelAdmin):
    list_display = ('group', 'user', 'role')
    ordering = ('group','user','role')
    search_fields = ['group__name', 'user__username']

class GroupRequestAdmin(admin.ModelAdmin):
    list_display = ('user','group', 'accepted')
    search_fields = ('user__username','group__name')

class MetricAdmin(admin.ModelAdmin):
    list_display = ('name', 'display_style', 'priority', 'show_by_default')
    ordering = ('-priority',)
    search_fields = ('name',)

admin.site.register(Allnighter, AllnighterAdmin)
admin.site.register(PartialSleep, PartialSleepAdmin)
admin.site.register(Sleep, SleepAdmin)
admin.site.register(SleeperProfile, SleeperProfileAdmin)
admin.site.register(FriendRequest, FriendRequestAdmin)
admin.site.register(SleeperGroup, SleeperGroupAdmin)
admin.site.register(Membership, MembershipAdmin)
admin.site.register(GroupInvite, GroupInviteAdmin)
admin.site.register(GroupRequest, GroupRequestAdmin)
admin.site.register(Metric,MetricAdmin)
<commit_msg>Makes admin panel more awesome<commit_after>from django.contrib import admin
from sleep.models import *

class AllnighterAdmin(admin.ModelAdmin):
    list_display = ('user', 'date', 'comments')
    list_filter = ('user', 'date')
    ordering = ('user',)
    search_fields = ['user__username']

class PartialSleepAdmin(admin.ModelAdmin):
    list_display = ('user','start_local_time')
    search_fields = ('user__username',)
    list_filter = ('user', 'start_time')

class SleepAdmin(admin.ModelAdmin):
    fieldsets = [
        (None, {'fields' : ['user']}),
        ('Date/Time info', {'fields' : ['date', 'start_time', 'end_time','timezone']}),
        ('Other', {'fields' : ['comments','quality','sleepcycles']})
         ]
    list_display = ('user', 'start_local_time', 'end_local_time','comments')
    list_filter = ('user','end_time')
    ordering = ('user','end_time',)
    search_fields = ['user__username','comments']

class SleeperProfileAdmin(admin.ModelAdmin):
    fieldsets = [
        (None, {'fields' : ['user','timezone','metrics']}),
        ('Friends', {'fields' : ['friends','follows']}),
        ('Privacy Settings', {'fields' : [ 'privacy', 'privacyLoggedIn', 'privacyFriends', 'autoAcceptGroups']}),
        ('Customizations', {'fields' : ['punchInDelay','useGravatar','moreMetrics', 'use12HourTime', 'mobile']}),
        ('Ideal Settings', {'fields' : ['idealSleep','idealSleepTimeWeekday', 'idealSleepTimeWeekend','idealWakeupWeekday','idealWakeupWeekend']}),
        ('Email Settings', {'fields' : ['emailreminders','emailTime','emailActivated']})
        ]
    list_display = ['user']
    ordering = ('user',)
    filter_horizontal = ('friends','follows','metrics')
    search_fields = ['user__username']

class FriendRequestAdmin(admin.ModelAdmin):
    list_display = ('requestor', 'requestee', 'accepted')
    search_fields = ['requestor__user__username','requestee__username']
    list_filter = ('requestor__user','requestee')

class MembershipInline(admin.TabularInline):
    model = Membership

class SleeperGroupAdmin(admin.ModelAdmin):
    fields = ['name', 'description', 'privacy', 'defunctMembers']
    list_display = ('name', 'description','privacy')
    inlines = [MembershipInline]
    search_fields = ['name','description']
    ordering = ('name',)

class GroupInviteAdmin(admin.ModelAdmin):
    list_display = ('group', 'user', 'accepted')
    ordering = ('accepted','group','user')
    search_fields = ['user__username','group__name']

class MembershipAdmin(admin.ModelAdmin):
    list_display = ('group', 'user', 'role')
    ordering = ('group','user','role')
    search_fields = ['group__name', 'user__username']
    list_filter = ('group', 'user')

class GroupRequestAdmin(admin.ModelAdmin):
    list_display = ('user','group', 'accepted')
    search_fields = ('user__username','group__name')

class MetricAdmin(admin.ModelAdmin):
    list_display = ('name', 'display_style', 'priority', 'show_by_default')
    ordering = ('-priority',)
    search_fields = ('name',)

admin.site.register(Allnighter, AllnighterAdmin)
admin.site.register(PartialSleep, PartialSleepAdmin)
admin.site.register(Sleep, SleepAdmin)
admin.site.register(SleeperProfile, SleeperProfileAdmin)
admin.site.register(FriendRequest, FriendRequestAdmin)
admin.site.register(SleeperGroup, SleeperGroupAdmin)
admin.site.register(Membership, MembershipAdmin)
admin.site.register(GroupInvite, GroupInviteAdmin)
admin.site.register(GroupRequest, GroupRequestAdmin)
admin.site.register(Metric,MetricAdmin)
<|endoftext|>


<commit_before>"""
Experimental sans-io implementation of mitmproxy's protocol stack.

Most important primitives:
    - layers: represent protocol layers, e.g. one for tcp, tls, and so on. Layers are stacked, so
      a typical configuration might be ReverseProxy/TLS/TCP.
    - server: the proxy server does all IO and communication with the mitmproxy master.
      It creates the top layer for each incoming client connection.
    - events: When IO actions occur at the proxy server, they are passed down to the top layer as events.
    - commands: In the other direction, layers can emit commands to higher layers or the proxy server.
      This is used to e.g. send data, request for new connections to be opened, or to use mitmproxy's
      script hooks.
    - context: The context is the connection context each layer is provided with. This is still very
      much WIP, but this should expose stuff like Server Name Indication to lower layers.
"""
<commit_msg>wording<commit_after>"""
Experimental sans-io implementation of mitmproxy's protocol stack.

Most important primitives:
    - layers: represent protocol layers, e.g. one for tcp, tls, and so on. Layers are nested, so
      a typical configuration might be ReverseProxy/TLS/TCP.
    - server: the proxy server does all IO and communication with the mitmproxy master.
      It creates the top/outermost layer for each incoming client connection.
    - events: When IO actions occur at the proxy server, they are passed down to the top layer as events.
    - commands: In the other direction, layers can emit commands to higher layers or the proxy server.
      This is used to e.g. send data, request for new connections to be opened, or to use mitmproxy's
      script hooks.
    - context: The context is the connection context each layer is provided with. This is still very
      much WIP, but this should expose stuff like Server Name Indication to lower layers.
"""
<|endoftext|>


<commit_before>import math

#Tilt pot setpoints
kMaxDown = .865
kMaxUp = .253
kTop = .300
kBottom = .700
kShootLevel = .500
kShootRamp = .400

class Settings():
    """Robot mapping. Values that are changed often go here."""

    #Numbers to be changed through drive station
    num_precision_one = 0.80
    num_precision_two = 0.50
    num_scaling = 1.25
    num_macro_timeout = 15
<commit_msg>More pot<commit_after>import math

#Tilt pot setpoints
kMaxDown = .848
kMaxUp = .253
kTop = .300
kBottom = .700
kShootLevel = .500
kShootRamp = .400

class Settings():
    """Robot mapping. Values that are changed often go here."""

    #Numbers to be changed through drive station
    num_precision_one = 0.80
    num_precision_two = 0.50
    num_scaling = 1.25
    num_macro_timeout = 15
<|endoftext|>


<commit_before>'''
Created on Mar 18, 2013

@author: max
'''

from spanningtree.helper.gurobi_helper import set_up_model
import gurobipy as grb
from gurobipy import quicksum
import math
from spanningtree.highdimgraph.crossing import has_crossing

def solve_lp(graph):
    points = range(0,graph.n)
    lines = graph.lines
    gamma_lp = set_up_model("connected_components_lp_2d")
    connected_components = graph.connected_components
    len_ccs = len(connected_components)
    edges = list(graph.edges.iter_subset(points))
    x = {}
    for (p,q) in edges:
        x[p,q] = gamma_lp.addVar(#obj=graph.euclidean_distance(p,q),
                                 name='edge|%s - %s|' % (p,q))

    t = gamma_lp.addVar(obj=1.0)
    gamma_lp.modelSense = grb.GRB.MINIMIZE

    gamma_lp.update()

    # correct number of edges
    gamma_lp.addConstr(quicksum(x[i,j] for (i,j) in edges) == (len_ccs-1))

    # crossing number range:
    gamma_lp.addConstr(0 <= t <= math.sqrt(len_ccs))

    # crossing constraints
    for line in lines:
        s = quicksum(x[p,q] for (p,q) in edges if has_crossing(line,
            graph.get_line_segment(p,q)))
        if s != 0.0:
                gamma_lp.addConstr(
                        #quicksum(x[p,q] for (p,q) in edges if has_crossing(line,
                        #LineSegment2D(p,q))) <= t)
                        s <= t)

    # connectivity constraint
    for cc1 in connected_components:
        gamma_lp.addConstr(quicksum(x[p,q] for (p,q) in edges if p < q and\
                         p in cc1 and q not in cc1) +
                quicksum(x[q,p] for (q,p) in edges if p > q and\
                         p in cc1 and q not in cc1)
                >= 1.)
    gamma_lp.optimize()

    if gamma_lp.status == grb.GRB.status.OPTIMAL:
        print gamma_lp.getVars()
        #print x[2,3].X
        add_best_edge(graph, x)
        return
    else:
        format_string = 'Vars:\n'
        for var in gamma_lp.getVars():
            format_string += "%s\n" % var
        print "number of lines = %s" % len(graph.lines)
        print '%s\nlp model=%s' % (format_string,gamma_lp)
        import spanningtree.plotting
        spanningtree.plotting.plot(graph)
        raise StandardError('Model infeasible')

def add_best_edge(graph, x):
    edges = graph.edges
    ccs = graph.connected_components
    solution = graph.solution
    (max_i, max_j) = (None, None)
    max_val = None
    for (i,j) in edges:
        cc_i = ccs.get_connected_component(i)
        cc_j = ccs.get_connected_component(j)
        print cc_i, cc_j, i, j
        if cc_i != cc_j and x[i,j].X > max_val:
            (max_i, max_j) = (i,j)
            max_val = x[i,j].X

    graph.merge_cc(max_i, max_j)
    solution.update(max_i, max_j, True)
    return

def compute_spanning_tree(graph):
    n = graph.n
    stored_lines = graph.lines[:]
    remaining_points = range(0,n)
    #lines = graph.lines
    iterations = 1
    while len(graph.connected_components) > 1:
        solve_lp(graph)
        #print x[2,3].X
        #add_best_edge(graph, x)
        #graph.compute_connected_components()
        #connected_components = graph.connected_components
        #lines = graph.preprocess_lines(remaining_points)
        iterations += 1
    #graph.lines = stored_lines
    return iterations

def main():
    # minimal example to find optimal spanning tree
    import numpy as np
    points = np.array([(2.,2.), (6.,4.), (3., 6.), (5., 7.), (4.25, 5.)])
    import spanningtree.highdimgraph.factories as factories
    graph = factories.create_graph(points, 5, 2, 'custom')
    from spanningtree.highdimgraph.lines import HighDimLine
    l1 = HighDimLine(np.array([(2., 6.), (3., 2.)])) # y = -4x + 14
    l2 = HighDimLine(np.array([(2., 3.), (6., 5.)])) # y = 0.5x + 2
    l3 = HighDimLine(np.array([(3., 5.5), (5., 6.5)])) # y = 0.5x + 4
    lines = [l1, l2, l3]
    graph.lines = lines
    graph.preprocess_lines()
    compute_spanning_tree(graph)
    print "crossing number = %s" % graph.crossing_number()
    import spanningtree.plotting as plotting
    plotting.plot(graph)

if __name__ == '__main__':
    main()<commit_msg>select an edge with highest value in fractional solution and minimal euclidean distance to avoid self-crossing edges<commit_after>'''
Created on Mar 18, 2013

@author: max
'''

from spanningtree.helper.gurobi_helper import set_up_model
import gurobipy as grb
from gurobipy import quicksum
import math
from spanningtree.highdimgraph.crossing import has_crossing

def solve_lp(graph):
    points = range(0,graph.n)
    lines = graph.lines
    gamma_lp = set_up_model("connected_components_lp_2d")
    connected_components = graph.connected_components
    len_ccs = len(connected_components)
    edges = list(graph.edges.iter_subset(points))
    x = {}
    for (p,q) in edges:
        x[p,q] = gamma_lp.addVar(#obj=graph.euclidean_distance(p,q),
                                 name='edge|%s - %s|' % (p,q))

    t = gamma_lp.addVar(obj=1.0)
    gamma_lp.modelSense = grb.GRB.MINIMIZE

    gamma_lp.update()

    # correct number of edges
    gamma_lp.addConstr(quicksum(x[i,j] for (i,j) in edges) == (len_ccs-1))

    # crossing number range:
    gamma_lp.addConstr(0 <= t <= math.sqrt(len_ccs))

    # crossing constraints
    for line in lines:
        s = quicksum(x[p,q] for (p,q) in edges if has_crossing(line,
            graph.get_line_segment(p,q)))
        if s != 0.0:
                gamma_lp.addConstr(
                        #quicksum(x[p,q] for (p,q) in edges if has_crossing(line,
                        #LineSegment2D(p,q))) <= t)
                        s <= t)

    # connectivity constraint
    for cc1 in connected_components:
        gamma_lp.addConstr(quicksum(x[p,q] for (p,q) in edges if p < q and\
                         p in cc1 and q not in cc1) +
                quicksum(x[q,p] for (q,p) in edges if p > q and\
                         p in cc1 and q not in cc1)
                >= 1.)
    gamma_lp.optimize()

    if gamma_lp.status == grb.GRB.status.OPTIMAL:
        #print gamma_lp.getVars()
        #print x[2,3].X
        add_best_edge(graph, x)
        return
    else:
        format_string = 'Vars:\n'
        for var in gamma_lp.getVars():
            format_string += "%s\n" % var
        print "number of lines = %s" % len(graph.lines)
        print '%s\nlp model=%s' % (format_string,gamma_lp)
        import spanningtree.plotting
        spanningtree.plotting.plot(graph)
        raise StandardError('Model infeasible')

def add_best_edge(graph, x):
    edges = graph.edges
    ccs = graph.connected_components
    solution = graph.solution
    (max_i, max_j) = (None, None)
    max_val = None
    min_dist = 10000000
    for (i,j) in edges:
        cc_i = ccs.get_connected_component(i)
        cc_j = ccs.get_connected_component(j)
        #print cc_i, cc_j, i, j
        dist = graph.euclidean_distance(i,j)
        if cc_i != cc_j and x[i,j].X > max_val and dist < min_dist:
            (max_i, max_j) = (i,j)
            max_val = x[i,j].X
            min_dist = dist

    graph.merge_cc(max_i, max_j)
    solution.update(max_i, max_j, True)
    return

def compute_spanning_tree(graph):
    n = graph.n
    stored_lines = graph.lines[:]
    remaining_points = range(0,n)
    #lines = graph.lines
    iterations = 1
    while len(graph.connected_components) > 1:
        solve_lp(graph)
        #print x[2,3].X
        #add_best_edge(graph, x)
        #graph.compute_connected_components()
        #connected_components = graph.connected_components
        #lines = graph.preprocess_lines(remaining_points)
        iterations += 1
    #graph.lines = stored_lines
    return iterations

def main():
    # minimal example to find optimal spanning tree
    import numpy as np
    points = np.array([(2.,2.), (6.,4.), (3., 6.), (5., 7.), (4.25, 5.)])
    import spanningtree.highdimgraph.factories as factories
    graph = factories.create_graph(points, 5, 2, 'custom')
    from spanningtree.highdimgraph.lines import HighDimLine
    l1 = HighDimLine(np.array([(2., 6.), (3., 2.)])) # y = -4x + 14
    l2 = HighDimLine(np.array([(2., 3.), (6., 5.)])) # y = 0.5x + 2
    l3 = HighDimLine(np.array([(3., 5.5), (5., 6.5)])) # y = 0.5x + 4
    lines = [l1, l2, l3]
    graph.lines = lines
    graph.preprocess_lines()
    compute_spanning_tree(graph)
    print "crossing number = %s" % graph.crossing_number()
    import spanningtree.plotting as plotting
    plotting.plot(graph)

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>import requests
import urllib
import csv
import os
import sys
from time import time, sleep

sys.path.append('/Users/connordog/Dropbox/Desktop_Files/Work_Things/CodeBase/Python_Scripts/Python_Projects/packages')
sys.path.append('/Users/connordog/Dropbox/Desktop_Files/Work_Things/maxdalury/sports/general')

from py_data_getter import data_getter
from py_db import db

db = db('NSBL')
getter = data_getter()

base_url = 'http://m.mlb.com/gen/players/prospects/%s/playerProspects.json'
player_base_url = 'http://m.mlb.com/gen/players/prospects/%s/%s.json'
player2_base_url = "http://mlb.com/lookup/json/named.player_info.bam?sport_code='mlb'&player_id=%s"

def initiate():
    start_time = time()

    for year in range(2013,2018):

        url = base_url % year
        json = getter.get_url_data(url, "json")
        prospect_lists = json["prospect_players"]

        scrape_prospects(year, prospect_lists)

    end_time = time()
    elapsed_time = float(end_time - start_time)
    print "time elapsed (in seconds): " + str(elapsed_time)
    print "time elapsed (in minutes): " + str(elapsed_time/60.0)

def scrape_prospects(year, prospect_lists):

    for list_type in prospect_lists:
        entries = []
        if list_type not in ('rule5', 'prospects', 'pdp', 'rhp', 'lhp', 'c', '1b', '2b', '3b', 'ss', 'of'):
        # if list_type in ('draft'):
            print year, list_type
            ind_list = prospect_lists[list_type]

            i = 0
            for player in ind_list:
                entry = {}
                i += 1
                sleep(0) # be nice
                player_id = player['player_id']
                player_url = player_base_url % (year, player_id)
                print '\t'+player_url
                player_json = getter.get_url_data(player_url, "json")

                try:
                    player_info = player_json["prospect_player"]
                except TypeError:
                    print '\tTYPE_ERROR'+str(year)+str(player_id)
                    continue

                entry["year"] = year
                entry["rank"] = i
                entry["player_id"] = player_id
                entry["fname"] = player_info["player_first_name"]
                entry["lname"] = player_info["player_last_name"]
                entry["position"] = player_info["positions"]

                if entry["player_id"] in (669160,'ryan_ryder',):
                    entry["position"] = "RHP"

                if entry["player_id"] in ('taylor,blake',):
                    entry["position"] = "LHP"

                if list_type in ('int','draft'):
                    bats = player_info["bats"]
                    throws = player_info["thrw"]
                    try:
                        height = player_info["height"].replace('\"','').split('\'')
                        height = int(height[0])*12+int(height[1])
                    except (IndexError, ValueError, AttributeError):
                        height = None
                    weight = player_info["weight"]
                else:
                    info_url = player2_base_url % player_id
                    print '\t\t'+info_url
                    try:
                        info_json = getter.get_url_data(info_url, "json")
                        info_info = info_json["player_info"]["queryResults"]["row"]
                        bats = info_info["bats"]
                        throws = info_info["throws"]
                        height = int(info_info["height_feet"])*12+int(info_info["height_inches"])
                        weight = int(info_info["weight"])
                    except UnicodeDecodeError:
                        bats, throws, height, weight = (None, None, None, None)

                entry["bats"] = bats
                entry["throws"] = throws
                entry["height"] = height
                entry["weight"] = weight

                entry["team"] = player['team_file_code']
                drafted = player_info["drafted"]

                if list_type == 'int':
                    drafted = None
                    try:
                        sign_text = player_info["signed"]
                        sign_value = sign_text.split(' - ')[1]
                        signed = sign_value
                    except IndexError:
                        signed = ''
                    try:
                        signed = int(signed.replace('$','').replace(',',''))
                    except ValueError:
                        signed = None
                elif list_type == 'draft':
                    try:
                        signed = player_info["preseason20"].replace(' ','').replace(',','').replace('$','').split('-')[1]
                    except (KeyError, IndexError):
                        signed = player_info["signed"].replace(' ','').replace(',','').replace('$','')
                    try:
                        signed = int(signed)
                    except ValueError:
                        signed = None
                else:
                    signed = player_info["signed"]
                entry["drafted"] = drafted
                entry["signed"] = signed

                if list_type not in ('int', 'draft'):
                    eta = player_info["eta"]
                    try:
                        pre_top100 = player_info["preseason100"]
                    except KeyError:
                        pre_top100 = None
                else:
                    pre_top100 = None
                    eta = None
                entry["pre_top100"] = pre_top100
                entry["eta"] = eta

                entry["twitter"] = player_info["twitter"]

                blurb = player_info["content"]["default"].replace('','').replace('','').replace('
','').replace('
','').replace('
','').replace('*','')
                entry["blurb"] = blurb

                entries.append(entry)

        if list_type == 'draft':
            table = '_draft_prospects'
        elif list_type == 'int':
            table = '_international_prospects'
        else:
            table = '_professional_prospects'

        if entries != []:
            for i in range(0, len(entries), 1000):
                db.insertRowDict(entries[i: i + 1000], table, insertMany=True, replace=True, rid=0,debug=1)
                db.conn.commit()

if __name__ == "__main__":
    initiate()

<commit_msg>bug fix<commit_after>import requests
import urllib
import csv
import os
import sys
from time import time, sleep

sys.path.append('/Users/connordog/Dropbox/Desktop_Files/Work_Things/CodeBase/Python_Scripts/Python_Projects/packages')
sys.path.append('/Users/connordog/Dropbox/Desktop_Files/Work_Things/maxdalury/sports/general')

from py_data_getter import data_getter
from py_db import db

db = db('NSBL')
getter = data_getter()

base_url = 'http://m.mlb.com/gen/players/prospects/%s/playerProspects.json'
player_base_url = 'http://m.mlb.com/gen/players/prospects/%s/%s.json'
player2_base_url = "http://mlb.com/lookup/json/named.player_info.bam?sport_code='mlb'&player_id=%s"

def initiate():
    start_time = time()

    for year in range(2013,2018):

        url = base_url % year
        json = getter.get_url_data(url, "json")
        prospect_lists = json["prospect_players"]

        scrape_prospects(year, prospect_lists)

    end_time = time()
    elapsed_time = float(end_time - start_time)
    print "time elapsed (in seconds): " + str(elapsed_time)
    print "time elapsed (in minutes): " + str(elapsed_time/60.0)

def scrape_prospects(year, prospect_lists):

    for list_type in prospect_lists:
        entries = []
        if list_type not in ('rule5', 'prospects', 'pdp', 'rhp', 'lhp', 'c', '1b', '2b', '3b', 'ss', 'of'):
        # if list_type in ('draft'):
            print year, list_type
            ind_list = prospect_lists[list_type]

            i = 0
            for player in ind_list:
                entry = {}
                i += 1
                sleep(0) # be nice
                player_id = player['player_id']
                player_url = player_base_url % (year, player_id)
                print '\t'+player_url
                player_json = getter.get_url_data(player_url, "json")

                try:
                    player_info = player_json["prospect_player"]
                except TypeError:
                    print '\tTYPE_ERROR'+str(year)+str(player_id)
                    continue

                entry["year"] = year
                entry["rank"] = i
                entry["player_id"] = player_id
                entry["fname"] = player_info["player_first_name"]
                entry["lname"] = player_info["player_last_name"]
                entry["position"] = player_info["positions"]

                if entry["player_id"] in (669160,'ryan_ryder',):
                    entry["position"] = "RHP"

                if entry["player_id"] in ('taylor_blake',):
                    entry["position"] = "LHP"

                if list_type in ('int','draft'):
                    bats = player_info["bats"]
                    throws = player_info["thrw"]
                    try:
                        height = player_info["height"].replace('\"','').split('\'')
                        height = int(height[0])*12+int(height[1])
                    except (IndexError, ValueError, AttributeError):
                        height = None
                    weight = player_info["weight"]
                else:
                    info_url = player2_base_url % player_id
                    print '\t\t'+info_url
                    try:
                        info_json = getter.get_url_data(info_url, "json")
                        info_info = info_json["player_info"]["queryResults"]["row"]
                        bats = info_info["bats"]
                        throws = info_info["throws"]
                        height = int(info_info["height_feet"])*12+int(info_info["height_inches"])
                        weight = int(info_info["weight"])
                    except UnicodeDecodeError:
                        bats, throws, height, weight = (None, None, None, None)

                entry["bats"] = bats
                entry["throws"] = throws
                entry["height"] = height
                entry["weight"] = weight

                entry["team"] = player['team_file_code']
                drafted = player_info["drafted"]

                if list_type == 'int':
                    drafted = None
                    try:
                        sign_text = player_info["signed"]
                        sign_value = sign_text.split(' - ')[1]
                        signed = sign_value
                    except IndexError:
                        signed = ''
                    try:
                        signed = int(signed.replace('$','').replace(',',''))
                    except ValueError:
                        signed = None
                elif list_type == 'draft':
                    try:
                        signed = player_info["preseason20"].replace(' ','').replace(',','').replace('$','').split('-')[1]
                    except (KeyError, IndexError):
                        signed = player_info["signed"].replace(' ','').replace(',','').replace('$','')
                    try:
                        signed = int(signed)
                    except ValueError:
                        signed = None
                else:
                    signed = player_info["signed"]
                entry["drafted"] = drafted
                entry["signed"] = signed

                if list_type not in ('int', 'draft'):
                    eta = player_info["eta"]
                    try:
                        pre_top100 = player_info["preseason100"]
                    except KeyError:
                        pre_top100 = None
                else:
                    pre_top100 = None
                    eta = None
                entry["pre_top100"] = pre_top100
                entry["eta"] = eta

                entry["twitter"] = player_info["twitter"]

                blurb = player_info["content"]["default"].replace('','').replace('','').replace('
','').replace('
','').replace('
','').replace('*','')
                entry["blurb"] = blurb

                entries.append(entry)

        if list_type == 'draft':
            table = '_draft_prospects'
        elif list_type == 'int':
            table = '_international_prospects'
        else:
            table = '_professional_prospects'

        if entries != []:
            for i in range(0, len(entries), 1000):
                db.insertRowDict(entries[i: i + 1000], table, insertMany=True, replace=True, rid=0,debug=1)
                db.conn.commit()

if __name__ == "__main__":
    initiate()

<|endoftext|>


<commit_before>from __future__ import division
from future.utils import with_metaclass
from builtins import super

import random
from abc import ABCMeta, abstractmethod
import logging

import numpy
import rlr

import dedupe.sampling as sampling
import dedupe.core as core
import dedupe.training as training

logger = logging.getLogger(__name__)

class ActiveLearner(with_metaclass(ABCMeta)):

    @abstractmethod
    def transform():
        pass

    @abstractmethod
    def pop():
        pass

    @abstractmethod
    def mark():
        pass

    @abstractmethod
    def __len__(self):
        pass

    def sample_combo(self, data, blocked_proportion, sample_size):
        blocked_sample_size = int(blocked_proportion * sample_size)
        predicates = list(self.data_model.predicates(index_predicates=False))

        data = sampling.randomDeque(data)
        blocked_sample_keys = sampling.dedupeBlockedSample(blocked_sample_size,
                                                           predicates,
                                                           data)

        random_sample_size = sample_size - len(blocked_sample_keys)
        random_sample_keys = set(core.randomPairs(len(data),
                                                  random_sample_size))
        data = dict(data)

        return [(data[k1], data[k2])
                for k1, k2
                in blocked_sample_keys | random_sample_keys]

    def sample_product(self, data_1, data_2, blocked_proportion, sample_size):
        offset = len(data_1)

        blocked_sample_size = int(blocked_proportion * sample_size)
        predicates = list(self.data_model.predicates(index_predicates=False))

        deque_1 = sampling.randomDeque(data_1)
        deque_2 = sampling.randomDeque(data_2)

        blocked_sample_keys = sampling.linkBlockedSample(blocked_sample_size,
                                                         predicates,
                                                         deque_1,
                                                         deque_2)

        random_sample_size = sample_size - len(blocked_sample_keys)
        random_sample_keys = core.randomPairsMatch(len(deque_1),
                                                   len(deque_2),
                                                   random_sample_size)

        random_sample_keys = {(a, b + offset)
                              for a, b in random_sample_keys}

        return [(data_1[k1], data_2[k2])
                for k1, k2
                in blocked_sample_keys | random_sample_keys]

class RLRLearner(ActiveLearner, rlr.RegularizedLogisticRegression):
    def __init__(self, data_model):
        super().__init__(alpha=1)

        self.data_model = data_model

    def transform(self, pairs):
        return self.data_model.distances(pairs)

    def fit(self, X, y):
        self.y = numpy.array(y)
        self.X = X

        super().fit(self.X, self.y, cv=False)

    def fit_transform(self, pairs, y):
        self.fit(self.transform(pairs), y)

    def pop(self):
        if not len(self.candidates):
            raise IndexError("No more unlabeled examples to label")

        target_uncertainty = self._bias()

        probabilities = self.candidate_scores()

        distance_to_target = numpy.abs(target_uncertainty - probabilities)
        uncertain_index = distance_to_target.argmin()

        self.distances = numpy.delete(self.distances, uncertain_index, axis=0)

        uncertain_pair = self.candidates.pop(uncertain_index)

        return [uncertain_pair]

    def remove(self, candidate):
        index = self.candidates.index(candidate)
        self.distances = numpy.delete(self.distances, index, axis=0)
        self.candidates.pop(index)

    def mark(self, pairs, y):

        self.y = numpy.concatenate([self.y, y])
        self.X = numpy.vstack([self.X, self.transform(pairs)])

        self.fit(self.X, self.y)

    def _bias(self):
        positive = numpy.sum(self.y == 1)
        n_examples = len(self.y)

        bias = 1 - (positive / n_examples if positive else 0)

        # When we have just a few examples we are okay with getting
        # examples where the model strongly believes the example is
        # going to be positive or negative. As we get more examples,
        # prefer to ask for labels of examples the model is more
        # uncertain of.
        uncertainty_weight = min(positive, n_examples - positive)
        bias_weight = 10

        weighted_bias = 0.5 * uncertainty_weight + bias * bias_weight
        weighted_bias /= uncertainty_weight + bias_weight

        return weighted_bias

    def candidate_scores(self):
        return self.predict_proba(self.distances)

    def __len__(self):
        return len(self.candidates)

    def _init(self, candidates, *args):
        # we should rethink this and have it happen in the __init__ method
        self.candidates = candidates
        self.distances = self.transform(candidates)
        random_pair = random.choice(self.candidates)
        exact_match = (random_pair[0], random_pair[0])
        self.fit_transform([exact_match, random_pair],
                           [1, 0])

    def sample_combo(self, *args):
        candidates = super().sample_combo(*args)
        self._init(candidates)

    def sample_product(self, *args):
        candidates = super().sample_product(*args)
        self._init(candidates)

class BlockLearner(object):
    def __init__(self, data_model):
        self.block_learner = None
        self.data_model = data_model
        self.current_predicates = ()

        self._cached_labels = None
        self._old_dupes = []

    def fit_transform(self, pairs, y):
        dupes = [pair for label, pair in zip(y, pairs) if label]

        new_dupes = [pair for pair in dupes if pair not in self._old_dupes]
        new_uncovered = (not all(self.predict(new_dupes)))

        if new_uncovered:
            self.current_predicates = self.block_learner.learn(dupes,
                                                               recall=1.0)
            self._cached_labels = None
            self._old_dupes = dupes

    def candidate_scores(self):
        if self._cached_labels is None:
            labels = self.predict(self.candidates)
            self._cached_labels = numpy.array(labels).reshape(-1, 1)

        return self._cached_labels

    def predict(self, candidates):
        labels = []
        for record_1, record_2 in candidates:

            for predicate in self.current_predicates:
                keys = predicate(record_1)
                if keys:
                    if set(predicate(record_2, target=True)) & set(keys):
                        labels.append(1)
                        break
            else:
                labels.append(0)

        return labels

    def remove(self, candidate):
        index = self.candidates.index(candidate)
        self.candidates.pop(index)
        if self._cached_labels is not None:
            self._cached_labels = numpy.delete(self._cached_labels,
                                               index,
                                               axis=0)

    def _init_combo(self, candidates, *args):
        preds = self.data_model.predicates()
        self.block_learner = training.DedupeBlockLearner(preds, *args)
        self.candidates = candidates

        self._freeze_index_predicates(self.candidates)

    def _init_product(self, candidates, *args):
        preds = self.data_model.predicates(canopies=False)
        self.block_learner = training.RecordLinkBlockLearner(preds, *args)
        self.candidates = candidates

        self._freeze_index_predicates(self.candidates)

    def _freeze_index_predicates(self, candidates):

        blocker = self.block_learner.blocker

        records = unique((record for pair in candidates for record in pair))

        for field in blocker.index_fields:
            unique_fields = {record[field] for record in records}
            blocker.index(unique_fields, field)

        for pred in blocker.index_predicates:
            pred.freeze(records)

class DisagreementLearner(ActiveLearner):

    def __init__(self, data_model):
        self.data_model = data_model

        self.classifier = RLRLearner(data_model)
        self.blocker = BlockLearner(data_model)

        self.learners = (self.classifier, self.blocker)
        self.y = numpy.array([])
        self.pairs = []

    def pop(self):
        if not len(self.candidates):
            raise IndexError("No more unlabeled examples to label")

        probs = []
        for learner in self.learners:
            probabilities = learner.candidate_scores()
            probs.append(probabilities)

        probs = numpy.concatenate(probs, axis=1)

        # where do the classifers disagree?
        disagreement = numpy.std(probs > 0.5, axis=1).astype(bool)

        if disagreement.any():
            conflicts = disagreement.nonzero()[0]
            uncertain_index = numpy.random.choice(conflicts, 1)[0]
        else:
            uncertain_index = numpy.std(probs, axis=1).argmax()

        logger.debug("Classifier: %.2f, Covered: %s",
                     probs[uncertain_index][0],
                     bool(probs[uncertain_index][1]))

        uncertain_pair = self.candidates.pop(uncertain_index)

        for learner in self.learners:
            learner.remove(uncertain_pair)

        return [uncertain_pair]

    def mark(self, pairs, y):

        self.y = numpy.concatenate([self.y, y])
        self.pairs.extend(pairs)

        for learner in self.learners:
            learner.fit_transform(self.pairs, self.y)

    def __len__(self):
        return len(self.candidates)

    def sample_combo(self, data, blocked_proportion,
                     sample_size, original_length=None):

        data = core.index(data)

        self.candidates = super().sample_combo(data,
                                               blocked_proportion,
                                               sample_size)

        self.classifier._init(self.candidates)

        index_data = Sample(data, 50000, original_length)
        sampled_records = Sample(index_data, 2000, original_length)

        self.blocker._init_combo(self.candidates,
                                 sampled_records,
                                 index_data)

        return sampled_records

    def sample_product(self, data_1, data_2, blocked_proportion,
                       sample_size, original_length_1=None,
                       original_length_2=None):

        data_1 = core.index(data_1)

        offset = len(data_1)
        data_2 = core.index(data_2, offset)

        self.candidates = super().sample_product(data_1,
                                                 data_2,
                                                 blocked_proportion,
                                                 sample_size)

        self.classifier._init(self.candidates)

        sampled_records_1 = Sample(data_1, 600, original_length_1)

        index_data = Sample(data_2, 50000, original_length_2)
        sampled_records_2 = Sample(index_data, 600, original_length_2)

        self.blocker._init_product(self.candidates,
                                   sampled_records_1,
                                   sampled_records_2,
                                   index_data)

        return sampled_records_1, sampled_records_2

    def transform(self):
        pass

    def learn_predicates(self, recall, index_predicates):
        dupes = [pair for label, pair in zip(self.y, self.pairs) if label]

        if not index_predicates:
            old_preds = self.blocker.block_learner.blocker.predicates.copy()

            no_index_predicates = [pred for pred in old_preds
                                   if not hasattr(pred, 'index')]
            self.blocker.block_learner.blocker.predicates = no_index_predicates

            learned_preds = self.blocker.block_learner.learn(dupes,
                                                             recall=recall)

            self.blocker.block_learner.blocker.predicates = old_preds

        else:
            learned_preds = self.blocker.block_learner.learn(dupes,
                                                             recall=recall)

        return learned_preds

class Sample(dict):

    def __init__(self, d, sample_size, original_length):
        if len(d) <= sample_size:
            super().__init__(d)
        else:
            _keys = tuple(d.keys())
            sample = (random.choice(_keys) for _ in range(sample_size))
            super().__init__({k: d[k] for k in sample})
        if original_length is None:
            self.original_length = len(d)
        else:
            self.original_length = original_length

def unique(seq):
    """Return the unique elements of a collection even if those elements are
       unhashable and unsortable, like dicts and sets"""
    cleaned = []
    for each in seq:
        if each not in cleaned:
            cleaned.append(each)
    return cleaned
<commit_msg>single candidates in active learner<commit_after>from __future__ import division
from future.utils import with_metaclass
from builtins import super

import random
from abc import ABCMeta, abstractmethod
import logging

import numpy
import rlr

import dedupe.sampling as sampling
import dedupe.core as core
import dedupe.training as training

logger = logging.getLogger(__name__)

class ActiveLearner(with_metaclass(ABCMeta)):

    @abstractmethod
    def transform():
        pass

    @abstractmethod
    def pop():
        pass

    @abstractmethod
    def mark():
        pass

    @abstractmethod
    def __len__(self):
        pass

    def sample_combo(self, data, blocked_proportion, sample_size):
        blocked_sample_size = int(blocked_proportion * sample_size)
        predicates = list(self.data_model.predicates(index_predicates=False))

        data = sampling.randomDeque(data)
        blocked_sample_keys = sampling.dedupeBlockedSample(blocked_sample_size,
                                                           predicates,
                                                           data)

        random_sample_size = sample_size - len(blocked_sample_keys)
        random_sample_keys = set(core.randomPairs(len(data),
                                                  random_sample_size))
        data = dict(data)

        return [(data[k1], data[k2])
                for k1, k2
                in blocked_sample_keys | random_sample_keys]

    def sample_product(self, data_1, data_2, blocked_proportion, sample_size):
        offset = len(data_1)

        blocked_sample_size = int(blocked_proportion * sample_size)
        predicates = list(self.data_model.predicates(index_predicates=False))

        deque_1 = sampling.randomDeque(data_1)
        deque_2 = sampling.randomDeque(data_2)

        blocked_sample_keys = sampling.linkBlockedSample(blocked_sample_size,
                                                         predicates,
                                                         deque_1,
                                                         deque_2)

        random_sample_size = sample_size - len(blocked_sample_keys)
        random_sample_keys = core.randomPairsMatch(len(deque_1),
                                                   len(deque_2),
                                                   random_sample_size)

        random_sample_keys = {(a, b + offset)
                              for a, b in random_sample_keys}

        return [(data_1[k1], data_2[k2])
                for k1, k2
                in blocked_sample_keys | random_sample_keys]

class RLRLearner(ActiveLearner, rlr.RegularizedLogisticRegression):
    def __init__(self, data_model):
        super().__init__(alpha=1)

        self.data_model = data_model

    def transform(self, pairs):
        return self.data_model.distances(pairs)

    def fit(self, X, y):
        self.y = numpy.array(y)
        self.X = X

        super().fit(self.X, self.y, cv=False)

    def fit_transform(self, pairs, y):
        self.fit(self.transform(pairs), y)

    def pop(self):
        if not len(self.candidates):
            raise IndexError("No more unlabeled examples to label")

        target_uncertainty = self._bias()

        probabilities = self.candidate_scores()

        distance_to_target = numpy.abs(target_uncertainty - probabilities)
        uncertain_index = distance_to_target.argmin()

        self.distances = numpy.delete(self.distances, uncertain_index, axis=0)

        uncertain_pair = self.candidates.pop(uncertain_index)

        return [uncertain_pair]

    def _remove(self, index):
        self.distances = numpy.delete(self.distances, index, axis=0)

    def mark(self, pairs, y):

        self.y = numpy.concatenate([self.y, y])
        self.X = numpy.vstack([self.X, self.transform(pairs)])

        self.fit(self.X, self.y)

    def _bias(self):
        positive = numpy.sum(self.y == 1)
        n_examples = len(self.y)

        bias = 1 - (positive / n_examples if positive else 0)

        # When we have just a few examples we are okay with getting
        # examples where the model strongly believes the example is
        # going to be positive or negative. As we get more examples,
        # prefer to ask for labels of examples the model is more
        # uncertain of.
        uncertainty_weight = min(positive, n_examples - positive)
        bias_weight = 10

        weighted_bias = 0.5 * uncertainty_weight + bias * bias_weight
        weighted_bias /= uncertainty_weight + bias_weight

        return weighted_bias

    def candidate_scores(self):
        return self.predict_proba(self.distances)

    def __len__(self):
        return len(self.candidates)

    def _init(self, candidates, *args):
        # we should rethink this and have it happen in the __init__ method
        self.candidates = candidates
        self.distances = self.transform(candidates)
        random_pair = random.choice(self.candidates)
        exact_match = (random_pair[0], random_pair[0])
        self.fit_transform([exact_match, random_pair],
                           [1, 0])

    def sample_combo(self, *args):
        candidates = super().sample_combo(*args)
        self._init(candidates)

    def sample_product(self, *args):
        candidates = super().sample_product(*args)
        self._init(candidates)

class BlockLearner(object):
    def __init__(self, data_model):
        self.block_learner = None
        self.data_model = data_model
        self.current_predicates = ()

        self._cached_labels = None
        self._old_dupes = []

    def fit_transform(self, pairs, y):
        dupes = [pair for label, pair in zip(y, pairs) if label]

        new_dupes = [pair for pair in dupes if pair not in self._old_dupes]
        new_uncovered = (not all(self.predict(new_dupes)))

        if new_uncovered:
            self.current_predicates = self.block_learner.learn(dupes,
                                                               recall=1.0)
            self._cached_labels = None
            self._old_dupes = dupes

    def candidate_scores(self):
        if self._cached_labels is None:
            labels = self.predict(self.candidates)
            self._cached_labels = numpy.array(labels).reshape(-1, 1)

        return self._cached_labels

    def predict(self, candidates):
        labels = []
        for record_1, record_2 in candidates:

            for predicate in self.current_predicates:
                keys = predicate(record_1)
                if keys:
                    if set(predicate(record_2, target=True)) & set(keys):
                        labels.append(1)
                        break
            else:
                labels.append(0)

        return labels

    def _remove(self, index):
        if self._cached_labels is not None:
            self._cached_labels = numpy.delete(self._cached_labels,
                                               index,
                                               axis=0)

    def _init_combo(self, candidates, *args):
        preds = self.data_model.predicates()
        self.block_learner = training.DedupeBlockLearner(preds, *args)
        self.candidates = candidates

        self._freeze_index_predicates(self.candidates)

    def _init_product(self, candidates, *args):
        preds = self.data_model.predicates(canopies=False)
        self.block_learner = training.RecordLinkBlockLearner(preds, *args)
        self.candidates = candidates

        self._freeze_index_predicates(self.candidates)

    def _freeze_index_predicates(self, candidates):

        blocker = self.block_learner.blocker

        records = unique((record for pair in candidates for record in pair))

        for field in blocker.index_fields:
            unique_fields = {record[field] for record in records}
            blocker.index(unique_fields, field)

        for pred in blocker.index_predicates:
            pred.freeze(records)

class DisagreementLearner(ActiveLearner):

    def __init__(self, data_model):
        self.data_model = data_model

        self.classifier = RLRLearner(data_model)
        self.blocker = BlockLearner(data_model)

        self.learners = (self.classifier, self.blocker)
        self.y = numpy.array([])
        self.pairs = []

    def pop(self):
        if not len(self.candidates):
            raise IndexError("No more unlabeled examples to label")

        probs = []
        for learner in self.learners:
            probabilities = learner.candidate_scores()
            probs.append(probabilities)

        probs = numpy.concatenate(probs, axis=1)

        # where do the classifers disagree?
        disagreement = numpy.std(probs > 0.5, axis=1).astype(bool)

        if disagreement.any():
            conflicts = disagreement.nonzero()[0]
            uncertain_index = numpy.random.choice(conflicts, 1)[0]
        else:
            uncertain_index = numpy.std(probs, axis=1).argmax()

        logger.debug("Classifier: %.2f, Covered: %s",
                     probs[uncertain_index][0],
                     bool(probs[uncertain_index][1]))

        uncertain_pair = self.candidates.pop(uncertain_index)

        for learner in self.learners:
            learner._remove(uncertain_index)

        return [uncertain_pair]

    def mark(self, pairs, y):

        self.y = numpy.concatenate([self.y, y])
        self.pairs.extend(pairs)

        for learner in self.learners:
            learner.fit_transform(self.pairs, self.y)

    def __len__(self):
        return len(self.candidates)

    def sample_combo(self, data, blocked_proportion,
                     sample_size, original_length=None):

        data = core.index(data)

        self.candidates = super().sample_combo(data,
                                               blocked_proportion,
                                               sample_size)

        self.classifier._init(self.candidates)

        index_data = Sample(data, 50000, original_length)
        sampled_records = Sample(index_data, 2000, original_length)

        self.blocker._init_combo(self.candidates,
                                 sampled_records,
                                 index_data)

        return sampled_records

    def sample_product(self, data_1, data_2, blocked_proportion,
                       sample_size, original_length_1=None,
                       original_length_2=None):

        data_1 = core.index(data_1)

        offset = len(data_1)
        data_2 = core.index(data_2, offset)

        self.candidates = super().sample_product(data_1,
                                                 data_2,
                                                 blocked_proportion,
                                                 sample_size)

        self.classifier._init(self.candidates)

        sampled_records_1 = Sample(data_1, 600, original_length_1)

        index_data = Sample(data_2, 50000, original_length_2)
        sampled_records_2 = Sample(index_data, 600, original_length_2)

        self.blocker._init_product(self.candidates,
                                   sampled_records_1,
                                   sampled_records_2,
                                   index_data)

        return sampled_records_1, sampled_records_2

    def transform(self):
        pass

    def learn_predicates(self, recall, index_predicates):
        dupes = [pair for label, pair in zip(self.y, self.pairs) if label]

        if not index_predicates:
            old_preds = self.blocker.block_learner.blocker.predicates.copy()

            no_index_predicates = [pred for pred in old_preds
                                   if not hasattr(pred, 'index')]
            self.blocker.block_learner.blocker.predicates = no_index_predicates

            learned_preds = self.blocker.block_learner.learn(dupes,
                                                             recall=recall)

            self.blocker.block_learner.blocker.predicates = old_preds

        else:
            learned_preds = self.blocker.block_learner.learn(dupes,
                                                             recall=recall)

        return learned_preds

class Sample(dict):

    def __init__(self, d, sample_size, original_length):
        if len(d) <= sample_size:
            super().__init__(d)
        else:
            _keys = tuple(d.keys())
            sample = (random.choice(_keys) for _ in range(sample_size))
            super().__init__({k: d[k] for k in sample})
        if original_length is None:
            self.original_length = len(d)
        else:
            self.original_length = original_length

def unique(seq):
    """Return the unique elements of a collection even if those elements are
       unhashable and unsortable, like dicts and sets"""
    cleaned = []
    for each in seq:
        if each not in cleaned:
            cleaned.append(each)
    return cleaned
<|endoftext|>


<commit_before>from __future__ import division
import numpy as np
import math
import sys

# Calculate the entropy of label array y
def calculate_entropy(y):
    log2 = lambda x: math.log(x) / math.log(2)
    unique_labels = np.unique(y)
    entropy = 0
    for label in unique_labels:
        count = len(y[y == label])
        p = count / len(y)
        entropy += -p * log2(p)
    return entropy

# Returns the mean squared error between y_true and y_pred
def mean_squared_error(y_true, y_pred):
    mse = np.mean(np.power(y_true - y_pred, 2))
    return mse

# Return the variance of the features in dataset X
def calculate_variance(X):
    mean = np.ones(np.shape(X)) * X.mean(0)
    n_samples = np.shape(X)[0]
    variance = (1 / n_samples) * np.diag((X - mean).T.dot(X - mean))

    return variance

# Calculate the standard deviations of the features in dataset X
def calculate_std_dev(X):
    std_dev = np.sqrt(calculate_variance(X))

    return std_dev

# Calculates the l2 distance between two vectors
def euclidean_distance(x1, x2):
    distance = 0
    # Squared distance between each coordinate
    for i in range(len(x1)):
        distance += pow((x1[i] - x2[i]), 2)

    return math.sqrt(distance)

# Compare y_true to y_pred and return the accuracy
def accuracy_score(y_true, y_pred):
    accuracy = np.sum(y_true == y_pred) / len(y_true)
    return accuracy

# Calculate the covariance matrix for the dataset X
def calculate_covariance_matrix(X, Y=np.empty([(0,0)])):
    if not Y.any():
        Y = X
    n_samples = np.shape(X)[0]
    covariance_matrix = (1 / (n_samples-1)) * (X - X.mean(0)).T.dot(Y - Y.mean(0))

    return np.array(covariance_matrix, dtype=float)

# Calculate the correlation matrix for the dataset X
def calculate_correlation_matrix(X, Y=np.empty((0,0))):
    if not Y.any():
        Y = X
    n_samples = np.shape(X)[0]
    covariance = (1 / n_samples) * (X - X.mean(0)).T.dot(Y - Y.mean(0))
    std_dev_X = np.expand_dims(calculate_std_dev(X), 1)
    std_dev_y = np.expand_dims(calculate_std_dev(Y), 1)
    correlation_matrix = np.divide(covariance, std_dev_X.dot(std_dev_y.T))

    return np.array(correlation_matrix, dtype=float)
<commit_msg>Logistic Regression fix<commit_after>from __future__ import division
import numpy as np
import math
import sys

# Calculate the entropy of label array y
def calculate_entropy(y):
    log2 = lambda x: math.log(x) / math.log(2)
    unique_labels = np.unique(y)
    entropy = 0
    for label in unique_labels:
        count = len(y[y == label])
        p = count / len(y)
        entropy += -p * log2(p)
    return entropy

# Returns the mean squared error between y_true and y_pred
def mean_squared_error(y_true, y_pred):
    mse = np.mean(np.power(y_true - y_pred, 2))
    return mse

# Return the variance of the features in dataset X
def calculate_variance(X):
    mean = np.ones(np.shape(X)) * X.mean(0)
    n_samples = np.shape(X)[0]
    variance = (1 / n_samples) * np.diag((X - mean).T.dot(X - mean))

    return variance

# Calculate the standard deviations of the features in dataset X
def calculate_std_dev(X):
    std_dev = np.sqrt(calculate_variance(X))

    return std_dev

# Calculates the l2 distance between two vectors
def euclidean_distance(x1, x2):
    distance = 0
    # Squared distance between each coordinate
    for i in range(len(x1)):
        distance += pow((x1[i] - x2[i]), 2)

    return math.sqrt(distance)

# Compare y_true to y_pred and return the accuracy
def accuracy_score(y_true, y_pred):
    accuracy = np.sum(y_true == y_pred) / len(y_true)
    return accuracy

# Calculate the covariance matrix for the dataset X
def calculate_covariance_matrix(X, Y=np.empty([0])):
    if not Y.any():
        Y = X
    n_samples = np.shape(X)[0]
    covariance_matrix = (1 / (n_samples-1)) * (X - X.mean(axis=0)).T.dot(Y - Y.mean(axis=0))

    return np.array(covariance_matrix, dtype=float)

# Calculate the correlation matrix for the dataset X
def calculate_correlation_matrix(X, Y=np.empty([0])):
    if not Y.any():
        Y = X
    n_samples = np.shape(X)[0]
    covariance = (1 / n_samples) * (X - X.mean(0)).T.dot(Y - Y.mean(0))
    std_dev_X = np.expand_dims(calculate_std_dev(X), 1)
    std_dev_y = np.expand_dims(calculate_std_dev(Y), 1)
    correlation_matrix = np.divide(covariance, std_dev_X.dot(std_dev_y.T))

    return np.array(correlation_matrix, dtype=float)
<|endoftext|>