<commit_before>"""Implement a peekable queue for gevent 0.13"""
from __future__ import absolute_import

import sys

from gevent.timeout import Timeout
from gevent.hub import Waiter, _NONE
from gevent.queue import (
    Empty,
    Full,
    ItemWaiter,
    Queue
)

# No peek method on queue in 0.13, so add one from gevent 1.0
class PeekableQueue(Queue):
    def put(self, item, block=True, timeout=None):
        """Put an item into the queue.

        If optional arg *block* is true and *timeout* is ``None`` (the default),
        block if necessary until a free slot is available. If *timeout* is
        a positive number, it blocks at most *timeout* seconds and raises
        the :class:`Full` exception if no free slot was available within that time.
        Otherwise (*block* is false), put an item on the queue if a free slot
        is immediately available, else raise the :class:`Full` exception (*timeout*
        is ignored in that case).
        """
        if self.maxsize is None or self.qsize() < self.maxsize:
            # there's a free slot, put an item right away
            self._put(item)
            if self.getters:
                self._schedule_unlock()
        elif block:
            waiter = ItemWaiter(item)
            self.putters.add(waiter)
            timeout = Timeout.start_new(timeout, Full)
            try:
                if self.getters:
                    self._schedule_unlock()
                result = waiter.get()
                assert result is waiter, "Invalid switch into Queue.put: %r" % (result, )
                if waiter.item is not _NONE:
                    self._put(item)
            finally:
                timeout.cancel()
                self.putters.discard(waiter)
        else:
            raise Full

    def get(self, block=True, timeout=None):
        """Remove and return an item from the queue.

        If optional args *block* is true and *timeout* is ``None`` (the default),
        block if necessary until an item is available. If *timeout* is a positive number,
        it blocks at most *timeout* seconds and raises the :class:`Empty` exception
        if no item was available within that time. Otherwise (*block* is false), return
        an item if one is immediately available, else raise the :class:`Empty` exception
        (*timeout* is ignored in that case).
        """
        if self.qsize():
            if self.putters:
                self._schedule_unlock()
            return self._get()
        elif block:
            waiter = Waiter()
            timeout = Timeout.start_new(timeout, Empty)
            try:
                self.getters.add(waiter)
                if self.putters:
                    self._schedule_unlock()
                result = waiter.get()
                assert result is waiter, 'Invalid switch into Queue.get: %r' % (result, )
                return self._get()
            finally:
                self.getters.discard(waiter)
                timeout.cancel()
        else:
            raise Empty

    def _peek(self):
        return self.queue[0]

    def peek(self, block=True, timeout=None):
        if self.qsize():
            if self.putters:
                self._schedule_unlock()
            return self._peek()
        elif block:
            waiter = Waiter()
            timeout = Timeout.start_new(timeout, Empty)
            try:
                self.getters.add(waiter)
                if self.putters:
                    self._schedule_unlock()
                result = waiter.get()
                assert result is waiter, "Invalid switch into Queue.put: %r" % (result, )
                return self._peek()
            finally:
                self.getters.discard(waiter)
                timeout.cancel()
        else:
            raise Empty

    def peek_nowait(self):
        return self.peek(False)

    def _unlock(self):
        while True:
            repeat = False
            if self.putters and (self.maxsize is None or self.qsize() < self.maxsize):
                repeat = True
                try:
                    putter = self.putters.pop()
                    self._put(putter.item)
                except:
                    putter.throw(*sys.exc_info())
                else:
                    putter.switch(putter)
            if self.getters and self.qsize():
                repeat = True
                getter = self.getters.pop()
                getter.switch(getter)
            if not repeat:
                return
<commit_msg>Retain full fledge peekable queue functionality. Don't add it to our coverage.<commit_after>"""Implement a peekable queue for gevent 0.13"""
from __future__ import absolute_import

import sys

from gevent.timeout import Timeout
from gevent.hub import get_hub, Waiter, getcurrent, _NONE
from gevent.queue import (
    Empty,
    Full,
    ItemWaiter,
    Queue
)

# No peek method on queue in 0.13, so add one from gevent 1.0
class PeekableQueue(Queue):  # pragma: nocover
    def put(self, item, block=True, timeout=None):
        """Put an item into the queue.

        If optional arg *block* is true and *timeout* is ``None`` (the default),
        block if necessary until a free slot is available. If *timeout* is
        a positive number, it blocks at most *timeout* seconds and raises
        the :class:`Full` exception if no free slot was available within that time.
        Otherwise (*block* is false), put an item on the queue if a free slot
        is immediately available, else raise the :class:`Full` exception (*timeout*
        is ignored in that case).
        """
        if self.maxsize is None or self.qsize() < self.maxsize:
            # there's a free slot, put an item right away
            self._put(item)
            if self.getters:
                self._schedule_unlock()
        elif not block and get_hub() is getcurrent():
            # we're in the mainloop, so we cannot wait; we can switch() to other greenlets though
            # find a getter and deliver an item to it

            while self.getters and self.qsize() and self.qsize() >= self.maxsize:
                getter = self.getters.pop()
                getter.switch(getter)
            if self.qsize() < self.maxsize:
                self._put(item)
                return
            raise Full
        elif block:
            waiter = ItemWaiter(item)
            self.putters.add(waiter)
            timeout = Timeout.start_new(timeout, Full)
            try:
                if self.getters:
                    self._schedule_unlock()
                result = waiter.get()
                assert result is waiter, "Invalid switch into Queue.put: %r" % (result, )
                if waiter.item is not _NONE:
                    self._put(item)
            finally:
                timeout.cancel()
                self.putters.discard(waiter)
        else:
            raise Full

    def get(self, block=True, timeout=None):
        """Remove and return an item from the queue.

        If optional args *block* is true and *timeout* is ``None`` (the default),
        block if necessary until an item is available. If *timeout* is a positive number,
        it blocks at most *timeout* seconds and raises the :class:`Empty` exception
        if no item was available within that time. Otherwise (*block* is false), return
        an item if one is immediately available, else raise the :class:`Empty` exception
        (*timeout* is ignored in that case).
        """
        if self.qsize():
            if self.putters:
                self._schedule_unlock()
            return self._get()
        elif not block and get_hub() is getcurrent():
            # special case to make get_nowait() runnable in the mainloop greenlet
            # there are no items in the queue; try to fix the situation by unlocking putters
            while self.putters:
                putter = self.putters.pop()
                if putter:
                    putter.switch(putter)
                    if self.qsize():
                        return self._get()
            raise Empty
        elif block:
            waiter = Waiter()
            timeout = Timeout.start_new(timeout, Empty)
            try:
                self.getters.add(waiter)
                if self.putters:
                    self._schedule_unlock()
                result = waiter.get()
                assert result is waiter, 'Invalid switch into Queue.get: %r' % (result, )
                return self._get()
            finally:
                self.getters.discard(waiter)
                timeout.cancel()
        else:
            raise Empty

    def _peek(self):
        return self.queue[0]

    def peek(self, block=True, timeout=None):
        if self.qsize():
            if self.putters:
                self._schedule_unlock()
            return self._peek()
        elif not block and get_hub() is getcurrent():
            # special case to make peek(False) runnable in the mainloop
            # greenlet there are no items in the queue; try to fix the
            # situation by unlocking putters
            while self.putters:
                putter = self.putters.pop()
                if putter:
                    putter.switch(putter)
                    if self.qsize():
                        return self._peek()
            raise Empty
        elif block:
            waiter = Waiter()
            timeout = Timeout.start_new(timeout, Empty)
            try:
                self.getters.add(waiter)
                if self.putters:
                    self._schedule_unlock()
                result = waiter.get()
                assert result is waiter, "Invalid switch into Queue.put: %r" % (result, )
                return self._peek()
            finally:
                self.getters.discard(waiter)
                timeout.cancel()
        else:
            raise Empty

    def peek_nowait(self):
        return self.peek(False)

    def _unlock(self):
        while True:
            repeat = False
            if self.putters and (self.maxsize is None or self.qsize() < self.maxsize):
                repeat = True
                try:
                    putter = self.putters.pop()
                    self._put(putter.item)
                except:
                    putter.throw(*sys.exc_info())
                else:
                    putter.switch(putter)
            if self.getters and self.qsize():
                repeat = True
                getter = self.getters.pop()
                getter.switch(getter)
            if not repeat:
                return
<|endoftext|>


<commit_before>import inspect
import warnings

def get_signature_start(function, clean_module_name):
    """For the Dense layer, it should return the string 'keras.layers.Dense'"""
    try:
        function_module = function.__module__
    except AttributeError:
        warnings.warn(f'function {function} has no module. '
                      f'It will not be included in the signature.')
        function_module = ''
    else:
        function_module = f'{clean_module_name(function_module)}.'
    return f'{function_module}{function.__name__}'

def get_signature_end(args, kwargs):
    kwargs_formatted = []
    for a, v in kwargs:
        if isinstance(v, str):
            v = f"'{v}'"
        kwargs_formatted.append(f'{a}={v}')

    all_args_str = map(str, args + kwargs_formatted)
    all_args_str = ', '.join(all_args_str)
    return f'({all_args_str})'

def get_function_signature(
    function, clean_module_name, post_process_signature, method=True
):
    original_function = getattr(function, "_original_function", function)
    signature = inspect.getfullargspec(original_function)
    args = list(signature.args)
    if method and args and args[0] == 'self':
        args.pop(0)
    defaults = signature.defaults
    if defaults:
        kwargs = zip(args[-len(defaults):], defaults)
        args = args[: -len(defaults)]
    else:
        kwargs = []

    signature_start = get_signature_start(function, clean_module_name)
    signature_end = get_signature_end(args, kwargs)
    return post_process_signature(signature_start + signature_end)

def get_class_signature(cls, clean_module_name, post_process_signature):
    try:
        init_method = cls.__init__
    except (TypeError, AttributeError):
        # in case the class inherits from object and does not
        # define __init__
        class_signature = f"{cls.__module__}.{cls.__name__}()"
    else:
        class_signature = get_function_signature(
            init_method, clean_module_name, post_process_signature
        )
        class_signature = class_signature.replace("__init__", cls.__name__)
    return post_process_signature(class_signature)
<commit_msg>Some simplification on the computing of args. (#10)<commit_after>import inspect
import warnings

def get_signature_start(function, clean_module_name):
    """For the Dense layer, it should return the string 'keras.layers.Dense'"""
    try:
        function_module = function.__module__
    except AttributeError:
        warnings.warn(f'function {function} has no module. '
                      f'It will not be included in the signature.')
        function_module = ''
    else:
        function_module = f'{clean_module_name(function_module)}.'
    return f'{function_module}{function.__name__}'

def get_signature_end(args, kwargs):
    kwargs_formatted = []
    for a, v in kwargs:
        if isinstance(v, str):
            v = f"'{v}'"
        kwargs_formatted.append(f'{a}={v}')

    all_args_str = map(str, args + kwargs_formatted)
    all_args_str = ', '.join(all_args_str)
    return f'({all_args_str})'

def get_function_signature(
    function, clean_module_name, post_process_signature, method=True
):
    original_function = getattr(function, "_original_function", function)
    signature = inspect.getfullargspec(original_function)
    args = list(signature.args)
    if method and args and args[0] == 'self':
        args.pop(0)
    defaults = signature.defaults or []
    kwargs = zip(args[-len(defaults):], defaults)
    args = args[: len(args)-len(defaults)]

    signature_start = get_signature_start(function, clean_module_name)
    signature_end = get_signature_end(args, kwargs)
    return post_process_signature(signature_start + signature_end)

def get_class_signature(cls, clean_module_name, post_process_signature):
    try:
        init_method = cls.__init__
    except (TypeError, AttributeError):
        # in case the class inherits from object and does not
        # define __init__
        class_signature = f"{cls.__module__}.{cls.__name__}()"
    else:
        class_signature = get_function_signature(
            init_method, clean_module_name, post_process_signature
        )
        class_signature = class_signature.replace("__init__", cls.__name__)
    return post_process_signature(class_signature)
<|endoftext|>


<commit_before>"""
Make sure we can find the binary inside an app bundle.
"""

from __future__ import print_function

import os
import time
import re
import lldb
from lldbsuite.test.decorators import *
import lldbsuite.test.lldbutil as lldbutil
import lldbsuite.test.lldbplatformutil as lldbplatformutil
from lldbsuite.test.lldbtest import *

class FindAppInMacOSAppBundle(TestBase):

    mydir = TestBase.compute_mydir(__file__)

    NO_DEBUG_INFO_TESTCASE = True

    def test_find_app_in_bundle(self):
        """There can be many tests in a test case - describe this test here."""
        self.build()
        self.main_source_file = lldb.SBFileSpec("main.c")
        self.find_app_in_bundle_test()

    def setUp(self):
        # Call super's setUp().
        TestBase.setUp(self)

    def find_app_in_bundle_test(self):
        """This reads in the .app, makes sure we get the right binary and can run it."""

        # This function starts a process, "a.out" by default, sets a source
        # breakpoint, runs to it, and returns the thread, process & target.
        # It optionally takes an SBLaunchOption argument if you want to pass
        # arguments or environment variables.
        exe = os.path.join(os.getcwd(), "TestApp.app")
        target = self.dbg.CreateTarget(exe)
        self.assertTrue(target, "Target: TestApp.app is not valid.")
        exe_module_spec = target.GetExecutable()
        self.assertTrue(exe_module_spec.GetFilename(), "TestApp")

        bkpt = target.BreakpointCreateBySourceRegex("Set a breakpoint here", self.main_source_file)
        self.assertTrue(bkpt.GetNumLocations() == 1, "Couldn't set a breakpoint in the main app")

        if lldbplatformutil.getPlatform() == "macosx":
            launch_info = lldb.SBLaunchInfo(None)
            launch_info.SetWorkingDirectory(self.get_process_working_directory())

            error = lldb.SBError()
            process = target.Launch(launch_info, error)

            self.assertTrue(process, "Could not create a valid process for TestApp: %s"%(error.GetCString()))

            # Frame #0 should be at our breakpoint.
            threads = lldbutil.get_threads_stopped_at_breakpoint(process, bkpt)

            self.assertTrue(len(threads) == 1, "Expected 1 thread to stop at breakpoint, %d did."%(len(threads)))

<commit_msg>Running this on other systems won't work because I don't know how to specifically build a MachO binary on other systems.<commit_after>"""
Make sure we can find the binary inside an app bundle.
"""

from __future__ import print_function

import os
import time
import re
import lldb
from lldbsuite.test.decorators import *
import lldbsuite.test.lldbutil as lldbutil
import lldbsuite.test.lldbplatformutil as lldbplatformutil
from lldbsuite.test.lldbtest import *

@decorators.skipUnlessDarwin
class FindAppInMacOSAppBundle(TestBase):

    mydir = TestBase.compute_mydir(__file__)

    NO_DEBUG_INFO_TESTCASE = True

    def test_find_app_in_bundle(self):
        """There can be many tests in a test case - describe this test here."""
        self.build()
        self.main_source_file = lldb.SBFileSpec("main.c")
        self.find_app_in_bundle_test()

    def setUp(self):
        # Call super's setUp().
        TestBase.setUp(self)

    def find_app_in_bundle_test(self):
        """This reads in the .app, makes sure we get the right binary and can run it."""

        # This function starts a process, "a.out" by default, sets a source
        # breakpoint, runs to it, and returns the thread, process & target.
        # It optionally takes an SBLaunchOption argument if you want to pass
        # arguments or environment variables.
        exe = os.path.join(os.getcwd(), "TestApp.app")
        target = self.dbg.CreateTarget(exe)
        self.assertTrue(target, "Target: TestApp.app is not valid.")
        exe_module_spec = target.GetExecutable()
        self.assertTrue(exe_module_spec.GetFilename(), "TestApp")

        bkpt = target.BreakpointCreateBySourceRegex("Set a breakpoint here", self.main_source_file)
        self.assertTrue(bkpt.GetNumLocations() == 1, "Couldn't set a breakpoint in the main app")

        if lldbplatformutil.getPlatform() == "macosx":
            launch_info = lldb.SBLaunchInfo(None)
            launch_info.SetWorkingDirectory(self.get_process_working_directory())

            error = lldb.SBError()
            process = target.Launch(launch_info, error)

            self.assertTrue(process, "Could not create a valid process for TestApp: %s"%(error.GetCString()))

            # Frame #0 should be at our breakpoint.
            threads = lldbutil.get_threads_stopped_at_breakpoint(process, bkpt)

            self.assertTrue(len(threads) == 1, "Expected 1 thread to stop at breakpoint, %d did."%(len(threads)))

<|endoftext|>


<commit_before># encoding: utf-8

###########################################################################################################
#
#
#	Reporter Plugin
#
#	Read the docs:
#	https://github.com/schriftgestalt/GlyphsSDK/tree/master/Python%20Templates/Reporter
#
#
###########################################################################################################

from GlyphsApp import GSControlLayer, subtractPoints
from GlyphsApp.plugins import *
import math

class ShowMarkPreview(ReporterPlugin):
	categoriesOnWhichToDrawAccents = ("Letter","Number","Punctuation")

	def transform(self, shiftX=0.0, shiftY=0.0, rotate=0.0, skew=0.0, scale=1.0):
		"""
		Returns an NSAffineTransform object for transforming layers.
		Apply an NSAffineTransform t object like this:
			Layer.transform_checkForSelection_doComponents_(t,False,True)
		Access its transformation matrix like this:
			tMatrix = t.transformStruct() # returns the 6-float tuple
		Apply the matrix tuple like this:
			Layer.applyTransform(tMatrix)
			Component.applyTransform(tMatrix)
			Path.applyTransform(tMatrix)
		Chain multiple NSAffineTransform objects t1, t2 like this:
			t1.appendTransform_(t2)
		"""
		myTransform = NSAffineTransform.transform()
		if rotate:
			myTransform.rotateByDegrees_(rotate)
		if scale != 1.0:
			myTransform.scaleBy_(scale)
		if not (shiftX == 0.0 and shiftY == 0.0):
			myTransform.translateXBy_yBy_(shiftX,shiftY)
		if skew:
			skewStruct = NSAffineTransformStruct()
			skewStruct.m11 = 1.0
			skewStruct.m22 = 1.0
			skewStruct.m21 = math.tan(math.radians(skew))
			skewTransform = NSAffineTransform.transform()
			skewTransform.setTransformStruct_(skewStruct)
			myTransform.appendTransform_(skewTransform)
		return myTransform

	def settings(self):
		self.menuName = Glyphs.localize({
			'en': u'Mark Preview',
			'de': u'Akzent-Vorschau',
			'es': u'previsualización de acentos',
		})

	def drawMarksOnLayer(self, layer, lineOfLayers, offset=NSPoint(0,0)):
		# draw only in letters:
		if layer.glyph().category in self.categoriesOnWhichToDrawAccents:
			anchorDict = {}
			for thisAnchor in layer.anchorsTraversingComponents():
				anchorDict[thisAnchor.name] = thisAnchor.position

			# continue if there are any anchors in the layer:
			if anchorDict:

				# continue if there is an Edit tab open:
				if lineOfLayers:
					marks = []
					for thisLayer in lineOfLayers:
						if thisLayer.glyph() and thisLayer.glyph().category == "Mark":
							marks.append( thisLayer )

					# continue if there are any marks in the Edit tab:
					if marks:
						for thisMark in marks:
							attachingAnchorNames = [a for a in thisMark.anchorNamesTraversingComponents() if a.startswith("_")]
							stackingAnchorNames = [a for a in thisMark.anchorNamesTraversingComponents() if not a.startswith("_")]
							if attachingAnchorNames:
								attachingAnchor = thisMark.anchorForName_traverseComponents_(attachingAnchorNames[0],True)
								relatedStackingAnchorName = attachingAnchor.name[1:]
								letterAnchor = anchorDict[relatedStackingAnchorName]

								if letterAnchor:
									# shift and draw bezier path:
									scale = self.getScale()
									shiftX = letterAnchor.x - attachingAnchor.x + offset.x / scale
									shiftY = letterAnchor.y - attachingAnchor.y + offset.y / scale
									displayShift = self.transform(shiftX=shiftX, shiftY=shiftY)
									displayMark = thisMark.completeBezierPath.copy()
									displayMark.transformUsingAffineTransform_(displayShift)
									displayMark.fill()

									# shift and store next anchor position (if exists)
									for stackingAnchorName in stackingAnchorNames:
										stackingAnchor = thisMark.anchorForName_traverseComponents_(stackingAnchorName,True)
										nextAnchorX = stackingAnchor.x + shiftX - offset.x / scale
										nextAnchorY = stackingAnchor.y + shiftY - offset.y / scale
										anchorDict[stackingAnchorName] = NSPoint( nextAnchorX, nextAnchorY )

	def foreground(self, layer):
		# define drawing colors
		activeColor = NSColor.colorWithRed_green_blue_alpha_(0.3, 0.0, 0.6, 0.4)
		inactiveColor = NSColor.colorWithRed_green_blue_alpha_(0.15, 0.05, 0.3, 0.5)

		# go through tab content
		glyph = layer.glyph()
		font = layer.font()
		if font: # sometimes font is empty, don't know why
			tabView = font.currentTab.graphicView()
			layerCount = tabView.cachedLayerCount()
			lineOfLayers = []
			lineOfOffsets = []

			# collect layers in the same line (to only draw the marks we need)
			for i in range(layerCount):
				thisLayer = tabView.cachedGlyphAtIndex_(i)

				# collect layers and their offsets except newlines
				if type(thisLayer) != GSControlLayer:
					lineOfLayers.append( thisLayer )
					lineOfOffsets.append( tabView.cachedPositionAtIndex_(i) )

				# if we reach end of line or end of text, draw with collected layers:
				if type(thisLayer) == GSControlLayer or i==layerCount-1:

					# step through all layers of the line:
					for j, thisLayerInLine in enumerate(lineOfLayers):

						# draw accents on them if they are Letter/Number/Punctuation
						if thisLayerInLine.parent.category in self.categoriesOnWhichToDrawAccents:
							activePosition = tabView.activePosition()
							lastLayerInLinePosition = tabView.cachedPositionAtIndex_(i)
							thisLayerInLinePosition = lineOfOffsets[j]
							offset = subtractPoints(thisLayerInLinePosition, activePosition)
							if offset == NSPoint(0,0):
								activeColor.set()
								self.drawMarksOnLayer(thisLayerInLine, lineOfLayers)
							else:
								inactiveColor.set()
								self.drawMarksOnLayer(thisLayerInLine, lineOfLayers, offset)

					# reset layer collection
					lineOfLayers = []
					lineOfOffsets = []

	def needsExtraMainOutlineDrawingForInactiveLayer_(self, layer):
		return True

	def shouldDrawAccentCloudForLayer_(self, layer):
		return False
	<commit_msg>Avoid noneType error<commit_after># encoding: utf-8

###########################################################################################################
#
#
#	Reporter Plugin
#
#	Read the docs:
#	https://github.com/schriftgestalt/GlyphsSDK/tree/master/Python%20Templates/Reporter
#
#
###########################################################################################################

from GlyphsApp import GSControlLayer, subtractPoints
from GlyphsApp.plugins import *
import math

class ShowMarkPreview(ReporterPlugin):
	categoriesOnWhichToDrawAccents = ("Letter","Number","Punctuation")

	def transform(self, shiftX=0.0, shiftY=0.0, rotate=0.0, skew=0.0, scale=1.0):
		"""
		Returns an NSAffineTransform object for transforming layers.
		Apply an NSAffineTransform t object like this:
			Layer.transform_checkForSelection_doComponents_(t,False,True)
		Access its transformation matrix like this:
			tMatrix = t.transformStruct() # returns the 6-float tuple
		Apply the matrix tuple like this:
			Layer.applyTransform(tMatrix)
			Component.applyTransform(tMatrix)
			Path.applyTransform(tMatrix)
		Chain multiple NSAffineTransform objects t1, t2 like this:
			t1.appendTransform_(t2)
		"""
		myTransform = NSAffineTransform.transform()
		if rotate:
			myTransform.rotateByDegrees_(rotate)
		if scale != 1.0:
			myTransform.scaleBy_(scale)
		if not (shiftX == 0.0 and shiftY == 0.0):
			myTransform.translateXBy_yBy_(shiftX,shiftY)
		if skew:
			skewStruct = NSAffineTransformStruct()
			skewStruct.m11 = 1.0
			skewStruct.m22 = 1.0
			skewStruct.m21 = math.tan(math.radians(skew))
			skewTransform = NSAffineTransform.transform()
			skewTransform.setTransformStruct_(skewStruct)
			myTransform.appendTransform_(skewTransform)
		return myTransform

	def settings(self):
		self.menuName = Glyphs.localize({
			'en': u'Mark Preview',
			'de': u'Akzent-Vorschau',
			'es': u'previsualización de acentos',
		})

	def drawMarksOnLayer(self, layer, lineOfLayers, offset=NSPoint(0,0)):
		# draw only in letters:
		if layer.glyph().category in self.categoriesOnWhichToDrawAccents:
			anchorDict = {}
			for thisAnchor in layer.anchorsTraversingComponents():
				anchorDict[thisAnchor.name] = thisAnchor.position

			# continue if there are any anchors in the layer:
			if anchorDict:

				# continue if there is an Edit tab open:
				if lineOfLayers:
					marks = []
					for thisLayer in lineOfLayers:
						if thisLayer.glyph() and thisLayer.glyph().category == "Mark":
							marks.append( thisLayer )

					# continue if there are any marks in the Edit tab:
					if marks:
						for thisMark in marks:
							attachingAnchorNames = [a for a in thisMark.anchorNamesTraversingComponents() if a.startswith("_")]
							stackingAnchorNames = [a for a in thisMark.anchorNamesTraversingComponents() if not a.startswith("_")]
							if attachingAnchorNames:
								attachingAnchor = thisMark.anchorForName_traverseComponents_(attachingAnchorNames[0],True)
								relatedStackingAnchorName = attachingAnchor.name[1:]
								letterAnchor = anchorDict[relatedStackingAnchorName]

								if letterAnchor:
									# shift and draw bezier path:
									scale = self.getScale()
									shiftX = letterAnchor.x - attachingAnchor.x + offset.x / scale
									shiftY = letterAnchor.y - attachingAnchor.y + offset.y / scale
									displayShift = self.transform(shiftX=shiftX, shiftY=shiftY)
									displayMark = thisMark.completeBezierPath.copy()
									displayMark.transformUsingAffineTransform_(displayShift)
									displayMark.fill()

									# shift and store next anchor position (if exists)
									for stackingAnchorName in stackingAnchorNames:
										if stackingAnchor:
											stackingAnchor = thisMark.anchorForName_traverseComponents_(stackingAnchorName,True)
											nextAnchorX = stackingAnchor.x + shiftX - offset.x / scale
											nextAnchorY = stackingAnchor.y + shiftY - offset.y / scale
											anchorDict[stackingAnchorName] = NSPoint( nextAnchorX, nextAnchorY )

	def foreground(self, layer):
		# define drawing colors
		activeColor = NSColor.colorWithRed_green_blue_alpha_(0.3, 0.0, 0.6, 0.4)
		inactiveColor = NSColor.colorWithRed_green_blue_alpha_(0.15, 0.05, 0.3, 0.5)

		# go through tab content
		glyph = layer.glyph()
		font = layer.font()
		if font: # sometimes font is empty, don't know why
			tabView = font.currentTab.graphicView()
			layerCount = tabView.cachedLayerCount()
			lineOfLayers = []
			lineOfOffsets = []

			# collect layers in the same line (to only draw the marks we need)
			for i in range(layerCount):
				thisLayer = tabView.cachedGlyphAtIndex_(i)

				# collect layers and their offsets except newlines
				if type(thisLayer) != GSControlLayer:
					lineOfLayers.append( thisLayer )
					lineOfOffsets.append( tabView.cachedPositionAtIndex_(i) )

				# if we reach end of line or end of text, draw with collected layers:
				if type(thisLayer) == GSControlLayer or i==layerCount-1:

					# step through all layers of the line:
					for j, thisLayerInLine in enumerate(lineOfLayers):

						# draw accents on them if they are Letter/Number/Punctuation
						if thisLayerInLine.parent.category in self.categoriesOnWhichToDrawAccents:
							activePosition = tabView.activePosition()
							lastLayerInLinePosition = tabView.cachedPositionAtIndex_(i)
							thisLayerInLinePosition = lineOfOffsets[j]
							offset = subtractPoints(thisLayerInLinePosition, activePosition)
							if offset == NSPoint(0,0):
								activeColor.set()
								self.drawMarksOnLayer(thisLayerInLine, lineOfLayers)
							else:
								inactiveColor.set()
								self.drawMarksOnLayer(thisLayerInLine, lineOfLayers, offset)

					# reset layer collection
					lineOfLayers = []
					lineOfOffsets = []

	def needsExtraMainOutlineDrawingForInactiveLayer_(self, layer):
		return True

	def shouldDrawAccentCloudForLayer_(self, layer):
		return False

<|endoftext|>


<commit_before>import numpy as np
import bisect

import scipy.signal

import fftfilt

lpf = scipy.signal.firwin(256,1/256.)

class DataBlock():
    def __init__(self, data, tone, fftbin,
                     nsamp, nfft, t0 = 0, fs = 512e6):
        self.data = data
        self.tone = tone
        self.fftbin = fftbin
        self.nsamp = nsamp
        self.nfft = nfft
        self.dt = 1/(fs/nfft)
        self.fs = fs
        self.t0 = t0
        self._mean = None
        self._lpf_data = None

    def mean(self):
        if self._mean is None:
            self._lpf_data = fftfilt.fftfilt(lpf,self.data)[len(lpf):]
            self._mean = self._lpf_data.mean(0)
        return self._mean

class SweepData():
    def __init__(self,sweep_id=1):
        self.sweep_id = sweep_id
        self.blocks = []
        self._freqs = []
    def add_block(self,block):
        f = (block.fs*block.tone)/block.nsamp
        idx = bisect.bisect(self._freqs, f)
        self._freqs.insert(idx,f)
        self.blocks.insert(idx,block)

    @property
    def freqs(self):
        return np.array(self._freqs)
    @property
    def data(self):
        return np.array([x.mean() for x in self.blocks])
<commit_msg>Add sweep_index to data block to make it easier to track fine sweeps around resonators<commit_after>import numpy as np
import bisect

import scipy.signal

import fftfilt

lpf = scipy.signal.firwin(256,1/256.)

class DataBlock():
    def __init__(self, data, tone, fftbin,
                     nsamp, nfft, t0 = 0, fs = 512e6,
                     sweep_index = 0):
        self.data = data
        self.tone = tone
        self.fftbin = fftbin
        self.nsamp = nsamp
        self.nfft = nfft
        self.dt = 1/(fs/nfft)
        self.fs = fs
        self.t0 = t0
        self.sweep_index = sweep_index
        self._mean = None
        self._lpf_data = None

    def mean(self):
        if self._mean is None:
            self._lpf_data = fftfilt.fftfilt(lpf,self.data)[len(lpf):]
            self._mean = self._lpf_data.mean(0)
        return self._mean

class SweepData():
    def __init__(self,sweep_id=1):
        self.sweep_id = sweep_id
        self.blocks = []
        self._freqs = []
        self._sweep_indexes = []
    def add_block(self,block):
        f = (block.fs*block.tone)/block.nsamp
        idx = bisect.bisect(self._freqs, f)
        self._freqs.insert(idx,f)
        self._sweep_indexes.insert(idx,block.sweep_index)
        self.blocks.insert(idx,block)

    def select_index(self,index):
        msk = self.sweep_indexes == index
        freqs = self.freqs[msk]
        data = self.data[msk]
        return freqs,data

    @property
    def freqs(self):
        return np.array(self._freqs)
    @property
    def data(self):
        return np.array([x.mean() for x in self.blocks])
    @property
    def sweep_indexes(self):
        return np.array(self._sweep_indexes)
<|endoftext|>


<commit_before>def break_words(stuff):
    """This function will break up words for us."""
    words = stuff.split(' ')
    return words

def sort_words(words):
    """Sorts the words."""
    return sorted(words)

def print_first_word(words)
    """Prints the first word after popping it off."""
    word = words.poop(0)
    print word

def print_last_word(words):
    """Prints the last word after popping it off."""
    word = words.pop(-1
    print word

def sort_sentence(sentence):
    """Takes in a full sentence and returns the sorted words."""
    words = break_words(sentence)
    return sort_words(words)

def print_first_and_last(sentence):
    """Prints the first and last words of the sentence."""
    words = break_words(sentence)
    print_first_word(words)
    print_last_word(words)

def print_first_and_last_sorted(sentence):
    """Sorts the words then prints the first and last one."""
    words = sort_sentence(sentence)
    print_first_word(words)
    print_last_word(words)

print "Let's practice everything."
print 'You\'d need to know \'bout escapes with \\ that do \n newlines and \t tabs.'

poem = """
\tThe lovely world
with logic so firmly planted
cannot discern \n the needs of love
nor comprehend passion from intuition
and requires an explantion
\n\t\twhere there is none.
"""

print "--------------"
print poem
print "--------------"

five = 10 - 2 + 3 - 5
print "This should be five: %s" % five

def secret_formula(started):
    jelly_beans = started * 500
    jars = jelly_beans \ 1000
    crates = jars / 100
    return jelly_beans, jars, crates

start_point = 10000
beans, jars, crates == secret_formula(start-point)

print "With a starting point of: %d" % start_point
print "We'd have %d jeans, %d jars, and %d crates." % (beans, jars, crates)

start_point = start_point / 10

print "We can also do that this way:"
print "We'd have %d beans, %d jars, and %d crabapples." % secret_formula(start_pont

sentence = "All god\tthings come to those who weight."

words = ex25.break_words(sentence)
sorted_words = ex25.sort_words(words)

print_first_word(words)
print_last_word(words)
.print_first_word(sorted_words)
print_last_word(sorted_words)
sorted_words = ex25.sort_sentence(sentence)
prin sorted_words

print_irst_and_last(sentence)

   print_first_a_last_sorted(senence)<commit_msg>Added a directory for the assignment 3.<commit_after>def break_words(stuff):
    """This function will break up words for us."""
    words = stuff.split(' ')
    return words

def sort_words(words):
    """Sorts the words."""
    return sorted(words)

def print_first_word(words): # missing ':', definition of function
    """Prints the first word after popping it off."""
    word = words.poop(0)
    print word

def print_last_word(words):
    """Prints the last word after popping it off."""
    word = words.pop(-1) # missing ')', closing the function parameters; the error reported on line 18
    print word

def sort_sentence(sentence):
    """Takes in a full sentence and returns the sorted words."""
    words = break_words(sentence)
    return sort_words(words)

def print_first_and_last(sentence):
    """Prints the first and last words of the sentence."""
    words = break_words(sentence)
    print_first_word(words)
    print_last_word(words)

def print_first_and_last_sorted(sentence):
    """Sorts the words then prints the first and last one."""
    words = sort_sentence(sentence)
    print_first_word(words)
    print_last_word(words)

print "Let's practice everything."
print 'You\'d need to know \'bout escapes with \\ that do \n newlines and \t tabs.'

poem = """
\tThe lovely world
with logic so firmly planted
cannot discern \n the needs of love
nor comprehend passion from intuition
and requires an explantion
\n\t\twhere there is none.
"""

print "--------------"
print poem
print "--------------"

five = 10 - 2 + 3 - 5
print "This should be five: %s" % five

def secret_formula(started):
    jelly_beans = started * 500
    jars = jelly_beans / 1000 # should be a '/' instead of '\' (line continuation char), but have to understand the logics to be sure
    crates = jars / 100
    return jelly_beans, jars, crates

start_point = 10000
jelly_beans, jars, crates == secret_formula(start_point) # change beans to jelly_beans ...?

print "With a starting point of: %d" % start_point
print "We'd have %d jeans, %d jars, and %d crates." % (beans, jars, crates)

start_point = start_point / 10

print "We can also do that this way:"
print "We'd have %d beans, %d jars, and %d crabapples." % secret_formula(start_point) # replace '...(start_pont #' with '...(start_point) #'; error reported on line 77

sentence = "All god\tthings come to those who weight."

words = ex25.break_words(sentence)
sorted_words = ex25.sort_words(words)

print_first_word(words)
print_last_word(words)
print_first_word(sorted_words)
print_last_word(sorted_words)
sorted_words = ex25.sort_sentence(sentence)
print sorted_words # print instead of prin

print_irst_and_last(sentence)

print_first_a_last_sorted(senence) # wrong/unexpected ident, move 4 places to the left<|endoftext|>


<commit_before>import six
from astropy.io import ascii

from . import XMATCH_URL, XMATCH_TIMEOUT
from ..query import BaseQuery
from ..utils import commons

class XMatchClass(BaseQuery):
    URL = XMATCH_URL()
    TIMEOUT = XMATCH_TIMEOUT()

    def query(self, cat1, cat2, max_distance, colRA1=None,
              colDec1=None, colRA2=None, colDec2=None):
        """
        Parameters
        ----------
        cat1 : str or file
            Identifier of the first table. It can either be a URL, the
            payload of a local file being uploaded or a CDS table
            identifier (either *simbad* for a view of SIMBAD data, or to
            point out a given VizieR table.
            If the table is uploaded or accessed through a URL, it must be
            in VOTable or CSV format with the positions in J2000
            equatorial frame and as decimal degrees numbers.
        cat2 : str or file
            Identifier of the second table. Follows the same rules as *cat1*.
        max_distance : int, float
            Maximum distance in arcsec to look for counterparts.
            Maximum allowed value is 180.
        colRA1 : str
            Name of the column holding the right ascension. Only required
            if `cat1` is an uploaded table or a pointer to a URL.
        colDec1 : str
            Name of the column holding the declination. Only required if
            `cat1` is an uploaded table or a pointer to a URL.
        colRA2 : str
            Name of the column holding the right ascension. Only required
            if `cat2` is an uploaded table or a pointer to a URL.
        colDec2 : str
            Name of the column holding the declination. Only required if
            `cat2` is an uploaded table or a pointer to a URL.

        Returns
        -------
        table : `~astropy.table.Table`
            Query results table

        """
        if max_distance > 180:
            raise ValueError(
                'max_distance argument must not be greater than 180')
        payload = {
            'request': 'xmatch',
            'distMaxArcsec': max_distance,
            'RESPONSEFORMAT': 'csv',
        }
        kwargs = {}
        if isinstance(cat1, six.string_types):
            payload['cat1'] = cat1
        else:
            # assume it's a file-like object, support duck-typing
            kwargs['files'] = {'cat1': cat1}
        if not self.is_table_available(cat1):
            # if `cat1` is not a VizieR table,
            # it is assumed it's either a URL or an uploaded table
            payload['colRA1'] = colRA1
            payload['colDec1'] = colDec1
        if isinstance(cat2, six.string_types):
            payload['cat2'] = cat2
        else:
            # assume it's a file-like object, support duck-typing
            kwargs['files'] = {'cat2': cat2}
        if not self.is_table_available(cat2):
            # if `cat2` is not a VizieR table,
            # it is assumed it's either a URL or an uploaded table
            payload['colRA2'] = colRA2
            payload['colDec2'] = colDec2
        response = commons.send_request(
            self.URL, payload, self.TIMEOUT, **kwargs)
        return ascii.read(response.text)

    def is_table_available(self, table_id):
        """Return True if the passed CDS table identifier is one of the
        available VizieR tables, otherwise False.

        """
        return table_id in self.get_available_tables()

    def get_available_tables(self):
        """Get the list of the VizieR tables which are available in the
        xMatch service and return them as a list of strings.

        """
        response = self.request(
            'GET',
            'http://cdsxmatch.u-strasbg.fr/xmatch/api/v1/sync/tables',
            {'action': 'getVizieRTableNames', 'RESPONSEFORMAT': 'txt'})
        return response.content.splitlines()

XMatch = XMatchClass()
<commit_msg>added docstring to XMatchClass.query<commit_after>import six
from astropy.io import ascii

from . import XMATCH_URL, XMATCH_TIMEOUT
from ..query import BaseQuery
from ..utils import commons

class XMatchClass(BaseQuery):
    URL = XMATCH_URL()
    TIMEOUT = XMATCH_TIMEOUT()

    def query(self, cat1, cat2, max_distance, colRA1=None,
              colDec1=None, colRA2=None, colDec2=None):
        """Query the `CDS cross-match service
        <http://cdsxmatch.u-strasbg.fr/xmatch>`_ by finding matches between
        two (potentially big) catalogues.

        Parameters
        ----------
        cat1 : str or file
            Identifier of the first table. It can either be a URL, the
            payload of a local file being uploaded or a CDS table
            identifier (either *simbad* for a view of SIMBAD data, or to
            point out a given VizieR table.
            If the table is uploaded or accessed through a URL, it must be
            in VOTable or CSV format with the positions in J2000
            equatorial frame and as decimal degrees numbers.
        cat2 : str or file
            Identifier of the second table. Follows the same rules as *cat1*.
        max_distance : int, float
            Maximum distance in arcsec to look for counterparts.
            Maximum allowed value is 180.
        colRA1 : str
            Name of the column holding the right ascension. Only required
            if `cat1` is an uploaded table or a pointer to a URL.
        colDec1 : str
            Name of the column holding the declination. Only required if
            `cat1` is an uploaded table or a pointer to a URL.
        colRA2 : str
            Name of the column holding the right ascension. Only required
            if `cat2` is an uploaded table or a pointer to a URL.
        colDec2 : str
            Name of the column holding the declination. Only required if
            `cat2` is an uploaded table or a pointer to a URL.

        Returns
        -------
        table : `~astropy.table.Table`
            Query results table

        """
        if max_distance > 180:
            raise ValueError(
                'max_distance argument must not be greater than 180')
        payload = {
            'request': 'xmatch',
            'distMaxArcsec': max_distance,
            'RESPONSEFORMAT': 'csv',
        }
        kwargs = {}
        if isinstance(cat1, six.string_types):
            payload['cat1'] = cat1
        else:
            # assume it's a file-like object, support duck-typing
            kwargs['files'] = {'cat1': cat1}
        if not self.is_table_available(cat1):
            # if `cat1` is not a VizieR table,
            # it is assumed it's either a URL or an uploaded table
            payload['colRA1'] = colRA1
            payload['colDec1'] = colDec1
        if isinstance(cat2, six.string_types):
            payload['cat2'] = cat2
        else:
            # assume it's a file-like object, support duck-typing
            kwargs['files'] = {'cat2': cat2}
        if not self.is_table_available(cat2):
            # if `cat2` is not a VizieR table,
            # it is assumed it's either a URL or an uploaded table
            payload['colRA2'] = colRA2
            payload['colDec2'] = colDec2
        response = commons.send_request(
            self.URL, payload, self.TIMEOUT, **kwargs)
        return ascii.read(response.text)

    def is_table_available(self, table_id):
        """Return True if the passed CDS table identifier is one of the
        available VizieR tables, otherwise False.

        """
        return table_id in self.get_available_tables()

    def get_available_tables(self):
        """Get the list of the VizieR tables which are available in the
        xMatch service and return them as a list of strings.

        """
        response = self.request(
            'GET',
            'http://cdsxmatch.u-strasbg.fr/xmatch/api/v1/sync/tables',
            {'action': 'getVizieRTableNames', 'RESPONSEFORMAT': 'txt'})
        return response.content.splitlines()

XMatch = XMatchClass()
<|endoftext|>


<commit_before>import asyncio

from .message import Message

class Connection:
    """
    Communicates with an IRC network.

    Incoming data is transformed into Message objects, and sent to `listeners`.
    """

    def __init__(self, listeners, host, port, ssl=True):
        self.listeners = listeners
        self.host = host
        self.port = port
        self.ssl = ssl

    @asyncio.coroutine
    def connect(self):
        """Connect to the server, and dispatch incoming messages."""
        connection = asyncio.open_connection(self.host, self.port, ssl=self.ssl)
        self.reader, self.writer = yield from connection

        self.on_connect()

        self._connected = True
        while self._connected:
            message = yield from self.reader.readline()
            if not message:
                self.disconnect()
                return
            self.handle(message)

    def disconnect(self):
        """Close the connection to the server."""
        self._connected = False
        self.writer.close()
        self.on_disconnect()

    def handle(self, raw_message):
        """Dispatch the message to all listeners."""
        message = Message(raw_message)
        for listener in self.listeners:
            listener.handle(self, message)

    def on_connect(self):
        """Upon connection to the network, send user's credentials."""
        self.send(b'USER meshybot 0 * :MeshyBot7')
        self.send(b'NICK meshybot')

    def on_disconnect(self):
        print('Connection closed')

    def send(self, message):
        message = message + b'\r\n'
        print('write', message)
        self.writer.write(message)
<commit_msg>Move disconnect decision into Connection.handle<commit_after>import asyncio

from .message import Message

class Connection:
    """
    Communicates with an IRC network.

    Incoming data is transformed into Message objects, and sent to `listeners`.
    """

    def __init__(self, listeners, host, port, ssl=True):
        self.listeners = listeners
        self.host = host
        self.port = port
        self.ssl = ssl

    @asyncio.coroutine
    def connect(self):
        """Connect to the server, and dispatch incoming messages."""
        connection = asyncio.open_connection(self.host, self.port, ssl=self.ssl)
        self.reader, self.writer = yield from connection

        self.on_connect()

        self._connected = True
        while self._connected:
            message = yield from self.reader.readline()
            self.handle(message)

    def disconnect(self):
        """Close the connection to the server."""
        self._connected = False
        self.writer.close()
        self.on_disconnect()

    def handle(self, raw_message):
        """Dispatch the message to all listeners."""
        if not raw_message:
            self.disconnect()
            return

        message = Message(raw_message)
        for listener in self.listeners:
            listener.handle(self, message)

    def on_connect(self):
        """Upon connection to the network, send user's credentials."""
        self.send(b'USER meshybot 0 * :MeshyBot7')
        self.send(b'NICK meshybot')

    def on_disconnect(self):
        print('Connection closed')

    def send(self, message):
        message = message + b'\r\n'
        print('write', message)
        self.writer.write(message)
<|endoftext|>


<commit_before>import os
import re

from jwt.api import PyJWT
import requests

class KeyIdentifier(object):

    """ This class represents a key identifier """

    def __init__(self, identifier):
        self.__key_id = validate_key_identifier(identifier)

    @property
    def key_id(self):
        return self.__key_id

def validate_key_identifier(identifier):
    """ returns a validated key identifier. """
    regex = re.compile('^[\w.\-\+/]*$')
    _error_msg = 'Invalid key identifier %s' % identifier
    if not identifier:
        raise ValueError(_error_msg)
    if not regex.match(identifier):
        raise ValueError(_error_msg)
    normalised = os.path.normpath(identifier)
    if normalised != identifier:
        raise ValueError(_error_msg)
    if normalised.startswith('/'):
        raise ValueError(_error_msg)
    if '..' in normalised:
        raise ValueError(_error_msg)
    return identifier

def _get_key_id_from_jwt_header(a_jwt):
    """ returns the key identifier from a jwt header. """
    payload, signing_input, header, signature = PyJWT()._load(a_jwt)
    return KeyIdentifier(header['kid'])

class HTTPSPublicKeyRetriever(object):

    """ This class retrieves public key from a https location based upon the
         given key id.
    """

    def __init__(self, base_url):
        if not base_url.startswith('https://'):
            raise ValueError('The base url must start with https://')
        if not base_url.endswith('/'):
            base_url += '/'
        self.base_url = base_url

    def retrieve(self, key_identifier, **requests_kwargs):
        """ returns the public key for given key_identifier. """
        if not isinstance(key_identifier, KeyIdentifier):
            key_identifier = KeyIdentifier(key_identifier)
        PEM_FILE_TYPE = 'application/x-pem-file'
        url = self.base_url + key_identifier.key_id
        resp = requests.get(url,
                            headers={'accept': PEM_FILE_TYPE},
                            **requests_kwargs)
        resp.raise_for_status()
        if resp.headers['content-type'] != PEM_FILE_TYPE:
            raise ValueError("Invalid content-type, '%s', for url '%s' ." %
                             (resp.headers['content-type'], url))
        return resp.text
<commit_msg>http headers are case insensitive - so the content-type check should be done in a case insensitive fashion.<commit_after>import os
import re

from jwt.api import PyJWT
import requests

class KeyIdentifier(object):

    """ This class represents a key identifier """

    def __init__(self, identifier):
        self.__key_id = validate_key_identifier(identifier)

    @property
    def key_id(self):
        return self.__key_id

def validate_key_identifier(identifier):
    """ returns a validated key identifier. """
    regex = re.compile('^[\w.\-\+/]*$')
    _error_msg = 'Invalid key identifier %s' % identifier
    if not identifier:
        raise ValueError(_error_msg)
    if not regex.match(identifier):
        raise ValueError(_error_msg)
    normalised = os.path.normpath(identifier)
    if normalised != identifier:
        raise ValueError(_error_msg)
    if normalised.startswith('/'):
        raise ValueError(_error_msg)
    if '..' in normalised:
        raise ValueError(_error_msg)
    return identifier

def _get_key_id_from_jwt_header(a_jwt):
    """ returns the key identifier from a jwt header. """
    payload, signing_input, header, signature = PyJWT()._load(a_jwt)
    return KeyIdentifier(header['kid'])

class HTTPSPublicKeyRetriever(object):

    """ This class retrieves public key from a https location based upon the
         given key id.
    """

    def __init__(self, base_url):
        if not base_url.startswith('https://'):
            raise ValueError('The base url must start with https://')
        if not base_url.endswith('/'):
            base_url += '/'
        self.base_url = base_url

    def retrieve(self, key_identifier, **requests_kwargs):
        """ returns the public key for given key_identifier. """
        if not isinstance(key_identifier, KeyIdentifier):
            key_identifier = KeyIdentifier(key_identifier)
        PEM_FILE_TYPE = 'application/x-pem-file'
        url = self.base_url + key_identifier.key_id
        resp = requests.get(url,
                            headers={'accept': PEM_FILE_TYPE},
                            **requests_kwargs)
        resp.raise_for_status()
        if resp.headers['content-type'].lower() != PEM_FILE_TYPE.lower():
            raise ValueError("Invalid content-type, '%s', for url '%s' ." %
                             (resp.headers['content-type'], url))
        return resp.text
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
""" \file identity/account/views.py
\brief Provides the views definitions for the accounts module.

\author Erich Healy (cactuscommander) ErichRHealy@gmail.com
\author Ryan Leckey (mehcode) leckey.ryan@gmail.com

\copyright Copyright 2012 © Concordus Applications, Inc.
           All Rights Reserved.
"""
from django.db import models
from django.contrib.auth.models import User
from django.contrib import auth
from django.contrib.auth import views
from identity.views import Protected
from django.conf import settings
from django.views.generic.edit import FormView, UpdateView
from . import forms
import ldap

class Profile(Protected, UpdateView):
    """TODO"""
    template_name = 'identity/account/profile.html'
    form_class = forms.Profile
    success_url = '/accounts/profile'  # FIXME: Make this dynamic

    def get_object(self):
        """TODO"""
        return self.request.user.get_profile()

    def form_valid(self, form):
        """TODO"""
        # Save the user model
        response = super(Profile, self).form_valid(form)

        # Hook in here to save to LDAP
        # FIXME: This can be abstracted
        # Construct the DN
        dn = "uid={},ou=people,{}".format(
            self.request.session['user'],
            settings.LDAP['DN']
        )

        # Initialize the LDAP connection
        connection = ldap.open(settings.LDAP['HOST'])
        connection.protocol_version = getattr(
            ldap, 'VERSION{}'.format(settings.LDAP['VERSION']))

        try:
            # Attempt to authenticate with the given credentials
            connection.simple_bind_s(dn, self.request.session['pass'])

            # Update LDAP attributes
            data = form.cleaned_data
            connection.modify_s(dn, [
                (ldap.MOD_REPLACE, 'givenName', [str(data['given_name'])]),
                (ldap.MOD_REPLACE, 'sn', [str(data['family_name'])]),
            ])

        except:
            # Something went wrong; we have no auth
            #
            raise

        finally:
            # Ensure we close our connection
            connection.unbind_s()

        # Return the response
        return response

def login(request, *args, **kwargs):
    """TODO"""
    if request.method == 'POST':
        # Store the username and password for later re-auth using LDAP
        request.session['user'] = request.POST['username']
        request.session['pass'] = request.POST['password']

    return views.login(request, *args, **kwargs)
<commit_msg>Fixed import error in account.<commit_after># -*- coding: utf-8 -*-
""" \file identity/account/views.py
\brief Provides the views definitions for the accounts module.

\author Erich Healy (cactuscommander) ErichRHealy@gmail.com
\author Ryan Leckey (mehcode) leckey.ryan@gmail.com

\copyright Copyright 2012 © Concordus Applications, Inc.
           All Rights Reserved.
"""
from django.db import models
from django.contrib.auth.models import User
from django.contrib import auth
from django.contrib.auth import views
from identity.common.views import Protected
from django.conf import settings
from django.views.generic.edit import FormView, UpdateView
from . import forms
import ldap

class Profile(Protected, UpdateView):
    """TODO"""
    template_name = 'identity/account/profile.html'
    form_class = forms.Profile
    success_url = '/accounts/profile'  # FIXME: Make this dynamic

    def get_object(self):
        """TODO"""
        return self.request.user.get_profile()

    def form_valid(self, form):
        """TODO"""
        # Save the user model
        response = super(Profile, self).form_valid(form)

        # Hook in here to save to LDAP
        # FIXME: This can be abstracted
        # Construct the DN
        dn = "uid={},ou=people,{}".format(
            self.request.session['user'],
            settings.LDAP['DN']
        )

        # Initialize the LDAP connection
        connection = ldap.open(settings.LDAP['HOST'])
        connection.protocol_version = getattr(
            ldap, 'VERSION{}'.format(settings.LDAP['VERSION']))

        try:
            # Attempt to authenticate with the given credentials
            connection.simple_bind_s(dn, self.request.session['pass'])

            # Update LDAP attributes
            data = form.cleaned_data
            connection.modify_s(dn, [
                (ldap.MOD_REPLACE, 'givenName', [str(data['given_name'])]),
                (ldap.MOD_REPLACE, 'sn', [str(data['family_name'])]),
            ])

        except:
            # Something went wrong; we have no auth
            #
            raise

        finally:
            # Ensure we close our connection
            connection.unbind_s()

        # Return the response
        return response

def login(request, *args, **kwargs):
    """TODO"""
    if request.method == 'POST':
        # Store the username and password for later re-auth using LDAP
        request.session['user'] = request.POST['username']
        request.session['pass'] = request.POST['password']

    return views.login(request, *args, **kwargs)
<|endoftext|>


<commit_before># =========================================================================
# Copyright 2012-present Yunify, Inc.
# -------------------------------------------------------------------------
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this work except in compliance with the License.
# You may obtain a copy of the License in the LICENSE file, or at:
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========================================================================

from qingcloud.cli.iaas_client.actions.base import BaseAction
from qingcloud.cli.misc.utils import explode_array

class ModifyLoadBalancerListenerAttributessAction(BaseAction):

    action = 'ModifyLoadBalancerListenerAttributes'
    command = 'modify-loadbalancer-listener-attributes'
    usage = '%(prog)s -s <lb_listener> [options] [-f <conf_file>]'
    description = 'Modify load balancer listener attributes.'

    @classmethod
    def add_ext_arguments(cls, parser):

        parser.add_argument('-s', '--lb_listener', dest='lb_listener',
                action='store', type=str, default='',
                help='the ID of load balancer listener.')

        parser.add_argument('-M', '--balance_mode', dest='balance_mode',
                action='store', type=str, default=None,
                help='the balance mode: roundrobin, leastconn.')

        parser.add_argument('-F', '--forwardfor', dest='forwardfor',
                action='store', type=str, default=None,
                help='comma separated headers the listener will pass, \
                      including: "X_FORWARD_FOR", "QC_LB_ID", "QC_LB_IP".')

        parser.add_argument('-m', '--healthy_check_method', dest='healthy_check_method',
                action='store', type=str, default=None,
                help='heathy check method: "tcp","http|/url","http|/url|host", empty string means disable check.')

        parser.add_argument('-o', '--healthy_check_option', dest='healthy_check_option',
                action='store', type=str, default=None,
                help='healthy check option, formatted as \
                      "inter|timeout|fall|rise"')

        parser.add_argument('-k', '--session_sticky', dest='session_sticky',
                action='store', type=str, default=None,
                help='session sticky: "insert|cookie_timeout","prefix|cookie_name",\
                      empty string means disable sticky.')

        parser.add_argument('-N', '--lb_listener_name', dest='lb_listener_name',
                action='store', type=str, default=None,
                help='new listener name.')

    @classmethod
    def build_directive(cls, options):
        if not options.lb_listener:
            print('error: listener should be specified')
            return None

        forwardfor = options.forwardfor
        if forwardfor:
            mapper = {
                    'X_FORWARD_FOR': 1,
                    'QC_LB_ID': 2,
                    'QC_LB_IP': 4,
                    }
            headers = [h for h in explode_array(forwardfor) if h in mapper]
            forwardfor = 0
            for header in headers:
                forwardfor |= mapper[header]

        return {
                'loadbalancer_listener': options.lb_listener,
                'loadbalancer_listener_name': options.lb_listener_name,
                'balance_mode': options.balance_mode,
                'forwardfor': forwardfor,
                'healthy_check_method': options.healthy_check_method,
                'healthy_check_option': options.healthy_check_option,
                'session_sticky': options.session_sticky,
                }
<commit_msg>Modify lb listner certification<commit_after># =========================================================================
# Copyright 2012-present Yunify, Inc.
# -------------------------------------------------------------------------
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this work except in compliance with the License.
# You may obtain a copy of the License in the LICENSE file, or at:
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========================================================================

from qingcloud.cli.iaas_client.actions.base import BaseAction
from qingcloud.cli.misc.utils import explode_array

class ModifyLoadBalancerListenerAttributessAction(BaseAction):

    action = 'ModifyLoadBalancerListenerAttributes'
    command = 'modify-loadbalancer-listener-attributes'
    usage = '%(prog)s -s <lb_listener> [options] [-f <conf_file>]'
    description = 'Modify load balancer listener attributes.'

    @classmethod
    def add_ext_arguments(cls, parser):

        parser.add_argument('-s', '--lb_listener', dest='lb_listener',
                action='store', type=str, default='',
                help='the ID of load balancer listener.')

        parser.add_argument('-M', '--balance_mode', dest='balance_mode',
                action='store', type=str, default=None,
                help='the balance mode: roundrobin, leastconn.')

        parser.add_argument('-F', '--forwardfor', dest='forwardfor',
                action='store', type=str, default=None,
                help='comma separated headers the listener will pass, \
                      including: "X_FORWARD_FOR", "QC_LB_ID", "QC_LB_IP".')

        parser.add_argument('-m', '--healthy_check_method', dest='healthy_check_method',
                action='store', type=str, default=None,
                help='heathy check method: "tcp","http|/url","http|/url|host", empty string means disable check.')

        parser.add_argument('-o', '--healthy_check_option', dest='healthy_check_option',
                action='store', type=str, default=None,
                help='healthy check option, formatted as \
                      "inter|timeout|fall|rise"')

        parser.add_argument('-k', '--session_sticky', dest='session_sticky',
                action='store', type=str, default=None,
                help='session sticky: "insert|cookie_timeout","prefix|cookie_name",\
                      empty string means disable sticky.')

        parser.add_argument('-N', '--lb_listener_name', dest='lb_listener_name',
                action='store', type=str, default=None,
                help='new listener name.')

        parser.add_argument('-S', '--server-certificate', dest='server_certificate_id',
                action='store', type=str, default=None,
                help='the ID of server certificate.')

    @classmethod
    def build_directive(cls, options):
        if not options.lb_listener:
            print('error: listener should be specified')
            return None

        forwardfor = options.forwardfor
        if forwardfor:
            mapper = {
                    'X_FORWARD_FOR': 1,
                    'QC_LB_ID': 2,
                    'QC_LB_IP': 4,
                    }
            headers = [h for h in explode_array(forwardfor) if h in mapper]
            forwardfor = 0
            for header in headers:
                forwardfor |= mapper[header]

        return {
                'loadbalancer_listener': options.lb_listener,
                'loadbalancer_listener_name': options.lb_listener_name,
                'balance_mode': options.balance_mode,
                'forwardfor': forwardfor,
                'healthy_check_method': options.healthy_check_method,
                'healthy_check_option': options.healthy_check_option,
                'session_sticky': options.session_sticky,
                'server_certificate_id': options.server_certificate_id,
                }
<|endoftext|>


<commit_before>"""
    Author: Jon Ander Gomez Adrian (jon@dsic.upv.es, http://personales.upv.es/jon)
    Version: 4.0
    Date: May 2021
    Universitat Politecnica de Valencia
    Technical University of Valencia TU.VLC

"""

# My KMeans implementation

import os
import sys
import random
import numpy
import gzip
import pickle

from sklearn import metrics

class KMeans:
    """
        Implementation of KMeans. This class provides several variants:
            1. Original KMeans
            2. Lloyd's algorithm with random or KMeans++ (Katsavounidis) initializations
            3. LBG incremental clustering
            4. K-Mediods - centroids are real samples, selected within each cluster
               as the sample that minimises the distance of all the samples in the
               cluster to it

        [LBG] Linde, Y.; Buzo, A.; Gray, R. (1980).
              "An Algorithm for Vector Quantizer Design".
              IEEE Transactions on Communications. 28: 84–95.
              doi:10.1109/TCOM.1980.1094577
    """

    # --------------------------------------------------------------------------------
    def __init__(self,  n_clusters = 2,
                        number_of_initializations = 1,
                        init = 'random',
                        max_iter = 1000,
                        modality = 'Lloyd',
                        verbosity = 0):
        self.max_iter = max_iter
        self.number_of_initializations = number_of_initializations
        self.init = init
        self.n_clusters = n_clusters
        self.cluster_centers_ = None
        self.cluster_pred = None
        self.J = 9.9e+300
        self.oldJ = 9.9e+300
        self.modality = modality
        if modality not in ['Lloyd', 'k-Mediods', 'k-Means', 'original-k-Means', 'LBG']:
            raise Exception('Wrong modality ' + modality)
        self.verbosity = verbosity
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def fit(self, X):

        if self.modality in ['Lloyd', 'k-Mediods']:
            #
            self.cluster_centers_ = numpy.zeros([self.n_clusters, X.shape[1]])
            if self.init in ['Katsavounidis', 'KMeans++']:
                self.katsavounidis(X)
            elif self.init == 'random':
                for c in range(self.n_clusters):
                    self.cluster_centers_[c][:] = X[numpy.random.randint(len(X))]
            else:
                print('ERROR: init method not specified!')
                sys.exit(1)
            #
            if self.modality == 'Lloyd':
                self.lloyd(X)
            else:
                self.k_mediods(X)
            #
        elif self.modality == 'LBG':
            #
            self.lgb(X, K = self.n_clusters)
            #
        elif self.modality in ['k-Means', 'original-k-Means']:
            #
            self.original_k_means(X, K = self.n_clusters)
            #
        else:
            raise Exception('Wrong modality ' + self.modality)
        #
        self.cluster_pred = None
        return self
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def lloyd(self, X):
        self.oldJ = self.J
        self.cluster_pred = self.predict(X)

        iteration = 0
        changes_counter = 1
        epsilon = 1.0e-4
        if self.verbosity > 0:
            print('iteration', iteration, 'max_iter', self.max_iter,
                    'changes', changes_counter, 'relative improvement', self.improvement())
        while iteration < self.max_iter and changes_counter > 0 and self.improvement() > epsilon:
            changes_counter = self.fit_iteration(X)
            iteration += 1
            if self.verbosity > 0:
                print('iteration', iteration, 'max_iter', self.max_iter,
                        'changes', changes_counter, 'relative improvement', self.improvement())

        return self
    # --------------------------------------------------------------------------------
    def k_mediods(self, X):
        self.oldJ = self.J
        self.cluster_pred = self.predict(X)

        iteration = 0
        changes_counter = 1
        epsilon = 1.0e-4

        if self.verbosity > 0:
            print('iteration', iteration, 'max_iter', self.max_iter,
                    'changes', changes_counter, 'relative improvement', self.improvement())
        while iteration < self.max_iter and changes_counter > 0 and self.improvement() > epsilon:
            changes_counter = self.fit_iteration(X)
            iteration += 1
            if self.verbosity > 0:
                print('iteration', iteration, 'max_iter', self.max_iter,
                        'changes', changes_counter, 'relative improvement', self.improvement())

        return self
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def fit_iteration(self, X):

        for c in range(len(self.cluster_centers_)):
            samples_in_cluster = X[self.cluster_pred == c]
            if len(samples_in_cluster) > 0:
                if self.modality == 'k-Mediods':
                    self.cluster_centers_[c, :] = self.mediod_from(samples_in_cluster)
                else:
                    self.cluster_centers_[c, :] = numpy.mean(samples_in_cluster, axis = 0)
            else:
                self.cluster_centers_[c, :] = 0.0

        self.oldJ = self.J
        y_pred = self.predict(X)

        changes_counter = numpy.abs(self.cluster_pred - y_pred).sum()
        self.cluster_pred[:] = y_pred[:]
        #print self.cluster_centers_ # For looking the evolution

        return changes_counter
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def mediod_from(self, samples):
        m = 0
        min_dist = sum(metrics.pairwise.euclidean_distances(samples, [samples[0]]))
        for i in range(1, len(samples)):
            dist = sum(metrics.pairwise.euclidean_distances(samples, [samples[i]]))
            if dist < min_dist:
                min_dist = dist
                m = i
        return samples[m]
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def lbg(self, X, K = None, verbose = 0):
        #
        if K is None : raise Exception('The number of desired clusters must be specified!')
        #
        #distortion = numpy.ones( X.shape[1] )
        distortion = X.std(axis = 0)
        #
        self.cluster_centers_ = numpy.array([X.mean(axis = 0)])
        #
        while len(self.cluster_centers_) < K :
            self.split(distortion)
            if verbose > 0: print("|codebook| = %d" % len(self.cluster_centers_))
            self.lloyd(X)
            if verbose > 0: print("distortion = %f" % self.J)
            #distortion = self.cluster_centers_.std(axis=0)
            distortion = 0.5 * distortion
        #
        self.n_clusters = len(self.cluster_centers_)
    # --------------------------------------------------------------------------------
    def split(self, distortion):
        new_codebook = numpy.zeros([2 * len(self.cluster_centers_), len(distortion)])
        for i in range(len(self.cluster_centers_)):
            new_codebook[2 * i    , :] = self.cluster_centers_[i] + distortion
            new_codebook[2 * i + 1, :] = self.cluster_centers_[i] - distortion
        self.cluster_centers_ = new_codebook
    # --------------------------------------------------------------------------------
    def drop_empty_clusters(self, X):
        y_pred = self.predict(X)
        centroids=list()
        for c in range(len(self.cluster_centers_)):
            if sum(y_pred == c) > 0:
                centroids.append(self.cluster_centers_[c].copy())
        self.cluster_centers_ = numpy.array(centroids)
        self.n_clusters = len(self.cluster_centers_)
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def original_k_means(self, X):
        self.original_k_means_init(X)
    # --------------------------------------------------------------------------------
    def original_k_means_init(self, X):
        if type(X) not in [numpy.ndarray, list]:
            raise Exception('X must be a numpy array or a list!')
        dim = None
        if type(X) is list:
            if type(X[0]) is not numpy.ndarray:
                raise Exception('Elements in X must numpy arrays!')
            dim = X[0].shape[0]
        else:
            dim = X.shape[1]

        if len(X) < self.n_clusters:
            raise Exception('Initialisation must be done with a minimum number of samples/vectors equal to the number of clusters!')

        K = self.n_clusters
        self.counters = numpy.ones(self.n_clusters)
        #self.cluster_centers_ = numpy.zeros([self.n_clusters, dim])
        #for i in range(K):
        #    self.cluster_centers_[i, :] = X[i, :]
        self.cluster_centers_ = numpy.copy(X[:K])

        self.original_k_means_iteration(X[K:])
    # --------------------------------------------------------------------------------
    def original_k_means_iteration(self, X):
        for n in range(len(X)):
            # Classification
            _diffs_ = self.cluster_centers_ - X[n]
            _distances_ = (_diffs_ * _diffs_).sum(axis = 1)
            k = _distances_.argmin()
            # Partition update
            self.counters[k] += 1
            # Codebook update
            self.cluster_centers_[k] = ((self.cluster_centers_[k] * (self.counters[k] - 1)) + X[n]) / self.counters[k]
            #
            if sum(numpy.isnan(self.cluster_centers_[k].ravel())) > 0:
                raise Exception(f'FATAL ERROR: update of cluster {k} by sample {X[n]} generates NaN')
            # alpha = 1.0e-6
            #self.cluster_centers_[k] = self.cluster_centers_[k] * (1 - alpha) + X[n] * alpha

    # --------------------------------------------------------------------------------
    def predict(self, X):
        distances = metrics.pairwise.euclidean_distances(X, self.cluster_centers_)
        Y = numpy.argmin(distances, axis = 1)
        self.J = numpy.min(distances, axis = 1).sum()
        '''
        self.J = 0.0
        Y = numpy.ones(len(X), dtype = int) * -1
        for n in range(len(X)):
            Y[n] = numpy.argmin(distances[n])
            self.J = self.J + distances[n][Y[n]]
        '''
        self.J = self.J / len(X)
        return Y
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def distances(self, X):
        return metrics.pairwise.euclidean_distances(X, self.cluster_centers_)
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def improvement(self):
        return abs((self.oldJ - self.J) / (abs(self.J) + 10e-5))
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def katsavounidis(self, X):
        if self.init == 'Katsavounidis':
            self.cluster_centers_[0] = numpy.mean(X, axis = 0)
        elif self.init == 'KMeans++':
            self.cluster_centers_[0] = X[random.randint(0, len(X) - 1)]
        else:
            raise Exception('Non-expected initialization' + self.init)
        c = 1
        while c < self.n_clusters:
            distances = metrics.pairwise.euclidean_distances(X, self.cluster_centers_[:c])
            distances = numpy.min(distances, axis = 1)
            self.cluster_centers_[c, :] = X[numpy.argmax(distances), :]
            c += 1
        distances = metrics.pairwise.euclidean_distances(X, self.cluster_centers_)
        distances = numpy.min(distances, axis = 1)
        self.cluster_centers_[0, :] = X[ numpy.argmax(distances), :]
        #print self.cluster_centers_
    # --------------------------------------------------------------------------------

    # -------------------------------------------------------------------------
    def save(self, filename = None):
        if filename is not None:
            with gzip.open(filename, 'wb') as f: pickle.dump(self, f)
            f.close()
    # -------------------------------------------------------------------------

def kmeans_load(filename=None):
    if filename is None : raise Exception('Impossible to load a codebook without the filename!')
    model = None
    if os.path.exists(filename) and os.path.isfile(filename):
        with gzip.open(filename, 'rb') as f: model = pickle.load(f)
        f.close()
    #
    return model
# -------------------------------------------------------------------------
<commit_msg>Correction of one error<commit_after>"""
    Author: Jon Ander Gomez Adrian (jon@dsic.upv.es, http://personales.upv.es/jon)
    Version: 4.0
    Date: May 2021
    Universitat Politecnica de Valencia
    Technical University of Valencia TU.VLC

"""

# My KMeans implementation

import os
import sys
import random
import numpy
import gzip
import pickle

from sklearn import metrics

class KMeans:
    """
        Implementation of KMeans. This class provides several variants:
            1. Original KMeans
            2. Lloyd's algorithm with random or KMeans++ (Katsavounidis) initializations
            3. LBG incremental clustering
            4. K-Mediods - centroids are real samples, selected within each cluster
               as the sample that minimises the distance of all the samples in the
               cluster to it

        [LBG] Linde, Y.; Buzo, A.; Gray, R. (1980).
              "An Algorithm for Vector Quantizer Design".
              IEEE Transactions on Communications. 28: 84–95.
              doi:10.1109/TCOM.1980.1094577
    """

    # --------------------------------------------------------------------------------
    def __init__(self,  n_clusters = 2,
                        number_of_initializations = 1,
                        init = 'random',
                        max_iter = 1000,
                        modality = 'Lloyd',
                        verbosity = 0):
        self.max_iter = max_iter
        self.number_of_initializations = number_of_initializations
        self.init = init
        self.n_clusters = n_clusters
        self.cluster_centers_ = None
        self.cluster_pred = None
        self.J = 9.9e+300
        self.oldJ = 9.9e+300
        self.modality = modality
        if modality not in ['Lloyd', 'k-Mediods', 'k-Means', 'original-k-Means', 'LBG']:
            raise Exception('Wrong modality ' + modality)
        self.verbosity = verbosity
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def fit(self, X):

        if self.modality in ['Lloyd', 'k-Mediods']:
            #
            self.cluster_centers_ = numpy.zeros([self.n_clusters, X.shape[1]])
            if self.init in ['Katsavounidis', 'KMeans++']:
                self.katsavounidis(X)
            elif self.init == 'random':
                for c in range(self.n_clusters):
                    self.cluster_centers_[c][:] = X[numpy.random.randint(len(X))]
            else:
                print('ERROR: init method not specified!')
                sys.exit(1)
            #
            if self.modality == 'Lloyd':
                self.lloyd(X)
            else:
                self.k_mediods(X)
            #
        elif self.modality == 'LBG':
            #
            self.lbg(X, K = self.n_clusters)
            #
        elif self.modality in ['k-Means', 'original-k-Means']:
            #
            self.original_k_means(X, K = self.n_clusters)
            #
        else:
            raise Exception('Wrong modality ' + self.modality)
        #
        self.cluster_pred = None
        return self
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def lloyd(self, X):
        self.oldJ = self.J
        self.cluster_pred = self.predict(X)

        iteration = 0
        changes_counter = 1
        epsilon = 1.0e-4
        if self.verbosity > 0:
            print('iteration', iteration, 'max_iter', self.max_iter,
                    'changes', changes_counter, 'relative improvement', self.improvement())
        while iteration < self.max_iter and changes_counter > 0 and self.improvement() > epsilon:
            changes_counter = self.fit_iteration(X)
            iteration += 1
            if self.verbosity > 0:
                print('iteration', iteration, 'max_iter', self.max_iter,
                        'changes', changes_counter, 'relative improvement', self.improvement())

        return self
    # --------------------------------------------------------------------------------
    def k_mediods(self, X):
        self.oldJ = self.J
        self.cluster_pred = self.predict(X)

        iteration = 0
        changes_counter = 1
        epsilon = 1.0e-4

        if self.verbosity > 0:
            print('iteration', iteration, 'max_iter', self.max_iter,
                    'changes', changes_counter, 'relative improvement', self.improvement())
        while iteration < self.max_iter and changes_counter > 0 and self.improvement() > epsilon:
            changes_counter = self.fit_iteration(X)
            iteration += 1
            if self.verbosity > 0:
                print('iteration', iteration, 'max_iter', self.max_iter,
                        'changes', changes_counter, 'relative improvement', self.improvement())

        return self
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def fit_iteration(self, X):

        for c in range(len(self.cluster_centers_)):
            samples_in_cluster = X[self.cluster_pred == c]
            if len(samples_in_cluster) > 0:
                if self.modality == 'k-Mediods':
                    self.cluster_centers_[c, :] = self.mediod_from(samples_in_cluster)
                else:
                    self.cluster_centers_[c, :] = numpy.mean(samples_in_cluster, axis = 0)
            else:
                self.cluster_centers_[c, :] = 0.0

        self.oldJ = self.J
        y_pred = self.predict(X)

        changes_counter = numpy.abs(self.cluster_pred - y_pred).sum()
        self.cluster_pred[:] = y_pred[:]
        #print self.cluster_centers_ # For looking the evolution

        return changes_counter
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def mediod_from(self, samples):
        m = 0
        min_dist = sum(metrics.pairwise.euclidean_distances(samples, [samples[0]]))
        for i in range(1, len(samples)):
            dist = sum(metrics.pairwise.euclidean_distances(samples, [samples[i]]))
            if dist < min_dist:
                min_dist = dist
                m = i
        return samples[m]
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def lbg(self, X, K = None, verbose = 0):
        #
        if K is None : raise Exception('The number of desired clusters must be specified!')
        #
        #distortion = numpy.ones( X.shape[1] )
        distortion = X.std(axis = 0)
        #
        self.cluster_centers_ = numpy.array([X.mean(axis = 0)])
        #
        while len(self.cluster_centers_) < K :
            self.split(distortion)
            if verbose > 0: print("|codebook| = %d" % len(self.cluster_centers_))
            self.lloyd(X)
            if verbose > 0: print("distortion = %f" % self.J)
            #distortion = self.cluster_centers_.std(axis=0)
            distortion = 0.5 * distortion
        #
        self.n_clusters = len(self.cluster_centers_)
    # --------------------------------------------------------------------------------
    def split(self, distortion):
        new_codebook = numpy.zeros([2 * len(self.cluster_centers_), len(distortion)])
        for i in range(len(self.cluster_centers_)):
            new_codebook[2 * i    , :] = self.cluster_centers_[i] + distortion
            new_codebook[2 * i + 1, :] = self.cluster_centers_[i] - distortion
        self.cluster_centers_ = new_codebook
    # --------------------------------------------------------------------------------
    def drop_empty_clusters(self, X):
        y_pred = self.predict(X)
        centroids=list()
        for c in range(len(self.cluster_centers_)):
            if sum(y_pred == c) > 0:
                centroids.append(self.cluster_centers_[c].copy())
        self.cluster_centers_ = numpy.array(centroids)
        self.n_clusters = len(self.cluster_centers_)
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def original_k_means(self, X):
        self.original_k_means_init(X)
    # --------------------------------------------------------------------------------
    def original_k_means_init(self, X):
        if type(X) not in [numpy.ndarray, list]:
            raise Exception('X must be a numpy array or a list!')
        dim = None
        if type(X) is list:
            if type(X[0]) is not numpy.ndarray:
                raise Exception('Elements in X must numpy arrays!')
            dim = X[0].shape[0]
        else:
            dim = X.shape[1]

        if len(X) < self.n_clusters:
            raise Exception('Initialisation must be done with a minimum number of samples/vectors equal to the number of clusters!')

        K = self.n_clusters
        self.counters = numpy.ones(self.n_clusters)
        #self.cluster_centers_ = numpy.zeros([self.n_clusters, dim])
        #for i in range(K):
        #    self.cluster_centers_[i, :] = X[i, :]
        self.cluster_centers_ = numpy.copy(X[:K])

        self.original_k_means_iteration(X[K:])
    # --------------------------------------------------------------------------------
    def original_k_means_iteration(self, X):
        for n in range(len(X)):
            # Classification
            _diffs_ = self.cluster_centers_ - X[n]
            _distances_ = (_diffs_ * _diffs_).sum(axis = 1)
            k = _distances_.argmin()
            # Partition update
            self.counters[k] += 1
            # Codebook update
            self.cluster_centers_[k] = ((self.cluster_centers_[k] * (self.counters[k] - 1)) + X[n]) / self.counters[k]
            #
            if sum(numpy.isnan(self.cluster_centers_[k].ravel())) > 0:
                raise Exception(f'FATAL ERROR: update of cluster {k} by sample {X[n]} generates NaN')
            # alpha = 1.0e-6
            #self.cluster_centers_[k] = self.cluster_centers_[k] * (1 - alpha) + X[n] * alpha

    # --------------------------------------------------------------------------------
    def predict(self, X):
        distances = metrics.pairwise.euclidean_distances(X, self.cluster_centers_)
        Y = numpy.argmin(distances, axis = 1)
        self.J = numpy.min(distances, axis = 1).sum()
        '''
        self.J = 0.0
        Y = numpy.ones(len(X), dtype = int) * -1
        for n in range(len(X)):
            Y[n] = numpy.argmin(distances[n])
            self.J = self.J + distances[n][Y[n]]
        '''
        self.J = self.J / len(X)
        return Y
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def distances(self, X):
        return metrics.pairwise.euclidean_distances(X, self.cluster_centers_)
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def improvement(self):
        return abs((self.oldJ - self.J) / (abs(self.J) + 10e-5))
    # --------------------------------------------------------------------------------

    # --------------------------------------------------------------------------------
    def katsavounidis(self, X):
        if self.init == 'Katsavounidis':
            self.cluster_centers_[0] = numpy.mean(X, axis = 0)
        elif self.init == 'KMeans++':
            self.cluster_centers_[0] = X[random.randint(0, len(X) - 1)]
        else:
            raise Exception('Non-expected initialization' + self.init)
        c = 1
        while c < self.n_clusters:
            distances = metrics.pairwise.euclidean_distances(X, self.cluster_centers_[:c])
            distances = numpy.min(distances, axis = 1)
            self.cluster_centers_[c, :] = X[numpy.argmax(distances), :]
            c += 1
        distances = metrics.pairwise.euclidean_distances(X, self.cluster_centers_)
        distances = numpy.min(distances, axis = 1)
        self.cluster_centers_[0, :] = X[ numpy.argmax(distances), :]
        #print self.cluster_centers_
    # --------------------------------------------------------------------------------

    # -------------------------------------------------------------------------
    def save(self, filename = None):
        if filename is not None:
            with gzip.open(filename, 'wb') as f: pickle.dump(self, f)
            f.close()
    # -------------------------------------------------------------------------

def kmeans_load(filename=None):
    if filename is None : raise Exception('Impossible to load a codebook without the filename!')
    model = None
    if os.path.exists(filename) and os.path.isfile(filename):
        with gzip.open(filename, 'rb') as f: model = pickle.load(f)
        f.close()
    #
    return model
# -------------------------------------------------------------------------
<|endoftext|>