<commit_before>import os
import zbar
from PIL import Image
import cv2
import numpy as np
import time
import bot.lib.lib as lib
import math

class QRCode2:
    def __init__(self, tvec, value, top_right):
        self.tvec = tvec
        self.value = value
        self.tr = top_right

class block:
    def __init__(self, size, color):
        self.size = size
        self.color = color

class Camera:
    def  __init__(self):
        # Image processing
        self.cam = cv2.VideoCapture(-1)

        # Camera dimensions had to be set manually
        self.cam.set(3,1280)
        self.cam.set(4,720)

        # QR scanning tools.
        self.scanner = zbar.ImageScanner()
        self.scanner.parse_config('enable')

        # Figure out what camera is being used
        self.bot_config = lib.get_config()
        cam_model = self.bot_config["camera_model"]

        # Constants based on calibration for image processing
        self.cam_matrix  = np.float32(
                        self.bot_config[cam_model]["camera_matrix"])
        self.dist_coeffs = np.float32(
                        self.bot_config[cam_model]["distortion_coefficients"])

        # initialize vertices of QR code
        l = 1.5
        self.qr_verts = np.float32([[-l/2, -l/2, 0],
                            [-l/2,  l/2, 0],
                            [ l/2, -l/2, 0],
                            [ l/2,  l/2, 0]])

    def readQR(self):

        while(True):
            QRList = []
            self.cam.grab()
            self.cam.grab()
            self.cam.grab()
            self.cam.grab()
            ret, frame = self.cam.read()
            # Direct conversion cv -> zbar images did not work.
            # Buffer file used to have native data structures.
            gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
            frame = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 11, 2)
            frame = cv2.bilateralFilter(frame, 9, 75, 75)
            cv2.imwrite('buffer.png', frame)

            # PIL -> zbar
            pil_im = Image.open('buffer.png').convert('L')
            width, height = pil_im.size
            raw = pil_im.tobytes()
            z_im = zbar.Image(width, height, 'Y800', raw)

            # Find codes in image
            self.scanner.scan(z_im)
            count = 0
            for symbol in z_im:
                tl, bl, br, tr = [item for item in symbol.location]
                points = np.float32([[tl[0], tl[1]],
                                     [tr[0], tr[1]],
                                     [bl[0], bl[1]],
                                     [br[0], br[1]]])

                ret, rvec, tvec = cv2.solvePnP(self.qr_verts, points
                                          , self.cam_matrix
                                          , self.dist_coeffs)

                #cv2.line(frame, tl, bl, (100,0,255), 8, 8)
                #cv2.line(frame, bl, br, (100,0,255), 8, 8)
                #cv2.line(frame, br, tr, (100,0,255), 8, 8)
                #cv2.line(frame, tr, tl, (100,0,255), 8, 8)

                QRList.append(QRCode(tvec, rvec, symbol.data, tr))
                count += 1
            if count == 0:
                #print "NO QR FOUND"
                #cv2.imshow('test', frame)
                if cv2.waitKey(1) & 0xFF == ord('q'):
                    break
                continue
            # find the best QRCode to grab (closest x then highest y)
            QRList.sort(key=lambda qr: qr.tvec[0], reverse=False)       # sort the list of qr codes by x, smallest to largest
            x_min = 100
            min_qr = 0
            count = 0
            for qr in QRList:
                #find smallest x
                if abs(qr.tvec[0]) < abs(x_min):
                    x_min = qr.tvec[0]
                    min_qr = count
                count += 1
            targetQR = None
            #check upper and lower qrs in list for height
            if (min_qr > 0):
                if (abs(QRList[min_qr -1].tvec[0] - x_min) < .1):
                    if QRList[min_qr -1].tvec[1] < QRList[min_qr].tvec[1]:
                        targetQR = QRList[min_qr-1]
                    else:
                        targetQR = QRList[min_qr]
            if ((min_qr < (len(QRList) - 1)) and (targetQR == None)):
                if (abs(QRList[min_qr +1].tvec[0] - x_min) < .1):
                    if QRList[min_qr +1].tvec[1] < QRList[min_qr].tvec[1]:
                        targetQR = QRList[min_qr+1]
                    else:
                        targetQR = QRList[min_qr]
            if targetQR == None:
                targetQR = QRList[min_qr]
            print "value: ", targetQR.value
            print "X:     ", targetQR.tvec[0]
            print "Y:     ", targetQR.tvec[1]
            print "====================================="
            cv2.circle(frame,targetQR.tr, 10, (0,255,0), -1)
            cv2.imshow('test', frame)
            if cv2.waitKey(1) & 0xFF == ord('q'):
                break
        self.cam.release()
        cv2.destroyAllWindows()

    def QRSweep(self):

        QRList = []
        self.cam.grab()
        self.cam.grab()
        self.cam.grab()
        self.cam.grab()
        ret, frame = self.cam.read()
        # Direct conversion cv -> zbar images did not work.
        # Buffer file used to have native data structures.

        # PIL -> zbar

        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        ret, frame = cv2.threshold(gray,95,255,cv2.THRESH_BINARY)
        cv2.imwrite('buffer.png', frame)
        pil_im = Image.open('buffer.png').convert('L')

        width, height = pil_im.size
        raw = pil_im.tobytes()
        z_im = zbar.Image(width, height, 'Y800', raw)

        # Find codes in image
        self.scanner.scan(z_im)
        count = 0
        for symbol in z_im:
            tl, bl, br, tr = [item for item in symbol.location]
            points = np.float32([[tl[0], tl[1]],
                                 [tr[0], tr[1]],
                                 [bl[0], bl[1]],
                                 [br[0], br[1]]])

            tvec = self.solveQR(tl, tr, bl, br)

            print symbol.data
            print "X_Displacement = ", tvec[0]
            print "Y_Displacement = ", tvec[1]
            print "Z_Displacement = ", tvec[2]
            print "========================================="

            #cv2.line(frame, tl, bl, (100,0,255), 8, 8)
            #cv2.line(frame, bl, br, (100,0,255), 8, 8)
            #cv2.line(frame, br, tr, (100,0,255), 8, 8)
            #cv2.line(frame, tr, tl, (100,0,255), 8, 8)

            QRList.append(QRCode2(tvec, symbol.data, tr))
            count += 1

        if count == 0:
            return None

        targetQR = self.selectQR(QRList)
        if targetQR == None:
            print "No QRCode Found"
        else:
            print "value: ", targetQR.value
            print "X:     ", targetQR.tvec[0]
            print "Y:     ", targetQR.tvec[1]
            print "X:     ", targetQR.tvec[2]
        #self.cam.release()
        return targetQR

    def selectQR(self, QRList):
        # find the best QRCode to grab (closest x then highest y)
        QRList.sort(key=lambda qr: qr.tvec[0], reverse=False)       # sort the list of qr codes by x, smallest to largest
        x_min = 100
        min_qr = 0
        count = 0
        for qr in QRList:
            #find smallest x
            if abs(qr.tvec[0]) < abs(x_min):
                x_min = qr.tvec[0]
                min_qr = count
            count += 1
        targetQR = None
        #check upper and lower qrs in list for height
        if (min_qr > 0):
            if (abs(QRList[min_qr -1].tvec[0] - x_min) < .1):
                if QRList[min_qr -1].tvec[1] < QRList[min_qr].tvec[1]:
                    targetQR = QRList[min_qr-1]
                else:
                    targetQR = QRList[min_qr]
        if ((min_qr < (len(QRList) - 1)) and (targetQR == None)):
            if (abs(QRList[min_qr +1].tvec[0] - x_min) < .1):
                if QRList[min_qr +1].tvec[1] < QRList[min_qr].tvec[1]:
                    targetQR = QRList[min_qr+1]
                else:
                    targetQR = QRList[min_qr]
        if targetQR == None:
            targetQR = QRList[min_qr]
        return targetQR

    #new solvepnp
    def solveQR(self, tl, tr, bl, br):
        QRSize = 1.5                # units = inches

        resX = 1280
        resY = 720
        center = [resX/2, resY/2]

        #find the center of the QRCode
        QRCenter = [int((tl[0] + tr[0] + bl[0] + br[0])/4), int((tl[1] + tr[1] + bl[1] + br[1])/4)]

        #find pixel displacement
        pixel_displacement = [(QRCenter[0] - center[0]),(QRCenter[1] - center[1])]

        #find Z distance to QRCode
        north = int(math.sqrt(pow(abs(tl[0] - tr[0]), 2) + pow(abs(tl[1] - tr[1]), 2)))
        south = int(math.sqrt(pow(abs(bl[0] - br[0]), 2) + pow(abs(bl[1] - br[1]), 2)))
        east = int(math.sqrt(pow(abs(tr[0] - br[0]), 2) + pow(abs(tr[1] - br[1]), 2)))
        west = int(math.sqrt(pow(abs(tl[0] - bl[0]), 2) + pow(abs(tl[1] - bl[1]), 2)))

        if (north >= south and north >= east and north >= west):
            largest_edge = north
        elif (south >= east and south >= west):
            largest_edge = south
        elif (east >= west):
            largest_edge = east
        else:
            largest_edge = west

        z_units = self.getDistance(largest_edge)

        #find x and y based on distance
        x_units = (QRSize)*(pixel_displacement[0]/float(largest_edge))
        y_units = -1 * (QRSize)*(pixel_displacement[1]/float(largest_edge))

        return [x_units, y_units, z_units]

    def getDistance(self, length):
        a = 2453.3
        n = -1.0235
        return a*math.pow(length, n)

    <commit_msg>Deleted old camera object.<commit_after>import os
import zbar
from PIL import Image
import cv2
import numpy as np
import time
import bot.lib.lib as lib
import math

class QRCode2:
    def __init__(self, tvec, value, top_right):
        self.tvec = tvec
        self.value = value
        self.tr = top_right

class block:
    def __init__(self, size, color):
        self.size = size
        self.color = color
        <|endoftext|>


<commit_before>import logging
import re

from discord.enums import Status

from bot.plugins.base import BasePlugin

from .models import GameNotification

logger = logging.getLogger(__name__)

class Plugin(BasePlugin):

    has_blocking_io = True
    subscribe_pattern = re.compile(r'!subscribe (?P<game>.+)', re.IGNORECASE)
    unsubscribe_pattern = re.compile(r'!unsubscribe (?P<game>.+)', re.IGNORECASE)

    help = (
        '`!subscribe <game>` sets up notifications for that game\n'
        '`!unsubscribe <game>` deletes your subscription\n'
        '`!unsubscribe !all` deletes all your subscriptions\n'
        'Commands are case-insensitive'
    )

    def _member_active(self, member):
        statuses = [Status.online, Status.idle]
        return member.status in statuses and not member.is_afk and not member.game

    def on_member_update(self, before, after):
        if after.game and after.game != before.game:
            member = after
            game = member.game.name
            subscribers = GameNotification.objects.filter(game_name__iexact=game).exclude(user=member.id)
            if not subscribers.exists():
                return

            ids = subscribers.values_list('user', flat=True)
            members = [m for m in member.server.members if m.id in ids and self._member_active(m)]
            if not members:
                return

            msg = '{name} started playing {game}'.format(name=member.name, game=game)
            for member in members:
                yield from self.client.send_message(member, msg)
                logger.info('Notified %s for %s', member.name, game)

    def on_message(self, message):
        user = message.author.id

        match = re.match(self.subscribe_pattern, message.content)
        if match:
            game = match.group('game').strip()
            notification, created = GameNotification.objects.get_or_create(game_name=game.lower(), user=user)
            msg = 'Subscribed you to {game}' if created else 'You were already subscribed to {game}'
            yield from self.client.send_message(message.channel, msg.format(game=game))
            return

        match = re.match(self.unsubscribe_pattern, message.content)
        if match:
            game = match.group('game').strip()
            if game.lower() == '!all':
                deleted, _ = GameNotification.objects.filter(user=user).delete()
                yield from self.client.send_message(
                    message.channel,
                    'Unsubscribed you from {num} games'.format(num=deleted)
                )
                return

            deleted, _ = GameNotification.objects.filter(user=user, game_name__iexact=game).delete()
            if deleted:
                yield from self.client.send_message(
                    message.channel,
                    'Unsubscribed you from {game} games'.format(game=game)
                )
            return
<commit_msg>Fixed notifications<commit_after>import logging
import re

from discord.enums import Status
from django.conf import settings

from bot.plugins.base import BasePlugin

from .models import GameNotification

logger = logging.getLogger(__name__)

class Plugin(BasePlugin):

    has_blocking_io = True
    subscribe_pattern = re.compile(r'!subscribe (?P<game>.+)', re.IGNORECASE)
    unsubscribe_pattern = re.compile(r'!unsubscribe (?P<game>.+)', re.IGNORECASE)

    help = (
        '`!subscribe <game>` sets up notifications for that game\n'
        '`!unsubscribe <game>` deletes your subscription\n'
        '`!unsubscribe !all` deletes all your subscriptions\n'
        'Commands are case-insensitive'
    )

    def _member_active(self, member):
        statuses = [Status.online, Status.idle]
        gaming = member.game and settings.DEBUG is False
        return member.status in statuses and not member.is_afk and not gaming

    def on_member_update(self, before, after):
        if not after.game:
            return

        member = after
        game = member.game.name
        subscribers = GameNotification.objects.filter(game_name__iexact=game)
        if settings.DEBUG:
            subscribers = subscribers.filter(user=member.id)
        else:
            subscribers = subscribers.exclude(user=member.id)
        if not subscribers.exists():
            return

        ids = subscribers.values_list('user', flat=True)
        members = [m for m in member.server.members if m.id in ids and self._member_active(m)]
        if not members:
            return

        msg = '{name} started playing {game}'.format(name=member.name, game=game)
        for member in members:
            logger.info('Notifying %s for %s', member.name, game)
            yield from self.client.send_message(member, msg)

    def on_message(self, message):
        user = message.author.id

        match = re.match(self.subscribe_pattern, message.content)
        if match:
            game = match.group('game').strip()
            notification, created = GameNotification.objects.get_or_create(game_name=game.lower(), user=user)
            msg = 'Subscribed you to {game}' if created else 'You were already subscribed to {game}'
            yield from self.client.send_message(message.channel, msg.format(game=game))
            return

        match = re.match(self.unsubscribe_pattern, message.content)
        if match:
            game = match.group('game').strip()
            if game.lower() == '!all':
                deleted, _ = GameNotification.objects.filter(user=user).delete()
                yield from self.client.send_message(
                    message.channel,
                    'Unsubscribed you from {num} games'.format(num=deleted)
                )
                return

            deleted, _ = GameNotification.objects.filter(user=user, game_name__iexact=game).delete()
            if deleted:
                yield from self.client.send_message(
                    message.channel,
                    'Unsubscribed you from {game} games'.format(game=game)
                )
            return
<|endoftext|>


<commit_before>import logging
import re

from discord.enums import Status
from django.conf import settings

from bot.plugins.base import BasePlugin
from bot.plugins.commands import command

from .models import GameNotification

logger = logging.getLogger(__name__)

class Plugin(BasePlugin):

    has_blocking_io = True

    # TODO: refactor into the command API
    help = (
        '`!subscribe <game>` sets up notifications for that game\n'
        '`!unsubscribe <game>` deletes your subscription\n'
        '`!unsubscribe !all` deletes all your subscriptions\n'
        '`!list` lists your current subscriptions\b'
        'Commands are case-insensitive'
    )

    def _member_active(self, member):
        statuses = [Status.online, Status.idle]
        gaming = member.game and settings.DEBUG is False
        return member.status in statuses and not member.is_afk and not gaming

    def on_member_update(self, before, after):
        if not after.game:
            return

        member = after
        game = member.game.name
        subscribers = GameNotification.objects.filter(game_name__iexact=game)
        if settings.DEBUG:
            subscribers = subscribers.filter(user=member.id)
        else:
            subscribers = subscribers.exclude(user=member.id)
        if not subscribers.exists():
            return

        ids = subscribers.values_list('user', flat=True)
        members = [m for m in member.server.members if m.id in ids and self._member_active(m)]
        if not members:
            return

        msg = '{name} started playing {game}'.format(name=member.name, game=game)
        for member in members:
            logger.info('Notifying %s for %s', member.name, game)
            yield from self.client.send_message(member, msg)

    @command(pattern=re.compile(r'(?P<game>.+)', re.IGNORECASE))
    def subscribe2(self, command):
        user = command.message.author.id
        game = command.args.game
        notification, created = GameNotification.objects.get_or_create(game_name=game.lower(), user=user)
        msg = 'Subscribed you to {game}' if created else 'You were already subscribed to {game}'
        yield from command.reply(msg.format(game=game))

    @command(pattern=re.compile(r'(?P<game>.+)', re.IGNORECASE))
    def unsubscribe(self, command):
        user = command.message.author.id
        game = command.args.game
        deleted, _ = GameNotification.objects.filter(user=user, game_name__iexact=game).delete()
        if deleted:
            yield from command.reply('Unsubscribed you from {game}'.format(game=game))

    @command('unsubscribe !all')
    def unsubscribe_all(self, command):
        user = command.message.author.id
        deleted, _ = GameNotification.objects.filter(user=user).delete()
        yield from command.reply('Unsubscribed you from {num} games'.format(num=deleted))

    @command()
    def list(self, command):
        user = command.message.author.id
        games = GameNotification.objects.filter(user=user).values_list('game_name', flat=True)
        msg = 'You\'re susbcribed to: {games}'.format(games=', '.join(games))
        yield from command.reply(msg)
<commit_msg>Revert debug command name<commit_after>import logging
import re

from discord.enums import Status
from django.conf import settings

from bot.plugins.base import BasePlugin
from bot.plugins.commands import command

from .models import GameNotification

logger = logging.getLogger(__name__)

class Plugin(BasePlugin):

    has_blocking_io = True

    # TODO: refactor into the command API
    help = (
        '`!subscribe <game>` sets up notifications for that game\n'
        '`!unsubscribe <game>` deletes your subscription\n'
        '`!unsubscribe !all` deletes all your subscriptions\n'
        '`!list` lists your current subscriptions\b'
        'Commands are case-insensitive'
    )

    def _member_active(self, member):
        statuses = [Status.online, Status.idle]
        gaming = member.game and settings.DEBUG is False
        return member.status in statuses and not member.is_afk and not gaming

    def on_member_update(self, before, after):
        if not after.game:
            return

        member = after
        game = member.game.name
        subscribers = GameNotification.objects.filter(game_name__iexact=game)
        if settings.DEBUG:
            subscribers = subscribers.filter(user=member.id)
        else:
            subscribers = subscribers.exclude(user=member.id)
        if not subscribers.exists():
            return

        ids = subscribers.values_list('user', flat=True)
        members = [m for m in member.server.members if m.id in ids and self._member_active(m)]
        if not members:
            return

        msg = '{name} started playing {game}'.format(name=member.name, game=game)
        for member in members:
            logger.info('Notifying %s for %s', member.name, game)
            yield from self.client.send_message(member, msg)

    @command(pattern=re.compile(r'(?P<game>.+)', re.IGNORECASE))
    def subscribe(self, command):
        user = command.message.author.id
        game = command.args.game
        notification, created = GameNotification.objects.get_or_create(game_name=game.lower(), user=user)
        msg = 'Subscribed you to {game}' if created else 'You were already subscribed to {game}'
        yield from command.reply(msg.format(game=game))

    @command(pattern=re.compile(r'(?P<game>.+)', re.IGNORECASE))
    def unsubscribe(self, command):
        user = command.message.author.id
        game = command.args.game
        deleted, _ = GameNotification.objects.filter(user=user, game_name__iexact=game).delete()
        if deleted:
            yield from command.reply('Unsubscribed you from {game}'.format(game=game))

    @command('unsubscribe !all')
    def unsubscribe_all(self, command):
        user = command.message.author.id
        deleted, _ = GameNotification.objects.filter(user=user).delete()
        yield from command.reply('Unsubscribed you from {num} games'.format(num=deleted))

    @command()
    def list(self, command):
        user = command.message.author.id
        games = GameNotification.objects.filter(user=user).values_list('game_name', flat=True)
        msg = 'You\'re susbcribed to: {games}'.format(games=', '.join(games))
        yield from command.reply(msg)
<|endoftext|>


<commit_before>"""

Copyright 2017, Institute for Systems Biology

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

from bq_data_access.data_types.definitions import FEATURE_ID_TO_TYPE_MAP
from bq_data_access.v2.feature_id_utils import FeatureDataTypeHelper, PlottableDataType

class SearchableFieldHelper(object):
    @classmethod
    def get_fields_for_all_datatypes(cls, genomic_build):
        """
        Return searchable fields for all data types for which data is available
        for the given genomic build.

        The data availability for given build for a data type is resolved by finding
        tables with matching build in the table configuration of the data type in
        the bq_data_access/data_types module. If no matching tables are found,
        no entry is returned for the data type.

        An exception to the above is clinical data, for which the searchable fields will
        always be returned. This is because the clinical data tables are not associated
        to a genomic build.

        Since the genomic build is stored in lower case in the BigQuery table
        configurations, the genomic_build parameter will be lowercased as well.

        Returns:
            Array of objects, with 'datatype' field and 'fields' key that contains an
            array of searchable_fields objects from the feature searcher classes.
        """
        genomic_build = genomic_build.lower()

        result = []
        for _, data_type in FEATURE_ID_TO_TYPE_MAP.iteritems():
            config_class = FeatureDataTypeHelper.get_feature_def_config_from_data_type(data_type)
            data_type_config_dict = FeatureDataTypeHelper.get_feature_def_default_config_dict_from_data_type(data_type)
            config_instance = config_class.from_dict(data_type_config_dict)

            # Check if any tables for the data type match the build.
            # Clinical data type will always be returned, as the clinical data tables are
            # not tied to a genomic build.
            found = False
            if data_type != PlottableDataType.CLIN:
                for table_config in config_instance.data_table_list:
                    # This would fail for a CLINTableConfig object, as it has no "genomic_build" member.
                    if table_config.genomic_build == genomic_build:
                        found = True
                        break
                if not found:
                    continue

            searcher_class = FeatureDataTypeHelper.get_feature_searcher_class_from_data_type(data_type)
            datatype_prefix = searcher_class.get_datatype_identifier()
            searchable_fields = searcher_class.get_searchable_fields()

            result.append({
                'datatype': datatype_prefix,
                'fields': searchable_fields
            })

        return result
<commit_msg>Updated import<commit_after>"""

Copyright 2017, Institute for Systems Biology

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

from bq_data_access.data_types.definitions import FEATURE_ID_TO_TYPE_MAP
from bq_data_access.v2.feature_id_utils import FeatureDataTypeHelper, PlottableDataType
from bq_data_access.v2.feature_search.searcher_mapping import get_feature_searcher_class_from_data_type

class SearchableFieldHelper(object):
    @classmethod
    def get_fields_for_all_datatypes(cls, genomic_build):
        """
        Return searchable fields for all data types for which data is available
        for the given genomic build.

        The data availability for given build for a data type is resolved by finding
        tables with matching build in the table configuration of the data type in
        the bq_data_access/data_types module. If no matching tables are found,
        no entry is returned for the data type.

        An exception to the above is clinical data, for which the searchable fields will
        always be returned. This is because the clinical data tables are not associated
        to a genomic build.

        Since the genomic build is stored in lower case in the BigQuery table
        configurations, the genomic_build parameter will be lowercased as well.

        Returns:
            Array of objects, with 'datatype' field and 'fields' key that contains an
            array of searchable_fields objects from the feature searcher classes.
        """
        genomic_build = genomic_build.lower()

        result = []
        for _, data_type in FEATURE_ID_TO_TYPE_MAP.iteritems():
            config_class = FeatureDataTypeHelper.get_feature_def_config_from_data_type(data_type)
            data_type_config_dict = FeatureDataTypeHelper.get_feature_def_default_config_dict_from_data_type(data_type)
            config_instance = config_class.from_dict(data_type_config_dict)

            # Check if any tables for the data type match the build.
            # Clinical data type will always be returned, as the clinical data tables are
            # not tied to a genomic build.
            found = False
            if data_type != PlottableDataType.CLIN:
                for table_config in config_instance.data_table_list:
                    # This would fail for a CLINTableConfig object, as it has no "genomic_build" member.
                    if table_config.genomic_build == genomic_build:
                        found = True
                        break
                if not found:
                    continue

            searcher_class = get_feature_searcher_class_from_data_type(data_type)
            datatype_prefix = searcher_class.get_datatype_identifier()
            searchable_fields = searcher_class.get_searchable_fields()

            result.append({
                'datatype': datatype_prefix,
                'fields': searchable_fields
            })

        return result
<|endoftext|>


<commit_before>from ..nvd3js import Nvd3js

class DiscreteBarChart(Nvd3js):
    _chart = "discreteBarChart"

    def options(self):
        return '''
            .x(function(d) { return d.label })
            .y(function(d) { return d.value })
            .staggerLabels('%(stagger_labels)s')
            .showValues('%(show_values)s')
            .margin({top: %(top_margin)i, right: %(right_margin)i, bottom: %(bottom_margin)i, left: %(left_margin)i})
            .width(%(width)s)
            .height(%(height)s)
            .showXAxis('%(show_xaxis)s')
            .showYAxis('%(show_yaxis)s')
            .duration(%(duration)i)
        ''' % dict(
                stagger_labels=self.stagger_labels,
                show_values=self.show_values,
                top_margin=self.top_margin,
                right_margin=self.right_margin,
                bottom_margin=self.bottom_margin,
                left_margin=self.left_margin,
                width=self.width,
                height=self.height,
                show_xaxis=self.show_xaxis,
                show_yaxis=self.show_yaxis,
                duration=self.duration)

    def __init__(self, **kwargs):
        self.stagger_labels = 'false'
        self.show_values = 'true'
        self.top_margin = 15
        self.right_margin = 10
        self.bottom_margin = 50
        self.left_margin = 60
        self.width = 'null'
        self.height = 'null'
        self.show_xaxis= 'true'
        self.show_yaxis= 'true'
        self.duration = 250

        super(DiscreteBarChart, self).__init__(**kwargs)

    def __html__(self):
        ''' used by tal:replace '''
        return self._js_ % dict(
                                    chart = self._chart,
                                    options = self.options(),
                                    data = self.data
                                )
<commit_msg>nvd3: fix booleans<commit_after>from ..nvd3js import Nvd3js

class DiscreteBarChart(Nvd3js):
    _chart = "discreteBarChart"

    def options(self):
        return '''
            .x(function(d) { return d.label })
            .y(function(d) { return d.value })
            .staggerLabels(%(stagger_labels)s)
            .showValues(%(show_values)s)
            .margin({top: %(top_margin)i, right: %(right_margin)i, bottom: %(bottom_margin)i, left: %(left_margin)i})
            .width(%(width)s)
            .height(%(height)s)
            .showXAxis(%(show_xaxis)s)
            .showYAxis(%(show_yaxis)s)
            .duration(%(duration)i)
            .tooltips(%(tooltips)s)
        ''' % dict(
                stagger_labels=self.stagger_labels,
                show_values=self.show_values,
                top_margin=self.top_margin,
                right_margin=self.right_margin,
                bottom_margin=self.bottom_margin,
                left_margin=self.left_margin,
                width=self.width,
                height=self.height,
                show_xaxis=self.show_xaxis,
                show_yaxis=self.show_yaxis,
                duration=self.duration,
                tooltips=self.tooltips,
            )

    def __init__(self, **kwargs):
        self.stagger_labels = 'false'
        self.show_values = 'false'
        self.top_margin = 15
        self.right_margin = 10
        self.bottom_margin = 50
        self.left_margin = 60
        self.width = 'null'
        self.height = 'null'
        self.show_xaxis = 'true'
        self.show_yaxis = 'true'
        self.duration = 250
        self.tooltips = 'true'

        super(DiscreteBarChart, self).__init__(**kwargs)

    def __html__(self):
        ''' used by tal:replace '''
        return self._js_ % dict(
                                    chart = self._chart,
                                    options = self.options(),
                                    data = self.data
                                )
<|endoftext|>


<commit_before>'''
Created on Dec 25, 2012

__author__ = "Elizabeth 'pidge' Flanagan"
__copyright__ = "Copyright 2013, Intel Corp."
__credits__ = ["Elizabeth Flanagan"]
__license__ = "GPL"
__version__ = "2.0"
__maintainer__ = "Elizabeth Flanagan"
__email__ = "elizabeth.flanagan@intel.com"
'''

from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, SKIPPED, \
     EXCEPTION, RETRY, worst_status
import os, datetime
import cPickle as pickle
from autobuilder.config import *
from twisted.python import log
from buildbot.steps.shell import ShellCommand

class SetDest(ShellCommand):
    name = "SetDest"
    haltOnFailure = True
    flunkOnFailure = True
    name = "Destination"
    description = ["Setting %s" % name]
    descriptionDone = [name]

    def __init__(self, factory, argdict=None, **kwargs):
        self.target=""
        for k, v in argdict.iteritems():
            setattr(self, k, v)
        self.slaveworkdir=os.path.join(os.path.join(YOCTO_ABBASE, "yocto-slave"))
        self.description = ["Setting", "Destination"]

        for k, v in argdict.iteritems():
            setattr(self, k, v)
        self.description = "Setting Destination"
        # Timeout needs to be passed to LoggingBuildStep as a kwarg
        self.timeout = 100
        kwargs['timeout']=self.timeout
        ShellCommand.__init__(self, **kwargs)

    def start(self):
        DEST = self.getProperty("DEST")
        log.msg(os.environ.get("BUILD_PUBLISH_DIR"))
        if DEST is not None:
            self.finished(SUCCESS)
        else:
            buildername=self.getProperty("buildername")
            self.workdir=os.path.join(self.slaveworkdir, buildername)
            if str(self.getProperty("custom_release_me_" + buildername)) == "True":
                rel_name = self.getProperty("custom_release_name_"  + buildername)
                if not rel_name or rel_name == "":
                    rel_name == self.getProperty(commit-description)
                DEST=os.path.normpath(os.path.join(
                                      os.environ.get("RELEASE_PUBLISH_DIR"),rel_name))
                if os.environ.get("RELEASE_PUBLISH_DIR") not in DEST:
                    self.step_status.setText(["Failed Setting Destination:", "Bad Release Name."])
                    self.finished(FAILURE)
            else:
                DEST = os.path.normpath(os.path.join(
                                        os.environ.get("BUILD_PUBLISH_DIR"), buildername))
                if os.environ.get("BUILD_PUBLISH_DIR") not in DEST:
                    self.step_status.setText(["Failed Setting Destination:", "Bad Release Name."])
                    self.finished(FAILURE)
                DEST_DATE=datetime.datetime.now().strftime("%Y%m%d")
                DATA_FILE = os.path.join(YOCTO_ABBASE, self.target + "_dest.dat")
                try:
                    pfile = open(DATA_FILE, 'rb')
                    data = pickle.load(pfile)
                except:
                    pfile = open(DATA_FILE, 'wb')
                    data = {}
                    pickle.dump(data, pfile)
                    pfile.close()
                # we can't os.path.exists here as we don't neccessarily have
                # access to the slave dest from master. So we keep a cpickle of
                # the dests.
                try:
                    # if the dictionary entry exists, we increment value by one, then repickle
                    REV=data[os.path.join(DEST, DEST_DATE)]
                    REV=int(REV) + 1
                except:
                    REV=1
                data[os.path.join(DEST, DEST_DATE)] = REV
                pfile = open(DATA_FILE, 'wb')
                pickle.dump(data, pfile)
                pfile.close()

                DEST = os.path.join(DEST, DEST_DATE + "-" + str(REV))
            self.setProperty("DEST", DEST)
            self.setProperty("DESTDIR", os.path.basename(os.path.normpath(DEST)))
            self.finished(SUCCESS)
<commit_msg>SetDest.py: Use new properties from nightly<commit_after>'''
Created on Dec 25, 2012

__author__ = "Elizabeth 'pidge' Flanagan"
__copyright__ = "Copyright 2013, Intel Corp."
__credits__ = ["Elizabeth Flanagan"]
__license__ = "GPL"
__version__ = "2.0"
__maintainer__ = "Elizabeth Flanagan"
__email__ = "elizabeth.flanagan@intel.com"
'''

from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, SKIPPED, \
     EXCEPTION, RETRY, worst_status
import os, datetime
import cPickle as pickle
from autobuilder.config import *
from twisted.python import log
from buildbot.steps.shell import ShellCommand

class SetDest(ShellCommand):
    name = "SetDest"
    haltOnFailure = True
    flunkOnFailure = True
    name = "Destination"
    description = ["Setting %s" % name]
    descriptionDone = [name]

    def __init__(self, factory, argdict=None, **kwargs):
        self.target=""
        for k, v in argdict.iteritems():
            setattr(self, k, v)
        self.slaveworkdir=os.path.join(os.path.join(YOCTO_ABBASE, "yocto-slave"))
        self.description = ["Setting", "Destination"]

        for k, v in argdict.iteritems():
            setattr(self, k, v)
        self.description = "Setting Destination"
        # Timeout needs to be passed to LoggingBuildStep as a kwarg
        self.timeout = 100
        kwargs['timeout']=self.timeout
        ShellCommand.__init__(self, **kwargs)

    def start(self):
        DEST = self.getProperty("DEST")
        log.msg(os.environ.get("BUILD_PUBLISH_DIR"))
        if DEST is not None:
            self.finished(SUCCESS)
        else:
            buildername=self.getProperty("buildername")
            self.workdir=os.path.join(self.slaveworkdir, buildername)
            if str(self.getProperty("custom_release_me_" + buildername)) == "True":
                rel_name = self.getProperty("custom_release_name_"  + buildername)
                if not rel_name or rel_name == "":
                    rel_name == self.getProperty(commit-description)
                DEST=os.path.normpath(os.path.join(
                                      os.environ.get("RELEASE_PUBLISH_DIR"),rel_name))
                if os.environ.get("RELEASE_PUBLISH_DIR") not in DEST:
                    self.step_status.setText(["Failed Setting Destination:", "Bad Release Name."])
                    self.finished(FAILURE)
            else:
                DEST = os.path.normpath(os.path.join(
                                        os.environ.get("BUILD_PUBLISH_DIR"), buildername))
                if os.environ.get("BUILD_PUBLISH_DIR") not in DEST:
                    self.step_status.setText(["Failed Setting Destination:", "Bad Release Name."])
                    self.finished(FAILURE)
                DEST_DATE=datetime.datetime.now().strftime("%Y%m%d")
                DATA_FILE = os.path.join(YOCTO_ABBASE, self.target + "_dest.dat")
                try:
                    pfile = open(DATA_FILE, 'rb')
                    data = pickle.load(pfile)
                except:
                    pfile = open(DATA_FILE, 'wb')
                    data = {}
                    pickle.dump(data, pfile)
                    pfile.close()
                # we can't os.path.exists here as we don't neccessarily have
                # access to the slave dest from master. So we keep a cpickle of
                # the dests.
                try:
                    # if the dictionary entry exists, we increment value by one, then repickle
                    REV=data[os.path.join(DEST, DEST_DATE)]
                    REV=int(REV) + 1
                except:
                    REV=1
                data[os.path.join(DEST, DEST_DATE)] = REV
                pfile = open(DATA_FILE, 'wb')
                pickle.dump(data, pfile)
                pfile.close()

                DEST = os.path.join(DEST, DEST_DATE + "-" + str(REV))
            self.setProperty("DEST", DEST)
            self.finished(SUCCESS)
<|endoftext|>


<commit_before># Copyright 2019 D-Wave Systems Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#
# =============================================================================
import itertools

import numpy as np
import dimod

from benchmarks.common import Benchmark

class NumpyArray(Benchmark):
    def setup(self):
        self.empty = np.asarray([], dtype=np.int8)
        self.arr100x100 = np.ones((100, 100), dtype=np.int8)
        self.labels100 = list(range(100))

    def time_empty(self):
        dimod.as_samples(self.empty)

    def time_100x100(self):
        dimod.as_samples(self.arr100x100)

    def time_empty_labelled(self):
        dimod.as_samples((self.empty, self.labels100))

    def time_100x100_labelled(self):
        dimod.as_samples((self.arr100x100, self.labels100))

class Dicts(Benchmark):
    def setup(self):
        # 1000 variables
        variables = list(''.join(letters) for letters in itertools.product('abcdefghij', repeat=3))

        self.dict1000 = {v: 1 for v in variables}
        self.dict1x1000 = [{v: 1 for v in variables}]
        self.dict1000x1000 = [{v: 1 for v in variables} for _ in range(1000)]

    def time_empty(self):
        dimod.as_samples({})

    def time_dict1000(self):
        dimod.as_samples(self.dict1000)

    def time_dict1x1000(self):
        dimod.as_samples(self.dict1x1000)

    def time_dict1000x1000(self):
        dimod.as_samples(self.dict1000x1000)

class Lists(Benchmark):
    def setup(self):
        self.list1000 = [1]*1000
        self.list1x1000 = [[1]*1000]
        self.list1000x1000 = [[1]*1000 for _ in range(1000)]

    def time_empty(self):
        dimod.as_samples([])

    def time_list1000(self):
        dimod.as_samples(self.list1000)

    def time_list1x1000(self):
        dimod.as_samples(self.list1x1000)

    def time_list1000x1000(self):
        dimod.as_samples(self.list1000x1000)
<commit_msg>Fix benchmark for as_samples<commit_after># Copyright 2019 D-Wave Systems Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#
# =============================================================================
import itertools

import numpy as np
import dimod

from benchmarks.common import Benchmark

class NumpyArray(Benchmark):
    def setup(self):
        self.empty = np.asarray([], dtype=np.int8)
        self.arr100x100 = np.ones((100, 100), dtype=np.int8)
        self.labels100 = list(range(100))

    def time_empty(self):
        dimod.as_samples(self.empty)

    def time_100x100(self):
        dimod.as_samples(self.arr100x100)

    def time_empty_labelled(self):
        dimod.as_samples((self.empty, []))

    def time_100x100_labelled(self):
        dimod.as_samples((self.arr100x100, self.labels100))

class Dicts(Benchmark):
    def setup(self):
        # 1000 variables
        variables = list(''.join(letters) for letters in itertools.product('abcdefghij', repeat=3))

        self.dict1000 = {v: 1 for v in variables}
        self.dict1x1000 = [{v: 1 for v in variables}]
        self.dict1000x1000 = [{v: 1 for v in variables} for _ in range(1000)]

    def time_empty(self):
        dimod.as_samples({})

    def time_dict1000(self):
        dimod.as_samples(self.dict1000)

    def time_dict1x1000(self):
        dimod.as_samples(self.dict1x1000)

    def time_dict1000x1000(self):
        dimod.as_samples(self.dict1000x1000)

class Lists(Benchmark):
    def setup(self):
        self.list1000 = [1]*1000
        self.list1x1000 = [[1]*1000]
        self.list1000x1000 = [[1]*1000 for _ in range(1000)]

    def time_empty(self):
        dimod.as_samples([])

    def time_list1000(self):
        dimod.as_samples(self.list1000)

    def time_list1x1000(self):
        dimod.as_samples(self.list1x1000)

    def time_list1000x1000(self):
        dimod.as_samples(self.list1000x1000)
<|endoftext|>


<commit_before>"""Feature extractor module."""
import itertools
import sys
import time

import numpy as np

from laserchicken import keys, utils
from laserchicken.keys import point
from laserchicken.feature_extractor.base_feature_extractor import FeatureExtractor
from laserchicken.feature_extractor.feature_map import create_default_feature_map, _create_name_extractor_pairs

FEATURES = create_default_feature_map()

def list_feature_names():
    return FEATURES  # [feature_name for feature_name in FEATURES]

def register_new_feature_extractor(extractor: FeatureExtractor):
    for name, extractor in _create_name_extractor_pairs([extractor]):
        FEATURES[name] = extractor

def compute_features(env_point_cloud, neighborhoods, target_point_cloud, feature_names, volume, verbose=True, **kwargs):
    """
    Compute features for each target and store result as point attributes in target point cloud.

    Example:
    >>> point_cloud = read_ply.read('data1.ply')
    >>> target_point_cloud = read_ply.read('data2.ply')
    >>> volume = volume_specification.InfiniteCylinder(4)
    >>> neighbors = compute_neighborhoods(point_cloud, target_point_cloud, volume)
    >>> neighborhoods = []
    >>> for x in neighbors:
    >>>   neighborhoods += x
    >>> compute_features(point_cloud,neighborhoods,['eigenv_1', 'kurto_z'],volume,)
    >>> eigenv_1 = target_point_cloud[point]['eigenv_1']['data']

    Results of the example above are stored in the target point cloud as extra point attributes.

    :param env_point_cloud: environment point cloud
    :param neighborhoods: list of neighborhoods which are themselves lists of indices referring to the environment
    :param target_point_cloud: point cloud of targets
    :param feature_names: list of features that are to be calculated
    :param volume: object describing the volume that contains the neighborhood points
    :param kwargs: keyword arguments for the individual feature extractors
    :param verbose: if true, output extra information
    :return: None, results are stored in attributes of the target point cloud
    """
    _verify_feature_names(feature_names)
    wanted_feature_names = feature_names + [existing_feature for existing_feature in target_point_cloud[point]]
    extended_features = _make_extended_feature_list(feature_names)

    for feature_name in extended_features:
        target_point_cloud[point][feature_name] = {"type": 'float64',
                                                   "data": np.zeros_like(target_point_cloud[point]['x']['data'],
                                                                         dtype=np.float64)}

    _add_features(extended_features, env_point_cloud, neighborhoods, target_point_cloud, volume, verbose, kwargs)

    _keep_only_wanted_features(target_point_cloud, wanted_feature_names)

def _add_features(extended_features, env_point_cloud, neighborhoods, target_point_cloud, volume, verbose, kwargs):
    chunk_size = 100000
    n_targets = _get_point_cloud_size(target_point_cloud)
    for chunk_no in range(_calculate_number_of_chunks(chunk_size, n_targets)):
        i_start = chunk_no * chunk_size
        i_end = min((chunk_no + 1) * chunk_size, n_targets)
        target_indices = np.arange(i_start, i_end)
        current_neighborhoods = list(itertools.islice(neighborhoods, i_end - i_start))

        features_to_do = list(extended_features)

        _compute_features_for_chunk(features_to_do, env_point_cloud, current_neighborhoods, target_point_cloud,
                                    target_indices, volume, verbose, kwargs)

def _get_point_cloud_size(target_point_cloud):
    return len(target_point_cloud[point]['x']['data'])

def _calculate_number_of_chunks(chunk_size, n_targets):
    return int(np.math.ceil(n_targets / chunk_size))

def _compute_features_for_chunk(features_to_do, env_point_cloud, current_neighborhoods, target_point_cloud,
                                target_indices, volume,
                                verbose, kwargs):
    while features_to_do:
        feature_name = features_to_do[0]
        extractor = FEATURES[feature_name]

        if verbose:
            sys.stdout.write('Extracting feature(s) "{}"'.format(extractor.provides()))
            start = time.time()

        for key_word in kwargs:
            setattr(extractor, key_word, kwargs[key_word])
        _add_features_from_single_extractor(extractor, env_point_cloud, current_neighborhoods, target_point_cloud,
                                            target_indices, volume)
        utils.add_metadata(target_point_cloud, type(extractor).__module__, extractor.get_params())

        if verbose:
            elapsed = time.time() - start
            sys.stdout.write('Extracting feature(s) "{}" took {:.2f} seconds\n'.format(extractor.provides(), elapsed))

        for provided_feature in extractor.provides():
            if provided_feature in features_to_do:
                features_to_do.remove(provided_feature)

def _add_features_from_single_extractor(extractor, env_point_cloud, current_neighborhoods, target_point_cloud,
                                        target_indices, volume):
    provided_features = extractor.provides()
    n_features = len(provided_features)
    point_values = extractor.extract(env_point_cloud, current_neighborhoods, target_point_cloud,
                                     target_indices, volume)

    n_targets = len(target_indices)
    feature_values = [np.empty(n_targets, dtype=np.float64) for _ in range(n_features)]
    if n_features > 1:
        for i in range(n_features):
            feature_values[i] = point_values[i]
    else:
        feature_values[0] = point_values
    for i in range(n_features):
        feature = provided_features[i]
        target_point_cloud[point][feature]['data'][target_indices] = feature_values[i]

def _keep_only_wanted_features(target_point_cloud, wanted_feature_names):
    redundant_features = [f for f in target_point_cloud[point] if f not in wanted_feature_names]
    if redundant_features:
        print('The following unrequested features were calculated as a side effect, but will not be returned:',
              redundant_features)
    for f in redundant_features:
        target_point_cloud[point].pop(f)

def _verify_feature_names(feature_names):
    unknown_features = [f for f in feature_names if f not in FEATURES]
    if any(unknown_features):
        raise ValueError('Unknown features selected: {}. Available feature are: {}'
                         .format(', '.join(unknown_features), ', '.join(FEATURES.keys())))

def _make_extended_feature_list(feature_names):
    feature_list = reversed(_make_extended_feature_list_helper(feature_names))
    return _remove_duplicates(feature_list)

def _remove_duplicates(feature_list):
    seen = set()
    return [f for f in feature_list if not (f in seen or seen.add(f))]

def _make_extended_feature_list_helper(feature_names):
    feature_list = list(feature_names)
    for feature_name in feature_names:
        extractor = FEATURES[feature_name]
        dependencies = extractor.requires()
        feature_list.extend(dependencies)
        feature_list.extend(_make_extended_feature_list_helper(dependencies))
        feature_list.extend(extractor.provides())
    return feature_list
<commit_msg>Update laserchicken/feature_extractor/__init__.py<commit_after>"""Feature extractor module."""
import itertools
import sys
import time

import numpy as np

from laserchicken import keys, utils
from laserchicken.keys import point
from laserchicken.feature_extractor.base_feature_extractor import FeatureExtractor
from laserchicken.feature_extractor.feature_map import create_default_feature_map, _create_name_extractor_pairs

FEATURES = create_default_feature_map()

def list_feature_names():
    return FEATURES  # [feature_name for feature_name in FEATURES]

def register_new_feature_extractor(extractor: FeatureExtractor):
    for name, extractor in _create_name_extractor_pairs([extractor]):
        FEATURES[name] = extractor

def compute_features(env_point_cloud, neighborhoods, target_point_cloud, feature_names, volume, verbose=True, **kwargs):
    """
    Compute features for each target and store result as point attributes in target point cloud.

    Example:
    >>> point_cloud = read_ply.read('data1.ply')
    >>> target_point_cloud = read_ply.read('data2.ply')
    >>> volume = volume_specification.InfiniteCylinder(4)
    >>> neighbors = compute_neighborhoods(point_cloud, target_point_cloud, volume)
    >>> neighborhoods = []
    >>> for x in neighbors:
    >>>   neighborhoods += x
    >>> compute_features(point_cloud, neighborhoods, target_point_cloud, ['eigenv_1', 'kurto_z'], volume)
    >>> eigenv_1 = target_point_cloud[point]['eigenv_1']['data']

    Results of the example above are stored in the target point cloud as extra point attributes.

    :param env_point_cloud: environment point cloud
    :param neighborhoods: list of neighborhoods which are themselves lists of indices referring to the environment
    :param target_point_cloud: point cloud of targets
    :param feature_names: list of features that are to be calculated
    :param volume: object describing the volume that contains the neighborhood points
    :param kwargs: keyword arguments for the individual feature extractors
    :param verbose: if true, output extra information
    :return: None, results are stored in attributes of the target point cloud
    """
    _verify_feature_names(feature_names)
    wanted_feature_names = feature_names + [existing_feature for existing_feature in target_point_cloud[point]]
    extended_features = _make_extended_feature_list(feature_names)

    for feature_name in extended_features:
        target_point_cloud[point][feature_name] = {"type": 'float64',
                                                   "data": np.zeros_like(target_point_cloud[point]['x']['data'],
                                                                         dtype=np.float64)}

    _add_features(extended_features, env_point_cloud, neighborhoods, target_point_cloud, volume, verbose, kwargs)

    _keep_only_wanted_features(target_point_cloud, wanted_feature_names)

def _add_features(extended_features, env_point_cloud, neighborhoods, target_point_cloud, volume, verbose, kwargs):
    chunk_size = 100000
    n_targets = _get_point_cloud_size(target_point_cloud)
    for chunk_no in range(_calculate_number_of_chunks(chunk_size, n_targets)):
        i_start = chunk_no * chunk_size
        i_end = min((chunk_no + 1) * chunk_size, n_targets)
        target_indices = np.arange(i_start, i_end)
        current_neighborhoods = list(itertools.islice(neighborhoods, i_end - i_start))

        features_to_do = list(extended_features)

        _compute_features_for_chunk(features_to_do, env_point_cloud, current_neighborhoods, target_point_cloud,
                                    target_indices, volume, verbose, kwargs)

def _get_point_cloud_size(target_point_cloud):
    return len(target_point_cloud[point]['x']['data'])

def _calculate_number_of_chunks(chunk_size, n_targets):
    return int(np.math.ceil(n_targets / chunk_size))

def _compute_features_for_chunk(features_to_do, env_point_cloud, current_neighborhoods, target_point_cloud,
                                target_indices, volume,
                                verbose, kwargs):
    while features_to_do:
        feature_name = features_to_do[0]
        extractor = FEATURES[feature_name]

        if verbose:
            sys.stdout.write('Extracting feature(s) "{}"'.format(extractor.provides()))
            start = time.time()

        for key_word in kwargs:
            setattr(extractor, key_word, kwargs[key_word])
        _add_features_from_single_extractor(extractor, env_point_cloud, current_neighborhoods, target_point_cloud,
                                            target_indices, volume)
        utils.add_metadata(target_point_cloud, type(extractor).__module__, extractor.get_params())

        if verbose:
            elapsed = time.time() - start
            sys.stdout.write('Extracting feature(s) "{}" took {:.2f} seconds\n'.format(extractor.provides(), elapsed))

        for provided_feature in extractor.provides():
            if provided_feature in features_to_do:
                features_to_do.remove(provided_feature)

def _add_features_from_single_extractor(extractor, env_point_cloud, current_neighborhoods, target_point_cloud,
                                        target_indices, volume):
    provided_features = extractor.provides()
    n_features = len(provided_features)
    point_values = extractor.extract(env_point_cloud, current_neighborhoods, target_point_cloud,
                                     target_indices, volume)

    n_targets = len(target_indices)
    feature_values = [np.empty(n_targets, dtype=np.float64) for _ in range(n_features)]
    if n_features > 1:
        for i in range(n_features):
            feature_values[i] = point_values[i]
    else:
        feature_values[0] = point_values
    for i in range(n_features):
        feature = provided_features[i]
        target_point_cloud[point][feature]['data'][target_indices] = feature_values[i]

def _keep_only_wanted_features(target_point_cloud, wanted_feature_names):
    redundant_features = [f for f in target_point_cloud[point] if f not in wanted_feature_names]
    if redundant_features:
        print('The following unrequested features were calculated as a side effect, but will not be returned:',
              redundant_features)
    for f in redundant_features:
        target_point_cloud[point].pop(f)

def _verify_feature_names(feature_names):
    unknown_features = [f for f in feature_names if f not in FEATURES]
    if any(unknown_features):
        raise ValueError('Unknown features selected: {}. Available feature are: {}'
                         .format(', '.join(unknown_features), ', '.join(FEATURES.keys())))

def _make_extended_feature_list(feature_names):
    feature_list = reversed(_make_extended_feature_list_helper(feature_names))
    return _remove_duplicates(feature_list)

def _remove_duplicates(feature_list):
    seen = set()
    return [f for f in feature_list if not (f in seen or seen.add(f))]

def _make_extended_feature_list_helper(feature_names):
    feature_list = list(feature_names)
    for feature_name in feature_names:
        extractor = FEATURES[feature_name]
        dependencies = extractor.requires()
        feature_list.extend(dependencies)
        feature_list.extend(_make_extended_feature_list_helper(dependencies))
        feature_list.extend(extractor.provides())
    return feature_list
<|endoftext|>


<commit_before>from .env import YowsupEnv
import base64
import hashlib
class S40YowsupEnv(YowsupEnv):
    _VERSION = "2.12.89"
    _OS_NAME= "S40"
    _OS_VERSION = "14.26"
    _DEVICE_NAME = "Nokia302"
    _TOKEN_STRING  = "PdA2DJyKoUrwLw1Bg6EIhzh502dF9noR9uFCllGk1435688727801{phone}"
    _AXOLOTL = True

    def getVersion(self):
        return self.__class__._VERSION

    def getOSName(self):
        return self.__class__._OS_NAME

    def getOSVersion(self):
        return self.__class__._OS_VERSION

    def getDeviceName(self):
        return self.__class__._DEVICE_NAME

    def isAxolotlEnabled(self):
        return self.__class__._AXOLOTL

    def getToken(self, phoneNumber):
        return hashlib.md5(self.__class__._TOKEN_STRING.format(phone = phoneNumber).encode()).hexdigest()

    def getUserAgent(self):
        return self.__class__._USERAGENT_STRING.format(
            WHATSAPP_VERSION = self.getVersion(),
            OS_NAME = self.getOSName() + "Version",
            OS_VERSION = self.getOSVersion(),
            DEVICE_NAME = self.getDeviceName()
        )

<commit_msg>Updated s40 env to 2.12.96<commit_after>from .env import YowsupEnv
import base64
import hashlib
class S40YowsupEnv(YowsupEnv):
    _VERSION = "2.12.96"
    _OS_NAME= "S40"
    _OS_VERSION = "14.26"
    _DEVICE_NAME = "Nokia302"
    _TOKEN_STRING  = "PdA2DJyKoUrwLw1Bg6EIhzh502dF9noR9uFCllGk1439921717185{phone}"
    _AXOLOTL = True

    def getVersion(self):
        return self.__class__._VERSION

    def getOSName(self):
        return self.__class__._OS_NAME

    def getOSVersion(self):
        return self.__class__._OS_VERSION

    def getDeviceName(self):
        return self.__class__._DEVICE_NAME

    def isAxolotlEnabled(self):
        return self.__class__._AXOLOTL

    def getToken(self, phoneNumber):
        return hashlib.md5(self.__class__._TOKEN_STRING.format(phone = phoneNumber).encode()).hexdigest()

    def getUserAgent(self):
        return self.__class__._USERAGENT_STRING.format(
            WHATSAPP_VERSION = self.getVersion(),
            OS_NAME = self.getOSName() + "Version",
            OS_VERSION = self.getOSVersion(),
            DEVICE_NAME = self.getDeviceName()
        )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Holds the zazu repo subcommand."""
import zazu.build
import zazu.git_helper
import zazu.github_helper
import zazu.util
zazu.util.lazy_import(locals(), [
    'click',
    'functools',
    'git',
    'os'
])

__author__ = "Nicholas Wiles"
__copyright__ = "Copyright 2016"

@click.group()
@click.pass_context
def repo(ctx):
    """Manage repository."""
    pass

@repo.group()
@click.pass_context
def setup(ctx):
    """Handle repository services setup."""
    ctx.obj.check_repo()
    pass

@setup.command()
@click.pass_context
def hooks(ctx):
    """Install default git hooks."""
    zazu.git_helper.install_git_hooks(ctx.obj.repo_root)

@setup.command()
@click.pass_context
def ci(ctx):
    """Post CI configurations to the CI server based on a zazu.yaml file."""
    ctx.obj.check_repo()
    build_server = ctx.obj.build_server()
    project_config = ctx.obj.project_config()
    if click.confirm("Post build configuration to {}?".format(build_server.type())):
        scm_url = ctx.obj.repo.remotes.origin.url
        _, scm_name = zazu.github_helper.parse_github_url(scm_url)
        components = project_config['components']
        for c in components:
            component = zazu.build.ComponentConfiguration(c)
            build_server.setup_component(component, scm_name, scm_url)

@repo.command()
@click.pass_context
def list(ctx):
    for name, host in ctx.obj.scm_hosts().iteritems():
        print('Host {}: {}'.format(name, zazu.util.pprint_list(host.repos())))

@repo.command()
@click.argument('repository_url')
@click.option('--nohooks', is_flag=True, help='does not install git hooks in the cloned repo')
@click.option('--nosubmodules', is_flag=True, help='does not update submodules')
def clone(repository_url, nohooks, nosubmodules):
    """Clone and initialize a repo.

    Args:
        repository_url (str): url of the repository to clone.
        nohooks (bool): if True, git hooks are not installed.
        nosubmodules (bool): if True submodules are not initialized.
    """
    try:
        destination = '{}/{}'.format(os.getcwd(), repository_url.rsplit('/', 1)[-1].replace('.git', ''))
        repo = git.Repo.clone_from(repository_url, destination)
        click.echo('Repository successfully cloned')

        if not nohooks:
            click.echo('Installing Git Hooks')
            zazu.git_helper.install_git_hooks(repo.working_dir)

        if not nosubmodules:
            click.echo('Updating all submodules')
            repo.submodule_update(init=True, recursive=True)

    except git.GitCommandError as err:
        raise click.ClickException(str(err))

@repo.command()
def init():
    """Initialize repo directory structure."""
    raise NotImplementedError

@repo.command()
@click.option('-r', '--remote', is_flag=True, help='Also clean up remote branches')
@click.option('-b', '--target_branch', default='origin/master', help='Delete branches merged with this branch')
@click.option('-y', '--yes', is_flag=True, help='Don\'t ask to before deleting branches')
@click.pass_context
def cleanup(ctx, remote, target_branch, yes):
    """Clean up merged branches that have been merged or are associated with closed/resolved tickets."""
    ctx.obj.check_repo()
    repo_obj = ctx.obj.repo
    try:
        repo_obj.git.checkout('develop')
    except git.exc.GitCommandError:
        raise click.ClickException('unable to checkout "develop"')
    try:
        issue_tracker = ctx.obj.issue_tracker()
    except click.ClickException:
        issue_tracker = None
    closed_branches = set([])
    if remote:
        repo_obj.git.fetch('--prune')
        remote_branches = zazu.git_helper.filter_undeletable([b.name for b in repo_obj.remotes.origin.refs])
        if issue_tracker is not None:
            closed_branches = set(get_closed_branches(issue_tracker, remote_branches))
        merged_remote_branches = zazu.git_helper.filter_undeletable(zazu.git_helper.get_merged_branches(repo_obj, target_branch, remote=True))
        merged_remote_branches = [b.replace('origin/', '') for b in merged_remote_branches]
        branches_to_delete = set(merged_remote_branches) | closed_branches
        if branches_to_delete:
            confirmation = 'These remote branches will be deleted: {} Proceed?'.format(zazu.util.pprint_list(branches_to_delete))
            if yes or click.confirm(confirmation):
                for b in branches_to_delete:
                    click.echo('Deleting {}'.format(b))
                repo_obj.git.push('-df', 'origin', *branches_to_delete)
    merged_branches = zazu.git_helper.filter_undeletable(zazu.git_helper.get_merged_branches(repo_obj, target_branch))
    local_branches = set(zazu.git_helper.filter_undeletable([b.name for b in repo_obj.heads]))
    if issue_tracker is not None:
        branches_to_check = local_branches - closed_branches
        closed_branches |= set(get_closed_branches(issue_tracker, branches_to_check))
    branches_to_delete = (closed_branches & local_branches) | set(merged_branches)
    if branches_to_delete:
        confirmation = 'These local branches will be deleted: {}\n Proceed?'.format(zazu.util.pprint_list(branches_to_delete))
        if yes or click.confirm(confirmation):
            for b in branches_to_delete:
                click.echo('Deleting {}'.format(b))
            repo_obj.git.branch('-D', *branches_to_delete)

def descriptors_from_branches(branches):
    """Generate IssueDescriptors from a branch names."""
    for b in branches:
        try:
            yield zazu.dev.commands.make_issue_descriptor(b)
        except click.ClickException:
            pass

def get_closed_branches(issue_tracker, branches):
    """Get descriptors of branches that refer to closed branches."""
    def descriptor_if_closed(tracker, descriptor):
        return descriptor if ticket_is_closed(issue_tracker, descriptor) else None

    work = [functools.partial(descriptor_if_closed, issue_tracker, d) for d in descriptors_from_branches(branches)]
    closed_tickets = zazu.util.dispatch(work)
    return [t.get_branch_name() for t in closed_tickets if t is not None]

def ticket_is_closed(issue_tracker, descriptor):
    """Determine if a ticket is closed or not, defaults to False in case the ticket isn't found by the issue tracker."""
    try:
        return issue_tracker.issue(descriptor.id).closed
    except zazu.issue_tracker.IssueTrackerError:
        return False
<commit_msg>remove repo list command<commit_after># -*- coding: utf-8 -*-
"""Holds the zazu repo subcommand."""
import zazu.build
import zazu.git_helper
import zazu.github_helper
import zazu.util
zazu.util.lazy_import(locals(), [
    'click',
    'functools',
    'git',
    'os'
])

__author__ = "Nicholas Wiles"
__copyright__ = "Copyright 2016"

@click.group()
@click.pass_context
def repo(ctx):
    """Manage repository."""
    pass

@repo.group()
@click.pass_context
def setup(ctx):
    """Handle repository services setup."""
    ctx.obj.check_repo()
    pass

@setup.command()
@click.pass_context
def hooks(ctx):
    """Install default git hooks."""
    zazu.git_helper.install_git_hooks(ctx.obj.repo_root)

@setup.command()
@click.pass_context
def ci(ctx):
    """Post CI configurations to the CI server based on a zazu.yaml file."""
    ctx.obj.check_repo()
    build_server = ctx.obj.build_server()
    project_config = ctx.obj.project_config()
    if click.confirm("Post build configuration to {}?".format(build_server.type())):
        scm_url = ctx.obj.repo.remotes.origin.url
        _, scm_name = zazu.github_helper.parse_github_url(scm_url)
        components = project_config['components']
        for c in components:
            component = zazu.build.ComponentConfiguration(c)
            build_server.setup_component(component, scm_name, scm_url)

@repo.command()
@click.argument('repository_url')
@click.option('--nohooks', is_flag=True, help='does not install git hooks in the cloned repo')
@click.option('--nosubmodules', is_flag=True, help='does not update submodules')
def clone(repository_url, nohooks, nosubmodules):
    """Clone and initialize a repo.

    Args:
        repository_url (str): url of the repository to clone.
        nohooks (bool): if True, git hooks are not installed.
        nosubmodules (bool): if True submodules are not initialized.
    """
    try:
        destination = '{}/{}'.format(os.getcwd(), repository_url.rsplit('/', 1)[-1].replace('.git', ''))
        repo = git.Repo.clone_from(repository_url, destination)
        click.echo('Repository successfully cloned')

        if not nohooks:
            click.echo('Installing Git Hooks')
            zazu.git_helper.install_git_hooks(repo.working_dir)

        if not nosubmodules:
            click.echo('Updating all submodules')
            repo.submodule_update(init=True, recursive=True)

    except git.GitCommandError as err:
        raise click.ClickException(str(err))

@repo.command()
def init():
    """Initialize repo directory structure."""
    raise NotImplementedError

@repo.command()
@click.option('-r', '--remote', is_flag=True, help='Also clean up remote branches')
@click.option('-b', '--target_branch', default='origin/master', help='Delete branches merged with this branch')
@click.option('-y', '--yes', is_flag=True, help='Don\'t ask to before deleting branches')
@click.pass_context
def cleanup(ctx, remote, target_branch, yes):
    """Clean up merged branches that have been merged or are associated with closed/resolved tickets."""
    ctx.obj.check_repo()
    repo_obj = ctx.obj.repo
    try:
        repo_obj.git.checkout('develop')
    except git.exc.GitCommandError:
        raise click.ClickException('unable to checkout "develop"')
    try:
        issue_tracker = ctx.obj.issue_tracker()
    except click.ClickException:
        issue_tracker = None
    closed_branches = set([])
    if remote:
        repo_obj.git.fetch('--prune')
        remote_branches = zazu.git_helper.filter_undeletable([b.name for b in repo_obj.remotes.origin.refs])
        if issue_tracker is not None:
            closed_branches = set(get_closed_branches(issue_tracker, remote_branches))
        merged_remote_branches = zazu.git_helper.filter_undeletable(zazu.git_helper.get_merged_branches(repo_obj, target_branch, remote=True))
        merged_remote_branches = [b.replace('origin/', '') for b in merged_remote_branches]
        branches_to_delete = set(merged_remote_branches) | closed_branches
        if branches_to_delete:
            confirmation = 'These remote branches will be deleted: {} Proceed?'.format(zazu.util.pprint_list(branches_to_delete))
            if yes or click.confirm(confirmation):
                for b in branches_to_delete:
                    click.echo('Deleting {}'.format(b))
                repo_obj.git.push('-df', 'origin', *branches_to_delete)
    merged_branches = zazu.git_helper.filter_undeletable(zazu.git_helper.get_merged_branches(repo_obj, target_branch))
    local_branches = set(zazu.git_helper.filter_undeletable([b.name for b in repo_obj.heads]))
    if issue_tracker is not None:
        branches_to_check = local_branches - closed_branches
        closed_branches |= set(get_closed_branches(issue_tracker, branches_to_check))
    branches_to_delete = (closed_branches & local_branches) | set(merged_branches)
    if branches_to_delete:
        confirmation = 'These local branches will be deleted: {}\n Proceed?'.format(zazu.util.pprint_list(branches_to_delete))
        if yes or click.confirm(confirmation):
            for b in branches_to_delete:
                click.echo('Deleting {}'.format(b))
            repo_obj.git.branch('-D', *branches_to_delete)

def descriptors_from_branches(branches):
    """Generate IssueDescriptors from a branch names."""
    for b in branches:
        try:
            yield zazu.dev.commands.make_issue_descriptor(b)
        except click.ClickException:
            pass

def get_closed_branches(issue_tracker, branches):
    """Get descriptors of branches that refer to closed branches."""
    def descriptor_if_closed(tracker, descriptor):
        return descriptor if ticket_is_closed(issue_tracker, descriptor) else None

    work = [functools.partial(descriptor_if_closed, issue_tracker, d) for d in descriptors_from_branches(branches)]
    closed_tickets = zazu.util.dispatch(work)
    return [t.get_branch_name() for t in closed_tickets if t is not None]

def ticket_is_closed(issue_tracker, descriptor):
    """Determine if a ticket is closed or not, defaults to False in case the ticket isn't found by the issue tracker."""
    try:
        return issue_tracker.issue(descriptor.id).closed
    except zazu.issue_tracker.IssueTrackerError:
        return False
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""

launchconfigcleanup - Assists in cleaning up unused LaunchConfigurations

Usage: launchconfigcleanup.py [options]

Options:
    -a, --minage DAYS   Minimum number of days a LaunchConfiguration must be
                        to be considered a deletion candidate.

    -D, --delete        Delete a candidate if one is present

    -n, --maxlcs NUM    Maximum number of LaunchConfigurations allowed in the
                        AWS account. A candidate will only be presented if the
                        account is at its limit.
"""

from __future__ import print_function

import boto3
import logging
from botocore.exceptions import ClientError
from datetime import datetime
from datetime import timedelta
from datetime import tzinfo

log = logging.getLogger(__name__)
"""Create a logger"""

DEFAULT_MINIMUM_AGE = 5
"""Min age (days) a LaunchConfiguration must be to be a candidate"""

ZERO = timedelta(0)

class UTC(tzinfo):
    """
    Implements UTC timezone for datetime interaction
    """
    def utcoffset(self, dt):
        return ZERO

    def tzname(self, dt):
        return "UTC"

    def dst(self, dt):
        return ZERO

class AllCandidatesTooNew(Exception):
    """
    Defines a failure to find an old enough unused LC deletion candidate
    """
    pass

def get_max_launchconfigurations(asg_client):
    """
    Return the maximum number of LaunchConfigurations allowed in account
    """
    return asg_client.describe_account_limits()['MaxNumberOfLaunchConfigurations']

def get_all_launchconfigurations(asg_client):
    """
    Generator to find all LaunchConfigurations
    """
    lc_pager = asg_client.get_paginator('describe_launch_configurations')
    lc_iter = lc_pager.paginate()
    for lc_page in lc_iter:
        for lc in lc_page['LaunchConfigurations']:
            yield lc

def get_inuse_launchconfigurations(asg_client):
    """
    Generator to find LaunchConfigurations in-use by ASGs

    :param asg_client: boto3 autoscaling client
    :type asg_client: botocore.client.AutoScaling
    """
    asg_pager = asg_client.get_paginator('describe_auto_scaling_groups')
    asg_iter = asg_pager.paginate()
    for asg_page in asg_iter:
        for asg in asg_page['AutoScalingGroups']:
            yield asg['LaunchConfigurationName']

def get_launchconfigurations_delete_candidate(asg_client,
        lc_limit=None,
        min_age=DEFAULT_MINIMUM_AGE):
    """
    Return the name of a LaunchConfiguration to delete if space is needed

    :param asg_client: boto3 autoscaling client
    :type asg_client: botocore.client.AutoScaling
    :param lc_limit: Max number of LaunchConfigurations allowed in account
    :type lc_limit: int
    :param min_age: Days old a LaunchConfiguration must be to be a candidate
    :type min_age: int
    """
    if lc_limit is None:
        # If not explicit LaunchConfiguration limit provided, determine the
        # max number allowed for the client account
        lc_limit = get_max_launchconfigurations(asg_client)

    all_lcs = [lc for lc in get_all_launchconfigurations(asg_client)]

    if len(all_lcs) < lc_limit:
        # No LC needs to be deleted
        log.info("{lc} < {limit}, no LaunchConfiguration deletion needed".format(
            lc=len(all_lcs),
            limit=lc_limit))
        return None

    log.warning("At LaunchConfiguration limit! ({limit})".format(
        limit=lc_limit))

    # Find LCs that are in-use
    used_lcs = [lc_name for lc_name in get_inuse_launchconfigurations(asg_client)]
    log.debug("Found {} LaunchConfigurations in-use".format(len(used_lcs)))

    # Determine unused LCs
    unused = [lc for lc in all_lcs if lc['LaunchConfigurationName'] not in used_lcs]
    log.debug("Found {} LaunchConfigurations unused".format(len(unused)))

    # Find old enough candidates
    min_created_time = datetime.now(UTC()) - timedelta(days=min_age)
    log.warning("Finding LCs from before {}".format(min_created_time.isoformat()))
    unused = [lc for lc in unused
              if lc['CreatedTime'] <= min_created_time]

    log.debug("Found {c} LaunchConfiguration candidate{s}".format(
        c=len(unused),
        s="" if len(unused) == 1 else "s"))

    # If there are no old enough candidates, raise an exception
    if len(unused) == 0:
        raise AllCandidatesTooNew(
            "No candidates at least {d} day{s} old!".format(
                d=min_age,
                s="" if min_age == 1 else "s"))

    return sorted(unused, key=lambda k: k['CreatedTime'])[0]

def lambda_handler(event, context):
    """
    Delete ASG LaunchConfiguration if at the account limit
    """

    # Setup logging
    logging.basicConfig(**{
        'level': logging.INFO,
        'format': '%(levelname)s - %(message)s',
    })

    # DryRun is not supported by the boto3 autoscaling client
    if 'DryRun' in event:
        raise NotImplementedError(
            "The boto3 autoscaling client does not support DryRun")

    candidate_params = {
        'lc_limit': None,
        'min_age': DEFAULT_MINIMUM_AGE,
    }

    # If the event includes a specified minimum age, use it
    if 'MinAge' in event:
        candidate_params.update({
            'min_page': int(event['MinAge']),
        })

    # Create boto client for ASG API
    client = boto3.client('autoscaling')

    # Find a candidate
    candidate = get_launchconfigurations_delete_candidate(
        client,
        **candidate_params)

    # Delete the candidate
    log.info("Candidate:")
    log.info(json.dumps(
        candidate,
        indent=4,
        separators=(',', ': ')))

    # Execute the deletion
    log.info("Attempting to delete the LaunchConfiguration")
    client.delete_launch_configuration(
        LaunchConfigurationName=candidate['LaunchConfigurationName'])

    log.info("Successfully deleted LaunchConfiguration")

if __name__ == '__main__':
    import json
    import sys
    from docopt import docopt

    # Setup logging
    logging.basicConfig(**{
        'level': logging.INFO,
        'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
    })

    # Parse CLI arguments
    args = docopt(__doc__, version='dev')

    do_delete = args['--delete']

    max_lcs = int(args['--maxlcs'] or -1)
    min_age = int(args['--minage'] or DEFAULT_MINIMUM_AGE)

    # Create boto client for ASG API
    client = boto3.client('autoscaling')

    # If no maxlcs was passed, determine the accounts maximum allowed
    if max_lcs < 1:
        max_lcs = get_max_launchconfigurations(client)

    # Find a deletion candidate
    candidate = get_launchconfigurations_delete_candidate(
        client,
        max_lcs,
        min_age)

    # If no candidate is present, exit
    if candidate is None:
        log.warning("No deletion candidate found")
        sys.exit(0)

    # Convert the candidates CreatedTime field to a string for serialization
    if 'CreatedTime' in candidate:
        candidate.update({
            'CreatedTime': candidate['CreatedTime'].isoformat()
        })

    log.info("Candidate:")
    log.info(json.dumps(
        candidate,
        indent=4,
        separators=(',', ': ')))

    if do_delete:
        # Execution the deletion
        log.info("Attempting to delete the LaunchConfiguration")

        try:
            client.delete_launch_configuration(
                LaunchConfigurationName=candidate['LaunchConfigurationName'])

        except AllCandidatesTooNew as e:
            log.critical("Could not find an old enough deletion candidate!")
            sys.exit(1)

        except ClientError as e:
            log.critical("Failed to delete LaunchConfiguration", exc_info=True)
            sys.exit(254)

        except Exception as e:
            log.critical("Unknown failure!", exc_info=True)
            sys.exit(254)

        log.info("Successfully deleted LaunchConfiguration")
<commit_msg>Add support make space for N LaunchConfigs DO-4059 This also significantly refactors: - Decoupling candidate enumeration & deletion - Parameter passing between CLI & Lambda - Adds mock DryRun support<commit_after>#!/usr/bin/env python
"""

launchconfigcleanup - Assists in cleaning up unused LaunchConfigurations

Usage: launchconfigcleanup.py [options]

Options:
    -a, --minage DAYS   Minimum number of days a LaunchConfiguration must be
                        to be considered a deletion candidate. [default: 5]

    -D, --delete NUM    Number of candidates to delete [default: 1]
                        If this number is negative, this script determines how
                        many LaunchConfigurations need to be deleted to ensure
                        that number of slots is available away from the
                        maximum.

    --dry-run           Only log what would happen, do not delete any
                        LaunchConfigurations

    -n, --maxlcs NUM    Maximum number of LaunchConfigurations allowed in the
                        AWS account. This parameter should only be used during
                        testing, otherwise the script will determine what your
                        accounts limitation is.

    -l, --log-level LEVEL   What logging level to use [default: warning]
"""

from __future__ import print_function

import boto3
import logging
from botocore.exceptions import ClientError
from datetime import datetime
from datetime import timedelta
from datetime import tzinfo

log = logging.getLogger(__name__)
"""Create a logger"""

ZERO = timedelta(0)

LAMBDA_DEFAULTS = {
    # --dry-run
    'DryRun': False,
    # --delete
    'LC_Delete': 1,
    # --maxlcs
    'LC_Limit': lambda c: get_max_launchconfigurations(c),
    # --log-level
    'LogLevel': 'info',
    # --minage
    'MinAge': 5,
}
"""Default parameters for Lambda, these mimic the docopt arguments"""

class UTC(tzinfo):
    """
    Implements UTC timezone for datetime interaction
    """
    def utcoffset(self, dt):
        return ZERO

    def tzname(self, dt):
        return "UTC"

    def dst(self, dt):
        return ZERO

class InsufficientCandidates(Exception):
    """
    Defines a failure to delete the requested number of LaunchConfigurations
    """
    pass

def get_max_launchconfigurations(asg_client):
    """
    Return the maximum number of LaunchConfigurations allowed in account
    """
    return asg_client.describe_account_limits()['MaxNumberOfLaunchConfigurations']

def get_all_launchconfigurations(asg_client):
    """
    Generator to find all LaunchConfigurations
    """
    lc_pager = asg_client.get_paginator('describe_launch_configurations')
    lc_iter = lc_pager.paginate()
    for lc_page in lc_iter:
        for lc in lc_page['LaunchConfigurations']:
            yield lc

def get_inuse_launchconfigurations(asg_client):
    """
    Generator to find LaunchConfigurations in-use by ASGs

    :param asg_client: boto3 autoscaling client
    :type asg_client: botocore.client.AutoScaling
    """
    asg_pager = asg_client.get_paginator('describe_auto_scaling_groups')
    asg_iter = asg_pager.paginate()
    for asg_page in asg_iter:
        for asg in asg_page['AutoScalingGroups']:
            yield asg['LaunchConfigurationName']

def get_launchconfiguration_deletion_candidates(asg_client, min_age):
    """
    Get a list of LaunchConfigurations that may be deleted

    :param asg_client: boto3 autoscaling client
    :type asg_client: botocore.client.AutoScaling
    :param min_age: Days old a LaunchConfiguration must be to be a candidate
    :type min_age: int
    :return: Tuple of (deletion_candidates, all_launch_configs)
    :rtype: tuple
    """
    all_lcs = [lc for lc in get_all_launchconfigurations(asg_client)]

    # Find LCs that are in-use
    used_lcs = [lc_name for lc_name in get_inuse_launchconfigurations(asg_client)]
    log.info("Found {} LaunchConfigurations in-use".format(len(used_lcs)))

    # Determine unused LCs
    unused = [lc for lc in all_lcs if lc['LaunchConfigurationName'] not in used_lcs]
    log.info("Found {} LaunchConfigurations unused".format(len(unused)))

    # Find old enough candidates
    min_created_time = datetime.now(UTC()) - timedelta(days=min_age)
    log.warning("Finding LCs from before {}".format(min_created_time.isoformat()))
    candidates = [lc for lc in unused
                  if lc['CreatedTime'] <= min_created_time]

    return sorted(candidates, key=lambda k: k['CreatedTime']), all_lcs

def json_serial(obj):
    """
    Handles formatting datetimes
    Source: http://stackoverflow.com/a/22238613
    """
    if isinstance(obj, datetime):
        return obj.isoformat()

    raise TypeError("{t} is not serializeable".format(t=type(obj)))

def lambda_handler(event, context):
    """
    Delete ASG LaunchConfiguration if at the account limit
    """

    # Setup logging
    log_level = event.get('LogLevel', False) or LAMBDA_DEFAULTS['LogLevel']
    logging.basicConfig(**{
        'level': getattr(logging, log_level.upper()),
        'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
    })

    # Create boto client for ASG API
    client = boto3.client('autoscaling')

    # Determine runtime parameters
    dry_run = bool(event.get('DryRun', LAMBDA_DEFAULTS['DryRun']))
    delete_count = int(event.get('LC_Delete', False) or LAMBDA_DEFAULTS['LC_Delete'])
    resource_limit = int(event.get('LC_Limit', False) or LAMBDA_DEFAULTS['LC_Limit'](client))
    min_age = int(event.get('MinAge', False) or LAMBDA_DEFAULTS['MinAge'])

    # DryRun is not supported by the boto3 autoscaling client
    if dry_run:
        log.warning("DryRun not supported by boto3 or the API!"
                    " A log line will be emitted instead")

    # Find candidates
    candidates, all_lcs = get_launchconfiguration_deletion_candidates(
        client,
        min_age=min_age)

    # Delete the candidate
    log.warning("Found {c} candidate{s}".format(
        c=len(candidates),
        s="" if len(candidates) == 1 else "s"))
    log.debug(json.dumps(
        candidates,
        indent=4,
        default=json_serial))

    if delete_count < 0:
        # We want to delete as many as necessary to be this many below the
        # resource limit
        delete_count = ((resource_limit - len(all_lcs)) + delete_count) * -1

    # Execute the deletion
    log.warning("Attempting to delete {c} LaunchConfiguration resource{s}".format(
        c=delete_count,
        s="" if delete_count == 1 else "s"))

    while delete_count > 0:
        try:
            lc = candidates.pop(0)
        except IndexError as e:
            raise InsufficientCandidates(
                "Failed to delete {c} remaining resource{s}".format(
                    c=delete_count,
                    s="" if delete_count == 1 else "s")
            )

        if not dry_run:
            client.delete_launch_configuration(
                LaunchConfigurationName=lc['LaunchConfigurationName'])
        else:
            log.warning(
                "DryRun: Would have deleted this LaunchConfiguration:\n"
                + json.dumps(lc,
                             indent=4,
                             default=json_serial)
            )

        delete_count -= 1

    log.warning("Successfully deleted LaunchConfiguration(s)")

if __name__ == '__main__':
    import json
    import sys
    from docopt import docopt

    # Parse CLI arguments
    args = docopt(__doc__, version='dev')

    try:
        lambda_handler(
            event={
                'DryRun': args['--dry-run'],
                'LC_Delete': args['--delete'],
                'LC_Limit': args['--maxlcs'],
                'LogLevel': args['--log-level'],
                'MinAge': args['--minage'],
            },
            context={}
        )

    except InsufficientCandidates as e:
        log.critical(str(e))
        log.info(str(e), exc_info=True)
        sys.exit(1)
<|endoftext|>


<commit_before>import tensorflow as tf
import numpy as np
import hyperchamber as hc
from hypergan.generators.common import *

import operator
from functools import reduce

from .base_generator import BaseGenerator

class ConfigurableGenerator(BaseGenerator):
    def __init__(self, gan, config, name=None, input=None, reuse=False):

        self.layer_ops = {
            "deconv": self.layer_deconv,
            "linear": self.layer_linear
            }
        BaseGenerator.__init__(self, gan, config, name=name, reuse=reuse,input=input)

    def required(self):
        return "layers defaults".split()

    def build(self, net):
        gan = self.gan
        config = self.config
        ops = self.ops

        for layer in config.layers:
            net = self.parse_layer(net, layer)

        pe_layers = self.gan.skip_connections.get_array("progressive_enhancement")
        s = ops.shape(net)
        img_dims = [s[1],s[2]]
        self.pe_layers = [tf.image.resize_images(elem, img_dims) for i, elem in enumerate(pe_layers)] + [net]
        if gan.config.progressive_growing:
            last_layer = net * self.progressive_growing_mask(len(pe_layers))
            self.debug_pe = [self.progressive_growing_mask(i) for i, elem in enumerate(pe_layers)]
        #    net = tf.add_n(nets + [last_layer])

        return net

    def parse_args(self, strs):
        options = {}
        args = []
        print("STRS", strs)
        for x in strs:
            if '=' in x:
                print("=Found ", strs)
                lhs, rhs = x.split('=')
                options[lhs]=rhs
            else:
                print("Found ", strs)
                args.append(x)
        return args, options

    def parse_layer(self, net, layer):
        config = self.config

        d = layer.split(' ')
        op = d[0]
        print("layer", layer, d)
        args, options = self.parse_args(d[1:])

        return self.build_layer(net, op, args, options)

    def build_layer(self, net, op, args, options):
        if self.layer_ops[op]:
            net = self.layer_ops[op](net, args, options)
        else:
            print("ConfigurableGenerator Op not defined", op)

        return net

    def layer_deconv(self, net, args, options):
        options = hc.Config(options)
        config = self.config
        ops = self.ops

        activation_s = options.activation or config.defaults.activation or "lrelu"
        activation = self.ops.lookup(activation_s)

        stride = config.stride or [2,2]
        fltr = config.filter or [5,5]
        print("ARGS", args)
        depth = int(args[0])

        net = ops.deconv2d(net, fltr[0], fltr[1], stride[0], stride[1], depth)
        self.add_progressive_enhancement(net)
        if activation:
            #net = self.layer_regularizer(net)
            net = activation(net)
        return net

    def layer_linear(self, net, args, options):
        options = hc.Config(options)
        ops = self.ops
        config = self.config
        fltr = options.filter or config.defaults.filter

        activation_s = options.activation or config.defaults.activation or "lrelu"
        activation = self.ops.lookup(activation_s)

        print("ARGS", args)
        dims = [int(x) for x in args[0].split("*")]
        size = reduce(operator.mul, dims, 1)

        net = ops.linear(net, size)

        if len(dims) > 1:
            net = ops.reshape(net, [ops.shape(net)[0], dims[0], dims[1], dims[2]])

        self.add_progressive_enhancement(net)
        if activation:
            #net = self.layer_regularizer(net)
            net = activation(net)
        return net
<commit_msg>[feature] resizeconv in configurable generator<commit_after>import tensorflow as tf
import numpy as np
import hyperchamber as hc
from hypergan.generators.common import *

import operator
from functools import reduce

from .base_generator import BaseGenerator

class ConfigurableGenerator(BaseGenerator):
    def __init__(self, gan, config, name=None, input=None, reuse=False):

        self.layer_ops = {
            "deconv": self.layer_deconv,
            "resize_conv": self.layer_resize_conv,
            "linear": self.layer_linear
            }
        BaseGenerator.__init__(self, gan, config, name=name, reuse=reuse,input=input)

    def required(self):
        return "layers defaults".split()

    def build(self, net):
        gan = self.gan
        config = self.config
        ops = self.ops

        for layer in config.layers:
            net = self.parse_layer(net, layer)

        pe_layers = self.gan.skip_connections.get_array("progressive_enhancement")
        s = ops.shape(net)
        img_dims = [s[1],s[2]]
        self.pe_layers = [tf.image.resize_images(elem, img_dims) for i, elem in enumerate(pe_layers)] + [net]
        if gan.config.progressive_growing:
            last_layer = net * self.progressive_growing_mask(len(pe_layers))
            self.debug_pe = [self.progressive_growing_mask(i) for i, elem in enumerate(pe_layers)]
        #    net = tf.add_n(nets + [last_layer])

        return net

    def parse_args(self, strs):
        options = {}
        args = []
        print("STRS", strs)
        for x in strs:
            if '=' in x:
                print("=Found ", strs)
                lhs, rhs = x.split('=')
                options[lhs]=rhs
            else:
                print("Found ", strs)
                args.append(x)
        return args, options

    def parse_layer(self, net, layer):
        config = self.config

        d = layer.split(' ')
        op = d[0]
        print("layer", layer, d)
        args, options = self.parse_args(d[1:])

        return self.build_layer(net, op, args, options)

    def build_layer(self, net, op, args, options):
        if self.layer_ops[op]:
            net = self.layer_ops[op](net, args, options)
        else:
            print("ConfigurableGenerator Op not defined", op)

        return net

    def layer_deconv(self, net, args, options):
        options = hc.Config(options)
        config = self.config
        ops = self.ops

        activation_s = options.activation or config.defaults.activation or "lrelu"
        activation = self.ops.lookup(activation_s)

        stride = config.stride or [2,2]
        fltr = config.filter or [5,5]
        print("ARGS", args)
        depth = int(args[0])

        net = ops.deconv2d(net, fltr[0], fltr[1], stride[0], stride[1], depth)
        self.add_progressive_enhancement(net)
        if activation:
            #net = self.layer_regularizer(net)
            net = activation(net)
        return net

    def layer_linear(self, net, args, options):
        options = hc.Config(options)
        ops = self.ops
        config = self.config
        fltr = options.filter or config.defaults.filter

        activation_s = options.activation or config.defaults.activation or "lrelu"
        activation = self.ops.lookup(activation_s)

        print("ARGS", args)
        dims = [int(x) for x in args[0].split("*")]
        size = reduce(operator.mul, dims, 1)

        net = ops.linear(net, size)

        if len(dims) > 1:
            net = ops.reshape(net, [ops.shape(net)[0], dims[0], dims[1], dims[2]])

        self.add_progressive_enhancement(net)
        if activation:
            #net = self.layer_regularizer(net)
            net = activation(net)
        return net

    def layer_resize_conv(self, net, args, options):
        options = hc.Config(options)
        config = self.config
        ops = self.ops

        activation_s = options.activation or config.defaults.activation or "lrelu"
        activation = self.ops.lookup(activation_s)

        stride = config.stride or [1,1]
        fltr = config.filter or [5,5]
        depth = int(args[0])

        print("NET", net)
        net = tf.image.resize_images(net, [ops.shape(net)[1]*2, ops.shape(net)[2]*2],1)
        net = ops.conv2d(net, fltr[0], fltr[1], stride[0], stride[1], depth)
        print("POTNET", net)
        self.add_progressive_enhancement(net)
        if activation:
            #net = self.layer_regularizer(net)
            net = activation(net)
        return net

<|endoftext|>