<commit_before>import numpy as np
import os
from xml.etree import ElementTree
from .utils import get_class_names
from pycocotools.coco import COCO

class XMLParser(object):
    """ Preprocess the VOC2007 xml annotations data.

    # TODO: Add background label

    # Arguments
        data_path: Data path to VOC2007 annotations

    # Return
        data: Dictionary which keys correspond to the image names
        and values are numpy arrays of shape (num_objects, 4 + num_classes)
        num_objects refers to the number of objects in that specific image
    """

    def __init__(self, data_path, class_names=None, dataset_name='VOC2007'):
        self.path_prefix = data_path
        self.dataset_name = dataset_name
        self.class_names = class_names
        if self.class_names == None:
            self.class_names = get_class_names(self.dataset_name)
        self.num_classes = len(self.class_names)
        keys = np.arange(self.num_classes)
        self.arg_to_class = dict(zip(keys, self.class_names))
        self.class_to_arg = {value: key for key, value
                             in self.arg_to_class.items()}
        self.data = dict()
        self._preprocess_XML()

    def get_data(self):
        return self.data

    def _preprocess_XML(self):
        filenames = os.listdir(self.path_prefix)
        for filename in filenames:
            tree = ElementTree.parse(self.path_prefix + filename)
            root = tree.getroot()
            bounding_boxes = []
            one_hot_classes = []
            size_tree = root.find('size')
            width = float(size_tree.find('width').text)
            height = float(size_tree.find('height').text)
            for object_tree in root.findall('object'):
                class_name = object_tree.find('name').text
                if class_name in self.class_names:
                    one_hot_class = self._to_one_hot(class_name)
                    one_hot_classes.append(one_hot_class)
                    for bounding_box in object_tree.iter('bndbox'):
                        xmin = float(bounding_box.find('xmin').text) / width
                        ymin = float(bounding_box.find('ymin').text) / height
                        xmax = float(bounding_box.find('xmax').text) / width
                        ymax = float(bounding_box.find('ymax').text) / height
                    bounding_box = [xmin, ymin, xmax, ymax]
                    bounding_boxes.append(bounding_box)
            if len(one_hot_classes) == 0:
                continue
            image_name = root.find('filename').text
            bounding_boxes = np.asarray(bounding_boxes)
            one_hot_classes = np.asarray(one_hot_classes)
            image_data = np.hstack((bounding_boxes, one_hot_classes))
            if len(bounding_boxes.shape) == 1:
                image_data = np.expand_dims(image_data, axis=0)
            self.data[image_name] = image_data

    def _to_one_hot(self, class_name):
        one_hot_vector = [0] * self.num_classes
        class_arg = self.class_to_arg[class_name]
        one_hot_vector[class_arg] = 1
        return one_hot_vector

class COCOParser(object):
    def __init__(self, annotations_path, class_names='all'):
        self.coco = COCO(annotations_path)
        self.class_names = class_names
        if self.class_names == 'all':
            class_data = self.coco.loadCats(self.coco.getCatIds())
            self.class_names = [class_['name'] for class_ in class_data]
            coco_ids = [class_['id'] for class_ in class_data]
            one_hot_ids = list(range(1, len(coco_ids) + 1))
            self.coco_id_to_class_arg = dict(zip(coco_ids, one_hot_ids))
            self.class_names = ['background'] + self.class_names
            self.num_classes = len(self.class_names)
        elif len(self.class_names) > 1:
            raise NotImplementedError('Only one or all classes supported')
        self.data = dict()

    def get_data(self):
        self._get_data()
        return self.data

    def _get_data(self):
        image_ids = self.coco.getImgIds()
        for image_id in image_ids:
            image_data = self.coco.loadImgs(image_id)[0]
            image_file_name = image_data['file_name']
            width = float(image_data['width'])
            height = float(image_data['height'])
            annotation_ids = self.coco.getAnnIds(imgIds=image_data['id'])
            annotations = self.coco.loadAnns(annotation_ids)
            num_objects_in_image = len(annotations)
            if num_objects_in_image == 0:
                continue
            image_ground_truth = []
            for object_arg in range(num_objects_in_image):
                coco_id = annotations[object_arg]['category_id']
                class_arg = self.coco_id_to_class_arg[coco_id]
                one_hot_class = self._to_one_hot(class_arg)
                coco_coordinates = annotations[object_arg]['bbox']
                #print('coco_coordinates:', coco_coordinates)
                x_min = (coco_coordinates[0]) #/ width
                y_min = (coco_coordinates[1]) #/ height
                x_max = (x_min + coco_coordinates[2]) #/ width
                y_max = (y_min + coco_coordinates[3]) #/ height
                #print('transformed_coordinates:', [x_min, y_min, x_max, y_max])
                x_min = x_min / width
                y_min = y_min / height
                x_max = x_max / width
                y_max = y_max / height
                #print('normalized_coordinates:', [x_min, y_min, x_max, y_max])
                ground_truth = [x_min, y_min, x_max, y_max] + one_hot_class
                image_ground_truth.append(ground_truth)
            image_ground_truth = np.asarray(image_ground_truth)
            if len(image_ground_truth.shape) == 1:
                image_ground_truth = np.expand_dims(image_ground_truth, 0)
            self.data[image_file_name] = image_ground_truth

    def _to_one_hot(self, class_arg):
        one_hot_vector = [0] * self.num_classes
        one_hot_vector[class_arg] = 1
        return one_hot_vector

class DataManager(object):
    """Class for loading VOC2007 and COCO datasets or
    """
    def __init__(self, dataset_name='VOC2007', class_names=None,
                                    dataset_path_prefix=None):

        self.dataset_name = dataset_name
        self.dataset_path_prefix = dataset_path_prefix
        self.class_names = class_names
        self.ground_truth_data = None
        self.parser = None
        if self.dataset_path_prefix != None:
            self.dataset_path_prefix = dataset_path_prefix
        elif self.dataset_name == 'VOC2007':
            self.dataset_path_prefix = '../datasets/VOCdevkit/VOC2007/Annotations/'
            self._load_VOC2007()
        elif self.dataset_name == 'COCO':
            self.dataset_path_prefix = ('../datasets/COCO/annotations/' +
                                                'instances_train2014.json')
            self._load_COCO()
        elif self.dataset_name == 'all':
            raise NotImplementedError
        else:
            raise Exception('Incorrect dataset name:', self.dataset_name)

    def get_data(self):
        print('Deprecated function use load_data')
        return self.ground_truth_data

    def load_data(self):
        print('Deprecated function use load_data')
        return self.ground_truth_data

    def _load_VOC2007(self):
        self.parser = XMLParser(self.dataset_path_prefix, self.class_names)
        self.ground_truth_data = self.parser.get_data()
        self.class_names = self.parser.class_names

    def _load_COCO(self):
        self.parser = COCOParser(self.dataset_path_prefix)
        self.ground_truth_data = self.parser.get_data()
<commit_msg>Moved previous functions to get class names to utils datasets<commit_after>import glob
import os
from xml.etree import ElementTree

import numpy as np
try:
    from pycocotools.coco import COCO
except ImportError:
    COCO = None

class XMLParser(object):
    """ Preprocess the VOC2007 xml annotations data.

    # TODO: Add background label

    # Arguments
        data_path: Data path to VOC2007 annotations

    # Return
        data: Dictionary which keys correspond to the image names
        and values are numpy arrays of shape (num_objects, 4 + num_classes)
        num_objects refers to the number of objects in that specific image
    """

    def __init__(self, data_path, class_names=None, dataset_name='VOC2007'):
        self.path_prefix = data_path
        self.dataset_name = dataset_name
        self.class_names = class_names
        if self.class_names == None:
            self.class_names = get_class_names(self.dataset_name)
        self.num_classes = len(self.class_names)
        keys = np.arange(self.num_classes)
        self.arg_to_class = dict(zip(keys, self.class_names))
        self.class_to_arg = {value: key for key, value
                             in self.arg_to_class.items()}
        self.data = dict()
        self._preprocess_XML()

    def get_data(self):
        return self.data

    def _preprocess_XML(self):
        filenames = os.listdir(self.path_prefix)
        for filename in filenames:
            tree = ElementTree.parse(self.path_prefix + filename)
            root = tree.getroot()
            bounding_boxes = []
            one_hot_classes = []
            size_tree = root.find('size')
            width = float(size_tree.find('width').text)
            height = float(size_tree.find('height').text)
            for object_tree in root.findall('object'):
                class_name = object_tree.find('name').text
                if class_name in self.class_names:
                    one_hot_class = self._to_one_hot(class_name)
                    one_hot_classes.append(one_hot_class)
                    for bounding_box in object_tree.iter('bndbox'):
                        xmin = float(bounding_box.find('xmin').text) / width
                        ymin = float(bounding_box.find('ymin').text) / height
                        xmax = float(bounding_box.find('xmax').text) / width
                        ymax = float(bounding_box.find('ymax').text) / height
                    bounding_box = [xmin, ymin, xmax, ymax]
                    bounding_boxes.append(bounding_box)
            if len(one_hot_classes) == 0:
                continue
            image_name = root.find('filename').text
            bounding_boxes = np.asarray(bounding_boxes)
            one_hot_classes = np.asarray(one_hot_classes)
            image_data = np.hstack((bounding_boxes, one_hot_classes))
            if len(bounding_boxes.shape) == 1:
                image_data = np.expand_dims(image_data, axis=0)
            self.data[image_name] = image_data

    def _to_one_hot(self, class_name):
        one_hot_vector = [0] * self.num_classes
        class_arg = self.class_to_arg[class_name]
        one_hot_vector[class_arg] = 1
        return one_hot_vector

class COCOParser(object):
    def __init__(self, annotations_path, class_names='all'):
        self.coco = COCO(annotations_path)
        self.class_names = class_names
        if self.class_names == 'all':
            class_data = self.coco.loadCats(self.coco.getCatIds())
            self.class_names = [class_['name'] for class_ in class_data]
            coco_ids = [class_['id'] for class_ in class_data]
            one_hot_ids = list(range(1, len(coco_ids) + 1))
            self.coco_id_to_class_arg = dict(zip(coco_ids, one_hot_ids))
            self.class_names = ['background'] + self.class_names
            self.num_classes = len(self.class_names)
        elif len(self.class_names) > 1:
            raise NotImplementedError('Only one or all classes supported')
        self.data = dict()

    def get_data(self):
        self._get_data()
        return self.data

    def _get_data(self):
        image_ids = self.coco.getImgIds()
        for image_id in image_ids:
            image_data = self.coco.loadImgs(image_id)[0]
            image_file_name = image_data['file_name']
            width = float(image_data['width'])
            height = float(image_data['height'])
            annotation_ids = self.coco.getAnnIds(imgIds=image_data['id'])
            annotations = self.coco.loadAnns(annotation_ids)
            num_objects_in_image = len(annotations)
            if num_objects_in_image == 0:
                continue
            image_ground_truth = []
            for object_arg in range(num_objects_in_image):
                coco_id = annotations[object_arg]['category_id']
                class_arg = self.coco_id_to_class_arg[coco_id]
                one_hot_class = self._to_one_hot(class_arg)
                coco_coordinates = annotations[object_arg]['bbox']
                #print('coco_coordinates:', coco_coordinates)
                x_min = (coco_coordinates[0]) #/ width
                y_min = (coco_coordinates[1]) #/ height
                x_max = (x_min + coco_coordinates[2]) #/ width
                y_max = (y_min + coco_coordinates[3]) #/ height
                #print('transformed_coordinates:', [x_min, y_min, x_max, y_max])
                x_min = x_min / width
                y_min = y_min / height
                x_max = x_max / width
                y_max = y_max / height
                #print('normalized_coordinates:', [x_min, y_min, x_max, y_max])
                ground_truth = [x_min, y_min, x_max, y_max] + one_hot_class
                image_ground_truth.append(ground_truth)
            image_ground_truth = np.asarray(image_ground_truth)
            if len(image_ground_truth.shape) == 1:
                image_ground_truth = np.expand_dims(image_ground_truth, 0)
            self.data[image_file_name] = image_ground_truth

    def _to_one_hot(self, class_arg):
        one_hot_vector = [0] * self.num_classes
        one_hot_vector[class_arg] = 1
        return one_hot_vector

class DataManager(object):
    """Class for loading VOC2007 and COCO datasets or
    """
    def __init__(self, dataset_name='VOC2007', class_names=None,
                                    dataset_path_prefix=None):

        self.dataset_name = dataset_name
        self.dataset_path_prefix = dataset_path_prefix
        self.class_names = class_names
        self.ground_truth_data = None
        self.parser = None
        if self.dataset_path_prefix != None:
            self.dataset_path_prefix = dataset_path_prefix
        elif self.dataset_name == 'VOC2007':
            self.dataset_path_prefix = '../datasets/VOCdevkit/VOC2007/Annotations/'
            self._load_VOC2007()
        elif self.dataset_name == 'COCO':
            self.dataset_path_prefix = ('../datasets/COCO/annotations/' +
                                                'instances_train2014.json')
            self._load_COCO()
        elif self.dataset_name == 'all':
            raise NotImplementedError
        else:
            raise Exception('Incorrect dataset name:', self.dataset_name)

    def get_data(self):
        print('Deprecated function use load_data')
        return self.ground_truth_data

    def load_data(self):
        print('Deprecated function use load_data')
        return self.ground_truth_data

    def _load_VOC2007(self):
        self.parser = XMLParser(self.dataset_path_prefix, self.class_names)
        self.ground_truth_data = self.parser.get_data()
        self.class_names = self.parser.class_names

    def _load_COCO(self):
        self.parser = COCOParser(self.dataset_path_prefix)
        self.ground_truth_data = self.parser.get_data()

def get_class_names(dataset_name='VOC2007'):
    if dataset_name == 'VOC2007':
        class_names = ['background','aeroplane', 'bicycle', 'bird', 'boat',
                       'bottle', 'bus', 'car', 'cat', 'chair', 'cow',
                       'diningtable', 'dog', 'horse', 'motorbike', 'person',
                       'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor']
    elif dataset_name == 'COCO':
        class_names = ['background', 'person', 'bicycle', 'car', 'motorcycle',
                        'airplane', 'bus', 'train', 'truck', 'boat',
                        'traffic light', 'fire hydrant', 'stop sign',
                        'parking meter', 'bench', 'bird', 'cat', 'dog',
                        'horse', 'sheep', 'cow', 'elephant', 'bear',
                        'zebra', 'giraffe', 'backpack', 'umbrella',
                        'handbag', 'tie', 'suitcase', 'frisbee', 'skis',
                        'snowboard', 'sports ball', 'kite', 'baseball bat',
                        'baseball glove', 'skateboard', 'surfboard',
                        'tennis racket', 'bottle', 'wine glass',
                        'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana',
                        'apple', 'sandwich', 'orange', 'broccoli', 'carrot',
                        'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch',
                        'potted plant', 'bed', 'dining table', 'toilet',
                        'tv', 'laptop', 'mouse', 'remote', 'keyboard',
                        'cell phone', 'microwave', 'oven', 'toaster',
                        'sink', 'refrigerator', 'book', 'clock', 'vase',
                        'scissors', 'teddy bear', 'hair drier', 'toothbrush']
    else:
        raise Exception('Invalid dataset', dataset_name)
    return class_names

def get_arg_to_class(class_names):
    return dict(zip(list(range(len(class_names))), class_names))

def list_files_in_directory(path_name='*'):
    return glob.glob(path_name)

<|endoftext|>


<commit_before>"""

This module is not yet ready to be used for adapters.

Here the logic should be placed that allows the application to draw already
existing relationships on the diagram when a model element is added (e.g.
by drag and drop).
"""

from zope import component

class AssociationRelationship(Relationship):
    """Relationship for associations.
    """
    def relationship(self, line, head_subject = None, tail_subject = None):
        # First check if we do not already contain the right subject:
        if line.subject:
            end1 = line.subject.memberEnd[0]
            end2 = line.subject.memberEnd[1]
            if (end1.type is head_type and end2.type is tail_type) \
               or (end2.type is head_type and end1.type is tail_type):
                return

        # Find all associations and determine if the properties on the
        # association ends have a type that points to the class.
        Association = UML.Association
        for assoc in resource(UML.ElementFactory).itervalues():
            if isinstance(assoc, Association):
                #print 'assoc.memberEnd', assoc.memberEnd
                end1 = assoc.memberEnd[0]
                end2 = assoc.memberEnd[1]
                if (end1.type is head_type and end2.type is tail_type) \
                   or (end2.type is head_type and end1.type is tail_type):
                    # check if this entry is not yet in the diagram
                    # Return if the association is not (yet) on the canvas
                    for item in assoc.presentation:
                        if item.canvas is line.canvas:
                            break
                    else:
                        return assoc
        return None

# vim:sw=4:et:ai
<commit_msg>- it is not ready yet, therefore commented out to allow run unit tests   nicely<commit_after>"""

This module is not yet ready to be used for adapters.

Here the logic should be placed that allows the application to draw already
existing relationships on the diagram when a model element is added (e.g.
by drag and drop).
"""

from zope import component

###class AssociationRelationship(Relationship):
###    """Relationship for associations.
###    """
###    def relationship(self, line, head_subject = None, tail_subject = None):
###        # First check if we do not already contain the right subject:
###        if line.subject:
###            end1 = line.subject.memberEnd[0]
###            end2 = line.subject.memberEnd[1]
###            if (end1.type is head_type and end2.type is tail_type) \
###               or (end2.type is head_type and end1.type is tail_type):
###                return
###
###        # Find all associations and determine if the properties on the
###        # association ends have a type that points to the class.
###        Association = UML.Association
###        for assoc in resource(UML.ElementFactory).itervalues():
###            if isinstance(assoc, Association):
###                #print 'assoc.memberEnd', assoc.memberEnd
###                end1 = assoc.memberEnd[0]
###                end2 = assoc.memberEnd[1]
###                if (end1.type is head_type and end2.type is tail_type) \
###                   or (end2.type is head_type and end1.type is tail_type):
###                    # check if this entry is not yet in the diagram
###                    # Return if the association is not (yet) on the canvas
###                    for item in assoc.presentation:
###                        if item.canvas is line.canvas:
###                            break
###                    else:
###                        return assoc
###        return None

# vim:sw=4:et:ai
<|endoftext|>


<commit_before>from sqlalchemy import text
from sqlalchemy.exc import SQLAlchemyError

from bot.exceptions import (NotEnoughPrefixes, PrefixExists,
                            PrefixDoesntExist)

class GuildCache:
    def __init__(self, bot):
        self._bot = bot
        self.guilds = {}

    @property
    def bot(self):
        return self._bot

    def update_cached_guild(self, guild_id, **values):
        """
        Updates a servers cached values. None values are ignored
        """
        settings = self.get_settings(guild_id)
        for k, v in values.items():
            if v is None:
                continue

            settings[k] = v

        # Reverse sort prefixes so some prefixes don't get overlooked
        # e.q. if you add a prefix a and then a prefix aa if the a prefix is
        # first in the list it will always get invoked when aa is used
        self._set_internal_value(guild_id, 'prefixes', sorted(list(self.prefixes(guild_id, use_set=True)), reverse=True))

    async def set_value(self, guild_id, name, value):
        sql = 'INSERT INTO `guilds` (`guild`, `{0}`) VALUES ({1}, :{0}) ON DUPLICATE KEY UPDATE {0}=:{0}'.format(name, guild_id)
        try:
            await self.bot.dbutil.execute(text(sql), params={name: value}, commit=True)
            success = True
        except SQLAlchemyError:
            success = False
        settings = self.get_settings(guild_id)
        settings[name] = value
        return success

    # Used for setting cached values not present in the database or in a different form e.g. cache a set as a list
    def _set_internal_value(self, guild_id, name, value):
        settings = self.get_settings(guild_id)
        internals = settings.get('_internals')
        if internals is None:
            internals = {}
            settings['_internals'] = internals

        internals[name] = value

    def _get_internals(self, guild_id):
        settings = self.get_settings(guild_id)
        internals = settings.get('_internals')
        if internals is None:
            internals = {}
            settings['_internals'] = internals

        return internals

    # utils
    def prefixes(self, guild_id, use_set=False):
        if use_set:
            return self.get_settings(guild_id).get('prefixes', {self.bot.default_prefix})

        prefixes = self._get_internals(guild_id).get('prefixes')
        if prefixes is None:
            return tuple(self.bot.default_prefix, )

        return prefixes

    async def add_prefix(self, guild_id, prefix):
        settings = self.get_settings(guild_id)
        if 'prefixes' not in settings:
            prefixes = {self.bot.default_prefix}
            settings['prefixes'] = prefixes
        else:
            prefixes = self.prefixes(guild_id, use_set=True)

        if prefix in prefixes:
            raise PrefixExists('Prefix is already in use')

        success = await self.bot.dbutil.add_prefix(guild_id, prefix)
        if success:
            prefixes_list = self.prefixes(guild_id)
            prefixes_list.append(prefix)
            prefixes_list.sort(reverse=True)
            prefixes.add(prefix)

        return success

    async def remove_prefix(self, guild_id, prefix):
        prefixes = self.prefixes(guild_id, use_set=True)
        if prefix not in prefixes:
            raise PrefixDoesntExist("Prefix doesn't exist")

        if len(prefixes) == 1:
            raise NotEnoughPrefixes('Must have at least one prefix')

        success = await self.bot.dbutil.remove_prefix(guild_id, prefix)
        if success:
            prefixes.discard(prefix)
            try:
                self.prefixes(guild_id).remove(prefix)
            except ValueError:
                pass

        return success

    # moderation
    def modlog(self, guild_id):
        return self.get_settings(guild_id).get('modlog', None)

    async def set_modlog(self, guild_id, channel_id):
        return await self.set_value(guild_id, 'modlog', channel_id)

    def mute_role(self, guild_id):
        return self.get_settings(guild_id).get('mute_role', None)

    async def set_mute_role(self, guild_id, role_id):
        return await self.set_value(guild_id, 'mute_role', role_id)

    def keeproles(self, guild_id):
        if self.get_settings(guild_id).get('keeproles', 0):
            return True
        else:
            return False

    async def set_keeproles(self, guild_id, value):
        return await self.set_value(guild_id, 'keeproles', value)

    # automod
    def automute(self, guild_id):
        return self.get_settings(guild_id).get('automute', False)

    async def set_automute(self, guild_id, on: bool):
        return await self.set_value(guild_id, 'automute', on)

    def automute_limit(self, guild_id):
        return self.get_settings(guild_id).get('automute_limit', 10)

    async def set_automute_limit(self, guild_id, limit: int):
        return await self.set_value(guild_id, 'automute_limit', limit)

    def automute_time(self, guild_id):
        return self.get_settings(guild_id).get('automute_time')

    async def set_automute_time(self, guild_id, time):
        return await self.set_value(guild_id, 'automute_time', time)

    # join config
    def join_message(self, guild_id, default_message=False):
        message = self.get_settings(guild_id).get('on_join_message')
        if message is None and default_message:
            message = self.bot.config.join_message

        return message

    async def set_join_message(self, guild_id, message):
        return await self.set_value(guild_id, 'on_join_message', message)

    def join_channel(self, guild_id):
        return self.get_settings(guild_id).get('on_join_channel')

    async def set_join_channel(self, guild_id, channel):
        return await self.set_value(guild_id, 'on_join_channel', channel)

    # random color on join
    def random_color(self, guild_id):
        return self.get_settings(guild_id).get('color_on_join', False)

    async def set_random_color(self, guild_id, value):
        return await self.set_value(guild_id, 'color_on_join', value)

    # leave config
    def leave_message(self, guild_id, default_message=False):
        message = self.get_settings(guild_id).get('on_leave_message')
        if message is None and default_message:
            message = self.bot.config.leave_message

        return message

    async def set_leave_message(self, guild_id, message):
        return await self.set_value(guild_id, 'on_leave_message', message)

    def leave_channel(self, guild_id):
        return self.get_settings(guild_id).get('on_leave_channel')

    async def set_leave_channel(self, guild_id, channel):
        return await self.set_value(guild_id, 'on_leave_channel', channel)

    # On message edit
    def on_edit_message(self, guild_id, default_message=False):
        message = self.get_settings(guild_id).get('on_edit_message')
        if message is None and default_message:
            message = self.bot.config.edit_message

        return message

    async def set_on_edit_message(self, guild_id, message):
        return await self.set_value(guild_id, 'on_edit_message', message)

    def on_edit_channel(self, guild_id):
        return self.get_settings(guild_id).get('on_edit_channel')

    async def set_on_edit_channel(self, guild_id, channel):
        return await self.set_value(guild_id, 'on_edit_channel', channel)

    def on_edit_embed(self, guild_id):
        return self.get_settings(guild_id).get('on_edit_embed')

    async def set_on_edit_embed(self, guild_id, boolean):
        return await self.set_value(guild_id, 'on_edit_embed', boolean)

    # On message delete
    def on_delete_message(self, guild_id, default_message=False):
        message = self.get_settings(guild_id).get('on_delete_message')
        if message is None and default_message:
            message = self.bot.config.delete_message

        return message

    async def set_on_delete_message(self, guild_id, message):
        return await self.set_value(guild_id, 'on_delete_message', message)

    def on_delete_channel(self, guild_id):
        return self.get_settings(guild_id).get('on_delete_channel')

    async def set_on_delete_channel(self, guild_id, channel):
        return await self.set_value(guild_id, 'on_delete_channel', channel)

    def on_delete_embed(self, guild_id):
        return self.get_settings(guild_id).get('on_delete_embed')

    async def set_on_delete_embed(self, guild_id, boolean):
        return await self.set_value(guild_id, 'on_delete_embed', boolean)

    def dailygachi(self, guild_id):
        return self.get_settings(guild_id).get('dailygachi')

    async def set_dailygachi(self, guild_id, channel):
        return await self.set_value(guild_id, 'dailygachi', channel)

    def get_settings(self, guild_id):
        settings = self[guild_id]
        if not settings:
            settings = {}
            self[guild_id] = settings

        return settings

    def __getitem__(self, item):
        return self.guilds.get(item, None)

    def __setitem__(self, key, value):
        self.guilds[key] = value

    def __delitem__(self, key):
        try:
            del self.guilds[key]
        except:
            pass
<commit_msg>maybe fix error when append is used on tuple<commit_after>from sqlalchemy import text
from sqlalchemy.exc import SQLAlchemyError

from bot.exceptions import (NotEnoughPrefixes, PrefixExists,
                            PrefixDoesntExist)

class GuildCache:
    def __init__(self, bot):
        self._bot = bot
        self.guilds = {}

    @property
    def bot(self):
        return self._bot

    def update_cached_guild(self, guild_id, **values):
        """
        Updates a servers cached values. None values are ignored
        """
        settings = self.get_settings(guild_id)
        for k, v in values.items():
            if v is None:
                continue

            settings[k] = v

        # Reverse sort prefixes so some prefixes don't get overlooked
        # e.q. if you add a prefix a and then a prefix aa if the a prefix is
        # first in the list it will always get invoked when aa is used
        self._set_internal_value(guild_id, 'prefixes', sorted(list(self.prefixes(guild_id, use_set=True)), reverse=True))

    async def set_value(self, guild_id, name, value):
        sql = 'INSERT INTO `guilds` (`guild`, `{0}`) VALUES ({1}, :{0}) ON DUPLICATE KEY UPDATE {0}=:{0}'.format(name, guild_id)
        try:
            await self.bot.dbutil.execute(text(sql), params={name: value}, commit=True)
            success = True
        except SQLAlchemyError:
            success = False
        settings = self.get_settings(guild_id)
        settings[name] = value
        return success

    # Used for setting cached values not present in the database or in a different form
    # e.g. cache a set as a list. Used only for prefixes as we need them stored in a
    # sorted list when checking for usable prefixes and a set when adding more
    # to the list
    def _set_internal_value(self, guild_id, name, value):
        settings = self.get_settings(guild_id)
        internals = settings.get('_internals')
        if internals is None:
            internals = {}
            settings['_internals'] = internals

        internals[name] = value

    def _get_internals(self, guild_id):
        settings = self.get_settings(guild_id)
        internals = settings.get('_internals')
        if internals is None:
            internals = {}
            settings['_internals'] = internals

        return internals

    # utils
    def prefixes(self, guild_id, use_set=False):
        if use_set:
            return self.get_settings(guild_id).get('prefixes', {self.bot.default_prefix})

        prefixes = self._get_internals(guild_id).get('prefixes')
        if prefixes is None:
            prefixes = list(self.get_settings(guild_id).get('prefixes', [self.bot.default_prefix]))
            self._set_internal_value(guild_id, 'prefixes', prefixes)

        return prefixes

    async def add_prefix(self, guild_id, prefix):
        settings = self.get_settings(guild_id)
        if 'prefixes' not in settings:
            prefixes = {self.bot.default_prefix}
            settings['prefixes'] = prefixes
        else:
            prefixes = self.prefixes(guild_id, use_set=True)

        if prefix in prefixes:
            raise PrefixExists('Prefix is already in use')

        success = await self.bot.dbutil.add_prefix(guild_id, prefix)
        if success:
            prefixes_list = self.prefixes(guild_id)
            prefixes_list.append(prefix)
            prefixes_list.sort(reverse=True)
            prefixes.add(prefix)

        return success

    async def remove_prefix(self, guild_id, prefix):
        prefixes = self.prefixes(guild_id, use_set=True)
        if prefix not in prefixes:
            raise PrefixDoesntExist("Prefix doesn't exist")

        if len(prefixes) == 1:
            raise NotEnoughPrefixes('Must have at least one prefix')

        success = await self.bot.dbutil.remove_prefix(guild_id, prefix)
        if success:
            prefixes.discard(prefix)
            try:
                self.prefixes(guild_id).remove(prefix)
            except ValueError:
                pass

        return success

    # moderation
    def modlog(self, guild_id):
        return self.get_settings(guild_id).get('modlog', None)

    async def set_modlog(self, guild_id, channel_id):
        return await self.set_value(guild_id, 'modlog', channel_id)

    def mute_role(self, guild_id):
        return self.get_settings(guild_id).get('mute_role', None)

    async def set_mute_role(self, guild_id, role_id):
        return await self.set_value(guild_id, 'mute_role', role_id)

    def keeproles(self, guild_id):
        if self.get_settings(guild_id).get('keeproles', 0):
            return True
        else:
            return False

    async def set_keeproles(self, guild_id, value):
        return await self.set_value(guild_id, 'keeproles', value)

    # automod
    def automute(self, guild_id):
        return self.get_settings(guild_id).get('automute', False)

    async def set_automute(self, guild_id, on: bool):
        return await self.set_value(guild_id, 'automute', on)

    def automute_limit(self, guild_id):
        return self.get_settings(guild_id).get('automute_limit', 10)

    async def set_automute_limit(self, guild_id, limit: int):
        return await self.set_value(guild_id, 'automute_limit', limit)

    def automute_time(self, guild_id):
        return self.get_settings(guild_id).get('automute_time')

    async def set_automute_time(self, guild_id, time):
        return await self.set_value(guild_id, 'automute_time', time)

    # join config
    def join_message(self, guild_id, default_message=False):
        message = self.get_settings(guild_id).get('on_join_message')
        if message is None and default_message:
            message = self.bot.config.join_message

        return message

    async def set_join_message(self, guild_id, message):
        return await self.set_value(guild_id, 'on_join_message', message)

    def join_channel(self, guild_id):
        return self.get_settings(guild_id).get('on_join_channel')

    async def set_join_channel(self, guild_id, channel):
        return await self.set_value(guild_id, 'on_join_channel', channel)

    # random color on join
    def random_color(self, guild_id):
        return self.get_settings(guild_id).get('color_on_join', False)

    async def set_random_color(self, guild_id, value):
        return await self.set_value(guild_id, 'color_on_join', value)

    # leave config
    def leave_message(self, guild_id, default_message=False):
        message = self.get_settings(guild_id).get('on_leave_message')
        if message is None and default_message:
            message = self.bot.config.leave_message

        return message

    async def set_leave_message(self, guild_id, message):
        return await self.set_value(guild_id, 'on_leave_message', message)

    def leave_channel(self, guild_id):
        return self.get_settings(guild_id).get('on_leave_channel')

    async def set_leave_channel(self, guild_id, channel):
        return await self.set_value(guild_id, 'on_leave_channel', channel)

    # On message edit
    def on_edit_message(self, guild_id, default_message=False):
        message = self.get_settings(guild_id).get('on_edit_message')
        if message is None and default_message:
            message = self.bot.config.edit_message

        return message

    async def set_on_edit_message(self, guild_id, message):
        return await self.set_value(guild_id, 'on_edit_message', message)

    def on_edit_channel(self, guild_id):
        return self.get_settings(guild_id).get('on_edit_channel')

    async def set_on_edit_channel(self, guild_id, channel):
        return await self.set_value(guild_id, 'on_edit_channel', channel)

    def on_edit_embed(self, guild_id):
        return self.get_settings(guild_id).get('on_edit_embed')

    async def set_on_edit_embed(self, guild_id, boolean):
        return await self.set_value(guild_id, 'on_edit_embed', boolean)

    # On message delete
    def on_delete_message(self, guild_id, default_message=False):
        message = self.get_settings(guild_id).get('on_delete_message')
        if message is None and default_message:
            message = self.bot.config.delete_message

        return message

    async def set_on_delete_message(self, guild_id, message):
        return await self.set_value(guild_id, 'on_delete_message', message)

    def on_delete_channel(self, guild_id):
        return self.get_settings(guild_id).get('on_delete_channel')

    async def set_on_delete_channel(self, guild_id, channel):
        return await self.set_value(guild_id, 'on_delete_channel', channel)

    def on_delete_embed(self, guild_id):
        return self.get_settings(guild_id).get('on_delete_embed')

    async def set_on_delete_embed(self, guild_id, boolean):
        return await self.set_value(guild_id, 'on_delete_embed', boolean)

    def dailygachi(self, guild_id):
        return self.get_settings(guild_id).get('dailygachi')

    async def set_dailygachi(self, guild_id, channel):
        return await self.set_value(guild_id, 'dailygachi', channel)

    def get_settings(self, guild_id):
        settings = self[guild_id]
        if not settings:
            settings = {}
            self[guild_id] = settings

        return settings

    def __getitem__(self, item):
        return self.guilds.get(item, None)

    def __setitem__(self, key, value):
        self.guilds[key] = value

    def __delitem__(self, key):
        try:
            del self.guilds[key]
        except:
            pass
<|endoftext|>


<commit_before># coding: utf-8
from config import pi_command

@pi_command
def take_photo(msg=None, debug=False):
    '''拍摄一张照片'''
    # The actual operation is done on the RaspberryPi side
    return

@pi_command
def ping_pi(msg=None, debug=False):
    '''检测树莓派在线状态'''
    return
<commit_msg>No underscore<commit_after># coding: utf-8
from config import pi_command

@pi_command
def takephoto(msg=None, debug=False):
    '''拍摄一张照片'''
    # The actual operation is done on the RaspberryPi side
    return

@pi_command
def pingpi(msg=None, debug=False):
    '''检测树莓派在线状态'''
    return
<|endoftext|>


<commit_before>from django.conf import settings
from django.db.models import F, Q
from django.db.models.aggregates import Count
from django.http import Http404

from rest_framework.response import Response

from geotrek.api.v2 import serializers as api_serializers, \
    viewsets as api_viewsets, filters as api_filters
from geotrek.api.v2.functions import Transform, Length, Length3D
from geotrek.trekking import models as trekking_models

class TrekViewSet(api_viewsets.GeotrekGeometricViewset):
    filter_backends = api_viewsets.GeotrekGeometricViewset.filter_backends + (api_filters.GeotrekTrekQueryParamsFilter,)
    serializer_class = api_serializers.TrekSerializer
    queryset = trekking_models.Trek.objects.existing() \
        .select_related('topo_object') \
        .prefetch_related('topo_object__aggregations', 'accessibilities', 'attachments') \
        .annotate(geom3d_transformed=Transform(F('geom_3d'), settings.API_SRID),
                  length_2d_m=Length('geom'),
                  length_3d_m=Length3D('geom_3d')) \
        .order_by('pk')  # Required for reliable pagination

    def retrieve(self, request, pk=None):
        # Return detail view even for unpublished treks that are childrens of other published treks
        trek = self.queryset.filter((Q(published=True) | Q(trek_parents__parent__published=True)) & Q(pk=pk)).distinct().first()
        if not trek:
            raise Http404('No %s matches the given query.' % self.queryset.model._meta.object_name)
        serializer = api_serializers.TrekSerializer(trek, many=False, context={'request': request})
        return Response(serializer.data)

class TourViewSet(TrekViewSet):
    serializer_class = api_serializers.TourSerializer
    queryset = TrekViewSet.queryset.annotate(count_children=Count('trek_children')) \
        .filter(count_children__gt=0)

class PracticeViewSet(api_viewsets.GeotrekViewSet):
    filter_backends = api_viewsets.GeotrekViewSet.filter_backends + (api_filters.GeotrekRelatedPortalTrekFilter,)
    serializer_class = api_serializers.PracticeSerializer
    queryset = trekking_models.Practice.objects.all()

class NetworksViewSet(api_viewsets.GeotrekViewSet):
    filter_backends = api_viewsets.GeotrekViewSet.filter_backends + (api_filters.GeotrekRelatedPortalTrekFilter,)
    serializer_class = api_serializers.NetworkSerializer
    queryset = trekking_models.TrekNetwork.objects.all()

class DifficultyViewSet(api_viewsets.GeotrekViewSet):
    filter_backends = api_viewsets.GeotrekViewSet.filter_backends + (api_filters.GeotrekRelatedPortalTrekFilter,)
    serializer_class = api_serializers.TrekDifficultySerializer
    queryset = trekking_models.DifficultyLevel.objects.all()

class POIViewSet(api_viewsets.GeotrekGeometricViewset):
    filter_backends = api_viewsets.GeotrekGeometricViewset.filter_backends + (api_filters.GeotrekPOIFilter,)
    serializer_class = api_serializers.POISerializer
    queryset = trekking_models.POI.objects.existing() \
        .select_related('topo_object', 'type', ) \
        .prefetch_related('topo_object__aggregations', 'attachments') \
        .annotate(geom3d_transformed=Transform(F('geom_3d'), settings.API_SRID)) \
        .order_by('pk')  # Required for reliable pagination

class POITypeViewSet(api_viewsets.GeotrekViewSet):
    serializer_class = api_serializers.POITypeSerializer
    queryset = trekking_models.POIType.objects.all()

class AccessibilityViewSet(api_viewsets.GeotrekViewSet):
    filter_backends = api_viewsets.GeotrekViewSet.filter_backends + (api_filters.GeotrekRelatedPortalTrekFilter,)
    serializer_class = api_serializers.AccessibilitySerializer
    queryset = trekking_models.Accessibility.objects.all()

class RouteViewSet(api_viewsets.GeotrekViewSet):
    filter_backends = api_viewsets.GeotrekViewSet.filter_backends + (api_filters.GeotrekRelatedPortalTrekFilter,)
    serializer_class = api_serializers.RouteSerializer
    queryset = trekking_models.Route.objects.all()
<commit_msg>Add published filter on trek when retrieving a single trek<commit_after>from django.conf import settings
from django.db.models import F, Q
from django.db.models.aggregates import Count
from django.http import Http404

from rest_framework.response import Response

from geotrek.api.v2 import serializers as api_serializers, \
    viewsets as api_viewsets, filters as api_filters
from geotrek.api.v2.functions import Transform, Length, Length3D
from geotrek.trekking import models as trekking_models

class TrekViewSet(api_viewsets.GeotrekGeometricViewset):
    filter_backends = api_viewsets.GeotrekGeometricViewset.filter_backends + (api_filters.GeotrekTrekQueryParamsFilter,)
    serializer_class = api_serializers.TrekSerializer
    queryset = trekking_models.Trek.objects.existing() \
        .select_related('topo_object') \
        .prefetch_related('topo_object__aggregations', 'accessibilities', 'attachments') \
        .annotate(geom3d_transformed=Transform(F('geom_3d'), settings.API_SRID),
                  length_2d_m=Length('geom'),
                  length_3d_m=Length3D('geom_3d')) \
        .order_by('pk')  # Required for reliable pagination

    def retrieve(self, request, pk=None):
        # Return detail view even for unpublished treks that are childrens of other published treks
        qs_filtered = self.filter_published_lang_retrieve(request, self.queryset)
        trek = qs_filtered.get(pk=pk)
        if not trek:
            raise Http404('No %s matches the given query.' % self.queryset.model._meta.object_name)
        serializer = api_serializers.TrekSerializer(trek, many=False, context={'request': request})
        return Response(serializer.data)

    def filter_published_lang_retrieve(self, request, queryset):
        # filter trek by publication language (including parents publication language)
        qs = queryset
        language = request.GET.get('language', 'all')
        associated_published_fields = [f.name for f in qs.model._meta.get_fields() if f.name.startswith('published')]

        if language == 'all':
            # no language specified. Check for all.
            q = Q()
            for lang in settings.MODELTRANSLATION_LANGUAGES:
                field_name = 'published_{}'.format(lang)
                if field_name in associated_published_fields:
                    field_name_parent = 'trek_parents__parent__published_{}'.format(lang)
                    q |= Q(**{field_name: True}) | Q(**{field_name_parent: True})
            qs = qs.filter(q)
        else:
            # one language is specified
            field_name = 'published_{}'.format(language)
            field_name_parent = 'trek_parents__parent__published_{}'.format(language)
            qs = qs.filter(Q(**{field_name: True}) | Q(**{field_name_parent: True}))
        return qs.distinct()

class TourViewSet(TrekViewSet):
    serializer_class = api_serializers.TourSerializer
    queryset = TrekViewSet.queryset.annotate(count_children=Count('trek_children')) \
        .filter(count_children__gt=0)

class PracticeViewSet(api_viewsets.GeotrekViewSet):
    filter_backends = api_viewsets.GeotrekViewSet.filter_backends + (api_filters.GeotrekRelatedPortalTrekFilter,)
    serializer_class = api_serializers.PracticeSerializer
    queryset = trekking_models.Practice.objects.all()

class NetworksViewSet(api_viewsets.GeotrekViewSet):
    filter_backends = api_viewsets.GeotrekViewSet.filter_backends + (api_filters.GeotrekRelatedPortalTrekFilter,)
    serializer_class = api_serializers.NetworkSerializer
    queryset = trekking_models.TrekNetwork.objects.all()

class DifficultyViewSet(api_viewsets.GeotrekViewSet):
    filter_backends = api_viewsets.GeotrekViewSet.filter_backends + (api_filters.GeotrekRelatedPortalTrekFilter,)
    serializer_class = api_serializers.TrekDifficultySerializer
    queryset = trekking_models.DifficultyLevel.objects.all()

class POIViewSet(api_viewsets.GeotrekGeometricViewset):
    filter_backends = api_viewsets.GeotrekGeometricViewset.filter_backends + (api_filters.GeotrekPOIFilter,)
    serializer_class = api_serializers.POISerializer
    queryset = trekking_models.POI.objects.existing() \
        .select_related('topo_object', 'type', ) \
        .prefetch_related('topo_object__aggregations', 'attachments') \
        .annotate(geom3d_transformed=Transform(F('geom_3d'), settings.API_SRID)) \
        .order_by('pk')  # Required for reliable pagination

class POITypeViewSet(api_viewsets.GeotrekViewSet):
    serializer_class = api_serializers.POITypeSerializer
    queryset = trekking_models.POIType.objects.all()

class AccessibilityViewSet(api_viewsets.GeotrekViewSet):
    filter_backends = api_viewsets.GeotrekViewSet.filter_backends + (api_filters.GeotrekRelatedPortalTrekFilter,)
    serializer_class = api_serializers.AccessibilitySerializer
    queryset = trekking_models.Accessibility.objects.all()

class RouteViewSet(api_viewsets.GeotrekViewSet):
    filter_backends = api_viewsets.GeotrekViewSet.filter_backends + (api_filters.GeotrekRelatedPortalTrekFilter,)
    serializer_class = api_serializers.RouteSerializer
    queryset = trekking_models.Route.objects.all()
<|endoftext|>


<commit_before># This file is part of Indico.
# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from collections import defaultdict, OrderedDict
from itertools import groupby, chain, combinations, izip_longest

from flask import render_template_string

def group_list(data, key=None, sort_by=None):
    return {group: sorted(list(items), key=sort_by) for group, items in groupby(sorted(data, key=key), key=key)}

def group_nested(objects, accessor=lambda x: x, _tree=None, _child_of=None):
    """Groups the equipment list so children follow their their parents."""
    if _tree is None:
        _tree = defaultdict(list)
        for obj in objects:
            _tree[accessor(obj).parent_id].append(obj)

    for obj in _tree[_child_of]:
        yield obj
        for child in group_nested(objects, accessor, _tree, accessor(obj).id):
            yield child

NESTED_TEMPLATE = """
{%- for item in items.itervalues() recursive -%}
    {{- item.name -}}
    {%- if item.children %}
        ({{ loop(item.children.itervalues()) }})
    {%- endif -%}
    {%- if not loop.last %}, {% endif -%}
{%- endfor -%}
""".strip()

def render_nested(objects, template=NESTED_TEMPLATE):
    """Renders a nested structure such as room equipment to a nice HTML string.

    :param objects: An iterable containing the objects

    All objects in `objects` must have `name`, `id` and `parent_id` attributes.
    """
    # We can't use `obj.children` since that triggers SQL queries
    tree = OrderedDict()
    children = {}
    for obj in group_nested(objects):
        if obj.parent_id is None:
            tree[obj.id] = {'name': obj.name, 'children': OrderedDict()}
            children[obj.id] = tree[obj.id]['children']
        else:
            children[obj.parent_id][obj.id] = {'name': obj.name, 'children': OrderedDict()}
            children[obj.id] = children[obj.parent_id][obj.id]['children']
    return render_template_string(template, items=tree)

def powerset(iterable):
    """powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)"""
    # Taken from https://docs.python.org/2/library/itertools.html#recipes
    s = list(iterable)
    return chain.from_iterable(combinations(s, r) for r in range(len(s)+1))

def committing_iterator(iterable, n=100):
    """Iterates over *iterable* and commits every *n* items.

    It also issues a commit after the iterator has been exhausted.

    :param iterable: An iterable object
    :param n: Number of items to commit after
    """
    from indico.core.db import db

    for i, data in enumerate(iterable, 1):
        yield data
        if i % n == 0:
            db.session.commit()
    db.session.commit()

def grouper(iterable, n, fillvalue=None, skip_missing=False):
    """Collect data into fixed-length chunks or blocks

    :param iterable: an iterable object
    :param n: number of items per chunk
    :param fillvalue: value to pad the last chunk with if necessary
    :param skip_missing: if the last chunk should be smaller instead
                         of being padded with `fillvalue`
    """
    # Taken from https://docs.python.org/2/library/itertools.html#recipes
    args = [iter(iterable)] * n
    if not skip_missing:
        return izip_longest(fillvalue=fillvalue, *args)
    else:
        # skips the missing items in the last tuple instead of padding it
        fillvalue = object()
        return (tuple(x for x in chunk if x is not fillvalue)
                for chunk in izip_longest(fillvalue=fillvalue, *args))
<commit_msg>Add sort_reverse to group_list<commit_after># This file is part of Indico.
# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from collections import defaultdict, OrderedDict
from itertools import groupby, chain, combinations, izip_longest

from flask import render_template_string

def group_list(data, key=None, sort_by=None, sort_reverse=False):
    return {group: sorted(list(items), key=sort_by, reverse=sort_reverse)
            for group, items in groupby(sorted(data, key=key), key=key)}

def group_nested(objects, accessor=lambda x: x, _tree=None, _child_of=None):
    """Groups the equipment list so children follow their their parents."""
    if _tree is None:
        _tree = defaultdict(list)
        for obj in objects:
            _tree[accessor(obj).parent_id].append(obj)

    for obj in _tree[_child_of]:
        yield obj
        for child in group_nested(objects, accessor, _tree, accessor(obj).id):
            yield child

NESTED_TEMPLATE = """
{%- for item in items.itervalues() recursive -%}
    {{- item.name -}}
    {%- if item.children %}
        ({{ loop(item.children.itervalues()) }})
    {%- endif -%}
    {%- if not loop.last %}, {% endif -%}
{%- endfor -%}
""".strip()

def render_nested(objects, template=NESTED_TEMPLATE):
    """Renders a nested structure such as room equipment to a nice HTML string.

    :param objects: An iterable containing the objects

    All objects in `objects` must have `name`, `id` and `parent_id` attributes.
    """
    # We can't use `obj.children` since that triggers SQL queries
    tree = OrderedDict()
    children = {}
    for obj in group_nested(objects):
        if obj.parent_id is None:
            tree[obj.id] = {'name': obj.name, 'children': OrderedDict()}
            children[obj.id] = tree[obj.id]['children']
        else:
            children[obj.parent_id][obj.id] = {'name': obj.name, 'children': OrderedDict()}
            children[obj.id] = children[obj.parent_id][obj.id]['children']
    return render_template_string(template, items=tree)

def powerset(iterable):
    """powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)"""
    # Taken from https://docs.python.org/2/library/itertools.html#recipes
    s = list(iterable)
    return chain.from_iterable(combinations(s, r) for r in range(len(s)+1))

def committing_iterator(iterable, n=100):
    """Iterates over *iterable* and commits every *n* items.

    It also issues a commit after the iterator has been exhausted.

    :param iterable: An iterable object
    :param n: Number of items to commit after
    """
    from indico.core.db import db

    for i, data in enumerate(iterable, 1):
        yield data
        if i % n == 0:
            db.session.commit()
    db.session.commit()

def grouper(iterable, n, fillvalue=None, skip_missing=False):
    """Collect data into fixed-length chunks or blocks

    :param iterable: an iterable object
    :param n: number of items per chunk
    :param fillvalue: value to pad the last chunk with if necessary
    :param skip_missing: if the last chunk should be smaller instead
                         of being padded with `fillvalue`
    """
    # Taken from https://docs.python.org/2/library/itertools.html#recipes
    args = [iter(iterable)] * n
    if not skip_missing:
        return izip_longest(fillvalue=fillvalue, *args)
    else:
        # skips the missing items in the last tuple instead of padding it
        fillvalue = object()
        return (tuple(x for x in chunk if x is not fillvalue)
                for chunk in izip_longest(fillvalue=fillvalue, *args))
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
This file is a bootstrap for the C++ edition of the FontTools compreffor.
It prepares the input data for the executable and reads back in the results,
applying them to the input font.
"""

import StringIO
import argparse
import array
import ctypes
import struct
import subprocess
import sys
import time
import os
from fontTools.ttLib import TTFont

from pyCompressor import Compreffor, CandidateSubr, tokenCost, human_size
from testPyCompressor import test_compression_integrity, test_call_depth

NSUBRS_LIMIT = 65533
SUBR_NEST_LIMIT  = 10

class IdKeyMap(object):
    def __getitem__(self, tok):
        return tok

class SimpleCandidateSubr(CandidateSubr):
    def __init__(self, length, ref_loc):
        self.length = length
        self.location = ref_loc
        self.freq = 0
        self._flatten = False
        self._global = False

    def usages(self):
        return self.freq

    frequency = usages

    def cost(self):
        try:
            return self.__cost
        except AttributeError:
            self.__cost = sum(map(tokenCost, self.value()))
            return self.__cost

    def encoding(self):
        return self._encoding

def write_data(td):
    """Writes CharStrings and FDSelect from the TopDict td into a string buffer."""
    out = StringIO.StringIO()
    td.CharStrings.charStringsIndex.getCompiler(td.strings, None).toFile(out)
    try:
        fdselect = struct.pack('B', len(td.FDArray)) + array.array('B', list(td.FDSelect)).tostring()
    except AttributeError:
        fdselect = struct.pack('B', 1)
    out.write(fdselect)
    return out.getvalue()

def get_encoding(data_buffer, subrs):
    pos = 0
    num_calls = data_buffer[pos]
    pos += 1
    enc = []
    for j in range(num_calls):
        insertion_pos = struct.unpack_from('<I', data_buffer[pos:pos+4])[0]
        pos += 4
        subr_index = struct.unpack_from('<I', data_buffer[pos:pos+4])[0]
        pos += 4
        subrs[subr_index].freq += 1
        enc.append((insertion_pos, subrs[subr_index]))
    return enc, pos

def read_data(td, result_string):
    results = array.array("B", result_string)
    num_subrs = struct.unpack_from('<I', results[:4])[0]

    # process subrs
    subrs = []
    pos = 4
    for i in range(num_subrs):
        glyph_idx = struct.unpack_from('<I', results[pos:pos+4])[0]
        pos += 4
        tok_idx = struct.unpack_from('<I', results[pos:pos+4])[0]
        pos += 4
        subr_len = struct.unpack_from('<I', results[pos:pos+4])[0]
        pos += 4
        subrs.append(SimpleCandidateSubr(subr_len, (glyph_idx, tok_idx)))
    for i in range(num_subrs):
        enc, num_read = get_encoding(results[pos:], subrs)
        pos += num_read
        subrs[i]._encoding = enc

    # process glyph encodings
    glyph_encodings = []
    for i in range(len(td.CharStrings)):
        enc, num_read = get_encoding(results[pos:], subrs)
        pos += num_read
        glyph_encodings.append(enc)

    assert pos == len(results)
    return (subrs, glyph_encodings)

def interpret_data(td, results):
    class MutableSpace: pass
    MutableSpace.pos = 0
    def pop_result():
        ans = results[MutableSpace.pos]
        MutableSpace.pos += 1
        return ans

    num_subrs = pop_result()

    # process subrs
    subrs = []
    for i in range(num_subrs):
        glyph_idx = pop_result()
        tok_idx = pop_result()
        subr_len = pop_result()
        subrs.append(SimpleCandidateSubr(subr_len, (glyph_idx, tok_idx)))

    def pop_encoding():
        num_calls = pop_result()
        enc = []
        for j in range(num_calls):
            insertion_pos = pop_result()
            subr_index = pop_result()
            subrs[subr_index].freq += 1
            enc.append((insertion_pos, subrs[subr_index]))
        return enc

    for i in range(num_subrs):
        enc = pop_encoding()
        subrs[i]._encoding = enc

    # process glyph encodings
    glyph_encodings = []
    for i in range(len(td.CharStrings)):
        enc = pop_encoding()
        glyph_encodings.append(enc)

    return (subrs, glyph_encodings)

def compreff(font, verbose=False, use_lib=False, **kwargs):
    full_start_time = start_time = time.time()

    assert len(font['CFF '].cff.topDictIndex) == 1

    td = font['CFF '].cff.topDictIndex[0]

    if verbose:
        print("Preparing external call...")
        start_time = time.time()

    call = [os.path.join(os.path.dirname(__file__), "cffCompressor")]

    if 'nrounds' in kwargs and kwargs.get('nrounds') != None:
        call.extend(['--nrounds', str(kwargs.get('nrounds'))])

    max_subrs = NSUBRS_LIMIT
    if 'nsubrs_limit' in kwargs and kwargs.get('nsubrs_limit') != None:
        max_subrs = kwargs.get('nsubrs_limit')
        call.extend(['--maxsubrs', str(max_subrs)])

    if use_lib:
        lib_path = os.path.join(os.path.dirname(__file__), "libcompreff.so")
        libcompreff = ctypes.CDLL(lib_path)
        libcompreff.compreff.restype = ctypes.POINTER(ctypes.c_uint32)
        input_data = ctypes.c_char_p(write_data(td))
        if verbose:
            print("Produced data for C++ (delta %gs)" % (time.time() - start_time))
            start_time = time.time()
        results = libcompreff.compreff(input_data, 4, ctypes.c_uint(max_subrs))
        if verbose:
            print("Lib call returned (delta %gs)" % (time.time() - start_time))
            start_time = time.time()
        subrs, glyph_encodings = interpret_data(td, results)
    else:
        p = subprocess.Popen(
                            call,
                            stdin=subprocess.PIPE,
                            stdout=subprocess.PIPE)
        input_data = write_data(td)
        if verbose:
            print("Produced data for C++ (delta %gs)" % (time.time() - start_time))
            start_time = time.time()
        results, _ = p.communicate(input=input_data)
        if verbose:
            print("Executable returned (delta %gs)" % (time.time() - start_time))
            start_time = time.time()
        subrs, glyph_encodings = read_data(td, results)

    if verbose:
        print("Extracted results (delta %gs)" % (time.time() - start_time))
        start_time = time.time()

    for cs in td.CharStrings.values():
        cs.decompile()

    if verbose:
        print("Decompiled charstrings (delta %gs)" % (time.time() - start_time))
        start_time = time.time()

    # in order of charset
    chstrings = map(lambda x: x.program, td.CharStrings.values())
    map(lambda x: Compreffor.collapse_hintmask(x), chstrings)

    for s in subrs:
        s.chstrings = chstrings

    if hasattr(td, 'FDSelect'):
        fdselect = lambda g: td.CharStrings.getItemAndSelector(g)[1]
        fdlen = len(td.FDArray)
    else:
        fdselect = None
        fdlen = 1

    gsubrs, lsubrs = Compreffor.process_subrs(
                            td.charset,
                            glyph_encodings,
                            fdlen,
                            fdselect,
                            subrs,
                            IdKeyMap(),
                            max_subrs,
                            SUBR_NEST_LIMIT)

    encoding = dict(zip(td.charset, glyph_encodings))

    Compreffor.apply_subrs(td, encoding, gsubrs, lsubrs)

    if verbose:
        print("Finished post-processing (delta %gs)" % (time.time() - start_time))
        print("Total time: %gs" % (time.time() - full_start_time))

def main(filename=None, comp_fname=None, test=False, decompress=False,
         verbose=False, check=False, generate_cff=False, recursive=False,
         **comp_kwargs):
    if test:
        pass

    if filename and comp_fname == None:
        def handle_font(font_name):
            font = TTFont(font_name)

            td = font['CFF '].cff.topDictIndex[0]
            no_subrs = lambda fd: hasattr(fd, 'Subrs') and len(fd.Subrs) > 0
            priv_subrs = (hasattr(td, 'FDArray') and
                          any(no_subrs(fd) for fd in td.FDArray))
            if len(td.GlobalSubrs) > 0 or priv_subrs:
                print("Warning: There are subrs in %s" % font_name)

            orig_size = os.path.getsize(font_name)

            if decompress:
                from fontTools import subset
                options = subset.Options()
                options.decompress = True
                subsetter = subset.Subsetter(options=options)
                subsetter.populate(glyphs=font.getGlyphOrder())
                subsetter.subset(font)

            out_name = "%s.compressed%s" % os.path.splitext(font_name)

            compreff(font, verbose=verbose, **comp_kwargs)

            # save compressed font
            start_time = time.time()
            font.save(out_name)
            if verbose:
                print("Compiled and saved (took %gs)" % (time.time() - start_time))

            if generate_cff:
                # save CFF version
                font['CFF '].cff.compile(open("%s.cff" % os.path.splitext(out_name)[0], 'w'), None)

            comp_size = os.path.getsize(out_name)
            print("Compressed to %s -- saved %s" %
                    (os.path.basename(out_name), human_size(orig_size - comp_size)))

            if check:
                test_compression_integrity(filename, out_name)
                test_call_depth(out_name)

        if recursive:
            for root, dirs, files in os.walk(filename):
                for fname in files:
                    if os.path.splitext(fname)[1] == '.otf':
                        handle_font(fname)
        else:
            handle_font(filename)

    if check and comp_fname != None:
        test_compression_integrity(filename, comp_fname)
        test_call_depth(comp_fname)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
                        description="""FontTools Compreffor will take a CFF-flavored
                                       OpenType font and automatically detect
                                       repeated routines and generate subroutines
                                       to minimize the disk space needed to
                                       represent a font.""")
    parser.add_argument('filename', help="the path to the font file", nargs='?')
    parser.add_argument('comp_fname', nargs='?', metavar='compressed-file',
                        help="the path to the compressed file. if this is given"
                             " with the -c flag, it will be checked against "
                             " `filename`.")
    # parser.add_argument('-t', '--test', required=False, action='store_true',
    #                     default=False, help="run test cases")
    parser.add_argument('-v', '--verbose', required=False, action='store_true',
                        dest='verbose', default=False)
    parser.add_argument('-c', '--check', required=False, action='store_true',
                        help="verify that the outputted font is valid and "
                             "functionally equivalent to the input")
    parser.add_argument('-d', '--decompress', required=False, action='store_true',
                        help="decompress source before compressing (necessary if "
                             "there are subroutines in the source)")
    parser.add_argument('-r', '--recursive', required=False, action='store_true',
                        default=False)
    parser.add_argument('-n', '--nrounds', required=False, type=int,
                        help="the number of iterations to run the algorithm"
                             " (defaults to 4)")
    parser.add_argument('-m', '--maxsubrs', required=False, type=int,
                        dest='nsubrs_limit', help="limit to the number of "
                                                  " subroutines per INDEX"
                                                  " (defaults to 64K)")
    parser.add_argument('--generatecff', required=False, action='store_true',
                        dest='generate_cff', default=False)
    parser.add_argument('--uselib', required=False, action='store_true',
                        dest='use_lib', default=False)

    kwargs = vars(parser.parse_args())

    main(**kwargs)
<commit_msg>stop passing --maxsubrs to C++ executable<commit_after>#!/usr/bin/env python

"""
This file is a bootstrap for the C++ edition of the FontTools compreffor.
It prepares the input data for the executable and reads back in the results,
applying them to the input font.
"""

import StringIO
import argparse
import array
import ctypes
import struct
import subprocess
import sys
import time
import os
from fontTools.ttLib import TTFont

from pyCompressor import Compreffor, CandidateSubr, tokenCost, human_size
from testPyCompressor import test_compression_integrity, test_call_depth

NSUBRS_LIMIT = 65533
SUBR_NEST_LIMIT  = 10

class IdKeyMap(object):
    def __getitem__(self, tok):
        return tok

class SimpleCandidateSubr(CandidateSubr):
    def __init__(self, length, ref_loc):
        self.length = length
        self.location = ref_loc
        self.freq = 0
        self._flatten = False
        self._global = False

    def usages(self):
        return self.freq

    frequency = usages

    def cost(self):
        try:
            return self.__cost
        except AttributeError:
            self.__cost = sum(map(tokenCost, self.value()))
            return self.__cost

    def encoding(self):
        return self._encoding

def write_data(td):
    """Writes CharStrings and FDSelect from the TopDict td into a string buffer."""
    out = StringIO.StringIO()
    td.CharStrings.charStringsIndex.getCompiler(td.strings, None).toFile(out)
    try:
        fdselect = struct.pack('B', len(td.FDArray)) + array.array('B', list(td.FDSelect)).tostring()
    except AttributeError:
        fdselect = struct.pack('B', 1)
    out.write(fdselect)
    return out.getvalue()

def get_encoding(data_buffer, subrs):
    pos = 0
    num_calls = data_buffer[pos]
    pos += 1
    enc = []
    for j in range(num_calls):
        insertion_pos = struct.unpack_from('<I', data_buffer[pos:pos+4])[0]
        pos += 4
        subr_index = struct.unpack_from('<I', data_buffer[pos:pos+4])[0]
        pos += 4
        subrs[subr_index].freq += 1
        enc.append((insertion_pos, subrs[subr_index]))
    return enc, pos

def read_data(td, result_string):
    results = array.array("B", result_string)
    num_subrs = struct.unpack_from('<I', results[:4])[0]

    # process subrs
    subrs = []
    pos = 4
    for i in range(num_subrs):
        glyph_idx = struct.unpack_from('<I', results[pos:pos+4])[0]
        pos += 4
        tok_idx = struct.unpack_from('<I', results[pos:pos+4])[0]
        pos += 4
        subr_len = struct.unpack_from('<I', results[pos:pos+4])[0]
        pos += 4
        subrs.append(SimpleCandidateSubr(subr_len, (glyph_idx, tok_idx)))
    for i in range(num_subrs):
        enc, num_read = get_encoding(results[pos:], subrs)
        pos += num_read
        subrs[i]._encoding = enc

    # process glyph encodings
    glyph_encodings = []
    for i in range(len(td.CharStrings)):
        enc, num_read = get_encoding(results[pos:], subrs)
        pos += num_read
        glyph_encodings.append(enc)

    assert pos == len(results)
    return (subrs, glyph_encodings)

def interpret_data(td, results):
    class MutableSpace: pass
    MutableSpace.pos = 0
    def pop_result():
        ans = results[MutableSpace.pos]
        MutableSpace.pos += 1
        return ans

    num_subrs = pop_result()

    # process subrs
    subrs = []
    for i in range(num_subrs):
        glyph_idx = pop_result()
        tok_idx = pop_result()
        subr_len = pop_result()
        subrs.append(SimpleCandidateSubr(subr_len, (glyph_idx, tok_idx)))

    def pop_encoding():
        num_calls = pop_result()
        enc = []
        for j in range(num_calls):
            insertion_pos = pop_result()
            subr_index = pop_result()
            subrs[subr_index].freq += 1
            enc.append((insertion_pos, subrs[subr_index]))
        return enc

    for i in range(num_subrs):
        enc = pop_encoding()
        subrs[i]._encoding = enc

    # process glyph encodings
    glyph_encodings = []
    for i in range(len(td.CharStrings)):
        enc = pop_encoding()
        glyph_encodings.append(enc)

    return (subrs, glyph_encodings)

def compreff(font, verbose=False, use_lib=False, **kwargs):
    full_start_time = start_time = time.time()

    assert len(font['CFF '].cff.topDictIndex) == 1

    td = font['CFF '].cff.topDictIndex[0]

    if verbose:
        print("Preparing external call...")
        start_time = time.time()

    call = [os.path.join(os.path.dirname(__file__), "cffCompressor")]

    if 'nrounds' in kwargs and kwargs.get('nrounds') != None:
        call.extend(['--nrounds', str(kwargs.get('nrounds'))])

    max_subrs = NSUBRS_LIMIT
    if 'nsubrs_limit' in kwargs and kwargs.get('nsubrs_limit') != None:
        max_subrs = kwargs.get('nsubrs_limit')

    if use_lib:
        lib_path = os.path.join(os.path.dirname(__file__), "libcompreff.so")
        libcompreff = ctypes.CDLL(lib_path)
        libcompreff.compreff.restype = ctypes.POINTER(ctypes.c_uint32)
        input_data = ctypes.c_char_p(write_data(td))
        if verbose:
            print("Produced data for C++ (delta %gs)" % (time.time() - start_time))
            start_time = time.time()
        results = libcompreff.compreff(input_data, 4, ctypes.c_uint(max_subrs))
        if verbose:
            print("Lib call returned (delta %gs)" % (time.time() - start_time))
            start_time = time.time()
        subrs, glyph_encodings = interpret_data(td, results)
    else:
        p = subprocess.Popen(
                            call,
                            stdin=subprocess.PIPE,
                            stdout=subprocess.PIPE)
        input_data = write_data(td)
        if verbose:
            print("Produced data for C++ (delta %gs)" % (time.time() - start_time))
            start_time = time.time()
        results, _ = p.communicate(input=input_data)
        if verbose:
            print("Executable returned (delta %gs)" % (time.time() - start_time))
            start_time = time.time()
        subrs, glyph_encodings = read_data(td, results)

    if verbose:
        print("Extracted results (delta %gs)" % (time.time() - start_time))
        start_time = time.time()

    for cs in td.CharStrings.values():
        cs.decompile()

    if verbose:
        print("Decompiled charstrings (delta %gs)" % (time.time() - start_time))
        start_time = time.time()

    # in order of charset
    chstrings = map(lambda x: x.program, td.CharStrings.values())
    map(lambda x: Compreffor.collapse_hintmask(x), chstrings)

    for s in subrs:
        s.chstrings = chstrings

    if hasattr(td, 'FDSelect'):
        fdselect = lambda g: td.CharStrings.getItemAndSelector(g)[1]
        fdlen = len(td.FDArray)
    else:
        fdselect = None
        fdlen = 1

    gsubrs, lsubrs = Compreffor.process_subrs(
                            td.charset,
                            glyph_encodings,
                            fdlen,
                            fdselect,
                            subrs,
                            IdKeyMap(),
                            max_subrs,
                            SUBR_NEST_LIMIT)

    encoding = dict(zip(td.charset, glyph_encodings))

    Compreffor.apply_subrs(td, encoding, gsubrs, lsubrs)

    if verbose:
        print("Finished post-processing (delta %gs)" % (time.time() - start_time))
        print("Total time: %gs" % (time.time() - full_start_time))

def main(filename=None, comp_fname=None, test=False, decompress=False,
         verbose=False, check=False, generate_cff=False, recursive=False,
         **comp_kwargs):
    if test:
        pass

    if filename and comp_fname == None:
        def handle_font(font_name):
            font = TTFont(font_name)

            td = font['CFF '].cff.topDictIndex[0]
            no_subrs = lambda fd: hasattr(fd, 'Subrs') and len(fd.Subrs) > 0
            priv_subrs = (hasattr(td, 'FDArray') and
                          any(no_subrs(fd) for fd in td.FDArray))
            if len(td.GlobalSubrs) > 0 or priv_subrs:
                print("Warning: There are subrs in %s" % font_name)

            orig_size = os.path.getsize(font_name)

            if decompress:
                from fontTools import subset
                options = subset.Options()
                options.decompress = True
                subsetter = subset.Subsetter(options=options)
                subsetter.populate(glyphs=font.getGlyphOrder())
                subsetter.subset(font)

            out_name = "%s.compressed%s" % os.path.splitext(font_name)

            compreff(font, verbose=verbose, **comp_kwargs)

            # save compressed font
            start_time = time.time()
            font.save(out_name)
            if verbose:
                print("Compiled and saved (took %gs)" % (time.time() - start_time))

            if generate_cff:
                # save CFF version
                font['CFF '].cff.compile(open("%s.cff" % os.path.splitext(out_name)[0], 'w'), None)

            comp_size = os.path.getsize(out_name)
            print("Compressed to %s -- saved %s" %
                    (os.path.basename(out_name), human_size(orig_size - comp_size)))

            if check:
                test_compression_integrity(filename, out_name)
                test_call_depth(out_name)

        if recursive:
            for root, dirs, files in os.walk(filename):
                for fname in files:
                    if os.path.splitext(fname)[1] == '.otf':
                        handle_font(fname)
        else:
            handle_font(filename)

    if check and comp_fname != None:
        test_compression_integrity(filename, comp_fname)
        test_call_depth(comp_fname)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
                        description="""FontTools Compreffor will take a CFF-flavored
                                       OpenType font and automatically detect
                                       repeated routines and generate subroutines
                                       to minimize the disk space needed to
                                       represent a font.""")
    parser.add_argument('filename', help="the path to the font file", nargs='?')
    parser.add_argument('comp_fname', nargs='?', metavar='compressed-file',
                        help="the path to the compressed file. if this is given"
                             " with the -c flag, it will be checked against "
                             " `filename`.")
    # parser.add_argument('-t', '--test', required=False, action='store_true',
    #                     default=False, help="run test cases")
    parser.add_argument('-v', '--verbose', required=False, action='store_true',
                        dest='verbose', default=False)
    parser.add_argument('-c', '--check', required=False, action='store_true',
                        help="verify that the outputted font is valid and "
                             "functionally equivalent to the input")
    parser.add_argument('-d', '--decompress', required=False, action='store_true',
                        help="decompress source before compressing (necessary if "
                             "there are subroutines in the source)")
    parser.add_argument('-r', '--recursive', required=False, action='store_true',
                        default=False)
    parser.add_argument('-n', '--nrounds', required=False, type=int,
                        help="the number of iterations to run the algorithm"
                             " (defaults to 4)")
    parser.add_argument('-m', '--maxsubrs', required=False, type=int,
                        dest='nsubrs_limit', help="limit to the number of "
                                                  " subroutines per INDEX"
                                                  " (defaults to 64K)")
    parser.add_argument('--generatecff', required=False, action='store_true',
                        dest='generate_cff', default=False)
    parser.add_argument('--uselib', required=False, action='store_true',
                        dest='use_lib', default=False)

    kwargs = vars(parser.parse_args())

    main(**kwargs)
<|endoftext|>


<commit_before>import os
from igf_data.utils.fileutils import get_temp_dir,remove_dir
from ehive.runnable.IGFBaseProcess import IGFBaseProcess

class CleanupDirOrFile(IGFBaseProcess):
  def param_defaults(self):
    params_dict=super(IGFBaseProcess,self).param_defaults()
    params_dict.update({
      'cleanup_status':False,
      })
    return params_dict

  def run(self):
    try:
      seqrun_igf_id=self.param_required('seqrun_igf_id')
      path=self.param_required('path')
      cleanup_status=self.param_required('cleanup_status')

      message=None
      if cleanup_status:
        if not os.path.exists(path):
          raise IOError('path {0} is not accessible'.format(path))

        if os.path.isdir(path):
          remove_dir(path)
          message='removed dir {0}'.format(path)
        elif os.path.isfile(path):
          os.remove(path)
          message='removed file {0}'.format(path)
        else:
          message='path {0} is not file or directory, skipped removing'.\
                  format(path)

      else:
        message='Not removing path {0} as cleanup_status is not True'.\
                format(path)
      if message:
        self.post_message_to_slack(message,reaction='pass')                     # send msg to slack
        self.comment_asana_task(task_name=seqrun_igf_id, comment=message)       # send msg to asana
    except Exception as e:
      message='seqrun: {2}, Error in {0}: {1}'.format(self.__class__.__name__, \
                                                      e, \
                                                      seqrun_igf_id)
      self.warning(message)
      self.post_message_to_slack(message,reaction='fail')                       # post msg to slack for failed jobs
      raise<commit_msg>added dataflow params<commit_after>import os
from igf_data.utils.fileutils import get_temp_dir,remove_dir
from ehive.runnable.IGFBaseProcess import IGFBaseProcess

class CleanupDirOrFile(IGFBaseProcess):
  def param_defaults(self):
    params_dict=super(IGFBaseProcess,self).param_defaults()
    params_dict.update({
      'cleanup_status':False,
      })
    return params_dict

  def run(self):
    try:
      seqrun_igf_id=self.param_required('seqrun_igf_id')
      path=self.param_required('path')
      cleanup_status=self.param_required('cleanup_status')

      message=None
      if cleanup_status:
        if not os.path.exists(path):
          raise IOError('path {0} is not accessible'.format(path))

        if os.path.isdir(path):
          remove_dir(path)
          message='removed dir {0}'.format(path)
        elif os.path.isfile(path):
          os.remove(path)
          message='removed file {0}'.format(path)

        else:
          message='path {0} is not file or directory, skipped removing'.\
                  format(path)

      else:
        message='Not removing path {0} as cleanup_status is not True'.\
                format(path)

      self.param('dataflow_params',{'path':path,'cleanup_status':cleanup_status})# set dataflow params
      if message:
        self.post_message_to_slack(message,reaction='pass')                     # send msg to slack
        self.comment_asana_task(task_name=seqrun_igf_id, comment=message)       # send msg to asana
    except Exception as e:
      message='seqrun: {2}, Error in {0}: {1}'.format(self.__class__.__name__, \
                                                      e, \
                                                      seqrun_igf_id)
      self.warning(message)
      self.post_message_to_slack(message,reaction='fail')                       # post msg to slack for failed jobs
      raise<|endoftext|>


<commit_before>import random

def test_prediction(br_dist, predictor):
	# distribution of predictions
	pred_dist = []
	# number of branches taken
	num_taken = 0

	# for each branch in the distribution
	for b in br_dist:
		# predict the branch
		pred_dist.append(predictor.predict())
		# update the predictor with whether the branch was taken or not
		predictor.update(b)
		# update num_taken
		if(b == 1):
			num_taken += 1

	# calculate % predictions correct
	num_correct = 0
	for i in range(len(br_dist)):
		if (br_dist[i] == pred_dist[i]):
			num_correct += 1

	# prints
	# - the number of predictions tested
	# - the percentage of those predictions that were correct
	# - the number of branches taken in the distribution (and %)
	# - the number of branches not taken (and %)
	proportion_correct = (float(num_correct) / float(len(br_dist))) * 100
	print(str(len(br_dist)) + " predictions were tested, "
			+ str(proportion_correct) + "% were correct.")
	print("In this distribution, " + str(num_taken) + " branches were taken ("
			+ str(float(num_taken * 100) / float(num_branches)) + "%), "
			+ str(num_branches - num_taken) + " were not taken ("
			+ str(((num_branches - num_taken) * 100) / float(num_branches))
			+ "%).")

def generate_distribution(num_branches):
	dist = []
	value = random.randint(0,1000)
	for i in dist:
		if (value <= 200):
			dist.append(0)
		elif (value <= 1000):
			dist.append(1)

	return dist

class twobit_saturated_counter:
	counter = 2
	def predict():
		if (counter >= 2):
			return 1
		if (counter <= 1):
			return 0
	def update(taken):
		if (taken == 1 and counter <3):
			counter += 1
		if (taken == 0 and counter >0):
			counter -= 1

<commit_msg>Separated toplevel functions to conform to PEP8<commit_after>import random

def test_prediction(br_dist, predictor):
	# distribution of predictions
	pred_dist = []
	# number of branches taken
	num_taken = 0

	# for each branch in the distribution
	for b in br_dist:
		# predict the branch
		pred_dist.append(predictor.predict())
		# update the predictor with whether the branch was taken or not
		predictor.update(b)
		# update num_taken
		if(b == 1):
			num_taken += 1

	# calculate % predictions correct
	num_correct = 0
	for i in range(len(br_dist)):
		if (br_dist[i] == pred_dist[i]):
			num_correct += 1

	# prints
	# - the number of predictions tested
	# - the percentage of those predictions that were correct
	# - the number of branches taken in the distribution (and %)
	# - the number of branches not taken (and %)
	proportion_correct = (float(num_correct) / float(len(br_dist))) * 100
	print(str(len(br_dist)) + " predictions were tested, "
			+ str(proportion_correct) + "% were correct.")
	print("In this distribution, " + str(num_taken) + " branches were taken ("
			+ str(float(num_taken * 100) / float(num_branches)) + "%), "
			+ str(num_branches - num_taken) + " were not taken ("
			+ str(((num_branches - num_taken) * 100) / float(num_branches))
			+ "%).")

def generate_distribution(num_branches):
	dist = []
	value = random.randint(0,1000)
	for i in dist:
		if (value <= 200):
			dist.append(0)
		elif (value <= 1000):
			dist.append(1)

	return dist

class twobit_saturated_counter:
	counter = 2
	def predict():
		if (counter >= 2):
			return 1
		if (counter <= 1):
			return 0
	def update(taken):
		if (taken == 1 and counter <3):
			counter += 1
		if (taken == 0 and counter >0):
			counter -= 1<|endoftext|>