<commit_before>"""
Support for models' internal Translation class.
"""

## TO DO: this is messy and needs to be cleaned up

from django.db import models
from django.db.models.base import ModelBase
from django.dispatch.dispatcher import connect
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import signals
from languages import *
from exceptions import TranslationDoesNotExist
import manager

from django.contrib.admin.templatetags.admin_modify import StackedBoundRelatedObject

class TransBoundRelatedObject(StackedBoundRelatedObject):
    """
    This class changes the template for translation objects.

    It also reorders the translation list to match the list of
    languages specified in settings file.
    """
    def __init__(self, related_object, field_mapping, original):
        super(TransBoundRelatedObject, self).__init__(related_object,
                                                      field_mapping,
                                                      original)

        # reorder field wrappers to match the ordering of languages
        # specified in settings

        # 1. find corresponding language for each mapping

        mappings_with_lang = {}
        mappings_without_lang = []
        for idx, mapping in self.field_mappings.items():
            if mapping['original']:
                mappings_with_lang[mapping['original'].language_id] = idx
            else:
                mappings_without_lang.append(idx)

        # 2. reorder the wrappers according to what you found in #1

        new_form_field_collection_wrappers = []
        ffcw = self.form_field_collection_wrappers
        for language_id in get_language_id_list():
            idx = mappings_with_lang.get(language_id, None)
            if idx is None:
                idx = mappings_without_lang.pop(0)
            new_form_field_collection_wrappers.append(ffcw[idx])
        self.form_field_collection_wrappers = new_form_field_collection_wrappers

    def template_name(self):
        return "admin/edit_inline_translations.html"

def translation_save_translated_fields(instance, **kwargs):
    """
    Save all the translations of instance in post_save signal handler.
    """
    if not hasattr(instance, '_translation_cache'):
        return
    instance.get_translation_qs().delete()
    for l_id, translation in instance._translation_cache.iteritems():
        # set the translation ID just in case the translation was
        # created while instance was not stored in the DB yet
        translation.master_id = instance.id
        translation.save()

def fill_translation_cache(instance):
    """
    Fill the translation cache using information received in the
    instance objects as extra fields.

    You can not do this in post_init because the extra fields are
    assigned by QuerySet.iterator after model initialization.
    """

    if hasattr(instance, '_translation_cache'):
        # do not refill the cache
        return

    instance._translation_cache = {}
    for language_id in get_language_id_list():
        # see if translation for language_id was in the query
        if hasattr(instance, get_translated_field_alias('id', language_id)):
            field_names = [f.attname for f in instance._meta.translation_model._meta.fields]

            # if so, create a translation object and put it in the cache
            field_data = {}
            for fname in field_names:
                field_data[fname] = getattr(instance,
                                            get_translated_field_alias(fname, language_id))

            translation = instance._meta.translation_model(**field_data)
            instance._translation_cache[language_id] = translation

    # In some situations an (existing in the DB) object is loaded
    # without using the normal QuerySet.  In such case fallback to
    # loading the translations using a separate query.

    # Unfortunately, this is indistinguishable from the situation when
    # an object does not have any translations.  Oh well, we'll have
    # to live with this for the time being.
    if len(instance._translation_cache.keys()) == 0:
        for translation in instance.get_translation_qs():
            instance._translation_cache[translation.language_id] = translation

class TranslatedFieldProxy(property):
    def __init__(self, field_name, alias, field, language_id=None):
        self.field_name = field_name
        self.field = field
        self.admin_order_field = alias
        self.language_id = language_id

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self

        return getattr(obj, 'get_' + self.field_name)(self.language_id)

    def __set__(self, obj, value):
        language_id = self.language_id

        return getattr(obj, 'set_' + self.field_name)(value, self.language_id)

    short_description = property(lambda self: self.field.short_description)

def getter_generator(field_name, short_description):
    """
    Generate get_'field name' method for field field_name.
    """
    def get_translation_field(self, language_id_or_code=None):
        try:
            return getattr(self.get_translation(language_id_or_code), field_name)
        except TranslationDoesNotExist:
            return "-translation-not-available-"
    get_translation_field.short_description = short_description
    return get_translation_field

def setter_generator(field_name):
    """
    Generate set_'field name' method for field field_name.
    """
    def set_translation_field(self, value, language_id_or_code=None):
        setattr(self.get_translation(language_id_or_code, True),
                field_name, value)
    set_translation_field.short_description = "set " + field_name
    return set_translation_field

def get_translation(self, language_id_or_code,
                    create_if_necessary=False):
    """
    Get a translation instance for the given language_id_or_code.

    If it does not exist, either create one or raise the
    TranslationDoesNotExist exception, depending on the
    create_if_necessary argument.
    """

    # fill the cache if necessary
    self.fill_translation_cache()

    language_id = get_language_id_from_id_or_code(language_id_or_code)
    if language_id is None:
        language_id = getattr(self, '_default_language', None)
    if language_id is None:
        language_id = get_default_language()

    if language_id not in self._translation_cache:
        if not create_if_necessary:
            raise TranslationDoesNotExist
        new_translation = self._meta.translation_model(master=self,
                                                       language_id=language_id)
        self._translation_cache[language_id] = new_translation
    return self._translation_cache.get(language_id, None)

def get_translation_qs(self):
    """
    Return the queryset with all translations of self.
    """
    return self._meta.translation_model.objects.filter(master=self)

class Translation:
    """
    A superclass for translatablemodel.Translation inner classes.
    """

    def contribute_to_class(cls, main_cls, name):
        """
        Handle the inner 'Translation' class.
        """

        # delay the creation of the *Translation until the master model is
        # fully created
        connect(cls.finish_multilingual_class, signal=signals.class_prepared,
                sender=main_cls, weak=False)

        # connect the post_save signal to a handler that saves translations
        connect(translation_save_translated_fields, signal=signals.post_save,
                sender=main_cls)
    contribute_to_class = classmethod(contribute_to_class)

    def create_translation_attrs(cls, main_cls):
        """
        Creates get_'field name'(language_id) and set_'field
        name'(language_id) methods for all the translation fields.
        Adds the 'field name' properties too.

        Returns the translated_fields hash used in field lookups, see
        multilingual.query.  It maps field names to (field,
        language_id) tuples.
        """
        translated_fields = {}

        for fname, field in cls.__dict__.items():
            if isinstance(field, models.fields.Field):
                translated_fields[fname] = (field, None)

                # add get_'fname' and set_'fname' methods to main_cls
                getter = getter_generator(fname, getattr(field, 'verbose_name', fname))
                setattr(main_cls, 'get_' + fname, getter)

                setter = setter_generator(fname)
                setattr(main_cls, 'set_' + fname, setter)

                # add the 'fname' proxy property that allows reads
                # from and writing to the appropriate translation
                setattr(main_cls, fname,
                        TranslatedFieldProxy(fname, fname, field))

                # create the 'fname'_'language_code' proxy properties
                for language_id in get_language_id_list():
                    language_code = get_language_code(language_id)
                    fname_lng = fname + '_' + language_code
                    translated_fields[fname_lng] = (field, language_id)
                    setattr(main_cls, fname_lng,
                            TranslatedFieldProxy(fname, fname_lng, field,
                                                 language_id))
        return translated_fields
    create_translation_attrs = classmethod(create_translation_attrs)

    def finish_multilingual_class(cls, *args, **kwargs):
        """
        Create a model with translations of a multilingual class.
        """

        main_cls = kwargs['sender']
        translation_model_name = main_cls.__name__ + "Translation"

        # create the model with all the translatable fields
        class TransMeta:
            ordering = ('language_id',)
            unique_together = (('master', 'language_id'),)
            db_table = main_cls._meta.db_table + 'translation'

        trans_attrs = cls.__dict__.copy()
        trans_attrs['Meta'] = TransMeta
        trans_attrs['language_id'] = models.IntegerField(blank=False, null=False, core=True,
                                                         choices=get_language_choices(),
                                                         db_index=True)
        trans_attrs['master'] = models.ForeignKey(main_cls, blank=False, null=False,
                                                  edit_inline=TransBoundRelatedObject,
                                                  num_in_admin=get_language_count(),
                                                  min_num_in_admin=get_language_count(),
                                                  num_extra_on_change=0)
        trans_attrs['__str__'] = lambda self: ("%s object, language_code=%s"
                                               % (translation_model_name,
                                                  get_language_code(self.language_id)))

        trans_model = ModelBase(translation_model_name, (models.Model,), trans_attrs)
        trans_model._meta.translated_fields = cls.create_translation_attrs(main_cls)

        main_cls._meta.translation_model = trans_model
        main_cls.get_translation = get_translation
        main_cls.get_translation_qs = get_translation_qs
        main_cls.fill_translation_cache = fill_translation_cache

    finish_multilingual_class = classmethod(finish_multilingual_class)

def install_translation_library():
    # modify ModelBase.__new__ so that it understands how to handle the
    # 'Translation' inner class

    if getattr(ModelBase, '_multilingual_installed', False):
        # don't install it twice
        return

    _old_new = ModelBase.__new__

    def multilingual_modelbase_new(cls, name, bases, attrs):
        if 'Translation' in attrs:
            if not issubclass(attrs['Translation'], Translation):
                raise ValueError, ("%s.Translation must be a subclass "
                                   + " of multilingual.Translation.") % (name,)

            # Make sure that if the class specifies objects then it is
            # a subclass of our Manager.
            #
            # Don't check other managers since someone might want to
            # have a non-multilingual manager, but assigning a
            # non-multilingual manager to objects would be a common
            # mistake.
            if ('objects' in attrs) and (not isinstance(attrs['objects'], manager.Manager)):
                raise ValueError, ("Model %s specifies translations, " +
                                   "so its 'objects' manager must be " +
                                   "a subclass of multilingual.Manager.") % (name,)

            # Change the default manager to one that knows how to handle
            # translation fields.  Ideally this should not be necessary, so
            # that it would still be possible to use a custom manager here.
            attrs['objects'] = manager.Manager()

            # Override the admin manager as well, or the admin views will
            # not see the translation data.
            if 'Admin' in attrs:
                attrs['Admin'].manager = attrs['objects']

        return _old_new(cls, name, bases, attrs)

    ModelBase.__new__ = staticmethod(multilingual_modelbase_new)
    ModelBase._multilingual_installed = True

# install the library
install_translation_library()
<commit_msg>Committing a change that should be a part of [37], but I missed it.<commit_after>"""
Support for models' internal Translation class.
"""

## TO DO: this is messy and needs to be cleaned up

from django.db import models
from django.db.models.base import ModelBase
from django.dispatch.dispatcher import connect
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import signals
from languages import *
from exceptions import TranslationDoesNotExist
import manager

from django.contrib.admin.templatetags.admin_modify import StackedBoundRelatedObject

class TransBoundRelatedObject(StackedBoundRelatedObject):
    """
    This class changes the template for translation objects.

    It also reorders the translation list to match the list of
    languages specified in settings file.
    """
    def __init__(self, related_object, field_mapping, original):
        super(TransBoundRelatedObject, self).__init__(related_object,
                                                      field_mapping,
                                                      original)

        # reorder field wrappers to match the ordering of languages
        # specified in settings

        # 1. find corresponding language for each mapping

        mappings_with_lang = {}
        mappings_without_lang = []
        for idx, mapping in self.field_mappings.items():
            if mapping['original']:
                mappings_with_lang[mapping['original'].language_id] = idx
            else:
                mappings_without_lang.append(idx)

        # 2. reorder the wrappers according to what you found in #1

        new_form_field_collection_wrappers = []
        ffcw = self.form_field_collection_wrappers
        for language_id in get_language_id_list():
            idx = mappings_with_lang.get(language_id, None)
            if idx is None:
                idx = mappings_without_lang.pop(0)
            new_form_field_collection_wrappers.append(ffcw[idx])
        self.form_field_collection_wrappers = new_form_field_collection_wrappers

    def template_name(self):
        return "admin/edit_inline_translations.html"

def translation_save_translated_fields(instance, **kwargs):
    """
    Save all the translations of instance in post_save signal handler.
    """
    if not hasattr(instance, '_translation_cache'):
        return
    instance.get_translation_qs().delete()
    for l_id, translation in instance._translation_cache.iteritems():
        # set the translation ID just in case the translation was
        # created while instance was not stored in the DB yet
        translation.master_id = instance.id
        translation.save()

def fill_translation_cache(instance):
    """
    Fill the translation cache using information received in the
    instance objects as extra fields.

    You can not do this in post_init because the extra fields are
    assigned by QuerySet.iterator after model initialization.
    """

    if hasattr(instance, '_translation_cache'):
        # do not refill the cache
        return

    instance._translation_cache = {}
    for language_id in get_language_id_list():
        # see if translation for language_id was in the query
        if hasattr(instance, get_translated_field_alias('id', language_id)):
            field_names = [f.attname for f in instance._meta.translation_model._meta.fields]

            # if so, create a translation object and put it in the cache
            field_data = {}
            for fname in field_names:
                field_data[fname] = getattr(instance,
                                            get_translated_field_alias(fname, language_id))

            translation = instance._meta.translation_model(**field_data)
            instance._translation_cache[language_id] = translation

    # In some situations an (existing in the DB) object is loaded
    # without using the normal QuerySet.  In such case fallback to
    # loading the translations using a separate query.

    # Unfortunately, this is indistinguishable from the situation when
    # an object does not have any translations.  Oh well, we'll have
    # to live with this for the time being.
    if len(instance._translation_cache.keys()) == 0:
        for translation in instance.get_translation_qs():
            instance._translation_cache[translation.language_id] = translation

class TranslatedFieldProxy(property):
    def __init__(self, field_name, alias, field, language_id=None):
        self.field_name = field_name
        self.field = field
        self.admin_order_field = alias
        self.language_id = language_id

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self

        return getattr(obj, 'get_' + self.field_name)(self.language_id)

    def __set__(self, obj, value):
        language_id = self.language_id

        return getattr(obj, 'set_' + self.field_name)(value, self.language_id)

    short_description = property(lambda self: self.field.short_description)

def getter_generator(field_name, short_description):
    """
    Generate get_'field name' method for field field_name.
    """
    def get_translation_field(self, language_id_or_code=None):
        try:
            return getattr(self.get_translation(language_id_or_code), field_name)
        except TranslationDoesNotExist:
            return "-translation-not-available-"
    get_translation_field.short_description = short_description
    return get_translation_field

def setter_generator(field_name):
    """
    Generate set_'field name' method for field field_name.
    """
    def set_translation_field(self, value, language_id_or_code=None):
        setattr(self.get_translation(language_id_or_code, True),
                field_name, value)
    set_translation_field.short_description = "set " + field_name
    return set_translation_field

def get_translation(self, language_id_or_code,
                    create_if_necessary=False):
    """
    Get a translation instance for the given language_id_or_code.

    If it does not exist, either create one or raise the
    TranslationDoesNotExist exception, depending on the
    create_if_necessary argument.
    """

    # fill the cache if necessary
    self.fill_translation_cache()

    language_id = get_language_id_from_id_or_code(language_id_or_code)
    if language_id is None:
        language_id = getattr(self, '_default_language', None)
    if language_id is None:
        language_id = get_default_language()

    if language_id not in self._translation_cache:
        if not create_if_necessary:
            raise TranslationDoesNotExist
        new_translation = self._meta.translation_model(master=self,
                                                       language_id=language_id)
        self._translation_cache[language_id] = new_translation
    return self._translation_cache.get(language_id, None)

def get_translation_qs(self):
    """
    Return the queryset with all translations of self.
    """
    return self._meta.translation_model.objects.filter(master=self)

class Translation:
    """
    A superclass for translatablemodel.Translation inner classes.
    """

    def contribute_to_class(cls, main_cls, name):
        """
        Handle the inner 'Translation' class.
        """

        # delay the creation of the *Translation until the master model is
        # fully created
        connect(cls.finish_multilingual_class, signal=signals.class_prepared,
                sender=main_cls, weak=False)

        # connect the post_save signal to a handler that saves translations
        connect(translation_save_translated_fields, signal=signals.post_save,
                sender=main_cls)
    contribute_to_class = classmethod(contribute_to_class)

    def create_translation_attrs(cls, main_cls):
        """
        Creates get_'field name'(language_id) and set_'field
        name'(language_id) methods for all the translation fields.
        Adds the 'field name' properties too.

        Returns the translated_fields hash used in field lookups, see
        multilingual.query.  It maps field names to (field,
        language_id) tuples.
        """
        translated_fields = {}

        for fname, field in cls.__dict__.items():
            if isinstance(field, models.fields.Field):
                translated_fields[fname] = (field, None)

                # add get_'fname' and set_'fname' methods to main_cls
                getter = getter_generator(fname, getattr(field, 'verbose_name', fname))
                setattr(main_cls, 'get_' + fname, getter)

                setter = setter_generator(fname)
                setattr(main_cls, 'set_' + fname, setter)

                # add the 'fname' proxy property that allows reads
                # from and writing to the appropriate translation
                setattr(main_cls, fname,
                        TranslatedFieldProxy(fname, fname, field))

                # create the 'fname'_'language_code' proxy properties
                for language_id in get_language_id_list():
                    language_code = get_language_code(language_id)
                    fname_lng = fname + '_' + language_code
                    translated_fields[fname_lng] = (field, language_id)
                    setattr(main_cls, fname_lng,
                            TranslatedFieldProxy(fname, fname_lng, field,
                                                 language_id))
        return translated_fields
    create_translation_attrs = classmethod(create_translation_attrs)

    def finish_multilingual_class(cls, *args, **kwargs):
        """
        Create a model with translations of a multilingual class.
        """

        main_cls = kwargs['sender']
        translation_model_name = main_cls.__name__ + "Translation"

        # create the model with all the translatable fields
        class TransMeta:
            ordering = ('language_id',)
            unique_together = (('master', 'language_id'),)
            db_table = main_cls._meta.db_table + 'translation'

        trans_attrs = cls.__dict__.copy()
        trans_attrs['Meta'] = TransMeta
        trans_attrs['language_id'] = models.IntegerField(blank=False, null=False, core=True,
                                                         choices=get_language_choices(),
                                                         db_index=True)
        trans_attrs['master'] = models.ForeignKey(main_cls, blank=False, null=False,
                                                  edit_inline=TransBoundRelatedObject,
                                                  num_in_admin=get_language_count(),
                                                  min_num_in_admin=get_language_count(),
                                                  num_extra_on_change=0)
        trans_attrs['__str__'] = lambda self: ("%s object, language_code=%s"
                                               % (translation_model_name,
                                                  get_language_code(self.language_id)))

        trans_model = ModelBase(translation_model_name, (models.Model,), trans_attrs)
        trans_model._meta.translated_fields = cls.create_translation_attrs(main_cls)

        main_cls._meta.translation_model = trans_model
        main_cls.get_translation = get_translation
        main_cls.get_translation_qs = get_translation_qs
        main_cls.fill_translation_cache = fill_translation_cache

    finish_multilingual_class = classmethod(finish_multilingual_class)

def install_translation_library():
    # modify ModelBase.__new__ so that it understands how to handle the
    # 'Translation' inner class

    if getattr(ModelBase, '_multilingual_installed', False):
        # don't install it twice
        return

    _old_new = ModelBase.__new__

    def multilingual_modelbase_new(cls, name, bases, attrs):
        if 'Translation' in attrs:
            if not issubclass(attrs['Translation'], Translation):
                raise ValueError, ("%s.Translation must be a subclass "
                                   + " of multilingual.Translation.") % (name,)

            # Make sure that if the class specifies objects then it is
            # a subclass of our Manager.
            #
            # Don't check other managers since someone might want to
            # have a non-multilingual manager, but assigning a
            # non-multilingual manager to objects would be a common
            # mistake.
            if ('objects' in attrs) and (not isinstance(attrs['objects'], manager.Manager)):
                raise ValueError, ("Model %s specifies translations, " +
                                   "so its 'objects' manager must be " +
                                   "a subclass of multilingual.Manager.") % (name,)

            # Change the default manager to multilingual.Manager.
            if not 'objects' in attrs:
                attrs['objects'] = manager.Manager()

            # Override the admin manager as well, or the admin views will
            # not see the translation data.
            if 'Admin' in attrs:
                attrs['Admin'].manager = attrs['objects']

        return _old_new(cls, name, bases, attrs)

    ModelBase.__new__ = staticmethod(multilingual_modelbase_new)
    ModelBase._multilingual_installed = True

# install the library
install_translation_library()
<|endoftext|>


<commit_before>#coding:utf-8
"""
functionモジュールのretrospective_resetting関数をテストする
"""
import unittest

from functions import retrospective_resetting
import class_definition

robot1 = class_definition.Robot(sensor=4,choice=3,particle_num=12,step=2,threshold=0.5,reduction=0.5)

robot1.particles.alpha = 0.1

robot1.particles.distribution=[0,1,2,0,1,2,0,1,2,0,1,2]
robot1.particles.weight = [1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]

robot1.event.sensor = [1,1,1,1]
robot1.event.action = 1
robot1.event.reward = 1.0
robot1.episodes.setEvent(robot1.event)

robot1.event.sensor = [0,1,1,1]
robot1.event.action = 1
robot1.event.reward = 1.0
robot1.episodes.setEvent(robot1.event)

robot1.event.sensor = [0,0,1,1]
robot1.event.action = 1
robot1.event.reward = 1.0
robot1.episodes.setEvent(robot1.event)

robot1.event.sensor = [0,0,0,1]
robot1.event.action = 1
robot1.event.reward = 1.0
robot1.episodes.setEvent(robot1.event)

sen_val = [1,1,1,1]

robot1.particles = retrospective_resetting(robot1.particles,robot1.episodes,robot1.resetting_threshold,robot1.resetting_step,sen_val)

print "distribution:",robot1.particles.distribution
print "weight:",robot1.particles.weight
print "sum_of_weight:",sum(robot1.particles.weight)
<commit_msg>.test_retrospective_resettingの修正<commit_after>#coding:utf-8
"""
functionモジュールのretrospective_resetting関数をテストする

retrospective_resetting(particles,episodes,resetting_threshold,resetting_step,sensor_val):
    処理：
        Particles.alphaがRobot.resetting_thresholdより小さく、
        かつ
        Robot.Episode.sets[] の数が充分存在している場合
        回想に基づくリセッティングを行う
    引数: particlesクラス,episodesクラス,resetting_threshold,resetting_step,sensor_val
    引数: resetting_threshold
    戻り値:
"""
from functions import retrospective_resetting
import pfoe

### ケース1 ###
robot1 = pfoe.Robot(sensor=4,choice=3,particle_num=12,step=3,threshold=0.5,reduction=0.5)
robot1.particles.alpha = 0.1
robot1.particles.distribution=[0,1,2,0,1,2,0,1,2,0,1,2]
robot1.particles.weight = [1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]

robot1.event.sensor = [1,1,1,1]
robot1.event.action = 1
robot1.event.reward = 1.0
robot1.episodes.setEvent(robot1.event)

robot1.event.sensor = [0,1,1,1]
robot1.event.action = 1
robot1.event.reward = 1.0
robot1.episodes.setEvent(robot1.event)

robot1.event.sensor = [0,0,1,1]
robot1.event.action = 1
robot1.event.reward = 1.0
robot1.episodes.setEvent(robot1.event)

robot1.event.sensor = [0,0,0,1]
robot1.event.action = 1
robot1.event.reward = 1.0
robot1.episodes.setEvent(robot1.event)

sen_val = [1,1,1,1]

robot1.particles = retrospective_resetting(robot1.particles,robot1.episodes,robot1.resetting_threshold,robot1.resetting_step,sen_val)

print "distribution:",robot1.particles.distribution
print "weight:",robot1.particles.weight
print "sum_of_weight:",sum(robot1.particles.weight)

### ケース2 ###
robot2 = pfoe.Robot(sensor=4,choice=3,particle_num=12,step=3,threshold=0.5,reduction=0.5)
robot2.particles.alpha = 0.1
robot2.particles.distribution=[0,1,2,0,1,2,0,1,2,0,1,2]
robot2.particles.weight = [1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]

robot2.particles = retrospective_resetting(robot2.particles,robot2.episodes,robot2.resetting_threshold,robot2.resetting_step,sen_val)

print "distribution:",robot2.particles.distribution
print "weight:",robot2.particles.weight
print "sum_of_weight:",sum(robot2.particles.weight)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import warnings

import django
from django.contrib.contenttypes.generic import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.core import exceptions
from django.db.models import fields
from django.utils.translation import ugettext_lazy as _

class FieldDefinitionTypeField(fields.related.ForeignKey):
    def __init__(self, *args, **kwargs):
        defaults = {'to': ContentType}
        defaults.update(kwargs)
        super(FieldDefinitionTypeField, self).__init__(*args, **defaults)

    def validate(self, value, model_instance):
        super(FieldDefinitionTypeField, self).validate(value, model_instance)
        if value is None:
            return
        if isinstance(value, int):
            try:
                value = ContentType.objects.get(id=value)
            except ContentType.DoesNotExist:
                msg = self.error_messages['invalid'] % {
                'model': self.rel.to._meta.verbose_name, 'pk': value}
                raise exceptions.ValidationError(msg)
        # Lazily import to avoid circular reference
        from mutant.models import FieldDefinition
        cls = value.model_class()
        if not issubclass(cls, FieldDefinition) or cls is FieldDefinition:
            msg = _('This field must be the ContentType of '
                    'an explicit FieldDefinition subclass.')
            raise exceptions.ValidationError(msg)

# TODO: Remove when support for Django 1.5 is dropped.
if django.VERSION < (1, 7):
    class ProxyAwareGenericForeignKey(GenericForeignKey):
        """
        Basically a GenericForeignKey that saves the actual ContentType of the
        object even if it's a proxy Model.
        """
        if django.VERSION >= (1, 6):
            def __init__(self, *args, **kwargs):
                warnings.warn(
                    '`ProxyAwareGenericForeignKey` is deprecated on Django >= '
                    '1.6. Use `GenericForeignKey` with the '
                    '`for_concrete_model=False` kwarg instead.',
                    DeprecationWarning, stacklevel=2
                )
                super(ProxyAwareGenericForeignKey, self).__init__(*args, **kwargs)

        def get_content_type(self, obj=None, **kwargs):
            if obj:
                return ContentType.objects.db_manager(obj._state.db.get_for_model(
                    obj.__class__, for_concrete_model=False)
                )
            else:
                return super(ProxyAwareGenericForeignKey, self).get_content_type(obj, **kwargs)
<commit_msg>Fixed ProxyAwareGenericForeignKey.get_content_type.<commit_after>from __future__ import unicode_literals

import warnings

import django
from django.contrib.contenttypes.generic import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.core import exceptions
from django.db.models import fields
from django.utils.translation import ugettext_lazy as _

class FieldDefinitionTypeField(fields.related.ForeignKey):
    def __init__(self, *args, **kwargs):
        defaults = {'to': ContentType}
        defaults.update(kwargs)
        super(FieldDefinitionTypeField, self).__init__(*args, **defaults)

    def validate(self, value, model_instance):
        super(FieldDefinitionTypeField, self).validate(value, model_instance)
        if value is None:
            return
        if isinstance(value, int):
            try:
                value = ContentType.objects.get(id=value)
            except ContentType.DoesNotExist:
                msg = self.error_messages['invalid'] % {
                'model': self.rel.to._meta.verbose_name, 'pk': value}
                raise exceptions.ValidationError(msg)
        # Lazily import to avoid circular reference
        from mutant.models import FieldDefinition
        cls = value.model_class()
        if not issubclass(cls, FieldDefinition) or cls is FieldDefinition:
            msg = _('This field must be the ContentType of '
                    'an explicit FieldDefinition subclass.')
            raise exceptions.ValidationError(msg)

# TODO: Remove when support for Django 1.5 is dropped.
if django.VERSION < (1, 7):
    class ProxyAwareGenericForeignKey(GenericForeignKey):
        """
        Basically a GenericForeignKey that saves the actual ContentType of the
        object even if it's a proxy Model.
        """
        if django.VERSION >= (1, 6):
            def __init__(self, *args, **kwargs):
                warnings.warn(
                    '`ProxyAwareGenericForeignKey` is deprecated on Django >= '
                    '1.6. Use `GenericForeignKey` with the '
                    '`for_concrete_model=False` kwarg instead.',
                    DeprecationWarning, stacklevel=2
                )
                super(ProxyAwareGenericForeignKey, self).__init__(*args, **kwargs)

        def get_content_type(self, obj=None, **kwargs):
            if obj:
                return ContentType.objects.db_manager(obj._state.db).get_for_model(
                    obj.__class__, for_concrete_model=False
                )
            else:
                return super(ProxyAwareGenericForeignKey, self).get_content_type(obj, **kwargs)
<|endoftext|>


<commit_before># Copyright 2017 Mycroft AI Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import json

from genericpath import exists, isfile

from mycroft.util.log import LOG

# The following lines are replaced during the release process.
# START_VERSION_BLOCK
CORE_VERSION_MAJOR = 0
CORE_VERSION_MINOR = 9
CORE_VERSION_BUILD = 13
# END_VERSION_BLOCK

CORE_VERSION_STR = (str(CORE_VERSION_MAJOR) + "." +
                    str(CORE_VERSION_MINOR) + "." +
                    str(CORE_VERSION_BUILD))

class VersionManager(object):
    __location = "/opt/mycroft/version.json"

    @staticmethod
    def get():
        if (exists(VersionManager.__location) and
                isfile(VersionManager.__location)):
            try:
                with open(VersionManager.__location) as f:
                    return json.load(f)
            except:
                LOG.error("Failed to load version from '%s'"
                          % VersionManager.__location)
        return {"coreVersion": None, "enclosureVersion": None}

def check_version(version_string):
    """
        Check if current version is equal or higher than the
        version string provided to the function

        Args:
            version_string (string): version string ('Major.Minor.Build')
    """
    major, minor, build = version_string.split('.')
    major = int(major)
    minor = int(minor)
    build = int(build)

    if CORE_VERSION_MAJOR > major:
        return True
    elif CORE_VERSION_MAJOR == major and CORE_VERSION_MINOR > minor:
        return True
    elif major == CORE_VERSION_MAJOR and minor == CORE_VERSION_MINOR and \
            CORE_VERSION_BUILD >= build:
        return True
    else:
        return False
<commit_msg>Version bump from 0.9.13 to 0.9.14<commit_after># Copyright 2017 Mycroft AI Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import json

from genericpath import exists, isfile

from mycroft.util.log import LOG

# The following lines are replaced during the release process.
# START_VERSION_BLOCK
CORE_VERSION_MAJOR = 0
CORE_VERSION_MINOR = 9
CORE_VERSION_BUILD = 14
# END_VERSION_BLOCK

CORE_VERSION_STR = (str(CORE_VERSION_MAJOR) + "." +
                    str(CORE_VERSION_MINOR) + "." +
                    str(CORE_VERSION_BUILD))

class VersionManager(object):
    __location = "/opt/mycroft/version.json"

    @staticmethod
    def get():
        if (exists(VersionManager.__location) and
                isfile(VersionManager.__location)):
            try:
                with open(VersionManager.__location) as f:
                    return json.load(f)
            except:
                LOG.error("Failed to load version from '%s'"
                          % VersionManager.__location)
        return {"coreVersion": None, "enclosureVersion": None}

def check_version(version_string):
    """
        Check if current version is equal or higher than the
        version string provided to the function

        Args:
            version_string (string): version string ('Major.Minor.Build')
    """
    major, minor, build = version_string.split('.')
    major = int(major)
    minor = int(minor)
    build = int(build)

    if CORE_VERSION_MAJOR > major:
        return True
    elif CORE_VERSION_MAJOR == major and CORE_VERSION_MINOR > minor:
        return True
    elif major == CORE_VERSION_MAJOR and minor == CORE_VERSION_MINOR and \
            CORE_VERSION_BUILD >= build:
        return True
    else:
        return False
<|endoftext|>


<commit_before>from django.contrib import admin
from restaurants.models import Restaurant, Food

class RestaurantAdmin(admin.ModelAdmin):
    list_display = ('name', 'phone_number', 'address')
    search_fields = ('name',)

class FoodAdmin(admin.ModelAdmin):
    list_display = ('name', 'restaurant', 'price')
    list_filter = ('is_spicy',)
    ordering = ('-price',)

admin.site.register(Restaurant, RestaurantAdmin)
admin.site.register(Food, FoodAdmin)
<commit_msg>Module09 - 使用ModelAdmin中的fields限制修改欄位<commit_after>from django.contrib import admin
from restaurants.models import Restaurant, Food

class RestaurantAdmin(admin.ModelAdmin):
    list_display = ('name', 'phone_number', 'address')
    search_fields = ('name',)

class FoodAdmin(admin.ModelAdmin):
    list_display = ('name', 'restaurant', 'price')
    list_filter = ('is_spicy',)
    ordering = ('-price',)
    fields = ('price', 'restaurant')

admin.site.register(Restaurant, RestaurantAdmin)
admin.site.register(Food, FoodAdmin)
<|endoftext|>


<commit_before>from django import forms
from django import forms

from dal import autocomplete
from . import models

class BigSearchForm(forms.Form):
	query = forms.CharField(
		widget=forms.TextInput(attrs={'size': '32', 'class': 'query'})
	)

class NewsForm(forms.ModelForm):
    class Meta:
        model = models.NewsObject

        # exclude translated fields
        exclude = ['title', 'annotation', 'annotation_source_1', 'annotation_source_2']

        widgets = {
            'source_1': autocomplete.ModelSelect2(url='source:source_public_autocomplete'),
            'source_2': autocomplete.ModelSelect2(url='source:source_public_autocomplete')
        }

<commit_msg>hide active field from news forms<commit_after>from django import forms
from django import forms

from dal import autocomplete
from . import models

class BigSearchForm(forms.Form):
	query = forms.CharField(
		widget=forms.TextInput(attrs={'size': '32', 'class': 'query'})
	)

class NewsForm(forms.ModelForm):
    class Meta:
        model = models.NewsObject

        # exclude translated fields
        exclude = ['active', 'title', 'annotation', 'annotation_source_1', 'annotation_source_2']

        widgets = {
            'source_1': autocomplete.ModelSelect2(url='source:source_public_autocomplete'),
            'source_2': autocomplete.ModelSelect2(url='source:source_public_autocomplete')
        }

<|endoftext|>


<commit_before>import socket
import sys
import time
import math
import sched

#Constants#
R_0 = 100
a = 3.9083 * (10**(-3))
b = -5.775 * (10**(-7))

#Creating a function to time sending of code #04
updateTimer = sched.scheduler(time.time, time.sleep)

# Create a TCP/IP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

# Connect the socket to the port where the server is listening
ip = "37.26.220.85"
port = 4002

print("Connecting...")
sock.connect((ip, port))
print("Connected to", ip, "with port", port)

def space():
    print()
    print()

def main(timer):
    try:

        # Send data
        message = "#04" + str(chr(13))
        print('Sending "%s"' % message)
        sock.send(message.encode('utf-8'))

        # Look for the response
        amount_received = 0
        amount_expected = 9

        dChar = []
        while amount_received < amount_expected:
            data = sock.recv(1)
            amount_received += len(data)
            dataFull = str(data,'utf-8')

            dVal = dataFull
            dChar.append(dVal)

        dataRes = float(dChar[2] + dChar[3] + dChar[4] + dChar[5] + dChar[6] + dChar[7])
        print(dataRes)

        R = float(dataRes)
        t = ((((-R_0)*a) + math.sqrt((R_0 ** 2) * (a ** 2) - 4 * R_0 * b * (R_0 - R))) / (2 * R_0 * b))
        if(t == -0.0):
            print(int(0.0))
        else:
            currentTemp = ("%.2f" % t)
            print(currentTemp)
        updateTimer.enter(5, 1, main, (timer,))

    except:
        print('There was an error during the sending and receiving of data')
        print('Closing Socket...')
        sock.close()
        print('Socket closed!')
        main()

updateTimer.enter(5, 1, main, (updateTimer,))
updateTimer.run()
<commit_msg>Made systemshutdown in case of error in connection with server<commit_after>import socket
import sys
import time
import math
import sched

#Constants#
R_0 = 100
a = 3.9083 * (10**(-3))
b = -5.775 * (10**(-7))

#Creating a function to time sending of code #04
updateTimer = sched.scheduler(time.time, time.sleep)

# Create a TCP/IP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

# Connect the socket to the port where the server is listening
ip = "37.26.220.85"
port = 4002

def space():
    print()
    print()

def systemShutdown():
    sys.exit()

def main(timer):
    try:

        # Send data
        message = "#04" + str(chr(13))
        print('Sending "%s"' % message)
        sock.send(message.encode('utf-8'))

        # Look for the response
        amount_received = 0
        amount_expected = 9

        dChar = []
        while amount_received < amount_expected:
            data = sock.recv(1)
            amount_received += len(data)
            dataFull = str(data,'utf-8')

            dVal = dataFull
            dChar.append(dVal)

        dataRes = float(dChar[2] + dChar[3] + dChar[4] + dChar[5] + dChar[6] + dChar[7])
        print(dataRes)

        R = float(dataRes)
        t = ((((-R_0)*a) + math.sqrt((R_0 ** 2) * (a ** 2) - 4 * R_0 * b * (R_0 - R))) / (2 * R_0 * b))
        if(t == -0.0):
            print(int(0.0))
        else:
            currentTemp = ("%.2f" % t)
            print(currentTemp)
        updateTimer.enter(5, 1, main, (timer,))

    except:
        print('There was an error during the sending and receiving of data')
        print('Closing Socket...')
        sock.close()
        print('Socket closed!')
        main()
#Making sure that the connection is made before sending data
try:
    print("Connecting...")
    sock.connect((ip, port))
    print("Connected to", ip, "with port", port)
except ConnectionRefusedError:
    print()
    print()
    print('Connection failed')
    print('Could not connect to IP: ' + str(ip) + ' port: ' + str(port))
    time.sleep(1)
    systemShutdown()

updateTimer.enter(5, 1, main, (updateTimer,))
updateTimer.run()
<|endoftext|>


<commit_before># Imported modules
import socket
import sys
import pymysql
import time
import math
import datetime
import os
import threading

# Setting an array for holding all 12 measurements for one minute
minuteMeasurements = []

# Constants
R_0 = 100
a = 3.9083 * (10**(-3))
b = -5.775 * (10**(-7))

# Create a TCP/IP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

# Connect the socket to the port where the server is listening
ip = "37.26.220.85"
port = 4002

# Connect to the database
db = pymysql.connect("193.93.253.25","codespo","45Pvilfd","codespo_metinstitutt")

# Prepare a cursor object using cursor() method for the database
cursor = db.cursor()

# Function for shutting down system if ConnectionError
def systemShutdown():
    sys.exit()

# Restarts the entire script if restarting main function fails
def systemReboot():
    python = sys.executable
    os.execl(python, python, * sys.argv)

# Main function for sending and receiving data
def main():

    # Setting an alternative timer to run code every 5 seconds
    threading.Timer(5.0, main).start()

    try:

        # Send data
        message = "#04" + str(chr(13))
        #print('Sending "%s"' % message)
        sock.send(message.encode('utf-8'))

        # Look for the response
        amount_received = 0
        amount_expected = 9

        # Splitting characters from received data so only numbers remain
        dChar = []
        while amount_received < amount_expected:
            data = sock.recv(1)
            amount_received += len(data)
            dataFull = str(data,'utf-8')

            dVal = dataFull
            dChar.append(dVal)

        # Collecting specific integers from reveived data
        dataRes = float(dChar[2] + dChar[3] + dChar[4] + dChar[5] + dChar[6] + dChar[7])

        # Formula for finding temperature using a measurement of ohm
        R = float(dataRes)
        t = ((((-R_0)*a) + math.sqrt((R_0 ** 2) * (a ** 2) - 4 * R_0 * b * (R_0 - R))) / (2 * R_0 * b))
        if(t == -0.0):

            # Debugging given value for temperatur when provided ohm is 100.00
            currentTemp = 0.0
        else:

            # Rounding off to a two decimal number
            currentTemp = ("%.2f" % t)

        # Sending all measurements to array
        minuteMeasurements.append(float(currentTemp))

        # Checking if length of array is 12 or more
        if(len(minuteMeasurements) >= 12):

            # Adding timestamp to average temperature per minute|
            timestamp = '{:%d.%m.%Y  %H:%M:%S}'.format(datetime.datetime.now())

            # Finding temp average per minute
            avrgMeasurement = sum(minuteMeasurements) / len(minuteMeasurements)

            # Rounding off average temperature to two decimal
            avrgTemp = float("%.2f" % avrgMeasurement)

            # Prepare SQL query to INSERT a record into the database.
            sql = "INSERT INTO data_main_min(TIMESTAMP, OHM, CELSIUS, UPTIME, DOWNTIME) \
                     VALUES ('%s', '%f', '%f', '%i', '%i')" % \
                     (timestamp, R, avrgTemp, 1, 0)
            try:
               # Execute the SQL command
               cursor.execute(sql)
               # Commit your changes in the database
               db.commit()
               print("Data is sent to database")
            except:
               # Rollback in case there is any error
               db.rollback()
               print("Database Error: Not sending data to database")

            # Print result to console
            print(timestamp + "   " + str(avrgTemp))

            # Clearing array for measurements for next minute
            minuteMeasurements.clear()
        else:

            # If array is not longer than or equal to 12: ignore this is/else statement
            pass

        # Wait 5 seconds before restarting function
        time.sleep(5)

    # In case of error while sending or receiving data, try closing socket an rebooting
    except:

        # Closing the socket
        sock.close()
        try:

            # Trying to restart the main function without shutting down
            main()
        except:

            # Rebooting program when program fails
            systemReboot()

#------------------------Start of program------------------------#
print("Connecting...")

# Making sure that the connection is made before sending data
try:

    # Connecting to the server via socket
    sock.connect((ip, port))

    # Setting the time of connetion to the server if successful
    connectionTime = '{:%H:%M:%S}'.format(datetime.datetime.now())

    print("Connected to", ip, "with port", port, "@", connectionTime)
    print("---------------------------------------------------")
    main()

# In case of Connection Error, shutdown program
except ConnectionRefusedError:
    print()
    print()
    print('Connection failed')
    print('Could not connect to IP: ' + str(ip) + ' port: ' + str(port))

    # Shutting down program if connection to server fails
    systemShutdown()
<commit_msg>Setting timer to avoid WinError 10060 by forcing script to reboot every 3 hours<commit_after># Imported modules
import socket
import sys
import pymysql
import time
import math
import datetime
import os
import threading

# Setting an array for holding all 12 measurements for one minute
minuteMeasurements = []

# Constants
R_0 = 100
a = 3.9083 * (10**(-3))
b = -5.775 * (10**(-7))

# Create a TCP/IP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

# Connect the socket to the port where the server is listening
ip = "37.26.220.85"
port = 4002

# Connect to the database
db = pymysql.connect("193.93.253.25","codespo","45Pvilfd","codespo_metinstitutt")

# Prepare a cursor object using cursor() method for the database
cursor = db.cursor()

# Function for shutting down system if ConnectionError
def systemShutdown():
    sys.exit()

# Restarts the entire script if restarting main function fails
def systemReboot():
    python = sys.executable
    os.execl(python, python, * sys.argv)

# Main function for sending and receiving data
def main():

    # Setting an alternative timer to run code every 5 seconds
    threading.Timer(5.0, main).start()

    # Setting timer to force script reboot every 3 hours to prevent WinError 10060
    threading.Timer(10800.0, systemReboot).start()

    try:

        # Send data
        message = "#04" + str(chr(13))
        #print('Sending "%s"' % message)
        sock.send(message.encode('utf-8'))

        # Look for the response
        amount_received = 0
        amount_expected = 9

        # Splitting characters from received data so only numbers remain
        dChar = []
        while amount_received < amount_expected:
            data = sock.recv(1)
            amount_received += len(data)
            dataFull = str(data,'utf-8')

            dVal = dataFull
            dChar.append(dVal)

        # Collecting specific integers from reveived data
        dataRes = float(dChar[2] + dChar[3] + dChar[4] + dChar[5] + dChar[6] + dChar[7])

        # Formula for finding temperature using a measurement of ohm
        R = float(dataRes)
        t = ((((-R_0)*a) + math.sqrt((R_0 ** 2) * (a ** 2) - 4 * R_0 * b * (R_0 - R))) / (2 * R_0 * b))
        if(t == -0.0):

            # Debugging given value for temperatur when provided ohm is 100.00
            currentTemp = 0.0
        else:

            # Rounding off to a two decimal number
            currentTemp = ("%.2f" % t)

        # Sending all measurements to array
        minuteMeasurements.append(float(currentTemp))

        # Checking if length of array is 12 or more
        if(len(minuteMeasurements) >= 12):

            # Adding timestamp to average temperature per minute|
            timestamp = '{:%d.%m.%Y  %H:%M:%S}'.format(datetime.datetime.now())

            # Finding temp average per minute
            avrgMeasurement = sum(minuteMeasurements) / len(minuteMeasurements)

            # Rounding off average temperature to two decimal
            avrgTemp = float("%.2f" % avrgMeasurement)

            # Prepare SQL query to INSERT a record into the database.
            sql = "INSERT INTO data_main_min(TIMESTAMP, OHM, CELSIUS, UPTIME, DOWNTIME) \
                     VALUES ('%s', '%f', '%f', '%i', '%i')" % \
                     (timestamp, R, avrgTemp, 1, 0)
            try:
               # Execute the SQL command
               cursor.execute(sql)
               # Commit your changes in the database
               db.commit()
               print("Data is sent to database")
            except:
               # Rollback in case there is any error
               db.rollback()
               print("Database Error: Not sending data to database")

            # Print result to console
            print(timestamp + "   " + str(avrgTemp))

            # Clearing array for measurements for next minute
            minuteMeasurements.clear()
        else:

            # If array is not longer than or equal to 12: ignore this is/else statement
            pass

        # Wait 5 seconds before restarting function
        time.sleep(5)

    # In case of error while sending or receiving data, try closing socket an rebooting
    except:

        # Closing the socket
        sock.close()
        try:

            # Trying to restart the main function without shutting down
            main()
        except:

            # Rebooting program when program fails
            systemReboot()

#------------------------Start of program------------------------#
print("Connecting...")

# Making sure that the connection is made before sending data
try:

    # Connecting to the server via socket
    sock.connect((ip, port))

    # Setting the time of connetion to the server if successful
    connectionTime = '{:%H:%M:%S}'.format(datetime.datetime.now())

    print("Connected to", ip, "with port", port, "@", connectionTime)
    print("---------------------------------------------------")
    main()

# In case of Connection Error, shutdown program
except ConnectionRefusedError:
    print()
    print()
    print('Connection failed')
    print('Could not connect to IP: ' + str(ip) + ' port: ' + str(port))

    # Shutting down program if connection to server fails
    systemReboot()
<|endoftext|>


<commit_before>import socket
import select
from struct import *
import hashlib
import base64
import json
import gc
import Client

class server:

	sock = None
	clients = {}
	deadClients = []

	'''
		Checks and handles sockets connecting
	'''
	def __init__(self):
		self.startServer()
		global server
		cycle = 0

		while True:
			print '[Debug] Heartbeat #' + str(cycle) #for debugging purposes, counts each cycle
			gc.collect()

			# Handle the dead sockets
			if not int(len(self.deadClients)) == 0:
				for (i, u) in enumerate(self.deadClients):
					if self.deadClients[i] in self.clients:
						self.clients[self.deadClients[i]].died()
					del self.deadClients[i]

			# Add all the clients to the socket list
			_select = [self.sock]
			for i in self.clients:
				_select.append(i)
			read, _, error = select.select(_select, [], _select, None)
			if self.sock in error:
				self.startServer()
				continue

			for _socket in read:
				# New connection
				if _socket == self.sock:
					_client, _addr = self.sock.accept()
					self.clients[_client] = Client.Client(_client, self)
				# Handle all the users
				else:
					user = self.clients[_socket]

					try:
						data = _socket.recv(1024)
						#check if user has completed the websocket handshake
						if not user.handshake_completed:
							###websocket handshake
							headers = data
							headers = self.getSockKey(headers[headers.index(headers[headers.index('Sec-WebSocket-Key: ')+len('Sec-WebSocket-Key: '):]):].split("\r\n")[0])
							headers = "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: " + headers + "\r\n\r\n"
							user.usersock.send(headers)
							user.handshake_completed = True
					except: pass

					if not data:
						try: self.clients[_socket].died()
						except: continue
					else:
						try:
							parsed = json.loads(str(self.parseMessage(bytearray(data))))
							print '[Debug] '+ str(parsed)
							self.broadcast(str(self.parseMessage(bytearray(data))))
						except Exception, e:
							pass
			cycle +=1 #debugging purposes
	'''
		Initialize all the socket stuff here
	'''
	def startServer(self):
		self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
		self.sock.bind(('0.0.0.0', 1337))

		self.sock.listen(True)

	'''
		Used to encode the websocket key to complete the handshake
	'''
	def getSockKey(self,key):
		MAGIC = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
		m = hashlib.sha1(key+MAGIC)
		return base64.b64encode(m.digest())

	'''
		Broadcasts a message to every single client
	'''
	def broadcast(self, msg):
		for i in self.clients:
			for j in self.pack_message(msg):
				if isinstance(j, (int, long)):
					i.send(chr(j))
				else:
					i.send(j)

	'''
		Encode any messages to be sent to the client complying with websocket standards
	'''
	def pack_message(self,msg):
		payload = []
		payload.append(129)
		if(len(msg) < 126):
			payload.append(len(msg))
		elif len(msg) >= 126:
			payload.append((126))
			payload.append(pack("!H",len(msg)))
		payload.append(msg)
		return payload

	'''
		Unmasks any messages sent from the client to the server to be read
	'''
	def parseMessage(self,msg):
		startIndex = 2
		if (msg[1]&127) == 126:
			startIndex = 4
		elif (msg[1]&127) == 127:
			startIndex = 10
		ns = msg[startIndex:startIndex+4]
		a = 0
		decoded = bytearray()
		for i in range(startIndex+4,len(msg)):
			msg[i]
			ns[a%4]
			decoded.append(msg[i]^ns[a%4])
			a=a+1
		return decoded
<commit_msg>Update Server.py<commit_after>import socket
import select
from struct import *
import hashlib
import base64
import json
import gc
import Client

class server:

	sock = None
	clients = {}
	deadClients = []

	'''
		Checks and handles sockets connecting
	'''
	def __init__(self):
		self.startServer()
		global server
		cycle = 0

		while True:
			print '[Debug] Heartbeat #' + str(cycle) #for debugging purposes, counts each cycle
			gc.collect()

			# Handle the dead sockets
			if not int(len(self.deadClients)) == 0:
				for (i, u) in enumerate(self.deadClients):
					if self.deadClients[i] in self.clients:
						self.clients[self.deadClients[i]].died()
					del self.deadClients[i]

			# Add all the clients to the socket list
			_select = [self.sock]
			for i in self.clients:
				_select.append(i)
			read, _, error = select.select(_select, [], _select, None)
			if self.sock in error:
				self.startServer()
				continue

			for _socket in read:
				# New connection
				if _socket == self.sock:
					_client, _addr = self.sock.accept()
					self.clients[_client] = Client.Client(_client, self)
				# Handle all the users
				else:
					user = self.clients[_socket]

					try:
						data = _socket.recv(1024)
						#check if user has completed the websocket handshake
						if not user.handshake_completed:
							###websocket handshake
							headers = data
							headers = self.getSockKey(headers[headers.index(headers[headers.index('Sec-WebSocket-Key: ')+len('Sec-WebSocket-Key: '):]):].split("\r\n")[0])
							headers = "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: " + headers + "\r\n\r\n"
							user.usersock.send(headers)
							user.handshake_completed = True
					except: pass

					if not data:
						try: self.clients[_socket].died()
						except: continue
					else:
						try:
							parsed = json.loads(str(self.parseMessage(bytearray(data))))
							print '[Debug] '+ str(parsed)

							# Get the name from the user
							if parsed['type'] == "join":
								user.name = parsed['name']
								print user.name

								# No idea how to broadcast these types of messages from server -> client :(
								# TODO: Notify the user who all is currently in the chat room (Probably get the client to append them to the userlist)
								self.broadcast("{'message': 'Welcome to the chat server!', 'type': 'system', 'name': 'System', 'avi': 'profile.png'}")
							self.broadcast(str(self.parseMessage(bytearray(data))))
						except ValueError:
							# Typically when the user disconnects, python throws up a ValueError saying that No JSON could be decoded, so that is an indication that the user has left
							# TODO: Server notifies everyone that the user has left the chat room
							print "User has decided to disconnect, notify everyone that he left"
						except Exception, e:
							pass
			cycle +=1 #debugging purposes
	'''
		Initialize all the socket stuff here
	'''
	def startServer(self):
		self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
		self.sock.bind(('0.0.0.0', 1337))

		self.sock.listen(True)

	'''
		Used to encode the websocket key to complete the handshake
	'''
	def getSockKey(self,key):
		MAGIC = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
		m = hashlib.sha1(key+MAGIC)
		return base64.b64encode(m.digest())

	'''
		Broadcasts a message to every single client
	'''
	def broadcast(self, msg):
		for i in self.clients:
			for j in self.pack_message(msg):
				if isinstance(j, (int, long)):
					i.send(chr(j))
				else:
					i.send(j)

	'''
		Encode any messages to be sent to the client complying with websocket standards
	'''
	def pack_message(self,msg):
		payload = []
		payload.append(129)
		if(len(msg) < 126):
			payload.append(len(msg))
		elif len(msg) >= 126:
			payload.append((126))
			payload.append(pack("!H",len(msg)))
		payload.append(msg)
		return payload

	'''
		Unmasks any messages sent from the client to the server to be read
	'''
	def parseMessage(self,msg):
		startIndex = 2
		if (msg[1]&127) == 126:
			startIndex = 4
		elif (msg[1]&127) == 127:
			startIndex = 10
		ns = msg[startIndex:startIndex+4]
		a = 0
		decoded = bytearray()
		for i in range(startIndex+4,len(msg)):
			msg[i]
			ns[a%4]
			decoded.append(msg[i]^ns[a%4])
			a=a+1
		return decoded
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import datetime

from django.contrib import messages
from django.core.urlresolvers import reverse
from django.shortcuts import get_object_or_404
from digest.models import Issue, Item
from digg_paginator import DiggPaginator

from django.db.models import Q
from django.views.generic import TemplateView, ListView, DetailView, FormView
from django.conf import settings

from .forms import AddNewsForm
from frontend.models import EditorMaterial

class Sitemap(TemplateView):
    content_type = 'text/xml'
    template_name = 'sitemap.html'

    def get_context_data(self, **kwargs):
        ctx = super(Sitemap, self).get_context_data(**kwargs)
        items = [
            {
                'loc': '',
                'changefreq': 'weekly',
            },
            {
                'loc': reverse('frontend:issues'),
                'changefreq': 'weekly',
            },
            {
                'loc': reverse('frontend:feed'),
                'changefreq': 'daily',
            },
        ]

        for issue in Issue.objects.filter(status='active'):
            items.append({
                'loc': issue.link,
                'changefreq': 'weekly',
            })

        ctx.update({
            'records': items,
            'domain': 'http://%s' % settings.BASE_DOMAIN
        })
        return ctx

class Index(TemplateView):
    '''
    Главная страница
    '''
    template_name = 'index.html'
    model = Issue
    context_object_name = 'index'

    def get_context_data(self, **kwargs):
        context = super(Index, self).get_context_data(**kwargs)
        issue = False
        try:
            issue = self.model.objects.filter(status='active').latest('published_at')
        except Issue.DoesNotExist:
            pass

        items = []
        if issue:
            qs = issue.item_set.filter(status='active')
            items = qs.order_by('-section__priority', '-priority')

        context.update({
            'issue': issue,
            'items': items,
            'active_menu_item': 'home'
        })
        return context

class IssuesList(ListView):
    '''
    Список выпусков
    '''
    template_name = 'issues_list.html'
    queryset = Issue.objects.filter(status='active').order_by('-published_at')
    context_object_name = 'items'
    paginate_by = 9
    paginator_class = DiggPaginator

    def get_context_data(self, **kwargs):
        context = super(IssuesList, self).get_context_data(**kwargs)
        context['active_menu_item'] = 'issues_list'
        return context

class IssueView(DetailView):
    '''
    Просмотр выпуска
    '''
    template_name = 'issue.html'
    model = Issue

    def get_context_data(self, **kwargs):
        context = super(IssueView, self).get_context_data(**kwargs)

        items = self.object.item_set.filter(status='active').order_by('-section__priority', '-priority')

        context.update({
            'items': items,
            'active_menu_item': 'issue_view'
        })

        return context

class NewsList(ListView):
    '''
    Лента новостей
    '''
    template_name = 'news_list.html'
    context_object_name = 'items'
    paginate_by = 20
    paginator_class = DiggPaginator
    model = Item

    def get_queryset(self):
        items = super(NewsList, self).get_queryset().filter(status='active')
        lang = self.request.GET.get('lang')
        if lang in ['ru', 'en']:
            items = items.filter(language=lang)

        search = self.request.GET.get('q')
        if search:
            filters = Q(title__icontains=search) | Q(description__icontains=search)
            items = items.filter(filters)

        section = self.request.GET.get('section')
        if section:
            items = items.filter(section__pk=section)

        items = items.prefetch_related('issue', 'section')
        items = items.order_by('-created_at', '-related_to_date')
        return items

    def get_context_data(self, **kwargs):
        context = super(NewsList, self).get_context_data(**kwargs)
        context['active_menu_item'] = 'feed'
        return context

class AddNews(FormView):
    template_name = "add_news.html"
    form_class = AddNewsForm

    def get_success_url(self):
        title = self.request.POST['title'].strip() or 'Без заголовка'
        link = self.request.POST['link']
        description = self.request.POST['description']
        section = self.request.POST['section']
        Item.objects.create(title=title,
                            link=link,
                            description=description,
                            status='pending',
                            related_to_date=datetime.datetime.now(),
                            section_id=section
                            )
        messages.info(
            self.request, u'Ваша ссылка успешно добавлена на рассмотрение'
        )
        return '/'

class ViewEditorMaterial(TemplateView):
    template_name = 'editor_material_view.html'

    def get_context_data(self, **kwargs):
        section = kwargs.get('section', 'landing')
        slug = kwargs.get('slug')

        material = get_object_or_404(
            EditorMaterial,
            slug=slug,
            section=section,
            status='active'
        )

        return {
            'material': material
        }

class ItemView(DetailView):
    """
    Просмотр отдельной новости
    """
    template_name = 'news_item.html'
    context_object_name = 'item'
    model = Item
<commit_msg>Валидация id раздела. Ранее некорректная строка приводила к пятисотой.<commit_after># -*- coding: utf-8 -*-
import datetime

from django.contrib import messages
from django.core.urlresolvers import reverse
from django.shortcuts import get_object_or_404
from digest.models import Issue, Item
from digg_paginator import DiggPaginator

from django.db.models import Q
from django.views.generic import TemplateView, ListView, DetailView, FormView
from django.conf import settings

from .forms import AddNewsForm
from frontend.models import EditorMaterial

class Sitemap(TemplateView):
    content_type = 'text/xml'
    template_name = 'sitemap.html'

    def get_context_data(self, **kwargs):
        ctx = super(Sitemap, self).get_context_data(**kwargs)
        items = [
            {
                'loc': '',
                'changefreq': 'weekly',
            },
            {
                'loc': reverse('frontend:issues'),
                'changefreq': 'weekly',
            },
            {
                'loc': reverse('frontend:feed'),
                'changefreq': 'daily',
            },
        ]

        for issue in Issue.objects.filter(status='active'):
            items.append({
                'loc': issue.link,
                'changefreq': 'weekly',
            })

        ctx.update({
            'records': items,
            'domain': 'http://%s' % settings.BASE_DOMAIN
        })
        return ctx

class Index(TemplateView):
    '''
    Главная страница
    '''
    template_name = 'index.html'
    model = Issue
    context_object_name = 'index'

    def get_context_data(self, **kwargs):
        context = super(Index, self).get_context_data(**kwargs)
        issue = False
        try:
            issue = self.model.objects.filter(status='active').latest('published_at')
        except Issue.DoesNotExist:
            pass

        items = []
        if issue:
            qs = issue.item_set.filter(status='active')
            items = qs.order_by('-section__priority', '-priority')

        context.update({
            'issue': issue,
            'items': items,
            'active_menu_item': 'home'
        })
        return context

class IssuesList(ListView):
    '''
    Список выпусков
    '''
    template_name = 'issues_list.html'
    queryset = Issue.objects.filter(status='active').order_by('-published_at')
    context_object_name = 'items'
    paginate_by = 9
    paginator_class = DiggPaginator

    def get_context_data(self, **kwargs):
        context = super(IssuesList, self).get_context_data(**kwargs)
        context['active_menu_item'] = 'issues_list'
        return context

class IssueView(DetailView):
    '''
    Просмотр выпуска
    '''
    template_name = 'issue.html'
    model = Issue

    def get_context_data(self, **kwargs):
        context = super(IssueView, self).get_context_data(**kwargs)

        items = self.object.item_set.filter(status='active').order_by('-section__priority', '-priority')

        context.update({
            'items': items,
            'active_menu_item': 'issue_view'
        })

        return context

class NewsList(ListView):
    '''
    Лента новостей
    '''
    template_name = 'news_list.html'
    context_object_name = 'items'
    paginate_by = 20
    paginator_class = DiggPaginator
    model = Item

    def get_queryset(self):
        items = super(NewsList, self).get_queryset().filter(status='active')
        lang = self.request.GET.get('lang')
        if lang in ['ru', 'en']:
            items = items.filter(language=lang)

        search = self.request.GET.get('q')
        if search:
            filters = Q(title__icontains=search) | Q(description__icontains=search)
            items = items.filter(filters)

        section = self.request.GET.get('section', '')
        if section.isdigit():
            items = items.filter(section__pk=section)

        items = items.prefetch_related('issue', 'section')
        items = items.order_by('-created_at', '-related_to_date')
        return items

    def get_context_data(self, **kwargs):
        context = super(NewsList, self).get_context_data(**kwargs)
        context['active_menu_item'] = 'feed'
        return context

class AddNews(FormView):
    template_name = "add_news.html"
    form_class = AddNewsForm

    def get_success_url(self):
        title = self.request.POST['title'].strip() or 'Без заголовка'
        link = self.request.POST['link']
        description = self.request.POST['description']
        section = self.request.POST['section']
        Item.objects.create(title=title,
                            link=link,
                            description=description,
                            status='pending',
                            related_to_date=datetime.datetime.now(),
                            section_id=section
                            )
        messages.info(
            self.request, u'Ваша ссылка успешно добавлена на рассмотрение'
        )
        return '/'

class ViewEditorMaterial(TemplateView):
    template_name = 'editor_material_view.html'

    def get_context_data(self, **kwargs):
        section = kwargs.get('section', 'landing')
        slug = kwargs.get('slug')

        material = get_object_or_404(
            EditorMaterial,
            slug=slug,
            section=section,
            status='active'
        )

        return {
            'material': material
        }

class ItemView(DetailView):
    """
    Просмотр отдельной новости
    """
    template_name = 'news_item.html'
    context_object_name = 'item'
    model = Item
<|endoftext|>


<commit_before>import sys
from os import walk, path
from fnmatch import filter as fnfilter
from csv import reader
from collections import OrderedDict
from copy import deepcopy

sys.path.append("../")
from Model.Sample import Sample
from Model.SequenceFile import SequenceFile
from Exceptions.SampleSheetError import SampleSheetError
from Exceptions.SequenceFileError import SequenceFileError

def parse_metadata(sample_sheet_file):
    """
    Parse all lines under [Header], [Reads] and [Settings] in .csv file
    Lines under [Reads] are stored in a list with key name "readLengths"
    All other key names are translated according to the
        metadata_key_translation_dict

    arguments:
            sample_sheet_file -- path to SampleSheet.csv

    returns a dictionary containing the parsed key:pair values from .csv file
    """

    metadata_dict = {}
    metadata_dict["readLengths"] = []

    csv_reader = get_csv_reader(sample_sheet_file)
    add_next_line_to_dict = False

    metadata_key_translation_dict = {
        'Assay': 'assay',
        'Description': 'description',
        'Application': 'application',
        'Investigator Name': 'investigatorName',
        'Adapter': 'adapter',
        'Workflow': 'workflow',
        'ReverseComplement': 'reversecomplement',
        'IEMFileVersion': 'iemfileversion',
        'Date': 'date',
        'Experiment Name': 'experimentName',
        'Chemistry': 'chemistry'
    }

    for line in csv_reader:

        if any(["[Header]" in line, "[Reads]" in line, "[Settings]" in line]):
            add_next_line_to_dict = True

        elif add_next_line_to_dict:

            if len(line) == 2:
                key_name = metadata_key_translation_dict[line[0]]
                metadata_dict[key_name] = line[1]

            elif len(line) == 1:  # case for "[Reads]"

                metadata_dict["readLengths"].append(line[0])

            elif len(line) == 0:  # current line is blank; end of section
                add_next_line_to_dict = False

        elif "[Data]" in line:
            break

    return metadata_dict

def complete_parse_samples(sample_sheet_file):
    """
    Creates a complete Sample object:
    Sample dict will only have the required (and already translated) keys:
        'sampleName', 'description', 'sequencerSampleId' 'sampleProject'.
    SequenceFile parsed out and holds Sample metadata (other keys) +
        pair files for the sample.
    SequenceFile is then set as an attribute of Sample
    These Sample objects will be stored in a list.

    arguments:
            sample_sheet_file -- path to SampleSheet.csv

    returns list containing complete Sample objects
    """

    sample_list = parse_samples(sample_sheet_file)
    data_dir = path.dirname(sample_sheet_file)
    for sample in sample_list:

        properties_dict = parse_out_sequence_file(sample)
        pf_list = get_pair_files(data_dir, sample.get_id())
        sq = SequenceFile(properties_dict, pf_list)

        sample.set_seq_file(deepcopy(sq))

    return sample_list

def parse_samples(sample_sheet_file):
    """
    Parse all the lines under "[Data]" in .csv file
    Keys in sample_key_translation_dict have their values changed for
        uploading to REST API
    All other keys keep the same name that they have in .csv file

    arguments:
            sample_sheet_file -- path to SampleSheet.csv

    returns	a list containing Sample objects that have been created by a
        dictionary from the parsed out key:pair values from .csv file
    """

    csv_reader = get_csv_reader(sample_sheet_file)
    # start with an ordered dictionary so that keys are ordered in the same
    # way that they are inserted.
    sample_dict = OrderedDict()
    sample_list = []

    sample_key_translation_dict = {
        'Sample_Name': 'sampleName',
        'Description': 'description',
        'Sample_ID': 'sequencerSampleId',
        'Sample_Project': 'sampleProject'
    }

    # initilize dictionary keys from first line (data headers/attributes)
    set_attributes = False
    for line in csv_reader:

        if set_attributes:
            for item in line:

                if item in sample_key_translation_dict:
                    key_name = sample_key_translation_dict[item]
                else:
                    key_name = item

                sample_dict[key_name] = ""

            break

        if "[Data]" in line:
            set_attributes = True

    # fill in values for keys
    for line in csv_reader:

        i = 0

        for key in sample_dict.keys():
            sample_dict[key] = line[i]  # assumes values are never empty
            i = i + 1

        sample = Sample(deepcopy(sample_dict))
        sample_list.append(sample)

    return sample_list

def parse_out_sequence_file(sample):
    """
    Removes keys in argument sample that are not in sample_keys and
        stores them in sequence_file_dict

    arguments:
            sample -- Sample object
            the dictionary inside the Sample object is changed

    returns a dictionary containing keys not in sample_keys to be used to
        create a SequenceFile object
    """

    sample_keys = ["sampleName", "description",
                   "sequencerSampleId", "sampleProject"]
    sequence_file_dict = {}
    sample_dict = sample.get_dict()
    for key in sample_dict.keys()[:]:  # iterate through a copy
        if key not in sample_keys:
            sequence_file_dict[key] = sample_dict[key]
            del sample_dict[key]

    return sequence_file_dict

def get_csv_reader(sample_sheet_file):
    """
    tries to create a csv.reader object which will be used to
        parse through the lines in SampleSheet.csv
    raises an error if:
            sample_sheet_file is not an existing file
            sample_sheet_file contains null byte(s)

    arguments:
            data_dir -- the directory that has SampleSheet.csv in it

    returns a csv.reader object
    """

    csvFile = sample_sheet_file
    if path.isfile(csvFile) and '\0' not in open(csvFile).read():

        # open and read file in binary then send it to be parsed by csv's
        # reader
        csv_reader = reader(open(csvFile, "rb"))
    else:
        msg = sample_sheet_file + " is not a valid SampleSheet file"
        raise SampleSheetError(msg)

    return csv_reader

def get_pair_files(data_dir, sample_id):
    """
    find the pair sequence files for the given sample_id
    raises an error if no sequence pair files found

    arguments:
            data_dir -- the directory that has SampleSheet.csv in it
            sample_id -- ID of the sample for the pair files

    returns a list containing the path of the pair files starting from data_dir
    """

    pattern = sample_id + "*.fastq.gz"
    pair_file_list = recursive_find(data_dir, pattern)
    pair_file_list.sort()

    return pair_file_list

def recursive_find(top_dir, pattern):
    """
    Traverse through a directory and its subdirectories looking for files that
        match given pattern

    arguments:
            top_dir -- top level directory to start searching from
            pattern -- pattern to try and match using fnfilter/ fnmatch.filter

    returns list containing files that match pattern
    """
    result_list = []

    if path.isdir(top_dir):
        for root, dirs, files in walk(top_dir):
            for filename in fnfilter(files, pattern):
                res = path.join(root, filename)
                result_list.append(res)
    else:
        msg = "Invalid directory " + top_dir
        raise IOError(msg)

    return result_list
<commit_msg>minor space increase<commit_after>import sys
from os import walk, path
from fnmatch import filter as fnfilter
from csv import reader
from collections import OrderedDict
from copy import deepcopy

sys.path.append("../")
from Model.Sample import Sample
from Model.SequenceFile import SequenceFile
from Exceptions.SampleSheetError import SampleSheetError
from Exceptions.SequenceFileError import SequenceFileError

def parse_metadata(sample_sheet_file):

    """
    Parse all lines under [Header], [Reads] and [Settings] in .csv file
    Lines under [Reads] are stored in a list with key name "readLengths"
    All other key names are translated according to the
        metadata_key_translation_dict

    arguments:
            sample_sheet_file -- path to SampleSheet.csv

    returns a dictionary containing the parsed key:pair values from .csv file
    """

    metadata_dict = {}
    metadata_dict["readLengths"] = []

    csv_reader = get_csv_reader(sample_sheet_file)
    add_next_line_to_dict = False

    metadata_key_translation_dict = {
        'Assay': 'assay',
        'Description': 'description',
        'Application': 'application',
        'Investigator Name': 'investigatorName',
        'Adapter': 'adapter',
        'Workflow': 'workflow',
        'ReverseComplement': 'reversecomplement',
        'IEMFileVersion': 'iemfileversion',
        'Date': 'date',
        'Experiment Name': 'experimentName',
        'Chemistry': 'chemistry'
    }

    for line in csv_reader:

        if any(["[Header]" in line, "[Reads]" in line, "[Settings]" in line]):
            add_next_line_to_dict = True

        elif add_next_line_to_dict:

            if len(line) == 2:
                key_name = metadata_key_translation_dict[line[0]]
                metadata_dict[key_name] = line[1]

            elif len(line) == 1:  # case for "[Reads]"

                metadata_dict["readLengths"].append(line[0])

            elif len(line) == 0:  # current line is blank; end of section
                add_next_line_to_dict = False

        elif "[Data]" in line:
            break

    return metadata_dict

def complete_parse_samples(sample_sheet_file):

    """
    Creates a complete Sample object:
    Sample dict will only have the required (and already translated) keys:
        'sampleName', 'description', 'sequencerSampleId' 'sampleProject'.
    SequenceFile parsed out and holds Sample metadata (other keys) +
        pair files for the sample.
    SequenceFile is then set as an attribute of Sample
    These Sample objects will be stored in a list.

    arguments:
            sample_sheet_file -- path to SampleSheet.csv

    returns list containing complete Sample objects
    """

    sample_list = parse_samples(sample_sheet_file)
    data_dir = path.dirname(sample_sheet_file)
    for sample in sample_list:

        properties_dict = parse_out_sequence_file(sample)
        pf_list = get_pair_files(data_dir, sample.get_id())
        sq = SequenceFile(properties_dict, pf_list)

        sample.set_seq_file(deepcopy(sq))

    return sample_list

def parse_samples(sample_sheet_file):

    """
    Parse all the lines under "[Data]" in .csv file
    Keys in sample_key_translation_dict have their values changed for
        uploading to REST API
    All other keys keep the same name that they have in .csv file

    arguments:
            sample_sheet_file -- path to SampleSheet.csv

    returns	a list containing Sample objects that have been created by a
        dictionary from the parsed out key:pair values from .csv file
    """

    csv_reader = get_csv_reader(sample_sheet_file)
    # start with an ordered dictionary so that keys are ordered in the same
    # way that they are inserted.
    sample_dict = OrderedDict()
    sample_list = []

    sample_key_translation_dict = {
        'Sample_Name': 'sampleName',
        'Description': 'description',
        'Sample_ID': 'sequencerSampleId',
        'Sample_Project': 'sampleProject'
    }

    # initilize dictionary keys from first line (data headers/attributes)
    set_attributes = False
    for line in csv_reader:

        if set_attributes:
            for item in line:

                if item in sample_key_translation_dict:
                    key_name = sample_key_translation_dict[item]
                else:
                    key_name = item

                sample_dict[key_name] = ""

            break

        if "[Data]" in line:
            set_attributes = True

    # fill in values for keys
    for line in csv_reader:

        i = 0

        for key in sample_dict.keys():
            sample_dict[key] = line[i]  # assumes values are never empty
            i = i + 1

        sample = Sample(deepcopy(sample_dict))
        sample_list.append(sample)

    return sample_list

def parse_out_sequence_file(sample):

    """
    Removes keys in argument sample that are not in sample_keys and
        stores them in sequence_file_dict

    arguments:
            sample -- Sample object
            the dictionary inside the Sample object is changed

    returns a dictionary containing keys not in sample_keys to be used to
        create a SequenceFile object
    """

    sample_keys = ["sampleName", "description",
                   "sequencerSampleId", "sampleProject"]
    sequence_file_dict = {}
    sample_dict = sample.get_dict()
    for key in sample_dict.keys()[:]:  # iterate through a copy
        if key not in sample_keys:
            sequence_file_dict[key] = sample_dict[key]
            del sample_dict[key]

    return sequence_file_dict

def get_csv_reader(sample_sheet_file):

    """
    tries to create a csv.reader object which will be used to
        parse through the lines in SampleSheet.csv
    raises an error if:
            sample_sheet_file is not an existing file
            sample_sheet_file contains null byte(s)

    arguments:
            data_dir -- the directory that has SampleSheet.csv in it

    returns a csv.reader object
    """

    csvFile = sample_sheet_file
    if path.isfile(csvFile) and '\0' not in open(csvFile).read():

        # open and read file in binary then send it to be parsed by csv's
        # reader
        csv_reader = reader(open(csvFile, "rb"))
    else:
        msg = sample_sheet_file + " is not a valid SampleSheet file"
        raise SampleSheetError(msg)

    return csv_reader

def get_pair_files(data_dir, sample_id):

    """
    find the pair sequence files for the given sample_id
    raises an error if no sequence pair files found

    arguments:
            data_dir -- the directory that has SampleSheet.csv in it
            sample_id -- ID of the sample for the pair files

    returns a list containing the path of the pair files starting from data_dir
    """

    pattern = sample_id + "*.fastq.gz"
    pair_file_list = recursive_find(data_dir, pattern)
    pair_file_list.sort()

    return pair_file_list

def recursive_find(top_dir, pattern):

    """
    Traverse through a directory and its subdirectories looking for files that
        match given pattern

    arguments:
            top_dir -- top level directory to start searching from
            pattern -- pattern to try and match using fnfilter/ fnmatch.filter

    returns list containing files that match pattern
    """

    result_list = []

    if path.isdir(top_dir):
        for root, dirs, files in walk(top_dir):
            for filename in fnfilter(files, pattern):
                res = path.join(root, filename)
                result_list.append(res)
    else:
        msg = "Invalid directory " + top_dir
        raise IOError(msg)

    return result_list
<|endoftext|>