<commit_before>"""
Support for models' internal Translation class.
"""

## TO DO: this is messy and needs to be cleaned up

from django.db import models
from django.db.models.base import ModelBase
from django.dispatch.dispatcher import connect
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import signals
from multilingual.languages import *
from multilingual.exceptions import TranslationDoesNotExist
from multilingual.fields import TranslationForeignKey
from multilingual.manipulators import add_multilingual_manipulators
from multilingual import manager
from multilingual.compat import IS_NEWFORMS_ADMIN

if IS_NEWFORMS_ADMIN:
    from django.contrib.admin import StackedInline, ModelAdmin
    class MultilingualStackedInline(StackedInline):
        template = "admin/edit_inline_translations_newforms.html"
else:
    from django.contrib.admin.templatetags.admin_modify import StackedBoundRelatedObject
    class TransBoundRelatedObject(StackedBoundRelatedObject):
        """
        This class changes the template for translation objects.
        """

        def template_name(self):
            return "admin/edit_inline_translations.html"

def translation_save_translated_fields(instance, **kwargs):
    """
    Save all the translations of instance in post_save signal handler.
    """
    if not hasattr(instance, '_translation_cache'):
        return
    for l_id, translation in instance._translation_cache.iteritems():
        # set the translation ID just in case the translation was
        # created while instance was not stored in the DB yet
        translation.master_id = instance.id
        translation.save()

def translation_overwrite_previous(instance, **kwargs):
    """
    Delete previously existing translation with the same master and
    language_id values.  To be called by translation model's pre_save
    signal.

    This most probably means I am abusing something here trying to use
    Django inline editor.  Oh well, it will have to go anyway when we
    move to newforms.
    """
    qs = instance.__class__.objects
    try:
        qs = qs.filter(master=instance.master).filter(language_id=instance.language_id)
        qs.delete()
    except ObjectDoesNotExist:
        # We are probably loading a fixture that defines translation entities
	# before their master entity.
        pass

def fill_translation_cache(instance):
    """
    Fill the translation cache using information received in the
    instance objects as extra fields.

    You can not do this in post_init because the extra fields are
    assigned by QuerySet.iterator after model initialization.
    """

    if hasattr(instance, '_translation_cache'):
        # do not refill the cache
        return

    instance._translation_cache = {}
    for language_id in get_language_id_list():
        # see if translation for language_id was in the query
        field_alias = get_translated_field_alias('id', language_id)
        if getattr(instance, field_alias, None) is not None:
            field_names = [f.attname for f in instance._meta.translation_model._meta.fields]

            # if so, create a translation object and put it in the cache
            field_data = {}
            for fname in field_names:
                field_data[fname] = getattr(instance,
                                            get_translated_field_alias(fname, language_id))

            translation = instance._meta.translation_model(**field_data)
            instance._translation_cache[language_id] = translation

    # In some situations an (existing in the DB) object is loaded
    # without using the normal QuerySet.  In such case fallback to
    # loading the translations using a separate query.

    # Unfortunately, this is indistinguishable from the situation when
    # an object does not have any translations.  Oh well, we'll have
    # to live with this for the time being.
    if len(instance._translation_cache.keys()) == 0:
        for translation in instance.translations.all():
            instance._translation_cache[translation.language_id] = translation

class TranslatedFieldProxy(property):
    def __init__(self, field_name, alias, field, language_id=None):
        self.field_name = field_name
        self.field = field
        self.admin_order_field = alias
        self.language_id = language_id

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self

        return getattr(obj, 'get_' + self.field_name)(self.language_id)

    def __set__(self, obj, value):
        language_id = self.language_id

        return getattr(obj, 'set_' + self.field_name)(value, self.language_id)

    short_description = property(lambda self: self.field.short_description)

def getter_generator(field_name, short_description):
    """
    Generate get_'field name' method for field field_name.
    """
    def get_translation_field(self, language_id_or_code=None):
        try:
            return getattr(self.get_translation(language_id_or_code), field_name)
        except TranslationDoesNotExist:
            return None
    get_translation_field.short_description = short_description
    return get_translation_field

def setter_generator(field_name):
    """
    Generate set_'field name' method for field field_name.
    """
    def set_translation_field(self, value, language_id_or_code=None):
        setattr(self.get_translation(language_id_or_code, True),
                field_name, value)
    set_translation_field.short_description = "set " + field_name
    return set_translation_field

def get_translation(self, language_id_or_code,
                    create_if_necessary=False):
    """
    Get a translation instance for the given language_id_or_code.

    If it does not exist, either create one or raise the
    TranslationDoesNotExist exception, depending on the
    create_if_necessary argument.
    """

    # fill the cache if necessary
    self.fill_translation_cache()

    language_id = get_language_id_from_id_or_code(language_id_or_code, False)
    if language_id is None:
        language_id = getattr(self, '_default_language', None)
    if language_id is None:
        language_id = get_default_language()

    if language_id not in self._translation_cache:
        if not create_if_necessary:
            raise TranslationDoesNotExist(language_id)
        new_translation = self._meta.translation_model(master=self,
                                                       language_id=language_id)
        self._translation_cache[language_id] = new_translation
    return self._translation_cache.get(language_id, None)

class Translation:
    """
    A superclass for translatablemodel.Translation inner classes.
    """

    def contribute_to_class(cls, main_cls, name):
        """
        Handle the inner 'Translation' class.
        """

        # delay the creation of the *Translation until the master model is
        # fully created
        connect(cls.finish_multilingual_class, signal=signals.class_prepared,
                sender=main_cls, weak=False)

        # connect the post_save signal on master class to a handler
        # that saves translations
        connect(translation_save_translated_fields, signal=signals.post_save,
                sender=main_cls)

    contribute_to_class = classmethod(contribute_to_class)

    def create_translation_attrs(cls, main_cls):
        """
        Creates get_'field name'(language_id) and set_'field
        name'(language_id) methods for all the translation fields.
        Adds the 'field name' properties too.

        Returns the translated_fields hash used in field lookups, see
        multilingual.query.  It maps field names to (field,
        language_id) tuples.
        """
        translated_fields = {}

        for fname, field in cls.__dict__.items():
            if isinstance(field, models.fields.Field):
                translated_fields[fname] = (field, None)

                # add get_'fname' and set_'fname' methods to main_cls
                getter = getter_generator(fname, getattr(field, 'verbose_name', fname))
                setattr(main_cls, 'get_' + fname, getter)

                setter = setter_generator(fname)
                setattr(main_cls, 'set_' + fname, setter)

                # add the 'fname' proxy property that allows reads
                # from and writing to the appropriate translation
                setattr(main_cls, fname,
                        TranslatedFieldProxy(fname, fname, field))

                # create the 'fname'_'language_code' proxy properties
                for language_id in get_language_id_list():
                    language_code = get_language_code(language_id)
                    fname_lng = fname + '_' + language_code
                    translated_fields[fname_lng] = (field, language_id)
                    setattr(main_cls, fname_lng,
                            TranslatedFieldProxy(fname, fname_lng, field,
                                                 language_id))
        return translated_fields
    create_translation_attrs = classmethod(create_translation_attrs)

    def get_unique_fields(cls):
        """
        Return a list of fields with "unique" attribute, which needs to
        be augmented by the language.
        """
        unique_fields = []

        for fname, field in cls.__dict__.items():
            if isinstance(field, models.fields.Field):
                if getattr(field,'unique',False):
                    field.unique = False
                    unique_fields.append(fname)
        return unique_fields
    get_unique_fields = classmethod(get_unique_fields)

    def finish_multilingual_class(cls, *args, **kwargs):
        """
        Create a model with translations of a multilingual class.
        """

        main_cls = kwargs['sender']
        translation_model_name = main_cls.__name__ + "Translation"

        # create the model with all the translatable fields
        unique = [('language_id', 'master')]
        for f in cls.get_unique_fields():
            unique.append(('language_id',f))

        class TransMeta:
            ordering = ('language_id',)
            unique_together = tuple(unique)
            db_table = main_cls._meta.db_table + '_translation'
            app_label = main_cls._meta.app_label

        trans_attrs = cls.__dict__.copy()
        trans_attrs['Meta'] = TransMeta
        trans_attrs['language_id'] = models.IntegerField(blank=False, null=False, core=True,
                                                         choices=get_language_choices(),
                                                         db_index=True)

        if IS_NEWFORMS_ADMIN:
            edit_inline = True
        else:
            edit_inline = TransBoundRelatedObject

        trans_attrs['master'] = TranslationForeignKey(main_cls, blank=False, null=False,
                                                      edit_inline=edit_inline,
                                                      related_name='translations',
                                                      num_in_admin=get_language_count(),
                                                      min_num_in_admin=get_language_count(),
                                                      num_extra_on_change=0)
        trans_attrs['__str__'] = lambda self: ("%s object, language_code=%s"
                                               % (translation_model_name,
                                                  get_language_code(self.language_id)))

        trans_model = ModelBase(translation_model_name, (models.Model,), trans_attrs)
        trans_model._meta.translated_fields = cls.create_translation_attrs(main_cls)

        main_cls._meta.translation_model = trans_model
        main_cls.get_translation = get_translation
        main_cls.fill_translation_cache = fill_translation_cache

        # Note: don't fill the translation cache in post_init, as all
        # the extra values selected by QAddTranslationData will be
        # assigned AFTER init()
#        connect(fill_translation_cache, signal=signals.post_init,
#                sender=main_cls)

        # connect the pre_save signal on translation class to a
        # function removing previous translation entries.
        connect(translation_overwrite_previous, signal=signals.pre_save,
                sender=trans_model, weak=False)

    finish_multilingual_class = classmethod(finish_multilingual_class)

def install_translation_library():
    # modify ModelBase.__new__ so that it understands how to handle the
    # 'Translation' inner class

    if getattr(ModelBase, '_multilingual_installed', False):
        # don't install it twice
        return

    _old_new = ModelBase.__new__

    def multilingual_modelbase_new(cls, name, bases, attrs):
        if 'Translation' in attrs:
            if not issubclass(attrs['Translation'], Translation):
                raise ValueError, ("%s.Translation must be a subclass "
                                   + " of multilingual.Translation.") % (name,)

            # Make sure that if the class specifies objects then it is
            # a subclass of our Manager.
            #
            # Don't check other managers since someone might want to
            # have a non-multilingual manager, but assigning a
            # non-multilingual manager to objects would be a common
            # mistake.
            if ('objects' in attrs) and (not isinstance(attrs['objects'], manager.Manager)):
                raise ValueError, ("Model %s specifies translations, " +
                                   "so its 'objects' manager must be " +
                                   "a subclass of multilingual.Manager.") % (name,)

            # Change the default manager to multilingual.Manager.
            if not 'objects' in attrs:
                attrs['objects'] = manager.Manager()

            # Override the admin manager as well, or the admin views will
            # not see the translation data.
            if 'Admin' in attrs:
                attrs['Admin'].manager = attrs['objects']

            # Install a hack to let add_multilingual_manipulators know
            # this is a translatable model
            attrs['is_translation_model'] = lambda self: True

        return _old_new(cls, name, bases, attrs)
    ModelBase.__new__ = staticmethod(multilingual_modelbase_new)
    ModelBase._multilingual_installed = True

    if IS_NEWFORMS_ADMIN:
        # Override ModelAdmin.__new__ to create automatic inline
        # editor for multilingual models.
        class MultiType(type):
            pass
        _old_admin_new = ModelAdmin.__new__
        def multilingual_modeladmin_new(cls, model, admin_site):
            if isinstance(model.objects, manager.Manager):
                X = MultiType('X',(MultilingualStackedInline,),
                         {'model':model._meta.translation_model,
                          'fk_name':'master',
                          'extra':get_language_count(),
                          'max_num':get_language_count()})
                if cls.inlines:
                    for inline in cls.inlines:
                        if X.__class__ == inline.__class__:
                            cls.inlines.remove(inline)
                            break
                    cls.inlines.append(X)
                else:
                    cls.inlines = [X]
            return _old_admin_new(cls, model, admin_site)
        ModelAdmin.__new__ = staticmethod(multilingual_modeladmin_new)

# install the library
install_translation_library()
<commit_msg>Use inner Translation.Meta class if present.<commit_after>"""
Support for models' internal Translation class.
"""

## TO DO: this is messy and needs to be cleaned up

from django.db import models
from django.db.models.base import ModelBase
from django.dispatch.dispatcher import connect
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import signals
from multilingual.languages import *
from multilingual.exceptions import TranslationDoesNotExist
from multilingual.fields import TranslationForeignKey
from multilingual.manipulators import add_multilingual_manipulators
from multilingual import manager
from multilingual.compat import IS_NEWFORMS_ADMIN

if IS_NEWFORMS_ADMIN:
    from django.contrib.admin import StackedInline, ModelAdmin
    class MultilingualStackedInline(StackedInline):
        template = "admin/edit_inline_translations_newforms.html"
else:
    from django.contrib.admin.templatetags.admin_modify import StackedBoundRelatedObject
    class TransBoundRelatedObject(StackedBoundRelatedObject):
        """
        This class changes the template for translation objects.
        """

        def template_name(self):
            return "admin/edit_inline_translations.html"

def translation_save_translated_fields(instance, **kwargs):
    """
    Save all the translations of instance in post_save signal handler.
    """
    if not hasattr(instance, '_translation_cache'):
        return
    for l_id, translation in instance._translation_cache.iteritems():
        # set the translation ID just in case the translation was
        # created while instance was not stored in the DB yet
        translation.master_id = instance.id
        translation.save()

def translation_overwrite_previous(instance, **kwargs):
    """
    Delete previously existing translation with the same master and
    language_id values.  To be called by translation model's pre_save
    signal.

    This most probably means I am abusing something here trying to use
    Django inline editor.  Oh well, it will have to go anyway when we
    move to newforms.
    """
    qs = instance.__class__.objects
    try:
        qs = qs.filter(master=instance.master).filter(language_id=instance.language_id)
        qs.delete()
    except ObjectDoesNotExist:
        # We are probably loading a fixture that defines translation entities
	# before their master entity.
        pass

def fill_translation_cache(instance):
    """
    Fill the translation cache using information received in the
    instance objects as extra fields.

    You can not do this in post_init because the extra fields are
    assigned by QuerySet.iterator after model initialization.
    """

    if hasattr(instance, '_translation_cache'):
        # do not refill the cache
        return

    instance._translation_cache = {}
    for language_id in get_language_id_list():
        # see if translation for language_id was in the query
        field_alias = get_translated_field_alias('id', language_id)
        if getattr(instance, field_alias, None) is not None:
            field_names = [f.attname for f in instance._meta.translation_model._meta.fields]

            # if so, create a translation object and put it in the cache
            field_data = {}
            for fname in field_names:
                field_data[fname] = getattr(instance,
                                            get_translated_field_alias(fname, language_id))

            translation = instance._meta.translation_model(**field_data)
            instance._translation_cache[language_id] = translation

    # In some situations an (existing in the DB) object is loaded
    # without using the normal QuerySet.  In such case fallback to
    # loading the translations using a separate query.

    # Unfortunately, this is indistinguishable from the situation when
    # an object does not have any translations.  Oh well, we'll have
    # to live with this for the time being.
    if len(instance._translation_cache.keys()) == 0:
        for translation in instance.translations.all():
            instance._translation_cache[translation.language_id] = translation

class TranslatedFieldProxy(property):
    def __init__(self, field_name, alias, field, language_id=None):
        self.field_name = field_name
        self.field = field
        self.admin_order_field = alias
        self.language_id = language_id

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self

        return getattr(obj, 'get_' + self.field_name)(self.language_id)

    def __set__(self, obj, value):
        language_id = self.language_id

        return getattr(obj, 'set_' + self.field_name)(value, self.language_id)

    short_description = property(lambda self: self.field.short_description)

def getter_generator(field_name, short_description):
    """
    Generate get_'field name' method for field field_name.
    """
    def get_translation_field(self, language_id_or_code=None):
        try:
            return getattr(self.get_translation(language_id_or_code), field_name)
        except TranslationDoesNotExist:
            return None
    get_translation_field.short_description = short_description
    return get_translation_field

def setter_generator(field_name):
    """
    Generate set_'field name' method for field field_name.
    """
    def set_translation_field(self, value, language_id_or_code=None):
        setattr(self.get_translation(language_id_or_code, True),
                field_name, value)
    set_translation_field.short_description = "set " + field_name
    return set_translation_field

def get_translation(self, language_id_or_code,
                    create_if_necessary=False):
    """
    Get a translation instance for the given language_id_or_code.

    If it does not exist, either create one or raise the
    TranslationDoesNotExist exception, depending on the
    create_if_necessary argument.
    """

    # fill the cache if necessary
    self.fill_translation_cache()

    language_id = get_language_id_from_id_or_code(language_id_or_code, False)
    if language_id is None:
        language_id = getattr(self, '_default_language', None)
    if language_id is None:
        language_id = get_default_language()

    if language_id not in self._translation_cache:
        if not create_if_necessary:
            raise TranslationDoesNotExist(language_id)
        new_translation = self._meta.translation_model(master=self,
                                                       language_id=language_id)
        self._translation_cache[language_id] = new_translation
    return self._translation_cache.get(language_id, None)

class Translation:
    """
    A superclass for translatablemodel.Translation inner classes.
    """

    def contribute_to_class(cls, main_cls, name):
        """
        Handle the inner 'Translation' class.
        """

        # delay the creation of the *Translation until the master model is
        # fully created
        connect(cls.finish_multilingual_class, signal=signals.class_prepared,
                sender=main_cls, weak=False)

        # connect the post_save signal on master class to a handler
        # that saves translations
        connect(translation_save_translated_fields, signal=signals.post_save,
                sender=main_cls)

    contribute_to_class = classmethod(contribute_to_class)

    def create_translation_attrs(cls, main_cls):
        """
        Creates get_'field name'(language_id) and set_'field
        name'(language_id) methods for all the translation fields.
        Adds the 'field name' properties too.

        Returns the translated_fields hash used in field lookups, see
        multilingual.query.  It maps field names to (field,
        language_id) tuples.
        """
        translated_fields = {}

        for fname, field in cls.__dict__.items():
            if isinstance(field, models.fields.Field):
                translated_fields[fname] = (field, None)

                # add get_'fname' and set_'fname' methods to main_cls
                getter = getter_generator(fname, getattr(field, 'verbose_name', fname))
                setattr(main_cls, 'get_' + fname, getter)

                setter = setter_generator(fname)
                setattr(main_cls, 'set_' + fname, setter)

                # add the 'fname' proxy property that allows reads
                # from and writing to the appropriate translation
                setattr(main_cls, fname,
                        TranslatedFieldProxy(fname, fname, field))

                # create the 'fname'_'language_code' proxy properties
                for language_id in get_language_id_list():
                    language_code = get_language_code(language_id)
                    fname_lng = fname + '_' + language_code
                    translated_fields[fname_lng] = (field, language_id)
                    setattr(main_cls, fname_lng,
                            TranslatedFieldProxy(fname, fname_lng, field,
                                                 language_id))
        return translated_fields
    create_translation_attrs = classmethod(create_translation_attrs)

    def get_unique_fields(cls):
        """
        Return a list of fields with "unique" attribute, which needs to
        be augmented by the language.
        """
        unique_fields = []

        for fname, field in cls.__dict__.items():
            if isinstance(field, models.fields.Field):
                if getattr(field,'unique',False):
                    field.unique = False
                    unique_fields.append(fname)
        return unique_fields
    get_unique_fields = classmethod(get_unique_fields)

    def finish_multilingual_class(cls, *args, **kwargs):
        """
        Create a model with translations of a multilingual class.
        """

        main_cls = kwargs['sender']
        translation_model_name = main_cls.__name__ + "Translation"

        # create the model with all the translatable fields
        unique = [('language_id', 'master')]
        for f in cls.get_unique_fields():
            unique.append(('language_id',f))

        class TransMeta:
            pass

        try:
            meta = cls.Meta
        except AttributeError:
            meta = TransMeta
        meta.ordering = ('language_id',)
        meta.unique_together = tuple(unique)
        meta.app_label = main_cls._meta.app_label
        if not hasattr(meta, 'db_table'):
            db_table = main_cls._meta.db_table + '_translation'

        trans_attrs = cls.__dict__.copy()
        trans_attrs['Meta'] = meta
        trans_attrs['language_id'] = models.IntegerField(blank=False, null=False, core=True,
                                                         choices=get_language_choices(),
                                                         db_index=True)

        if IS_NEWFORMS_ADMIN:
            edit_inline = True
        else:
            edit_inline = TransBoundRelatedObject

        trans_attrs['master'] = TranslationForeignKey(main_cls, blank=False, null=False,
                                                      edit_inline=edit_inline,
                                                      related_name='translations',
                                                      num_in_admin=get_language_count(),
                                                      min_num_in_admin=get_language_count(),
                                                      num_extra_on_change=0)
        trans_attrs['__str__'] = lambda self: ("%s object, language_code=%s"
                                               % (translation_model_name,
                                                  get_language_code(self.language_id)))

        trans_model = ModelBase(translation_model_name, (models.Model,), trans_attrs)
        trans_model._meta.translated_fields = cls.create_translation_attrs(main_cls)

        main_cls._meta.translation_model = trans_model
        main_cls.get_translation = get_translation
        main_cls.fill_translation_cache = fill_translation_cache

        # Note: don't fill the translation cache in post_init, as all
        # the extra values selected by QAddTranslationData will be
        # assigned AFTER init()
#        connect(fill_translation_cache, signal=signals.post_init,
#                sender=main_cls)

        # connect the pre_save signal on translation class to a
        # function removing previous translation entries.
        connect(translation_overwrite_previous, signal=signals.pre_save,
                sender=trans_model, weak=False)

    finish_multilingual_class = classmethod(finish_multilingual_class)

def install_translation_library():
    # modify ModelBase.__new__ so that it understands how to handle the
    # 'Translation' inner class

    if getattr(ModelBase, '_multilingual_installed', False):
        # don't install it twice
        return

    _old_new = ModelBase.__new__

    def multilingual_modelbase_new(cls, name, bases, attrs):
        if 'Translation' in attrs:
            if not issubclass(attrs['Translation'], Translation):
                raise ValueError, ("%s.Translation must be a subclass "
                                   + " of multilingual.Translation.") % (name,)

            # Make sure that if the class specifies objects then it is
            # a subclass of our Manager.
            #
            # Don't check other managers since someone might want to
            # have a non-multilingual manager, but assigning a
            # non-multilingual manager to objects would be a common
            # mistake.
            if ('objects' in attrs) and (not isinstance(attrs['objects'], manager.Manager)):
                raise ValueError, ("Model %s specifies translations, " +
                                   "so its 'objects' manager must be " +
                                   "a subclass of multilingual.Manager.") % (name,)

            # Change the default manager to multilingual.Manager.
            if not 'objects' in attrs:
                attrs['objects'] = manager.Manager()

            # Override the admin manager as well, or the admin views will
            # not see the translation data.
            if 'Admin' in attrs:
                attrs['Admin'].manager = attrs['objects']

            # Install a hack to let add_multilingual_manipulators know
            # this is a translatable model
            attrs['is_translation_model'] = lambda self: True

        return _old_new(cls, name, bases, attrs)
    ModelBase.__new__ = staticmethod(multilingual_modelbase_new)
    ModelBase._multilingual_installed = True

    if IS_NEWFORMS_ADMIN:
        # Override ModelAdmin.__new__ to create automatic inline
        # editor for multilingual models.
        class MultiType(type):
            pass
        _old_admin_new = ModelAdmin.__new__
        def multilingual_modeladmin_new(cls, model, admin_site):
            if isinstance(model.objects, manager.Manager):
                X = MultiType('X',(MultilingualStackedInline,),
                         {'model':model._meta.translation_model,
                          'fk_name':'master',
                          'extra':get_language_count(),
                          'max_num':get_language_count()})
                if cls.inlines:
                    for inline in cls.inlines:
                        if X.__class__ == inline.__class__:
                            cls.inlines.remove(inline)
                            break
                    cls.inlines.append(X)
                else:
                    cls.inlines = [X]
            return _old_admin_new(cls, model, admin_site)
        ModelAdmin.__new__ = staticmethod(multilingual_modeladmin_new)

# install the library
install_translation_library()
<|endoftext|>


<commit_before>import time
import datetime
import urllib
import urlparse
import re
import sqlite

# create table series (id INTEGER PRIMARY KEY, uid TEXT UNIQUE, serie TEXT, season INTEGER, episode INTEGER, title TEXT, airdate DATE);
# insert into series values(null, 'Alias', 1, 1, 'Pilot', date('2006-01-01'));
# select * from series where airdate = date('2006-01-02', '-1 day');
# select * from series where airdate = date('now', '-1 day');

precache = ['NCIS', 'Boston Legal', 'battlestar galactica', 'veronica mars',
'smallville', '24', 'lost', 'stargate sg-1', 'stargate atlantis', 'viva la bam', 'bullshit',
'4400', 'mythbusters', 'house', 'last comic standing', 'deadwood', 'shield', 'blade -flashing',
'the unit', 'prison break', 'bones', 'the simpsons', 'survivor']

CREATETABLE = "CREATE TABLE series (id INTEGER PRIMARY KEY, uid TEXT UNIQUE ON CONFLICT REPLACE, serie TEXT, season INTEGER, episode INTEGER, title TEXT, airdate DATE);"

def command_sqlcache(bot, user, channel, args):
    """Cache common series into the database"""
    if not isAdmin(user): return

    for seriename in precache:
        url = _search_serie(seriename)
        if url != None:
            _cache_serie(url)
    bot.say(channel, "Caching done")

def NOTinit():
    con = sqlite.connect("/home/shrike/pyfibot/modules/series.db");
    cur = con.cursor()
    cur.execute(CREATETABLE)
    cur.close()
    con.commit()
    con.close()

def command_epdel(bot, user, channel, args):
    if not isAdmin(user): return

    con = sqlite.connect("/home/shrike/pyfibot/modules/series.db");
    cur = con.cursor()
    cur.execute("DELETE FROM series WHERE serie='%s';" % args)
    cur.close()
    con.commit()
    con.close()

    bot.say(channel, "Done");

def command_epinfo(bot, user, channel, args):
    """List episodes in the database"""

    con = sqlite.connect("/home/shrike/pyfibot/modules/series.db");
    cur = con.cursor()

    cur.execute("SELECT DISTINCT serie FROM series");

    res = []
    for serie in cur:
        res.append(serie[0])

    res.sort()
    bot.say(channel, "Known series ("+str(len(res))+"): " + ", ".join(res))

    cur.close()
    con.close()

def command_ep(bot, user, channel, args):
    """Usage: sqlep [today|yesterday|tomorrow] or [seriename]"""

    con = sqlite.connect("/home/shrike/pyfibot/modules/series.db");
    cur = con.cursor()

    if not args:
        bot.say(channel, "Usage: ep [today|yesterday|tomorrow] or [seriename]")
        return

    if args == "today":
        cur.execute("SELECT * FROM series WHERE airdate = date('now', 'localtime');")
        if cur.rowcount == 0:
            bot.say(channel, "No known releases today")
            return
    elif args == "yesterday":
        cur.execute("SELECT * FROM series WHERE airdate = date('now', 'localtime', '-1 day');")
        if cur.rowcount == 0:
            bot.say(channel, "No known releases yesterday")
            return
    elif args == "tomorrow":
        cur.execute("SELECT * FROM series WHERE airdate = date('now', 'localtime', '+1 day');")
        if cur.rowcount == 0:
            bot.say(channel, "No known releases tomorrow")
            return
    else:
        # try to find the serie
        cur.execute("SELECT * FROM series WHERE serie LIKE %s AND airdate > date('now', 'localtime') LIMIT 1", ("%"+args+"%",))
        # nothing found, get more data from the web
        if cur.rowcount == 0:
            url = _search_serie(args)
            if url != None:
                _cache_serie(url)
                cur.execute("SELECT * FROM series WHERE serie LIKE %s AND airdate > date('now', 'localtime') LIMIT 1", ("%"+args+"%",))
                # still nothing found, give up
                if cur.rowcount == 0:
                    bot.say(channel, "No future episodes of '%s' found" % args)
                    return

    episodes = []
    # go through the results
    for (id, uid, serie, season, episode, title, airdate) in cur:
        if episode < 10: episode = "0%d" % episode # pad ep with zeroes

        # YYYY-MM-DD -> datetime -> timedelta
        t = time.strptime(airdate, "%Y-%m-%d")
        ad = datetime.date(t.tm_year, t.tm_mon, t.tm_mday)
        now = datetime.date.today()
        tomorrow = now + datetime.timedelta(days=1)
        td = ad-now

        # change 0 and 1 to today & tomorrow, don't show date if we're asking stuff for a certain day
        airdatestr = ""
        if td.days > 0:
            if ad == now:
                if args != "today": airdatestr = "on %s (Today)" % airdate
            elif ad == tomorrow:
                if args != "tomorrow": airdatestr = "on %s (Tomorrow)" % airdate
            else:
                airdatestr = "on %s (%d days)" % (airdate, td.days)

        episodes.append("%s %sx%s '%s' %s" % (serie, season, episode, title, airdatestr))

    bot.say(channel, "-- ".join(episodes))

    cur.close()
    con.close()

def _cache_serie(url):
    """Parse the seriedata from the internets into sqlite"""

    serie, epdata, updatedate = _get_seriedata(url)
    con = sqlite.connect("/home/shrike/pyfibot/modules/series.db");

    for episode in epdata:
        episode['epname'] = episode['epname'].replace("'", "pier")
        cur = con.cursor()
        uid = "%s%d%d" % (serie, episode['season'], episode['episode'])
        cur.execute("INSERT INTO series (uid, serie, season, episode, title, airdate) VALUES (%s, %s, %d, %d, %s, date(%s));", (uid, serie, episode['season'], episode['episode'], episode['epname'], episode['airdate2'].strftime("%Y-%m-%d")))
        cur.close()

    # Commit & close
    con.commit()
    con.close()

def _search_serie(searchterms):
    """Search for a serie and return its episode list page URL"""

    # google power
    if not searchterms: return

    url = "http://www.google.com/search?hl=en&q=site:epguides.com&q=%s"
    searchterms = urllib.quote(searchterms)
    bs = getUrl(url % searchterms).getBS()

    if not bs: return

    results = []

    # tidy up the search results
    for url in bs.fetch("a", {"href":re.compile("http://epguides.com/")}):
        url = url['href']

        # only add serie summary pages (don't end with .html)
        if url.endswith("/"):
            results.append(url)

    if not results: return None

    # lessee if we are smarter than the search engine (look for an exact match):
    # mangle the searchterm string
    term = searchterms.replace(" ", "").lower()

    for url in results:
        seriename = urlparse.urlparse(url)[2].replace("/", "").lower()
        if term == seriename:
            return url

    # we weren't smarter, so just return what the search engine thought was best
    return results[0]

def _get_seriedata(url):
    """Get serie name and all episodes from the given url

    @return serie name, episode data, last update day"""

    epRe=re.compile("(?P<no>[\d]+)\.[\s]+(?P<season>[\d]+)-[ ]?(?P<episode>[\d]+)[\s]+(?P<prodno>[\w-]+)?[\s]+(?P<date>[\d]{1,2} [\w]{3} [\d]{2})[\s]+<[^>]+>(?P<name>.*)")

    filedata = getUrl(url).getContent()

    # tidy up the crappy html a bit
    r=[];
    for x in [x.split('\n') for x in filedata.split('\r')]:
        if x:
            r += x

    filedata = r
    r = []
    for line in filedata:
        if line:
            r.append(line.strip())

    filedata = r

    epdata = []

    seriename = getUrl(url).getBS().first("h1").renderContents()
    seriename = re.sub("<.*?>", "", seriename)

    for line in filedata:
        ###
        # find episodes
        ###
        m = epRe.match(line)
        if m:

            # convert datestring to gmtime format
            t = time.strptime(m.group('date'), '%d %b %y')

            # put episode data into a nifty dict
            data = {'epno'    : m.group('no'),
                    'season'  : int(m.group('season')),
                    'episode' : int(m.group('episode')),
                    'prodno'  : m.group('prodno'),
                    'airdate' : m.group('date'),
                    'airdate2': datetime.date(t[0], t[1], t[2]),
                    'epname'  : m.group('name')}

            epdata.append(data)

    return seriename, epdata, datetime.date.today()
<commit_msg>The Google search should be much more accurate now.<commit_after>import time
import datetime
import urllib
import urlparse
import re
import sqlite

# create table series (id INTEGER PRIMARY KEY, uid TEXT UNIQUE, serie TEXT, season INTEGER, episode INTEGER, title TEXT, airdate DATE);
# insert into series values(null, 'Alias', 1, 1, 'Pilot', date('2006-01-01'));
# select * from series where airdate = date('2006-01-02', '-1 day');
# select * from series where airdate = date('now', '-1 day');

precache = ['NCIS', 'Boston Legal', 'battlestar galactica', 'veronica mars',
'smallville', '24', 'lost', 'stargate sg-1', 'stargate atlantis', 'viva la bam', 'bullshit',
'4400', 'mythbusters', 'house', 'last comic standing', 'deadwood', 'shield', 'blade -flashing',
'the unit', 'prison break', 'bones', 'the simpsons', 'survivor']

CREATETABLE = "CREATE TABLE series (id INTEGER PRIMARY KEY, uid TEXT UNIQUE ON CONFLICT REPLACE, serie TEXT, season INTEGER, episode INTEGER, title TEXT, airdate DATE);"

def command_sqlcache(bot, user, channel, args):
    """Cache common series into the database"""
    if not isAdmin(user): return

    for seriename in precache:
        url = _search_serie(seriename)
        if url != None:
            _cache_serie(url)
    bot.say(channel, "Caching done")

def NOTinit():
    con = sqlite.connect("/home/shrike/pyfibot/modules/series.db");
    cur = con.cursor()
    cur.execute(CREATETABLE)
    cur.close()
    con.commit()
    con.close()

def command_epdel(bot, user, channel, args):
    """Delete all data about a serie"""
    if not isAdmin(user): return

    con = sqlite.connect("/home/shrike/pyfibot/modules/series.db");
    cur = con.cursor()
    cur.execute("DELETE FROM series WHERE serie='%s';" % args)
    cur.close()
    con.commit()
    con.close()

    bot.say(channel, "Done");

def command_epinfo(bot, user, channel, args):
    """List series in the database"""

    con = sqlite.connect("/home/shrike/pyfibot/modules/series.db");
    cur = con.cursor()

    cur.execute("SELECT DISTINCT serie FROM series");

    res = []
    for serie in cur:
        res.append(serie[0])

    res.sort()
    bot.say(channel, "Known series ("+str(len(res))+"): " + ", ".join(res))

    cur.close()
    con.close()

def command_ep(bot, user, channel, args):
    """Usage: sqlep [today|yesterday|tomorrow] or [seriename]"""

    con = sqlite.connect("/home/shrike/pyfibot/modules/series.db");
    cur = con.cursor()

    if not args:
        bot.say(channel, "Usage: ep [today|yesterday|tomorrow] or [seriename]")
        return

    if args == "today":
        cur.execute("SELECT * FROM series WHERE airdate = date('now', 'localtime');")
        if cur.rowcount == 0:
            bot.say(channel, "No known releases today")
            return
    elif args == "yesterday":
        cur.execute("SELECT * FROM series WHERE airdate = date('now', 'localtime', '-1 day');")
        if cur.rowcount == 0:
            bot.say(channel, "No known releases yesterday")
            return
    elif args == "tomorrow":
        cur.execute("SELECT * FROM series WHERE airdate = date('now', 'localtime', '+1 day');")
        if cur.rowcount == 0:
            bot.say(channel, "No known releases tomorrow")
            return
    else:
        # try to find the serie
        cur.execute("SELECT * FROM series WHERE serie LIKE %s AND airdate >= date('now', 'localtime') LIMIT 1", ("%"+args+"%",))
        # nothing found, get more data from the web
        if cur.rowcount == 0:
            url = _search_serie(args)
            if url != None:
                _cache_serie(url)
                cur.execute("SELECT * FROM series WHERE serie LIKE %s AND airdate >= date('now', 'localtime') LIMIT 1", ("%"+args+"%",))
                # still nothing found, give up
                if cur.rowcount == 0:
                    bot.say(channel, "No future episodes of '%s' found" % args)
                    return

    episodes = []
    # go through the results
    for (idno, uid, serie, season, episode, title, airdate) in cur:
        if episode < 10: episode = "0%d" % episode # pad ep with zeroes

        # YYYY-MM-DD -> datetime -> timedelta
        t = time.strptime(airdate, "%Y-%m-%d")
        ad = datetime.date(t.tm_year, t.tm_mon, t.tm_mday)
        now = datetime.date.today()
        tomorrow = now + datetime.timedelta(days=1)
        td = ad-now

        # change 0 and 1 to today & tomorrow, don't show date if we're asking stuff for a certain day
        airdatestr = ""
        if td.days > 0:
            if ad == now:
                if args != "today": airdatestr = "on %s (Today)" % airdate
            elif ad == tomorrow:
                if args != "tomorrow": airdatestr = "on %s (Tomorrow)" % airdate
            else:
                airdatestr = "on %s (%d days)" % (airdate, td.days)

        episodes.append("%s %sx%s '%s' %s" % (serie, season, episode, title, airdatestr))

    bot.say(channel, "-- ".join(episodes))

    cur.close()
    con.close()

def _cache_serie(url):
    """Parse the seriedata from the internets into sqlite"""

    serie, epdata, updatedate = _get_seriedata(url)
    con = sqlite.connect("/home/shrike/pyfibot/modules/series.db");

    for episode in epdata:
        episode['epname'] = episode['epname'].replace("'", "pier")
        cur = con.cursor()
        uid = "%s%d%d" % (serie, episode['season'], episode['episode'])
        cur.execute("INSERT INTO series (uid, serie, season, episode, title, airdate) VALUES (%s, %s, %d, %d, %s, date(%s));", (uid, serie, episode['season'], episode['episode'], episode['epname'], episode['airdate2'].strftime("%Y-%m-%d")))
        cur.close()

    # Commit & close
    con.commit()
    con.close()

def _search_serie(searchterms):
    """Search for a serie and return its episode list page URL"""

    if not searchterms: return

    # google power!
    # Try to find the exact page by adding "(a Titles and Air Dates Guide)" to the search
    url = "http://www.google.com/search?hl=en&q=site:epguides.com+%s+%%28a+Titles+and+Air+Dates+Guide%%29"
    searchterms = urllib.quote(searchterms)
    bs = getUrl(url % searchterms).getBS()

    if not bs: return

    results = []

    # tidy up the search results
    for url in bs.fetch("a", {"href":re.compile("http://epguides.com/")}):
        url = url['href']

        # only add serie summary pages (don't end with .html)
        if url.endswith("/"):
            results.append(url)

    if not results: return None

    # The first result is the correct one
    return results[0]

def _get_seriedata(url):
    """Get serie name and all episodes from the given url

    @return serie name, episode data, last update day"""

    epRe=re.compile("(?P<no>[\d]+)\.[\s]+(?P<season>[\d]+)-[ ]?(?P<episode>[\d]+)[\s]+(?P<prodno>[\w-]+)?[\s]+(?P<date>[\d]{1,2} [\w]{3} [\d]{2})[\s]+<[^>]+>(?P<name>.*)")

    filedata = getUrl(url).getContent()

    # tidy up the crappy html a bit
    r=[];
    for x in [x.split('\n') for x in filedata.split('\r')]:
        if x:
            r += x

    filedata = r
    r = []
    for line in filedata:
        if line:
            r.append(line.strip())

    filedata = r

    epdata = []

    seriename = getUrl(url).getBS().first("h1").renderContents()
    seriename = re.sub("<.*?>", "", seriename)

    for line in filedata:
        ###
        # find episodes
        ###
        m = epRe.match(line)
        if m:

            # convert datestring to gmtime format
            t = time.strptime(m.group('date'), '%d %b %y')

            # put episode data into a nifty dict
            data = {'epno'    : m.group('no'),
                    'season'  : int(m.group('season')),
                    'episode' : int(m.group('episode')),
                    'prodno'  : m.group('prodno'),
                    'airdate' : m.group('date'),
                    'airdate2': datetime.date(t[0], t[1], t[2]),
                    'epname'  : m.group('name')}

            epdata.append(data)

    return seriename, epdata, datetime.date.today()
<|endoftext|>


<commit_before>import csv
import os

#return: dict, key = uniq id
#val: dict, key = column name, val = val
#example: dict: {2066053: {'affiliation': 'KAIST', 'name': 'myname'}}
def load_single_file(input_file):
	with open(input_file, 'r', encoding='utf-8') as read_file:
		reader = csv.reader(read_file)
		column = reader.__next__()
		column = [x.strip().lower() for x in column]

		id_table = dict()

		for line in reader:
			if(len(line) == 0):
				continue
			val = dict()
			key = None
			prep_key = []
			for (col, item) in zip(column, line):
				if('id' in col):
					prep_key.append(int(item))
					continue
				val[col] = item
			if(len(prep_key) == 1):
				key = prep_key[0]
			else:
				key = tuple(prep_key)
			id_table[key] = val
		return id_table

def load_all(directory = '.', file_list = None):
	return_data = dict()
	if file_list == None:
		file_list = ['Author', 'Conference', 'Journal', 'PaperAuthor', 'Test', 'Train', 'Valid', 'ValidSolution']
	for name in file_list:
		file_name = os.path.join(directory, '{}.csv'.format(name))
		return_data[name] = load_single_file(file_name)
	return return_data

load_all()<commit_msg>remove debug stmt<commit_after>import csv
import os

#return: dict, key = uniq id
#val: dict, key = column name, val = val
#example: dict: {2066053: {'affiliation': 'KAIST', 'name': 'myname'}}
def load_single_file(input_file):
	with open(input_file, 'r', encoding='utf-8') as read_file:
		reader = csv.reader(read_file)
		column = reader.__next__()
		column = [x.strip().lower() for x in column]

		id_table = dict()

		for line in reader:
			if(len(line) == 0):
				continue
			val = dict()
			key = None
			prep_key = []
			for (col, item) in zip(column, line):
				if('id' in col):
					prep_key.append(int(item))
					continue
				val[col] = item
			if(len(prep_key) == 1):
				key = prep_key[0]
			else:
				key = tuple(prep_key)
			id_table[key] = val
		return id_table

def load_all(directory = '.', file_list = None):
	return_data = dict()
	if file_list == None:
		file_list = ['Author', 'Conference', 'Journal', 'PaperAuthor', 'Test', 'Train', 'Valid', 'ValidSolution']
	for name in file_list:
		file_name = os.path.join(directory, '{}.csv'.format(name))
		return_data[name] = load_single_file(file_name)
	return return_data<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
The MIT License (MIT)

Copyright (c) 2013 Jeremy Gillick

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

import re
import os
import shutil
import yaml
import json
from LendingClubInvestor import util

class Settings():
    """
    Handles all the user and investment settings
    """

    logger = None
    investor = None

    # Files
    settings_dir = None  # The directory that holds all the settings files
    settings_file = 'settings.yaml'  # User settings
    investing_file = 'investing.json'  # Investment settings

    # Auth settings
    auth = {
        'email': False,
        'pass': False
    }

    # Defines the investment funding settings
    investing = {
        'minCash': 500,
        'minPercent': False,
        'maxPercent': False,
        'portfolio': False,
        'filters': False
    }

    # Default investment filters
    filters = {
        'exclude_existing': True,
        'term36month': True,
        'term60month': True,
        'funding_progress': 0,
        'grades': {
            'All': True,
            'A': True,
            'B': True,
            'C': True,
            'D': True,
            'E': True,
            'F': True,
            'G': True
        }
    }

    # Default user settings
    user_settings = {
        'frequency': 60,
        'alert': False
    }

    def __init__(self, settings_dir=None, logger=False):
        """
        settings_dir is the directory that will be used to
        save the user and investment settings files
        """
        self.settings_dir = settings_dir

        # Create logger if none was passed in
        if not logger:
            self.logger = util.create_logger(False)
        else:
            self.logger = logger

        # Create the settings directory, if it doesn't exist
        if self.settings_dir and not os.path.exists(self.settings_dir):
            os.mkdir(self.settings_dir)

        self.get_user_settings()
        self.load_investment_settings_file()

    def __getitem__(self, arg):
        """
        Attempts to a value from one of the dictionaries
        """
        if arg in self.investing:
            return self.investing[arg]
        if arg in self.user_settings:
            return self.user_settings[arg]
        if arg in self.auth:
            return self.auth[arg]
        return None

    def get_user_settings(self):
        """
        Load the settings.yaml file into memory.
        If this file doesn't exist, create in
        """
        if not self.settings_dir:
            return

        file_path = os.path.join(self.settings_dir, self.settings_file)

        # Create the file, if it doesn't exist
        if not os.path.exists(file_path):
            this_path = os.path.dirname(os.path.realpath(__file__))
            default_file = os.path.join(this_path, 'settings.yaml')
            shutil.copy2(default_file, file_path)

        # Read file
        self.user_settings = yaml.load(open(file_path).read())
        return self.user_settings

    def save(self):
        """
        Save the investment settings dict to a file
        """
        if not self.settings_dir:
            return

        try:
            # Convert to JSON (and add email)
            to_save = self.investing.copy()
            to_save['email'] = self.auth['email']
            json_out = json.dumps(to_save)

            # Save
            self.logger.debug('Saving investment settings file: {0}'.format(json_out))
            investing_file = os.path.join(self.settings_dir, self.investing_file)
            f = open(investing_file, 'w')
            f.write(json_out)
            f.close()

            self.logger.debug('Saved')
        except Exception as e:
            self.logger.warning('Could not save the investment settings to file: {0}'.format(str(e)))

    def load_investment_settings_file(self):
        """
        Returned the saved settings used last time this program was run
        """
        if not self.settings_dir:
            return

        investing_file = os.path.join(self.settings_dir, self.investing_file)
        if os.path.exists(investing_file):
            self.logger.debug('Loading saved investment settings file')
            try:
                # Read file
                f = open(investing_file, 'r')
                jsonStr = f.read()
                f.close()

                self.logger.debug('Saved investment settings: {0}'.format(jsonStr))

                # Convert JSON to dictionary
                saved_settings = json.loads(jsonStr)

                # Add values to dictionary
                for key, value in self.investing.iteritems():
                    if key in saved_settings:
                        self.investing[key] = saved_settings[key]

                # Add email to auth
                if 'email' in saved_settings:
                    self.auth['email'] = saved_settings['email']

                # Migrations
                if '36month' in self.investing['filters']:
                    self.investing['filters']['term36month'] = self.investing['filters']['36month']
                    del self.investing['filters']['36month']

                if '60month' in self.investing['filters']:
                    self.investing['filters']['term60month'] = self.investing['filters']['60month']
                    del self.investing['filters']['60month']
                if 'funding_progress' not in self.investing['filters']:
                    self.investing['filters']['funding_progress'] = 0

            except Exception as e:
                self.logger.debug('Could not read investment settings file: {0}'.format(str(e)))
        else:
            self.logger.debug('No saved investment settings file to load')

    def get_auth_settings(self):
        """
        Get the email and password from the user
        """
        self.auth['email'] = util.prompt('LendingClub email', self.auth['email'])
        self.auth['pass'] = util.get_password()
        return self.auth

    def show_summary(self, title='Summary'):
        """
        Show a summary of the settings that will be used for auto investing
        """

        print '\n========= {0} ========='.format(title)
        print 'Invest ALL available funds with the following criteria\n'
        print 'With at LEAST ${0} available to invest'.format(self.investing['minCash'])
        print 'Select a portfolio with an average interest rate between {0}% - {1}%'.format(self.investing['minPercent'], self.investing['maxPercent'])

        if self.investing['portfolio']:
            print 'Add investments to: "{0}"'.format(self.investing['portfolio'])

        # Filters
        if self.investing['filters'] is not False:
            print '\nAdvanced filters:'

            # Exclude existing
            if self.investing['filters']['exclude_existing']:
                print '  + Exclude loans already invested in'

            # Funding progress
            if self.investing['filters']['funding_progress'] > 0:
                print '  + Include loans which are at least {0}% funded'.format(self.investing['filters']['funding_progress'])

            # Loan term
            terms = []
            if 'term36month' in self.investing['filters'] and self.investing['filters']['term36month']:
                terms.append('36')
            if 'term60month' in self.investing['filters'] and self.investing['filters']['term60month']:
                terms.append('60')
            print '  + Term: {0} months'.format(' & '.join(terms))

            # Interest rate grades
            if self.investing['filters']['grades']['All']:
                print '   + All interest rate grades'
            else:
                grades = []
                for grade in self.investing['filters']['grades']:
                    if grade != 'All' and self.investing['filters']['grades'][grade] is True:
                        grades.append(grade)
                print '  + Interest rates grades: {0}'.format(', '.join(sorted(grades)))

        print '=========={0}==========\n'.format(''.center(len(title), '='))

    def get_investment_settings(self):
        """
        Display a series of prompts to get how the user wants to invest their available cash.
        This fills out the investing dictionary.
        """

         # Minimum cash
        print '---------'
        print 'The auto investor will automatically try to invest ALL available cash into a diversified portfolio'
        while(True):
            self.investing['minCash'] = util.prompt_int('What\'s the MINIMUM amount of cash you want to invest each round?', self.investing['minCash'])
            if self.investing['minCash'] < 25:
                print '\nYou cannot invest less than $25. Please try again.'
            else:
                break

        # Min/max percent
        print '---------'
        while(True):
            print 'When auto investing, the LendingClub API will provide us a list of possible investment portfolios available at that moment.'
            print 'To pick the appropriate one for you, it needs to know what the minimum and maximum AVERAGE interest rate value you will accept.'
            print 'The investment option closest to the maximum value will be chosen and all your available cash will be invested in it.\n'

            self.investing['minPercent'] = util.prompt_float('What\'s MININUM average interest rate portfolio that you will accept?', self.investing['minPercent'])

            # Max percent should default to being larger than the min percent
            if self.investing['maxPercent'] is False or self.investing['maxPercent'] < self.investing['minPercent']:
                self.investing['maxPercent'] = self.investing['minPercent'] + 1
            self.investing['maxPercent'] = util.prompt_float('What\'s MAXIMUM average interest rate portfolio that you will accept?', self.investing['maxPercent'])

            # Validation
            if self.investing['maxPercent'] < self.investing['minPercent']:
                print 'The maximum value must be larger than, or equal to, the minimum value. Please try again.'
            elif self.investing['maxPercent'] == self.investing['minPercent']:
                print 'It\'s very uncommon to find an available portfolio that will match an exact percent.'
                if not util.prompt_yn('Would you like to specify a broader range?'):
                    break
            else:
                break

        # Portfolio
        print '---------'
        folioOption = False
        if self.investing['portfolio']:  # if saved settings has a portfolio set, default the prompt to 'Y' to choose
            folioOption = 'y'

        if util.prompt_yn('Do you want to put your new investments into a named portfolio?', folioOption):
            self.investing['portfolio'] = self.portfolio_picker(self.investing['portfolio'])
        else:
            self.investing['portfolio'] = False

        # Advanced filter settings
        print '\n---------'
        if util.prompt_yn('Would you like to set advanced filter settings?', self.investing['filters'] is not False):
            self.get_filter_settings()

        # Review summary
        self.show_summary()
        if util.prompt_yn('Would you like to continue with these settings?', 'y'):
            self.save()
        else:
            self.get_investment_settings()

        return True

    def get_filter_settings(self):
        """
        Setup the advanced portfolio filters (terms, grades, existing notes, etc.)
        """
        filters = self.investing['filters']
        if not filters:
            filters = self.filters

        print 'The following questions are from the filters section of the Invest page on LendingClub\n'

        # Existing loans
        filters['exclude_existing'] = util.prompt_yn('Exclude loans already invested in?', filters['exclude_existing'])

        # Funding progress rounded to the nearest tenth
        print '---------'
        print 'Funding progress'
        progress = util.prompt_float('Only include loans which already have at least __% funding (0 - 100)', filters['funding_progress'])
        filters['funding_progress'] = int(round(progress / 10) * 10)

        print '---------'
        print 'Choose term (36 - 60 month)'

        while(True):
            filters['term36month'] = util.prompt_yn('Include 36 month term loans?', filters['term36month'])
            filters['term60month'] = util.prompt_yn('Include 60 month term loans?', filters['term60month'])

            # Validate 1 was chosen
            if not filters['term36month'] and not filters['term60month']:
                print 'You have to AT LEAST choose one term length!'
            else:
                break

        print '---------'
        print 'Choose interest rate grades (7.4% - 24.84%)'
        while(True):
            if util.prompt_yn('Include ALL interest rate grades', filters['grades']['All']):
                filters['grades']['All'] = True
            else:
                filters['grades']['All'] = False
                filters['grades']['A'] = util.prompt_yn('A - ~7.41%', filters['grades']['A'])
                filters['grades']['B'] = util.prompt_yn('B - ~12.12%', filters['grades']['B'])
                filters['grades']['C'] = util.prompt_yn('C - ~15.80%', filters['grades']['C'])
                filters['grades']['D'] = util.prompt_yn('D - ~18.76%', filters['grades']['D'])
                filters['grades']['E'] = util.prompt_yn('E - ~21.49%', filters['grades']['E'])
                filters['grades']['F'] = util.prompt_yn('F - ~23.49%', filters['grades']['F'])
                filters['grades']['G'] = util.prompt_yn('G - ~24.84%', filters['grades']['G'])

            # Verify one was chosen
            gradeChosen = False
            for grade in filters['grades']:
                if filters['grades'][grade] is True:
                    gradeChosen = True
            if not gradeChosen:
                print 'You have to AT LEAST choose one interest rate grade!'
            else:
                break

        self.investing['filters'] = filters

    def portfolio_picker(self, default_folio=False):
        """
        Load existing portfolios and let the user choose one or create a new one
        default_folio is the name of the default portfolio, or the one that was used last time.
        """

        if not self.investor:
            return False

        print '\nPortfolios...'
        try:
            folios = self.investor.get_portfolio_list()

            # Add default portfolio to the list
            if default_folio is not False and default_folio not in folios:
                folios.append(default_folio)

            # Print out the portfolio list
            folios.sort()
            i = 1
            other_index = 0
            cancel_index = 0
            default_indicator = ''
            default_index = False
            for folio in folios:
                default_indicator = '  '
                if default_folio == folio:
                    default_indicator = '> '
                    default_index = str(i)

                print '{0}{1}: {2}'.format(default_indicator, i, folio)
                i += 1

            other_index = i
            print '  {0}: Other'.format(other_index)
            i += 1

            cancel_index = i
            print '  {0}: Cancel'.format(cancel_index)

            # Choose a portfolio
            while(True):
                choice = util.prompt('Choose one', default_index)

                # If no digit was chosen, ask again unless a default portfolio is present
                if not choice.isdigit():
                    if default_folio is not False:
                        return default_folio
                    else:
                        continue
                choice = int(choice)

                # No zero
                if choice == 0:
                    continue

                # Existing portfolio chosen
                if choice <= len(folios):
                    break

                # Other
                elif choice == other_index:
                    while(True):
                        other = util.prompt('Enter the name for your new portfolio')

                        # Empty string entered, show list again
                        if other.strip() == '':
                            break

                        # Invalid character
                        elif re.search('[^a-zA-Z0-9 ,_\-#\.]', other):
                            print 'The portfolio name \'{0}\' is not valid! Only alphanumeric, spaces , _ - # and . are allowed.'.format(other)

                        # Return custom portfolio name
                        else:
                            return other

                # Cancel
                else:
                    return False

            # Existing portfolio
            if choice < other_index:
                return folios[choice - 1]

        except Exception as e:
            self.logger.error(e)

<commit_msg>Update review output<commit_after>#!/usr/bin/env python

"""
The MIT License (MIT)

Copyright (c) 2013 Jeremy Gillick

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

import re
import os
import shutil
import yaml
import json
from LendingClubInvestor import util

class Settings():
    """
    Handles all the user and investment settings
    """

    logger = None
    investor = None

    # Files
    settings_dir = None  # The directory that holds all the settings files
    settings_file = 'settings.yaml'  # User settings
    investing_file = 'investing.json'  # Investment settings

    # Auth settings
    auth = {
        'email': False,
        'pass': False
    }

    # Defines the investment funding settings
    investing = {
        'minCash': 500,
        'minPercent': False,
        'maxPercent': False,
        'portfolio': False,
        'filters': False
    }

    # Default investment filters
    filters = {
        'exclude_existing': True,
        'term36month': True,
        'term60month': True,
        'funding_progress': 0,
        'grades': {
            'All': True,
            'A': True,
            'B': True,
            'C': True,
            'D': True,
            'E': True,
            'F': True,
            'G': True
        }
    }

    # Default user settings
    user_settings = {
        'frequency': 60,
        'alert': False
    }

    def __init__(self, settings_dir=None, logger=False):
        """
        settings_dir is the directory that will be used to
        save the user and investment settings files
        """
        self.settings_dir = settings_dir

        # Create logger if none was passed in
        if not logger:
            self.logger = util.create_logger(False)
        else:
            self.logger = logger

        # Create the settings directory, if it doesn't exist
        if self.settings_dir and not os.path.exists(self.settings_dir):
            os.mkdir(self.settings_dir)

        self.get_user_settings()
        self.load_investment_settings_file()

    def __getitem__(self, arg):
        """
        Attempts to a value from one of the dictionaries
        """
        if arg in self.investing:
            return self.investing[arg]
        if arg in self.user_settings:
            return self.user_settings[arg]
        if arg in self.auth:
            return self.auth[arg]
        return None

    def get_user_settings(self):
        """
        Load the settings.yaml file into memory.
        If this file doesn't exist, create in
        """
        if not self.settings_dir:
            return

        file_path = os.path.join(self.settings_dir, self.settings_file)

        # Create the file, if it doesn't exist
        if not os.path.exists(file_path):
            this_path = os.path.dirname(os.path.realpath(__file__))
            default_file = os.path.join(this_path, 'settings.yaml')
            shutil.copy2(default_file, file_path)

        # Read file
        self.user_settings = yaml.load(open(file_path).read())
        return self.user_settings

    def save(self):
        """
        Save the investment settings dict to a file
        """
        if not self.settings_dir:
            return

        try:
            # Convert to JSON (and add email)
            to_save = self.investing.copy()
            to_save['email'] = self.auth['email']
            json_out = json.dumps(to_save)

            # Save
            self.logger.debug('Saving investment settings file: {0}'.format(json_out))
            investing_file = os.path.join(self.settings_dir, self.investing_file)
            f = open(investing_file, 'w')
            f.write(json_out)
            f.close()

            self.logger.debug('Saved')
        except Exception as e:
            self.logger.warning('Could not save the investment settings to file: {0}'.format(str(e)))

    def load_investment_settings_file(self):
        """
        Returned the saved settings used last time this program was run
        """
        if not self.settings_dir:
            return

        investing_file = os.path.join(self.settings_dir, self.investing_file)
        if os.path.exists(investing_file):
            self.logger.debug('Loading saved investment settings file')
            try:
                # Read file
                f = open(investing_file, 'r')
                jsonStr = f.read()
                f.close()

                self.logger.debug('Saved investment settings: {0}'.format(jsonStr))

                # Convert JSON to dictionary
                saved_settings = json.loads(jsonStr)

                # Add values to dictionary
                for key, value in self.investing.iteritems():
                    if key in saved_settings:
                        self.investing[key] = saved_settings[key]

                # Add email to auth
                if 'email' in saved_settings:
                    self.auth['email'] = saved_settings['email']

                # Migrations
                if '36month' in self.investing['filters']:
                    self.investing['filters']['term36month'] = self.investing['filters']['36month']
                    del self.investing['filters']['36month']

                if '60month' in self.investing['filters']:
                    self.investing['filters']['term60month'] = self.investing['filters']['60month']
                    del self.investing['filters']['60month']
                if 'funding_progress' not in self.investing['filters']:
                    self.investing['filters']['funding_progress'] = 0

            except Exception as e:
                self.logger.debug('Could not read investment settings file: {0}'.format(str(e)))
        else:
            self.logger.debug('No saved investment settings file to load')

    def get_auth_settings(self):
        """
        Get the email and password from the user
        """
        self.auth['email'] = util.prompt('LendingClub email', self.auth['email'])
        self.auth['pass'] = util.get_password()
        return self.auth

    def show_summary(self, title='Summary'):
        """
        Show a summary of the settings that will be used for auto investing
        """

        print '\n========= {0} ========='.format(title)
        print 'Invest ALL available funds with the following criteria\n'
        print 'With at LEAST ${0} available to invest'.format(self.investing['minCash'])
        print 'Select a portfolio with an average interest rate between {0}% - {1}%'.format(self.investing['minPercent'], self.investing['maxPercent'])

        if self.investing['portfolio']:
            print 'Add investments to: "{0}"'.format(self.investing['portfolio'])

        # Filters
        if self.investing['filters'] is not False:
            print '\nAdvanced filters:'

            # Exclude existing
            if self.investing['filters']['exclude_existing']:
                print '  + Exclude loans already invested in'

            # Funding progress
            if self.investing['filters']['funding_progress'] > 0:
                print '  + Only loans which are at least {0}% funded'.format(self.investing['filters']['funding_progress'])

            # Loan term
            terms = []
            if 'term36month' in self.investing['filters'] and self.investing['filters']['term36month']:
                terms.append('36')
            if 'term60month' in self.investing['filters'] and self.investing['filters']['term60month']:
                terms.append('60')
            print '  + Term: {0} months'.format(' & '.join(terms))

            # Interest rate grades
            if self.investing['filters']['grades']['All']:
                print '   + All interest rate grades'
            else:
                grades = []
                for grade in self.investing['filters']['grades']:
                    if grade != 'All' and self.investing['filters']['grades'][grade] is True:
                        grades.append(grade)
                print '  + Interest rates grades: {0}'.format(', '.join(sorted(grades)))

        print '=========={0}==========\n'.format(''.center(len(title), '='))

    def get_investment_settings(self):
        """
        Display a series of prompts to get how the user wants to invest their available cash.
        This fills out the investing dictionary.
        """

         # Minimum cash
        print '---------'
        print 'The auto investor will automatically try to invest ALL available cash into a diversified portfolio'
        while(True):
            self.investing['minCash'] = util.prompt_int('What\'s the MINIMUM amount of cash you want to invest each round?', self.investing['minCash'])
            if self.investing['minCash'] < 25:
                print '\nYou cannot invest less than $25. Please try again.'
            else:
                break

        # Min/max percent
        print '---------'
        while(True):
            print 'When auto investing, the LendingClub API will provide us a list of possible investment portfolios available at that moment.'
            print 'To pick the appropriate one for you, it needs to know what the minimum and maximum AVERAGE interest rate value you will accept.'
            print 'The investment option closest to the maximum value will be chosen and all your available cash will be invested in it.\n'

            self.investing['minPercent'] = util.prompt_float('What\'s MININUM average interest rate portfolio that you will accept?', self.investing['minPercent'])

            # Max percent should default to being larger than the min percent
            if self.investing['maxPercent'] is False or self.investing['maxPercent'] < self.investing['minPercent']:
                self.investing['maxPercent'] = self.investing['minPercent'] + 1
            self.investing['maxPercent'] = util.prompt_float('What\'s MAXIMUM average interest rate portfolio that you will accept?', self.investing['maxPercent'])

            # Validation
            if self.investing['maxPercent'] < self.investing['minPercent']:
                print 'The maximum value must be larger than, or equal to, the minimum value. Please try again.'
            elif self.investing['maxPercent'] == self.investing['minPercent']:
                print 'It\'s very uncommon to find an available portfolio that will match an exact percent.'
                if not util.prompt_yn('Would you like to specify a broader range?'):
                    break
            else:
                break

        # Portfolio
        print '---------'
        folioOption = False
        if self.investing['portfolio']:  # if saved settings has a portfolio set, default the prompt to 'Y' to choose
            folioOption = 'y'

        if util.prompt_yn('Do you want to put your new investments into a named portfolio?', folioOption):
            self.investing['portfolio'] = self.portfolio_picker(self.investing['portfolio'])
        else:
            self.investing['portfolio'] = False

        # Advanced filter settings
        print '\n---------'
        if util.prompt_yn('Would you like to set advanced filter settings?', self.investing['filters'] is not False):
            self.get_filter_settings()

        # Review summary
        self.show_summary()
        if util.prompt_yn('Would you like to continue with these settings?', 'y'):
            self.save()
        else:
            self.get_investment_settings()

        return True

    def get_filter_settings(self):
        """
        Setup the advanced portfolio filters (terms, grades, existing notes, etc.)
        """
        filters = self.investing['filters']
        if not filters:
            filters = self.filters

        print 'The following questions are from the filters section of the Invest page on LendingClub\n'

        # Existing loans
        filters['exclude_existing'] = util.prompt_yn('Exclude loans already invested in?', filters['exclude_existing'])

        # Funding progress rounded to the nearest tenth
        print '---------'
        print 'Funding progress'
        progress = util.prompt_float('Only include loans which already have at least __% funding (0 - 100)', filters['funding_progress'])
        filters['funding_progress'] = int(round(progress / 10) * 10)

        print '---------'
        print 'Choose term (36 - 60 month)'

        while(True):
            filters['term36month'] = util.prompt_yn('Include 36 month term loans?', filters['term36month'])
            filters['term60month'] = util.prompt_yn('Include 60 month term loans?', filters['term60month'])

            # Validate 1 was chosen
            if not filters['term36month'] and not filters['term60month']:
                print 'You have to AT LEAST choose one term length!'
            else:
                break

        print '---------'
        print 'Choose interest rate grades (7.4% - 24.84%)'
        while(True):
            if util.prompt_yn('Include ALL interest rate grades', filters['grades']['All']):
                filters['grades']['All'] = True
            else:
                filters['grades']['All'] = False
                filters['grades']['A'] = util.prompt_yn('A - ~7.41%', filters['grades']['A'])
                filters['grades']['B'] = util.prompt_yn('B - ~12.12%', filters['grades']['B'])
                filters['grades']['C'] = util.prompt_yn('C - ~15.80%', filters['grades']['C'])
                filters['grades']['D'] = util.prompt_yn('D - ~18.76%', filters['grades']['D'])
                filters['grades']['E'] = util.prompt_yn('E - ~21.49%', filters['grades']['E'])
                filters['grades']['F'] = util.prompt_yn('F - ~23.49%', filters['grades']['F'])
                filters['grades']['G'] = util.prompt_yn('G - ~24.84%', filters['grades']['G'])

            # Verify one was chosen
            gradeChosen = False
            for grade in filters['grades']:
                if filters['grades'][grade] is True:
                    gradeChosen = True
            if not gradeChosen:
                print 'You have to AT LEAST choose one interest rate grade!'
            else:
                break

        self.investing['filters'] = filters

    def portfolio_picker(self, default_folio=False):
        """
        Load existing portfolios and let the user choose one or create a new one
        default_folio is the name of the default portfolio, or the one that was used last time.
        """

        if not self.investor:
            return False

        print '\nPortfolios...'
        try:
            folios = self.investor.get_portfolio_list()

            # Add default portfolio to the list
            if default_folio is not False and default_folio not in folios:
                folios.append(default_folio)

            # Print out the portfolio list
            folios.sort()
            i = 1
            other_index = 0
            cancel_index = 0
            default_indicator = ''
            default_index = False
            for folio in folios:
                default_indicator = '  '
                if default_folio == folio:
                    default_indicator = '> '
                    default_index = str(i)

                print '{0}{1}: {2}'.format(default_indicator, i, folio)
                i += 1

            other_index = i
            print '  {0}: Other'.format(other_index)
            i += 1

            cancel_index = i
            print '  {0}: Cancel'.format(cancel_index)

            # Choose a portfolio
            while(True):
                choice = util.prompt('Choose one', default_index)

                # If no digit was chosen, ask again unless a default portfolio is present
                if not choice.isdigit():
                    if default_folio is not False:
                        return default_folio
                    else:
                        continue
                choice = int(choice)

                # No zero
                if choice == 0:
                    continue

                # Existing portfolio chosen
                if choice <= len(folios):
                    break

                # Other
                elif choice == other_index:
                    while(True):
                        other = util.prompt('Enter the name for your new portfolio')

                        # Empty string entered, show list again
                        if other.strip() == '':
                            break

                        # Invalid character
                        elif re.search('[^a-zA-Z0-9 ,_\-#\.]', other):
                            print 'The portfolio name \'{0}\' is not valid! Only alphanumeric, spaces , _ - # and . are allowed.'.format(other)

                        # Return custom portfolio name
                        else:
                            return other

                # Cancel
                else:
                    return False

            # Existing portfolio
            if choice < other_index:
                return folios[choice - 1]

        except Exception as e:
            self.logger.error(e)

<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
#
# Here is implemented the preApplication and postApplication method
# along the Asynchronous event schedule

import os

from twisted.scripts._twistd_unix import UnixApplicationRunner
from twisted.internet import reactor, defer
from twisted.python.util import untilConcludes

from globaleaks.db import init_db, clean_untracked_files, \
    refresh_memory_variables, update_version

from globaleaks.db.appdata import update_appdata

from globaleaks.jobs import session_management_sched, statistics_sched, \
    notification_sched, delivery_sched, cleaning_sched, \
    pgp_check_sched

from globaleaks.settings import GLSettings
from globaleaks.utils.utility import log, datetime_now, disable_swap

test_reactor = None

class GlobaLeaksRunner(UnixApplicationRunner):
    """
    This runner is specific to Unix systems.
    """
    _reactor = reactor

    def start_asynchronous_jobs(self):
        """
        Initialize the asynchronous operation, scheduled in the system
        """
        if test_reactor:
            self._reactor = test_reactor

        # Scheduling the Delivery schedule to be executed every 2 seconds
        delivery_sched.DeliverySchedule().schedule(2, 1)

        # Scheduling the Anomalies Check schedule to be executed every 30 seconds
        statistics_sched.AnomaliesSchedule().schedule(30, 2)

        # Scheduling the Notification schedule to be executed every 60 seconds
        notification_sched.NotificationSchedule().schedule(60, 3)

        # Scheduling the Session Management schedule to be executed every minute
        session_management_sched.SessionManagementSchedule().schedule(60, 5)

        # Scheduling the Tip Cleaning scheduler to be executed every day at 00:00
        current_time = datetime_now()
        delay = (3600 * (24 + 0)) - (current_time.hour * 3600) - (current_time.minute * 60) - current_time.second
        clean = cleaning_sched.CleaningSchedule().schedule(3600 * 24, delay)

        # Scheduling the PGP Check scheduler to be executed every day at 02:00
        delay = (3600 * (24 + 2)) - (current_time.hour * 3600) - (current_time.minute * 60) - current_time.second
        pgp_check = pgp_check_sched.PGPCheckSchedule().schedule(3600 * 24, delay)

        # Scheduling the Statistics schedule to be executed every hour on the hour
        delay = (3600) - (current_time.minute * 60) - current_time.second
        stats = statistics_sched.StatisticsSchedule().schedule(3600, delay)

    @defer.inlineCallbacks
    def start_globaleaks(self):
        try:
            GLSettings.fix_file_permissions()
            GLSettings.drop_privileges()
            GLSettings.check_directories()

            if GLSettings.initialize_db:
                yield init_db()
            else:
                yield update_version()

            yield clean_untracked_files()

            yield refresh_memory_variables()

            self.start_asynchronous_jobs()

        except Exception as excep:
            log.err("ERROR: Cannot start GlobaLeaks; please manual check the error.")
            log.err("EXCEPTION: %s" % excep)
            self._reactor.stop()
<commit_msg>Fix against commit 8710a2ccca00b8860c25bb86e6746472b991f8cc<commit_after># -*- encoding: utf-8 -*-
#
# Here is implemented the preApplication and postApplication method
# along the Asynchronous event schedule

import os

from twisted.scripts._twistd_unix import UnixApplicationRunner
from twisted.internet import reactor, defer
from twisted.python.util import untilConcludes

from globaleaks.db import init_db, clean_untracked_files, \
    refresh_memory_variables, update_version

from globaleaks.db.appdata import update_appdata

from globaleaks.jobs import session_management_sched, statistics_sched, \
    notification_sched, delivery_sched, cleaning_sched, \
    pgp_check_sched

from globaleaks.settings import GLSettings
from globaleaks.utils.utility import log, datetime_now, disable_swap

test_reactor = None

class GlobaLeaksRunner(UnixApplicationRunner):
    """
    This runner is specific to Unix systems.
    """
    _reactor = reactor

    def start_asynchronous_jobs(self):
        """
        Initialize the asynchronous operation, scheduled in the system
        """
        if test_reactor:
            self._reactor = test_reactor

        # Scheduling the Delivery schedule to be executed every 2 seconds
        delivery_sched.DeliverySchedule().schedule(2, 1)

        # Scheduling the Anomalies Check schedule to be executed every 30 seconds
        statistics_sched.AnomaliesSchedule().schedule(30, 2)

        # Scheduling the Notification schedule to be executed every 60 seconds
        notification_sched.NotificationSchedule().schedule(60, 3)

        # Scheduling the Session Management schedule to be executed every minute
        session_management_sched.SessionManagementSchedule().schedule(60, 5)

        # Scheduling the Tip Cleaning scheduler to be executed every day at 00:00
        current_time = datetime_now()
        delay = (3600 * (24 + 0)) - (current_time.hour * 3600) - (current_time.minute * 60) - current_time.second
        clean = cleaning_sched.CleaningSchedule().schedule(3600 * 24, delay)

        # Scheduling the PGP Check scheduler to be executed every day at 02:00
        delay = (3600 * (24 + 2)) - (current_time.hour * 3600) - (current_time.minute * 60) - current_time.second
        pgp_check = pgp_check_sched.PGPCheckSchedule().schedule(3600 * 24, delay)

        # Scheduling the Statistics schedule to be executed every hour on the hour
        delay = (3600) - (current_time.minute * 60) - current_time.second
        stats = statistics_sched.StatisticsSchedule().schedule(3600, delay)

    @defer.inlineCallbacks
    def start_globaleaks(self):
        try:
            GLSettings.fix_file_permissions()
            GLSettings.drop_privileges()
            GLSettings.check_directories()

            if GLSettings.initialize_db:
                yield init_db()
            else:
                yield update_version()

            yield clean_untracked_files()

            yield refresh_memory_variables()

            self.start_asynchronous_jobs()

        except Exception as excep:
            log.err("ERROR: Cannot start GlobaLeaks; please manual check the error.")
            log.err("EXCEPTION: %s" % excep)
            self._reactor.stop()

    def postApplication(self):
        reactor.callLater(0, self.start_globaleaks)

        UnixApplicationRunner.postApplication(self)
<|endoftext|>