<commit_before># -*- coding: utf-8 -*-

import re
import utils
import fields
from copy import deepcopy
from lxml import etree
from unaccent import unaccent
from django.db import models
from django.contrib.admin.models import LogEntry
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes import generic
from django.core import urlresolvers
from django.core.exceptions import ValidationError, ObjectDoesNotExist
from django.core.urlresolvers import NoReverseMatch
from django.utils.html import conditional_escape
from django.utils.safestring import mark_safe

# -------------------------------------------------------------------------------
# Abstract base classes and interfaces.
# -------------------------------------------------------------------------------

class CreationMetadata(models.Model):
    creator = models.ForeignKey(User, editable=False, related_name='created_%(class)s_set')
    created = models.DateTimeField(auto_now_add=True)
    class Meta:
        abstract = True
        get_latest_by = 'created'

class LastUpdateMetadata(CreationMetadata):
    last_updater = models.ForeignKey(User, editable=False, related_name='last_to_update_%(class)s_set')
    last_updated = models.DateTimeField(auto_now=True)
    class Meta:
        abstract = True

class Administered():
    def get_admin_url(self):
        return urlresolvers.reverse(
            'admin:main_%s_change' % self._meta.module_name, args=(self.id,))

class URLAccessible():
    @models.permalink
    def get_absolute_url(self):
        return ('%s_view' % self._meta.module_name, [str(self.id)])
    def __unicode__(self):
        return utils.truncate(self.as_text())
    def as_text(self):
        raise Exception('Must implement %s.as_text()' % self._meta.module_name)
    def as_html(self):
        return '%s' % (
            self._meta.module_name, conditional_escape(self.as_text()))

# -------------------------------------------------------------------------------
# Primary models. These have their own URLs and administration interfaces.
# -------------------------------------------------------------------------------

class DocumentManager(models.Manager):
    # Include whether or not documents have scans/transcripts in default query.
    def get_query_set(self):
        return super(DocumentManager, self).get_query_set()\
            .select_related('_transcript')\
            .extra(select = { 'scan_count': '''SELECT COUNT(*)
FROM main_scan WHERE main_scan.document_id = main_document.id''',
                              '_has_transcript': '''EXISTS ( SELECT 1
FROM main_transcript WHERE main_transcript.document_id = main_document.id )''' })

class Document(LastUpdateMetadata, Administered, URLAccessible):
    u"""
    Anything that can be taken as evidence for (documentation of) something.

    Documents are best defined by example: letters, newspaper articles,
    contracts, photographs, database records, whole databases, etc. etc.
    """
    description = fields.XHTMLField()
    type = models.CharField(max_length=1, choices=(('P','primary source'),('S','secondary source')), default='S')
    ordering = models.CharField(max_length=32, editable=False)
    url = models.URLField(blank=True, verify_exists=True)
    objects = DocumentManager()
    @property
    def transcript(self):
        try:
            return self._transcript
        except Transcript.DoesNotExist:
            return None
    def get_scan_count(self):
        if hasattr(self, 'scan_count'):
            return self.scan_count
        return self.scans.count()
    def has_scans(self):
        return self.get_scan_count() > 0
    def has_transcript(self):
        if hasattr(self, '_has_transcript'):
            return self._has_transcript
        return self.transcript is not None
    def as_text(self):
        return utils.xhtml_to_text(self.description)
    def as_html(self):
        return mark_safe(
            '
%s
' % (
                (self.has_transcript() or self.has_scans())
                 and ' has-scans-or-transcript' or '',
                etree.tostring(self.description)))
    def save(self, *args, **kwargs):
        self.ordering = re.sub(r'[^\w\s]', '', utils.xhtml_to_text(self.description))[:32]
        super(Document, self).save(*args, **kwargs)
    class Meta:
        ordering = ['ordering']

class TranscriptManager(models.Manager):
    # Include related document in default query.
    def get_query_set(self):
        return super(TranscriptManager, self).get_query_set()\
            .select_related('document')

class Transcript(LastUpdateMetadata, Administered, URLAccessible):
    u"""
    A text transcript of a document.
    """
    document = models.OneToOneField(Document, related_name='_transcript', null=True)
    content = fields.XHTMLField()
    objects = TranscriptManager()
    def get_absolute_url(self):
        # Transcripts don't have their own URLs; use the document URL.
        return '%s#transcript' % self.document.get_absolute_url()
    def as_text(self):
        return self.document.as_text()
    def as_html(self):
        return self.document.as_html()

class Footnote(LastUpdateMetadata, Administered, URLAccessible):
    u"""
    A footnote attached to a transcript.
    """
    transcript = models.ForeignKey(Transcript, related_name='footnotes')
    content = fields.XHTMLField()
    def footnoted_text(self):
        try:
            selector = 'a.footnote[href="%s"]' % self.get_absolute_url()
            results = self.transcript.content.cssselect(selector)
            if len(results) == 1:
                return results[0].xpath('string()')
        except NoReverseMatch: # footnote has been deleted
            pass
        return None
    def as_text(self):
        footnoted_text = self.footnoted_text()
        if footnoted_text is None:
            return utils.xhtml_to_text(self.content)
        else:
            return u'"%s"' % footnoted_text
    def remove_self_from(self, transcript):
        selector = 'a.footnote[href="%s"]' % self.get_absolute_url()
        results = transcript.content.cssselect(selector)
        if len(results) == 1:
            results[0].drop_tag()
    def delete(self, *args, **kwargs):
        self.remove_self_from(self.transcript)
        self.transcript.save()
        super(Footnote, self).delete(*args, **kwargs)

class Topic(LastUpdateMetadata, Administered, URLAccessible):
    """
    A controlled topic such as a person name, an organization name, a
    place name, an event name, a publication name, or the name of a
    topic or theme.
    """
    preferred_name = models.CharField(max_length='80', unique=True)
    slug = models.CharField(max_length='80', unique=True, editable=False)
    related_topics = models.ManyToManyField('self', blank=True)
    summary = fields.XHTMLField(verbose_name='article')
    summary_citations = generic.GenericRelation('Citation')
    def __init__(self, *args, **kwargs):
        super(Topic, self).__init__(*args, **kwargs)
        if 'preferred_name' in kwargs:
            self.slug = self._make_slug(kwargs['preferred_name'])
    def _make_slug(self, preferred_name):
        return '-'.join(
            [ x for x in re.split('\W+', unaccent(unicode(preferred_name)))
              if len(x) > 0 ]).lower()
    def __setattr__(self, key, value):
        super(Topic, self).__setattr__(key, value)
        if key == 'preferred_name':
            self.slug = self._make_slug(value)
    def get_aliases(self):
        return u' '.join([ a.name for a in self.aliases.all() ])
    @models.permalink
    def get_absolute_url(self):
        return ('topic_view', [str(self.slug)])
    def as_text(self):
        return self.preferred_name
    def validate_unique(self, exclude=None):
        if 'slug' in exclude:
            exclude.remove('slug')
        try:
            super(Topic, self).validate_unique(exclude)
        except ValidationError as e:
            if ('slug' in e.message_dict and
                u'Topic with this Slug already exists.' in e.message_dict['slug']):
                e.message_dict['slug'].remove(
                    u'Topic with this Slug already exists.')
                if len(e.message_dict['slug']) == 0:
                    del e.message_dict['slug']
                if not ('preferred_name' in e.message_dict and
                        u'Topic with this Preferred name already exists.' in e.message_dict['preferred_name']):
                    if not 'preferred_name' in e.message_dict:
                        e.message_dict['preferred_name'] = []
                    e.message_dict['preferred_name'].append(u'Topic with a very similar Preferred name already exists.')
            raise e
    class Meta:
        ordering = ['slug']

class Note(LastUpdateMetadata, Administered, URLAccessible):
    u"""
    Text written by an editor or curator. The text is stored as XHTML,
    so it may have hyperlinks and all the other features that XHTML
    enables.
    """
    title = models.CharField(max_length='80', unique=True)
    content = fields.XHTMLField()
    topics = generic.GenericRelation('TopicAssignment')
    citations = generic.GenericRelation('Citation')
    def has_topic(self, topic):
        return topic.id in self.topics.values_list('topic_id', flat=True)
    def as_text(self):
        return self.title
    class Meta:
        ordering = ['-last_updated']

class UserProfile(models.Model, URLAccessible):
    user = models.ForeignKey(User, unique=True)
    def _get_display_name(self):
        "Returns the full name if available, or the username if not."
        display_name = self.user.username
        if self.user.first_name:
            display_name = self.user.first_name
            if self.user.last_name:
                display_name = '%s %s' % (self.user.first_name, self.user.last_name)
        return display_name
    display_name = property(_get_display_name)
    @models.permalink
    def get_absolute_url(self):
        return ('user_view', [str(self.user.username)])
    def as_text(self):
        return self.display_name
    @staticmethod
    def get_for(user):
        try:
            return user.get_profile()
        except UserProfile.DoesNotExist:
            return UserProfile.objects.create(user=user)
    @staticmethod
    def get_activity_for(user, max_count=50):
        object_urls = set()
        checked_object_ids = {
            'topic': [], 'note': [], 'document': [], 'transcript': [] }
        activity = []
        for entry in LogEntry.objects\
                .select_related('content_type__name')\
                .filter(content_type__app_label='main',
                        content_type__model__in=checked_object_ids.keys(),
                        user=user):
            if entry.object_id in checked_object_ids[entry.content_type.name]:
                continue
            checked_object_ids[entry.content_type.name].append(entry.object_id)
            if entry.is_deletion():
                continue
            try:
                obj = entry.get_edited_object()
            except ObjectDoesNotExist:
                continue
            object_url = obj.get_absolute_url().split('#')[0]
            if object_url in object_urls:
                continue
            object_urls.add(object_url)
            activity.append({ 'what': obj, 'when': entry.action_time })
            if len(activity) == max_count:
                break
        return activity, checked_object_ids

# -------------------------------------------------------------------------------
# Support models.
# -------------------------------------------------------------------------------

class Alias(CreationMetadata):
    u"""
    An alternate name for a topic.
    """
    topic = models.ForeignKey(Topic, related_name='aliases')
    name = models.CharField(max_length='80')
    def __unicode__(self):
        return self.name
    class Meta(CreationMetadata.Meta):
        unique_together = ('topic', 'name')
        verbose_name_plural = 'aliases'

class TopicAssignment(CreationMetadata):
    u"""
    An assignment of a topic to any other object.
    """
    topic = models.ForeignKey(Topic, related_name='assignments')
    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField()
    content_object = generic.GenericForeignKey()
    def __unicode__(self):
        return self.topic.preferred_name
    class Meta(CreationMetadata.Meta):
        unique_together = ('topic', 'object_id')

class Scan(CreationMetadata):
    u"""
    A scanned image of (part of) a dcument.
    """
    document = models.ForeignKey(Document, related_name='scans', null=True)
    image = models.ImageField(upload_to='scans/%Y/%m')
    ordering = models.IntegerField(blank=True, null=True)
    def __unicode__(self):
        return u'Scan for %s (order: %s)' % (self.document, self.ordering)
    class Meta:
        ordering = ['ordering']

class Citation(CreationMetadata):
    u"""
    A reference to or citation of a document.
    """
    document = models.ForeignKey(Document, related_name='citations', null=True)
    locator = models.CharField(max_length=16, blank=True)
    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField()
    content_object = generic.GenericForeignKey()
<commit_msg>Disallow NULLs.<commit_after># -*- coding: utf-8 -*-

import re
import utils
import fields
from copy import deepcopy
from lxml import etree
from unaccent import unaccent
from django.db import models
from django.contrib.admin.models import LogEntry
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes import generic
from django.core import urlresolvers
from django.core.exceptions import ValidationError, ObjectDoesNotExist
from django.core.urlresolvers import NoReverseMatch
from django.utils.html import conditional_escape
from django.utils.safestring import mark_safe

# -------------------------------------------------------------------------------
# Abstract base classes and interfaces.
# -------------------------------------------------------------------------------

class CreationMetadata(models.Model):
    creator = models.ForeignKey(User, editable=False, related_name='created_%(class)s_set')
    created = models.DateTimeField(auto_now_add=True)
    class Meta:
        abstract = True
        get_latest_by = 'created'

class LastUpdateMetadata(CreationMetadata):
    last_updater = models.ForeignKey(User, editable=False, related_name='last_to_update_%(class)s_set')
    last_updated = models.DateTimeField(auto_now=True)
    class Meta:
        abstract = True

class Administered():
    def get_admin_url(self):
        return urlresolvers.reverse(
            'admin:main_%s_change' % self._meta.module_name, args=(self.id,))

class URLAccessible():
    @models.permalink
    def get_absolute_url(self):
        return ('%s_view' % self._meta.module_name, [str(self.id)])
    def __unicode__(self):
        return utils.truncate(self.as_text())
    def as_text(self):
        raise Exception('Must implement %s.as_text()' % self._meta.module_name)
    def as_html(self):
        return '%s' % (
            self._meta.module_name, conditional_escape(self.as_text()))

# -------------------------------------------------------------------------------
# Primary models. These have their own URLs and administration interfaces.
# -------------------------------------------------------------------------------

class DocumentManager(models.Manager):
    # Include whether or not documents have scans/transcripts in default query.
    def get_query_set(self):
        return super(DocumentManager, self).get_query_set()\
            .select_related('_transcript')\
            .extra(select = { 'scan_count': '''SELECT COUNT(*)
FROM main_scan WHERE main_scan.document_id = main_document.id''',
                              '_has_transcript': '''EXISTS ( SELECT 1
FROM main_transcript WHERE main_transcript.document_id = main_document.id )''' })

class Document(LastUpdateMetadata, Administered, URLAccessible):
    u"""
    Anything that can be taken as evidence for (documentation of) something.

    Documents are best defined by example: letters, newspaper articles,
    contracts, photographs, database records, whole databases, etc. etc.
    """
    description = fields.XHTMLField()
    type = models.CharField(max_length=1, choices=(('P','primary source'),('S','secondary source')), default='S')
    ordering = models.CharField(max_length=32, editable=False)
    url = models.URLField(blank=True, verify_exists=True)
    objects = DocumentManager()
    @property
    def transcript(self):
        try:
            return self._transcript
        except Transcript.DoesNotExist:
            return None
    def get_scan_count(self):
        if hasattr(self, 'scan_count'):
            return self.scan_count
        return self.scans.count()
    def has_scans(self):
        return self.get_scan_count() > 0
    def has_transcript(self):
        if hasattr(self, '_has_transcript'):
            return self._has_transcript
        return self.transcript is not None
    def as_text(self):
        return utils.xhtml_to_text(self.description)
    def as_html(self):
        return mark_safe(
            '
%s
' % (
                (self.has_transcript() or self.has_scans())
                 and ' has-scans-or-transcript' or '',
                etree.tostring(self.description)))
    def save(self, *args, **kwargs):
        self.ordering = re.sub(r'[^\w\s]', '', utils.xhtml_to_text(self.description))[:32]
        super(Document, self).save(*args, **kwargs)
    class Meta:
        ordering = ['ordering']

class TranscriptManager(models.Manager):
    # Include related document in default query.
    def get_query_set(self):
        return super(TranscriptManager, self).get_query_set()\
            .select_related('document')

class Transcript(LastUpdateMetadata, Administered, URLAccessible):
    u"""
    A text transcript of a document.
    """
    document = models.OneToOneField(Document, related_name='_transcript')
    content = fields.XHTMLField()
    objects = TranscriptManager()
    def get_absolute_url(self):
        # Transcripts don't have their own URLs; use the document URL.
        return '%s#transcript' % self.document.get_absolute_url()
    def as_text(self):
        return self.document.as_text()
    def as_html(self):
        return self.document.as_html()

class Footnote(LastUpdateMetadata, Administered, URLAccessible):
    u"""
    A footnote attached to a transcript.
    """
    transcript = models.ForeignKey(Transcript, related_name='footnotes')
    content = fields.XHTMLField()
    def footnoted_text(self):
        try:
            selector = 'a.footnote[href="%s"]' % self.get_absolute_url()
            results = self.transcript.content.cssselect(selector)
            if len(results) == 1:
                return results[0].xpath('string()')
        except NoReverseMatch: # footnote has been deleted
            pass
        return None
    def as_text(self):
        footnoted_text = self.footnoted_text()
        if footnoted_text is None:
            return utils.xhtml_to_text(self.content)
        else:
            return u'"%s"' % footnoted_text
    def remove_self_from(self, transcript):
        selector = 'a.footnote[href="%s"]' % self.get_absolute_url()
        results = transcript.content.cssselect(selector)
        if len(results) == 1:
            results[0].drop_tag()
    def delete(self, *args, **kwargs):
        self.remove_self_from(self.transcript)
        self.transcript.save()
        super(Footnote, self).delete(*args, **kwargs)

class Topic(LastUpdateMetadata, Administered, URLAccessible):
    """
    A controlled topic such as a person name, an organization name, a
    place name, an event name, a publication name, or the name of a
    topic or theme.
    """
    preferred_name = models.CharField(max_length='80', unique=True)
    slug = models.CharField(max_length='80', unique=True, editable=False)
    related_topics = models.ManyToManyField('self', blank=True)
    summary = fields.XHTMLField(verbose_name='article')
    summary_citations = generic.GenericRelation('Citation')
    def __init__(self, *args, **kwargs):
        super(Topic, self).__init__(*args, **kwargs)
        if 'preferred_name' in kwargs:
            self.slug = self._make_slug(kwargs['preferred_name'])
    def _make_slug(self, preferred_name):
        return '-'.join(
            [ x for x in re.split('\W+', unaccent(unicode(preferred_name)))
              if len(x) > 0 ]).lower()
    def __setattr__(self, key, value):
        super(Topic, self).__setattr__(key, value)
        if key == 'preferred_name':
            self.slug = self._make_slug(value)
    def get_aliases(self):
        return u' '.join([ a.name for a in self.aliases.all() ])
    @models.permalink
    def get_absolute_url(self):
        return ('topic_view', [str(self.slug)])
    def as_text(self):
        return self.preferred_name
    def validate_unique(self, exclude=None):
        if 'slug' in exclude:
            exclude.remove('slug')
        try:
            super(Topic, self).validate_unique(exclude)
        except ValidationError as e:
            if ('slug' in e.message_dict and
                u'Topic with this Slug already exists.' in e.message_dict['slug']):
                e.message_dict['slug'].remove(
                    u'Topic with this Slug already exists.')
                if len(e.message_dict['slug']) == 0:
                    del e.message_dict['slug']
                if not ('preferred_name' in e.message_dict and
                        u'Topic with this Preferred name already exists.' in e.message_dict['preferred_name']):
                    if not 'preferred_name' in e.message_dict:
                        e.message_dict['preferred_name'] = []
                    e.message_dict['preferred_name'].append(u'Topic with a very similar Preferred name already exists.')
            raise e
    class Meta:
        ordering = ['slug']

class Note(LastUpdateMetadata, Administered, URLAccessible):
    u"""
    Text written by an editor or curator. The text is stored as XHTML,
    so it may have hyperlinks and all the other features that XHTML
    enables.
    """
    title = models.CharField(max_length='80', unique=True)
    content = fields.XHTMLField()
    topics = generic.GenericRelation('TopicAssignment')
    citations = generic.GenericRelation('Citation')
    def has_topic(self, topic):
        return topic.id in self.topics.values_list('topic_id', flat=True)
    def as_text(self):
        return self.title
    class Meta:
        ordering = ['-last_updated']

class UserProfile(models.Model, URLAccessible):
    user = models.ForeignKey(User, unique=True)
    def _get_display_name(self):
        "Returns the full name if available, or the username if not."
        display_name = self.user.username
        if self.user.first_name:
            display_name = self.user.first_name
            if self.user.last_name:
                display_name = '%s %s' % (self.user.first_name, self.user.last_name)
        return display_name
    display_name = property(_get_display_name)
    @models.permalink
    def get_absolute_url(self):
        return ('user_view', [str(self.user.username)])
    def as_text(self):
        return self.display_name
    @staticmethod
    def get_for(user):
        try:
            return user.get_profile()
        except UserProfile.DoesNotExist:
            return UserProfile.objects.create(user=user)
    @staticmethod
    def get_activity_for(user, max_count=50):
        object_urls = set()
        checked_object_ids = {
            'topic': [], 'note': [], 'document': [], 'transcript': [] }
        activity = []
        for entry in LogEntry.objects\
                .select_related('content_type__name')\
                .filter(content_type__app_label='main',
                        content_type__model__in=checked_object_ids.keys(),
                        user=user):
            if entry.object_id in checked_object_ids[entry.content_type.name]:
                continue
            checked_object_ids[entry.content_type.name].append(entry.object_id)
            if entry.is_deletion():
                continue
            try:
                obj = entry.get_edited_object()
            except ObjectDoesNotExist:
                continue
            object_url = obj.get_absolute_url().split('#')[0]
            if object_url in object_urls:
                continue
            object_urls.add(object_url)
            activity.append({ 'what': obj, 'when': entry.action_time })
            if len(activity) == max_count:
                break
        return activity, checked_object_ids

# -------------------------------------------------------------------------------
# Support models.
# -------------------------------------------------------------------------------

class Alias(CreationMetadata):
    u"""
    An alternate name for a topic.
    """
    topic = models.ForeignKey(Topic, related_name='aliases')
    name = models.CharField(max_length='80')
    def __unicode__(self):
        return self.name
    class Meta(CreationMetadata.Meta):
        unique_together = ('topic', 'name')
        verbose_name_plural = 'aliases'

class TopicAssignment(CreationMetadata):
    u"""
    An assignment of a topic to any other object.
    """
    topic = models.ForeignKey(Topic, related_name='assignments')
    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField()
    content_object = generic.GenericForeignKey()
    def __unicode__(self):
        return self.topic.preferred_name
    class Meta(CreationMetadata.Meta):
        unique_together = ('topic', 'object_id')

class Scan(CreationMetadata):
    u"""
    A scanned image of (part of) a dcument.
    """
    document = models.ForeignKey(Document, related_name='scans')
    image = models.ImageField(upload_to='scans/%Y/%m')
    ordering = models.IntegerField(blank=True, null=True)
    def __unicode__(self):
        return u'Scan for %s (order: %s)' % (self.document, self.ordering)
    class Meta:
        ordering = ['ordering']

class Citation(CreationMetadata):
    u"""
    A reference to or citation of a document.
    """
    document = models.ForeignKey(Document, related_name='citations')
    locator = models.CharField(max_length=16, blank=True)
    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField()
    content_object = generic.GenericForeignKey()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from arch.api.utils import log_utils
from federatedml.feature.feature_selection.filter_base import BaseFilterMethod
from federatedml.param.feature_selection_param import ManuallyFilterParam
from federatedml.protobuf.generated import feature_selection_meta_pb2

LOGGER = log_utils.getLogger()

class ManuallyFilter(BaseFilterMethod):
    def __init__(self, filter_param: ManuallyFilterParam):
        self.filter_out_indexes = None
        self.filter_out_names = None
        self.filter_param = None
        super().__init__(filter_param)

    def _parse_filter_param(self, filter_param):
        self.filter_param = filter_param

    def _transfer_params(self):
        header = self.selection_properties.header
        col_name_maps = self.selection_properties.col_name_maps
        if (self.filter_param.filter_out_names or self.filter_param.filter_out_names) is not None:
            self.filter_out_indexes = self.filter_param.filter_out_indexes
            self.filter_out_names = self.filter_param.filter_out_names
        else:
            filter_out_set = set([i for i in range(len(header))])
            if self.filter_param.left_col_indexes is not None:
                filter_out_set = filter_out_set.difference(self.filter_param.left_col_indexes)
            if self.filter_param.left_col_names is not None:
                left_idx = [col_name_maps.get(name) for name in self.filter_param.left_col_names]
                filter_out_set = filter_out_set.difference(left_idx)
            self.filter_out_indexes = list(filter_out_set)

        if self.filter_out_indexes is None:
            self.filter_out_indexes = []

        if self.filter_out_names is None:
            self.filter_out_names = []

    def fit(self, data_instances, suffix):
        self._transfer_params()
        all_filter_out_names = []
        for col_idx, col_name in zip(self.selection_properties.select_col_indexes,
                                     self.selection_properties.select_col_names):
            LOGGER.debug("Col_idx: {}, col_names: {}, filter_out_indexes: {}, filter_out_names: {}".format(
                col_idx, col_name, self.filter_out_indexes, self.filter_out_names
            ))
            if col_idx not in self.filter_out_indexes and col_name not in self.filter_out_names:
                self.selection_properties.add_left_col_name(col_name)
            else:
                all_filter_out_names.append(col_name)
        self._keep_one_feature()
        self.filter_out_names = all_filter_out_names
        return self

    def get_meta_obj(self, meta_dicts):
        result = feature_selection_meta_pb2.ManuallyFilterMeta(filter_out_names=self.filter_out_names)
        meta_dicts['manually_meta'] = result
        return meta_dicts
<commit_msg>fix manually filter error<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from arch.api.utils import log_utils
from federatedml.feature.feature_selection.filter_base import BaseFilterMethod
from federatedml.param.feature_selection_param import ManuallyFilterParam
from federatedml.protobuf.generated import feature_selection_meta_pb2

LOGGER = log_utils.getLogger()

class ManuallyFilter(BaseFilterMethod):
    def __init__(self, filter_param: ManuallyFilterParam):
        self.filter_out_indexes = None
        self.filter_out_names = None
        self.filter_param = None
        super().__init__(filter_param)

    def _parse_filter_param(self, filter_param):
        self.filter_param = filter_param

    def _transfer_params(self):
        header = self.selection_properties.header
        col_name_maps = self.selection_properties.col_name_maps
        if (self.filter_param.filter_out_indexes or self.filter_param.filter_out_names) is not None:
            self.filter_out_indexes = self.filter_param.filter_out_indexes
            self.filter_out_names = self.filter_param.filter_out_names
        else:
            filter_out_set = set([i for i in range(len(header))])
            if self.filter_param.left_col_indexes is not None:
                filter_out_set = filter_out_set.difference(self.filter_param.left_col_indexes)
            if self.filter_param.left_col_names is not None:
                left_idx = [col_name_maps.get(name) for name in self.filter_param.left_col_names]
                filter_out_set = filter_out_set.difference(left_idx)
            self.filter_out_indexes = list(filter_out_set)

        if self.filter_out_indexes is None:
            self.filter_out_indexes = []

        if self.filter_out_names is None:
            self.filter_out_names = []

    def fit(self, data_instances, suffix):
        self._transfer_params()
        all_filter_out_names = []
        for col_idx, col_name in zip(self.selection_properties.select_col_indexes,
                                     self.selection_properties.select_col_names):
            LOGGER.debug("Col_idx: {}, col_names: {}, filter_out_indexes: {}, filter_out_names: {}".format(
                col_idx, col_name, self.filter_out_indexes, self.filter_out_names
            ))
            if col_idx not in self.filter_out_indexes and col_name not in self.filter_out_names:
                self.selection_properties.add_left_col_name(col_name)
            else:
                all_filter_out_names.append(col_name)
        self._keep_one_feature()
        self.filter_out_names = all_filter_out_names
        return self

    def get_meta_obj(self, meta_dicts):
        result = feature_selection_meta_pb2.ManuallyFilterMeta(filter_out_names=self.filter_out_names)
        meta_dicts['manually_meta'] = result
        return meta_dicts
<|endoftext|>


<commit_before># pyinfra
# File: pyinfra/modules/iptables.py
# Desc: manage iptables rules

'''
The iptables modules handles iptables rules
'''

from __future__ import unicode_literals

from pyinfra.api import operation

@operation
def forward(state, host, dport, to_ip, to_port=None,
            interface='eth0', protocol='tcp', chain='PREROUTING',
            present=True):
    '''
    Manage an iptables rule to forward packets arriving on a given port and
    interface to another host.

    + dport: port on which the packets are arriving
    + to_ip: ip of the destination
    + to_port: port of the destination (default to dport)
    + interface: interface on which the packets are arriving
    + protocol: protocol (tcp or udp)
    + chain: iptables chain (PREROUTING, POSTROUTING, OUTPUT)
    + present: whether the rule should be present or removed
    '''
    to_port = to_port or dport
    definition = {
        'dport': str(dport),
        'destination': '{}:{}'.format(to_ip, to_port),
        'interface': interface,
        'protocol': protocol,
        'match': protocol,
        'chain': chain,
        'jump': 'DNAT',
    }

    if definition in host.fact.iptables('nat'):
        if present:
            # Not removing the rule if it should be present:
            return []
        else:
            # Command to remove the rule, note -D instead of -A:
            return [(
                'iptables -t nat -D {chain} '
                '-i {interface} -p {protocol} -m {protocol} '
                '--dport {dport} -j DNAT --to-destination {destination}'
                ).format(**definition)
            ]

    else:
        if not present:
            # Not adding the rule if it should be absent:
            return []
        else:
            # Command to add the rule:
            return [(
                'iptables -t nat -A {chain} '
                '-i {interface} -p {protocol} -m {protocol} '
                '--dport {dport} -j DNAT --to-destination {destination}'
                ).format(**definition)
            ]
<commit_msg>Add iptables jump rule as kwarg<commit_after># pyinfra
# File: pyinfra/modules/iptables.py
# Desc: manage iptables rules

'''
The iptables modules handles iptables rules
'''

from __future__ import unicode_literals

from pyinfra.api import operation

@operation
def forward(state, host, dport, to_ip, to_port=None,
            interface='eth0', protocol='tcp', chain='PREROUTING', jump='DNAT',
            present=True):
    '''
    Manage an iptables rule to forward packets arriving on a given port and
    interface to another host.

    + dport: port on which the packets are arriving
    + to_ip: ip of the destination
    + to_port: port of the destination (default to dport)
    + interface: interface on which the packets are arriving
    + protocol: protocol (tcp or udp)
    + chain: iptables chain (PREROUTING, POSTROUTING, OUTPUT)
    + jump: target of the rule (eg: DNAT)
    + present: whether the rule should be present or removed
    '''
    to_port = to_port or dport
    definition = {
        'dport': str(dport),
        'destination': '{}:{}'.format(to_ip, to_port),
        'interface': interface,
        'protocol': protocol,
        'match': protocol,
        'chain': chain,
        'jump': jump,
    }

    if definition in host.fact.iptables('nat'):
        if present:
            # Not removing the rule if it should be present:
            return []
        else:
            # Command to remove the rule, note -D instead of -A:
            return [(
                'iptables -t nat -D {chain} '
                '-i {interface} -p {protocol} -m {protocol} '
                '--dport {dport} -j DNAT --to-destination {destination}'
                ).format(**definition)
            ]

    else:
        if not present:
            # Not adding the rule if it should be absent:
            return []
        else:
            # Command to add the rule:
            return [(
                'iptables -t nat -A {chain} '
                '-i {interface} -p {protocol} -m {protocol} '
                '--dport {dport} -j DNAT --to-destination {destination}'
                ).format(**definition)
            ]
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
Sends slack notifications for alarms events
"""

import json
import pprint
import os

from botocore.vendored import requests

def main(event, _):
    print(f'Received event:\n{pprint.pformat(event)}')
    message = event['Records'][0]['Sns']['Message']
    message_data = json.loads(message)
    alarm_name = message_data['AlarmName']
    namespace = message_data['Trigger']['Namespace']
    metric_name = message_data['Trigger']['MetricName']
    dimensions = message_data['Trigger']['Dimensions']
    state_reason = message_data['NewStateReason']

    slack_data = {'username': 'cloudwatch-alert',
                  "icon_emoji": ":slack:",
                  "attachments": [{
                      'color': 'danger',
                      'fallback': alarm_name,
                      "title": alarm_name,
                      "fields": [
                          {
                              "title": "Metric",
                              "value": f"{namespace}/{metric_name}"
                          },
                          {
                              "title": "Dimensions",
                              "value": f"{pprint.pformat(dimensions)}"
                          },
                          {
                              "title": "Reason",
                              "value": state_reason
                          }
                      ]
                  }]}

    response = requests.post(
        os.environ["SLACK_INCOMING_WEBHOOK"],
        data=json.dumps(slack_data),
        headers={'Content-Type': 'application/json'}
    )
    if response.status_code != 200:
        raise ValueError(
            'Request to slack returned an error %s, the response is:\n%s'
            % (response.status_code, response.text)
        )
<commit_msg>Refactor lambda as per comments<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
Sends slack notifications for alarms events
"""

import json
import pprint
import os

from botocore.vendored import requests

class Alarm:
    def __init__(self, json_message):
        self.message = json.loads(json_message)

    @property
    def name(self):
        return self.message['AlarmName']

    @property
    def namespace(self):
        return self.message['Trigger']['Namespace']

    @property
    def metric_name(self):
        return self.message['Trigger']['MetricName']

    @property
    def dimensions(self):
        return self.message['Trigger']['Dimensions']

    @property
    def state_reason(self):
        return self.message['NewStateReason']

def main(event, _):
    print(f'Received event:\n{pprint.pformat(event)}')
    alarm = Alarm(event['Records'][0]['Sns']['Message'])

    slack_data = {'username': 'cloudwatch-alert',
                  "icon_emoji": ":slack:",
                  "attachments": [{
                      'color': 'danger',
                      'fallback': alarm.name,
                      "title": alarm.name,
                      "fields": [
                          {
                              "title": "Metric",
                              "value": f"{alarm.namespace}/{alarm.metric_name}"
                          },
                          {
                              "title": "Dimensions",
                              "value": f"{pprint.pformat(alarm.dimensions)}"
                          },
                          {
                              "title": "Reason",
                              "value": alarm.state_reason
                          }
                      ]
                  }]}

    response = requests.post(
        os.environ["SLACK_INCOMING_WEBHOOK"],
        data=json.dumps(slack_data),
        headers={'Content-Type': 'application/json'}
    )
    response.raise_for_status()
<|endoftext|>


<commit_before>"""
Iterators providing indices for different kinds of iteration over
datasets.
"""
from __future__ import division
import warnings
import numpy
from theano import config

class SubsetIterator(object):
    def __init__(self, dataset_size, batch_size, num_batches, rng=None):
        raise NotImplementedError()

    def next(self):
        raise NotImplementedError()

    def __iter__(self):
        return self

    # Class-level attributes that might hint the behaviour of
    # FiniteDatasetIterator.

    # Does this return subsets that need fancy indexing? (i.e. lists
    # of indices)
    fancy = False

    # Does this class make use of random number generators?
    stochastic = False

class SequentialSubsetIterator(SubsetIterator):
    def __init__(self, dataset_size, batch_size, num_batches, rng=None):
        if rng is not None:
            raise ValueError("non-None rng argument not supported for "
                             "sequential batch iteration")
        self.dataset_size = dataset_size
        if batch_size is None:
            if num_batches is not None:
                batch_size = numpy.ceil(self.X.shape[0] / num_batches)
            else:
                raise ValueError("need one of batch_size, num_batches "
                                 "for sequential batch iteration")
        elif batch_size is not None:
            if num_batches is not None:
                # TODO: some might prefer we just raise an exception here.
                # A warning seems like sensible enough behaviour to me,
                # but this may change.
                warnings.warn("Ignoring num_batches argument in presence "
                              "of batch_size argument for sequential "
                              "iteration")
        self.batch_size = batch_size
        self.current = 0

    def next(self):
        if self.current >= self.dataset_size:
            raise StopIteration()
        else:
            self._last = slice(self.current, self.current + self.batch_size)
            self.current += self.batch_size
            return self._last

    fancy = False
    stochastic = False

class RandomUniformSubsetIterator(SubsetIterator):
    def __init__(self, dataset_size, batch_size, num_batches, rng=None):
        if rng is not None and hasattr(rng, 'random_integers'):
            self._rng = rng
        else:
            self._rng = numpy.random.RandomState(rng)
        if batch_size is None:
            raise ValueError("batch_size cannot be None for random uniform "
                             "iteration")
        elif num_batches is None:
            raise ValueError("num_batches cannot be None for random uniform "
                             "iteration")
        self.dataset_size = dataset_size
        self.batch_size = batch_size
        self.num_batches = num_batches
        self._next_batch_no = 0

    def next(self):
        if self._next_batch_no >= self.num_batches:
            raise StopIteration()
        else:
            self._last = self._rng.random_integers(low=0,
                                                   high=self.dataset_size - 1,
                                                   size=(self.batch_size,))
            self._next_batch_no += 1
            return self._last

    fancy = True
    stochastic = True

class RandomSliceSubsetIterator(RandomUniformSubsetIterator):
    def __init__(self, dataset_size, batch_size, num_batches, rng=None):
        if batch_size is None:
            raise ValueError("batch_size cannot be None for random slice "
                             "iteration")
        elif num_batches is None:
            raise ValueError("num_batches cannot be None for random slice "
                             "iteration")
        super(RandomSliceSubsetIterator, self).__init__(dataset_size,
                                                        batch_size,
                                                        num_batches, rng)
        self._last_start = self.dataset_size - self.batch_size
        if self._last_start < 0:
            raise ValueError("batch_size > dataset_size not supported for "
                             "random slice iteration")

    def next(self):
        if self._next_batch_no >= self.num_batches:
            raise StopIteration()
        else:
            start = self._rng.random_integers(low=0, high=self._last_start)
            self._last = slice(start, start + self.batch_size)
            self._next_batch_no += 1
            return self._last

    fancy = False
    stochastic = True

_iteration_schemes = {
    'sequential': SequentialSubsetIterator,
    'random_slice': RandomSliceSubsetIterator,
    'random_uniform': RandomUniformSubsetIterator,
}

def resolve_iterator_class(mode):
    if isinstance(mode, basestring) and mode not in _iteration_schemes:
        raise ValueError("unknown iteration mode string: %s" % mode)
    elif mode in _iteration_schemes:
        subset_iter_class = _iteration_schemes[mode]
    else:
        subset_iter_class = mode
    return subset_iter_class

class FiniteDatasetIterator(object):
    """A thin wrapper around one of the mode iterators."""
    def __init__(self, dataset, subset_iterator, topo=False, targets=False):
        self._topo = topo
        self._targets = targets
        self._dataset = dataset
        self._subset_iterator = subset_iterator
        # TODO: More thought about how to handle things where this
        # fails (gigantic HDF5 files, etc.)
        if self._topo:
            self._raw_data = self._dataset.get_topological_view()
        else:
            self._raw_data = self._dataset.get_design_matrix()
        if self._targets:
            self._raw_targets = self._dataset.get_targets()

    def __iter__(self):
        return self

    def next(self):
        next_index = self._subset_iterator.next()
        # TODO: handle fancy-index copies by allocating a buffer and
        # using numpy.take()
        features = numpy.cast[config.floatX](self._raw_data[next_index])
        if self._targets:
            return features, self.targets[next_index]
        else:
            return features
<commit_msg>Typo in iteration.py.<commit_after>"""
Iterators providing indices for different kinds of iteration over
datasets.
"""
from __future__ import division
import warnings
import numpy
from theano import config

class SubsetIterator(object):
    def __init__(self, dataset_size, batch_size, num_batches, rng=None):
        raise NotImplementedError()

    def next(self):
        raise NotImplementedError()

    def __iter__(self):
        return self

    # Class-level attributes that might hint the behaviour of
    # FiniteDatasetIterator.

    # Does this return subsets that need fancy indexing? (i.e. lists
    # of indices)
    fancy = False

    # Does this class make use of random number generators?
    stochastic = False

class SequentialSubsetIterator(SubsetIterator):
    def __init__(self, dataset_size, batch_size, num_batches, rng=None):
        if rng is not None:
            raise ValueError("non-None rng argument not supported for "
                             "sequential batch iteration")
        self.dataset_size = dataset_size
        if batch_size is None:
            if num_batches is not None:
                batch_size = numpy.ceil(self.X.shape[0] / num_batches)
            else:
                raise ValueError("need one of batch_size, num_batches "
                                 "for sequential batch iteration")
        elif batch_size is not None:
            if num_batches is not None:
                # TODO: some might prefer we just raise an exception here.
                # A warning seems like sensible enough behaviour to me,
                # but this may change.
                warnings.warn("Ignoring num_batches argument in presence "
                              "of batch_size argument for sequential "
                              "iteration")
        self.batch_size = batch_size
        self.current = 0

    def next(self):
        if self.current >= self.dataset_size:
            raise StopIteration()
        else:
            self._last = slice(self.current, self.current + self.batch_size)
            self.current += self.batch_size
            return self._last

    fancy = False
    stochastic = False

class RandomUniformSubsetIterator(SubsetIterator):
    def __init__(self, dataset_size, batch_size, num_batches, rng=None):
        if rng is not None and hasattr(rng, 'random_integers'):
            self._rng = rng
        else:
            self._rng = numpy.random.RandomState(rng)
        if batch_size is None:
            raise ValueError("batch_size cannot be None for random uniform "
                             "iteration")
        elif num_batches is None:
            raise ValueError("num_batches cannot be None for random uniform "
                             "iteration")
        self.dataset_size = dataset_size
        self.batch_size = batch_size
        self.num_batches = num_batches
        self._next_batch_no = 0

    def next(self):
        if self._next_batch_no >= self.num_batches:
            raise StopIteration()
        else:
            self._last = self._rng.random_integers(low=0,
                                                   high=self.dataset_size - 1,
                                                   size=(self.batch_size,))
            self._next_batch_no += 1
            return self._last

    fancy = True
    stochastic = True

class RandomSliceSubsetIterator(RandomUniformSubsetIterator):
    def __init__(self, dataset_size, batch_size, num_batches, rng=None):
        if batch_size is None:
            raise ValueError("batch_size cannot be None for random slice "
                             "iteration")
        elif num_batches is None:
            raise ValueError("num_batches cannot be None for random slice "
                             "iteration")
        super(RandomSliceSubsetIterator, self).__init__(dataset_size,
                                                        batch_size,
                                                        num_batches, rng)
        self._last_start = self.dataset_size - self.batch_size
        if self._last_start < 0:
            raise ValueError("batch_size > dataset_size not supported for "
                             "random slice iteration")

    def next(self):
        if self._next_batch_no >= self.num_batches:
            raise StopIteration()
        else:
            start = self._rng.random_integers(low=0, high=self._last_start)
            self._last = slice(start, start + self.batch_size)
            self._next_batch_no += 1
            return self._last

    fancy = False
    stochastic = True

_iteration_schemes = {
    'sequential': SequentialSubsetIterator,
    'random_slice': RandomSliceSubsetIterator,
    'random_uniform': RandomUniformSubsetIterator,
}

def resolve_iterator_class(mode):
    if isinstance(mode, basestring) and mode not in _iteration_schemes:
        raise ValueError("unknown iteration mode string: %s" % mode)
    elif mode in _iteration_schemes:
        subset_iter_class = _iteration_schemes[mode]
    else:
        subset_iter_class = mode
    return subset_iter_class

class FiniteDatasetIterator(object):
    """A thin wrapper around one of the mode iterators."""
    def __init__(self, dataset, subset_iterator, topo=False, targets=False):
        self._topo = topo
        self._targets = targets
        self._dataset = dataset
        self._subset_iterator = subset_iterator
        # TODO: More thought about how to handle things where this
        # fails (gigantic HDF5 files, etc.)
        if self._topo:
            self._raw_data = self._dataset.get_topological_view()
        else:
            self._raw_data = self._dataset.get_design_matrix()
        if self._targets:
            self._raw_targets = self._dataset.get_targets()

    def __iter__(self):
        return self

    def next(self):
        next_index = self._subset_iterator.next()
        # TODO: handle fancy-index copies by allocating a buffer and
        # using numpy.take()
        features = numpy.cast[config.floatX](self._raw_data[next_index])
        if self._targets:
            return features, self._raw_targets[next_index]
        else:
            return features
<|endoftext|>


<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

#=========================================================================
#
#    Copyright © 2016 BIREME/PAHO/WHO
#
#    This file is part of API-NLM.
#
#    API-NLM is free software: you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public License as
#    published by the Free Software Foundation, either version 2.1 of
#    the License, or (at your option) any later version.
#
#    API-NLM is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#    GNU Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with API-NLM. If not, see <http://www.gnu.org/licenses/>.
#
#=========================================================================

from datetime import datetime
import os
from os.path import join
import xmltodict
from NLM_API import NLM_API
from MongoDb import MyMongo
from XML import MyXML
from DocIterator import DocIterator
import Tools

__date__ = 20160418

class NLM_AheadOfPrint:
    def __init__(self,
                 factory):
        """
        mongoHost - mongodb server host
        dbName - mongodb database name
        idColName - name of mongodb collection of ids
        docColName - name of mongodb collection of documents
        xmlOutDir - directory path where xml files will be created
        nowDate - current date/time
        encoding - xml files encoding
        mongoPort - mongodb server port
        """

        factory.check()

        self.api = NLM_API()
        self.mid = MyMongo(factory.dbName, factory.idColName, factory.mongoHost,
                           factory.mongoPort)
        self.mdoc = MyMongo(factory.dbName, factory.docColName,
                            factory.mongoHost, factory.mongoPort)

        self.xmlOutDir = factory.xmlOutDir
        self.xmlProcDir = factory.xmlProcDir
        self.date = factory.date
        self.hour = factory.hour
        self.encoding = factory.encoding

    def __getNewDoc(self,
                    id_,
                    status="aheadofprint"):
        """
        id_ - document id
        status - document status: aheadofprint or no_aheadofprint
        Returns a new document
        """
        doc = {"id": id_}
        doc["data"] = self.date
        doc["hour"] = self.hour
        doc["process"] = "api-nlm"
        doc["status"] = status
        doc["owner"] = "unknown"

        return doc

    def insert(self, docId):
        return self.__insertDocId(docId)

    def __insertDocId(self,
                      docId):
        """
            Inserts an id document into collection "id"
            docId - NLM document id
            Returns True is it a new document False is it was already saved
        """

        query = {"_id": docId}

        # Check if the new document is already stored in Mongo and physical file
        cursor = self.mdoc.search(query)

        isNew = (cursor.count() == 0)
        #print("isNew1=" + str(isNew))
        if not isNew:
            #print("arquivo: " + join(self.xmlOutDir, docId + ".xml"))
             # Last processing failed
            isNew = not Tools.existFile(join(self.xmlOutDir, docId + ".xml"))
            #print("isNew2=" + str(isNew))
            if isNew:
                #print("antes da delecao:" + docId)
                if not self.mdoc.deleteDoc(docId):  # forces mongo xml document delete
                    raise Exception("Document id:" + str(docId) + " deletion failed")

        if isNew:
            doc = self.__getNewDoc(docId)
            self.mid.saveDoc(doc) # Save document into mongo

        return isNew

    def __insertDocs(self,
                     ids,
                     xdir=".",
                     encoding="UTF-8",
                     verbose=False):
        """
        For each id from a list of ids, adds a new id document into mongo id
        collection, add a new document into mongo xml collection and creates a
        new file with xml content.
        ids - a list of NLM document ids
        xdir - output file directory
        encoding - output file encoding
        verbose - if True prints the document is inserted
        """

        newDocs = []
        id_size = len(ids)

        if verbose:
            print("Checking " + str(id_size) + " documents: ", end='', flush=True)
        for id_ in ids:
            # Insert id document into collection "id"
            isNewDoc = self.__insertDocId(id_)
            if isNewDoc:
                newDocs.append(id_)
            if verbose:
                ch = '+' if isNewDoc else '.'
                print(ch, end='', flush=True)

        newDocLen = len(newDocs)
        if newDocLen > 0:
            if verbose:
                print("\nDownloading and saving " + str(newDocLen) + " documents: ",
                      end='', flush=True)
            diter = DocIterator(newDocs, verbose=verbose)

            for dId in diter:
                docId = dId[0]
                xml = dId[1]

                # Save xml content into mongo and file
                Tools.xmlToFile(docId, xml, xdir, encoding)    # save into file
                docDict = xmltodict.parse(xml)
                doc = {"_id":docId, "doc":docDict}
                self.mdoc.saveDoc(doc)                         # save into mongo

            if verbose:
                print()  # to print a new line

    def __changeDocStatus(self,
                          ids,
                          verbose=False):
        """
        Changes the document status from "aheadofprint" to "no_aheadofprint" if
        MongoDb lastHarvesting document field is older than current date/hour.
        ids - list of aheadofprint document ids
        verbose - if True prints document id into standard output
        """

        # Searches all documents in the doc collection belongs to the ids list.
        # if not delete it.
        query = {"_id" : {"$nin" : ids}}
        cursor = self.mdoc.search(query)

        for oldDoc in cursor:
            id_ = oldDoc["_id"]

            # Update id mongo doc
            self.mid.saveDoc(self.__getNewDoc(id_, "no_aheadofprint"))

            # Deletes the xml physical file
            fpath = join(self.xmlOutDir, id_ + ".xml")
            try:
                Tools.removeFile(fpath)
            except OSError:
                pass

            # Deletes document from collection "xml"
            self.mdoc.deleteDoc(id_)

        if verbose:
            print("Total: " + str(cursor.count()) + " xml files were deleted.")

    def __getDocId(self,
                   filePath,
                   idXPath="PubmedArticle/MedlineCitation/PMID",
                   encoding="UFT-8"):
        """
        filePath - the xml file path
        encoding - xml file encoding
        Returns the id tag of a xml document.
        """
        xml = Tools.readFile(filePath, encoding=encoding)
        xpath = MyXML(xml)
        xlist = xpath.getXPath(idXPath)

        if len(xlist) == 0:
            id_ = None
        else:
            id_ = xlist[0][0]

        return id_

    def __changeDocStatus2(self,
                           workDir,
                           verbose=False):
        """
        Changes the document status from "aheadofprint" to "no_aheadofprint" if
        the xml document is also present in 'workDir'
        workDir - directory in where the xml files will be searched
        verbose - if True prints document id into standard output
        """

        # For all documents in workDir
        for f in os.listdir(workDir):
            id_ = self.__getDocId(join(workDir, f))

            # Searches if there is a mongo document with same id and updates/deletes it.
            query = {"_id": id}
            cursor = self.mdoc.search(query)

            if len(cursor > 0):
                doc = self.__getNewDoc(id, "no_aheadofprint")

                self.mid.saveDoc(doc)    # create new id mongo doc

                filename = join(self.xmlOutDir, id_ + ".xml") # delete xml physical file
                try:
                    os.remove(filename)
                except OSError:
                    pass

                self.mdoc.deleteDoc(id)  # delete xml mongo doc

                if verbose:
                    print("Doc[" + id + "] status changed. Xml deleted.")

    def process(self,
                verbose=True):
        """
        Downloads 'ahead of print' xlm documents and saves then into mongo and files
        verbose - True if processing progress should be printed into standard
                  output
        """

        nowDate = datetime.now()

        # Retrive all ahead of print document ids
        if verbose:
            print("\nRetrieving document ids: ", end='')

        idTuple = self.api.getAllIds()
        numOfDocs = int(idTuple[0])
        idList = idTuple[1]

        if verbose:
            print("Total: " + str(numOfDocs) + " ahead of print documents.",
                  flush=True)

        # Remove no more ahead of print documents
        if verbose:
            print("\nRemoving no ahead of print documents.")
        self.__changeDocStatus(idList, verbose)

        # Insert new ahead of print documents
        from_ = 0
        loop = 1000 #10000

        while from_ < numOfDocs:
            if verbose:
                print("\n" + str(from_+1) + "/" + str(numOfDocs))

            to = from_ + loop
            self.__insertDocs(idList[from_:to], self.xmlOutDir, self.encoding, verbose)
            from_ += loop

        if verbose:
            elapsedTime = datetime.now() - nowDate
            print("\nElapsed time: " + str(elapsedTime))

    def syncWorkDir(self,
                    verbose=True):
        """
        Removes from processing directory all documents which are also in the
        xmlProcDir. After that, moves all documents from processing directory
        into xmlProcDir.
        """
        nowDate = datetime.now()

        # Removes duplicated documents from processing directory and workDir
        self.__changeDocStatus2(self.xmlProcDir, verbose)

        # Moves all xml files from processing directory to workDir directory
        Tools.moveFiles(self.xmlOutDir, self.xmlProcDir, fileFilter="*.xml",
                        createToDir=False)

        if verbose:
            elapsedTime = datetime.now() - nowDate
            print("\nElapsed time: " + str(elapsedTime))
<commit_msg> Some bugs were fixed<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

#=========================================================================
#
#    Copyright © 2016 BIREME/PAHO/WHO
#
#    This file is part of API-NLM.
#
#    API-NLM is free software: you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public License as
#    published by the Free Software Foundation, either version 2.1 of
#    the License, or (at your option) any later version.
#
#    API-NLM is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#    GNU Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with API-NLM. If not, see <http://www.gnu.org/licenses/>.
#
#=========================================================================

from datetime import datetime
import os
from os.path import join
import xmltodict
from NLM_API import NLM_API
from MongoDb import MyMongo
from XML import MyXML
from DocIterator import DocIterator
import Tools

__date__ = 20160418

class NLM_AheadOfPrint:
    def __init__(self,
                 factory):
        """
        mongoHost - mongodb server host
        dbName - mongodb database name
        idColName - name of mongodb collection of ids
        docColName - name of mongodb collection of documents
        xmlOutDir - directory path where xml files will be created
        nowDate - current date/time
        encoding - xml files encoding
        mongoPort - mongodb server port
        """

        factory.check()

        self.api = NLM_API()
        self.mid = MyMongo(factory.dbName, factory.idColName, factory.mongoHost,
                           factory.mongoPort)
        self.mdoc = MyMongo(factory.dbName, factory.docColName,
                            factory.mongoHost, factory.mongoPort)

        self.xmlOutDir = factory.xmlOutDir
        self.xmlProcDir = factory.xmlProcDir
        self.date = factory.date
        self.hour = factory.hour
        self.encoding = factory.encoding

    def __getNewDoc(self,
                    id_,
                    status="aheadofprint"):
        """
        id_ - document id
        status - document status: aheadofprint or no_aheadofprint
        Returns a new document
        """
        doc = {"id": id_}
        doc["data"] = self.date
        doc["hour"] = self.hour
        doc["process"] = "api-nlm"
        doc["status"] = status
        doc["owner"] = "unknown"

        return doc

    def insert(self, docId):
        return self.__insertDocId(docId)

    def __insertDocId(self,
                      docId):
        """
            Inserts an id document into collection "id"
            docId - NLM document id
            Returns True is it a new document False is it was already saved
        """

        query = {"_id": docId}

        # Check if the new document is already stored in Mongo and physical file
        cursor = self.mdoc.search(query)

        isNew = (cursor.count() == 0)
        #print("isNew1=" + str(isNew))
        if not isNew:
            #print("arquivo: " + join(self.xmlOutDir, docId + ".xml"))
             # Last processing failed
            isNew = not Tools.existFile(join(self.xmlOutDir, docId + ".xml"))
            #print("isNew2=" + str(isNew))
            if isNew:
                #print("antes da delecao:" + docId)
                if not self.mdoc.deleteDoc(docId):  # forces mongo xml document delete
                    raise Exception("Document id:" + str(docId) + " deletion failed")

        if isNew:
            doc = self.__getNewDoc(docId)
            self.mid.saveDoc(doc) # Save document into mongo

        return isNew

    def __insertDocs(self,
                     ids,
                     xdir=".",
                     encoding="UTF-8",
                     verbose=False):
        """
        For each id from a list of ids, adds a new id document into mongo id
        collection, add a new document into mongo xml collection and creates a
        new file with xml content.
        ids - a list of NLM document ids
        xdir - output file directory
        encoding - output file encoding
        verbose - if True prints the document is inserted
        """

        newDocs = []
        id_size = len(ids)

        if verbose:
            print("Checking " + str(id_size) + " documents: ", end='', flush=True)
        for id_ in ids:
            # Insert id document into collection "id"
            isNewDoc = self.__insertDocId(id_)
            if isNewDoc:
                newDocs.append(id_)
            if verbose:
                ch = '+' if isNewDoc else '.'
                print(ch, end='', flush=True)

        newDocLen = len(newDocs)
        if newDocLen > 0:
            if verbose:
                print("\nDownloading and saving " + str(newDocLen) + " documents: ",
                      end='', flush=True)
            diter = DocIterator(newDocs, verbose=verbose)

            for dId in diter:
                docId = dId[0]
                xml = dId[1]

                # Save xml content into mongo and file
                Tools.xmlToFile(docId, xml, xdir, encoding)    # save into file
                docDict = xmltodict.parse(xml)
                doc = {"_id":docId, "doc":docDict}
                self.mdoc.saveDoc(doc)                         # save into mongo

            if verbose:
                print()  # to print a new line

    def __changeDocStatus(self,
                          ids,
                          verbose=False):
        """
        Changes the document status from "aheadofprint" to "no_aheadofprint" if
        MongoDb lastHarvesting document field is older than current date/hour.
        ids - list of aheadofprint document ids
        verbose - if True prints document id into standard output
        """

        # Searches all documents in the doc collection belongs to the ids list.
        # if not delete it.
        query = {"_id" : {"$nin" : ids}}
        cursor = self.mdoc.search(query)

        for oldDoc in cursor:
            id_ = oldDoc["_id"]

            # Update id mongo doc
            self.mid.saveDoc(self.__getNewDoc(id_, "no_aheadofprint"))

            # Deletes the xml physical file
            fpath = join(self.xmlOutDir, id_ + ".xml")
            try:
                Tools.removeFile(fpath)
            except OSError:
                pass

            # Deletes document from collection "xml"
            self.mdoc.deleteDoc(id_)

        if verbose:
            print("Total: " + str(cursor.count()) + " xml files were deleted.")

    def __getDocId(self,
                   filePath,
                   idXPath="PubmedArticle/MedlineCitation/PMID",
                   encoding="UFT-8"):
        """
        filePath - the xml file path
        idXPath - the xml path to the document id
        encoding - xml file encoding
        Returns the id tag of a xml document.
        """
        xml = Tools.readFile(filePath, encoding=encoding)
        xpath = MyXML(xml)
        xlist = xpath.getXPath(idXPath)

        if len(xlist) == 0:
            id_ = None
        else:
            id_ = xlist[0][0]

        return id_

    def __getDocStatus(self,
                       filePath,
                       idXPath="PubmedArticle/PubmedData/PublicationStatus",
                       encoding="UFT-8"):
        """
        filePath - the xml file path
        idXPath - the xml path to the document status
        encoding - xml file encoding
        Returns the id tag of a xml document.
        """

        xml = Tools.readFile(filePath, encoding=encoding)
        xpath = MyXML(xml)
        xlist = xpath.getXPath(idXPath)

        if len(xlist) == 0:
            status = None
        else:
            status = xlist[0][0]

        return status

    def __changeDocStatus2(self,
                           workDir,
                           verbose=False):
        """
        Changes the document status from "aheadofprint" to "no_aheadofprint" if
        the xml document is also present in 'workDir'
        workDir - directory in where the xml files will be searched
        verbose - if True prints document id into standard output
        """

        # For all documents in workDir
        listDir = os.listdir(workDir)

        for f in listDir:
            status = self.__getDocStatus(join(workDir, f))

            # Searches if there is a mongo document with same id and updates/deletes it.
            if status == "aheadofprint":
                id_ = self.__getDocId(join(workDir, f))
                query = {"_id": id}
                cursor = self.mdoc.search(query)

                if len(cursor > 0):
                    doc = self.__getNewDoc(id, "no_aheadofprint")

                    self.mid.saveDoc(doc)    # create new id mongo doc

                    filename = join(self.xmlOutDir, id_ + ".xml") # delete xml physical file
                    try:
                        os.remove(filename)
                    except OSError:
                        pass

                    self.mdoc.deleteDoc(id)  # delete xml mongo doc

        if verbose:
            print("Total: " + str(len(listDir)) + " xml files were deleted.")

    def process(self,
                verbose=True):
        """
        Downloads 'ahead of print' xlm documents and saves then into mongo and files
        verbose - True if processing progress should be printed into standard
                  output
        """

        nowDate = datetime.now()

        # Retrive all ahead of print document ids
        if verbose:
            print("\nRetrieving document ids: ", end='')

        idTuple = self.api.getAllIds()
        numOfDocs = int(idTuple[0])
        idList = idTuple[1]

        if verbose:
            print("Total: " + str(numOfDocs) + " ahead of print documents.",
                  flush=True)

        # Remove no more ahead of print documents
        if verbose:
            print("\nRemoving no ahead of print documents.")
        self.__changeDocStatus(idList, verbose)

        # Insert new ahead of print documents
        from_ = 0
        loop = 1000 #10000

        while from_ < numOfDocs:
            if verbose:
                print("\n" + str(from_+1) + "/" + str(numOfDocs))

            to = from_ + loop
            self.__insertDocs(idList[from_:to], self.xmlOutDir, self.encoding, verbose)
            from_ += loop

        if verbose:
            elapsedTime = datetime.now() - nowDate
            print("\nElapsed time: " + str(elapsedTime))

    def syncWorkDir(self,
                    verbose=True):
        """
        Removes from processing directory all documents which are also in the
        xmlProcDir. After that, moves all documents from processing directory
        into xmlProcDir.
        """
        nowDate = datetime.now()

        # Removes duplicated documents from processing directory and workDir
        if verbose:
            print("\nRemoving duplicated xml files.")
        self.__changeDocStatus2(self.xmlProcDir, verbose)

        # Moves all xml files to the oficial processing directory
        if verbose:
            print("\nMoving xml files to the processing directory.")
        Tools.moveFiles(self.xmlOutDir, self.xmlProcDir, fileFilter="*.xml",
                        createToDir=False)

        if verbose:
            elapsedTime = datetime.now() - nowDate
            print("\nElapsed time: " + str(elapsedTime))
<|endoftext|>


<commit_before>"""Leetcode 5. Longest Palindromic Substring
Medium

Given a string s, find the longest palindromic substring in s.
You may assume that the maximum length of s is 1000.

Example 1:
Input: "babad"
Output: "bab"
Note: "aba" is also a valid answer.

Example 2:
Input: "cbbd"
Output: "bb"
"""

class SolutionNaive(object):
    def longestPalindrome(self, s):
        """
        :type s: str
        :rtype: str

        Time limit exceeded.
        """
        n = len(s)
        max_len = 1
        max_i = 0
        max_j = 0

        for i in range(n):
            for j in range(i + 1, n):
                if s[i:(j + 1)] == s[i:(j + 1)][::-1]:
                    ij_len = j - i + 1
                    if ij_len > max_len:
                        max_len = ij_len
                        max_i = i
                        max_j = j
        return s[max_i:(max_j + 1)]

class SolutionDP(object):
    def longestPalindrome(self, s):
        """
        :type s: str
        :rtype: str
        """
        pass

def main():
    s = 'babad'    # Ans: bab.
    print(SolutionNaive().longestPalindrome(s))

    s = 'cbbd'     # Ans: bb.
    print(SolutionNaive().longestPalindrome(s))

if __name__ == '__main__':
    main()
<commit_msg>Complete longest palindromic substr by DP<commit_after>"""Leetcode 5. Longest Palindromic Substring
Medium

Given a string s, find the longest palindromic substring in s.
You may assume that the maximum length of s is 1000.

Example 1:
Input: "babad"
Output: "bab"
Note: "aba" is also a valid answer.

Example 2:
Input: "cbbd"
Output: "bb"
"""

class SolutionNaive(object):
    def longestPalindrome(self, s):
        """
        :type s: str
        :rtype: str

        Time limit exceeded.
        """
        n = len(s)
        max_len = 1
        max_i = 0
        max_j = 0

        for i in range(n):
            for j in range(i + 1, n):
                if s[i:(j + 1)] == s[i:(j + 1)][::-1]:
                    ij_len = j - i + 1
                    if ij_len > max_len:
                        max_len = ij_len
                        max_i = i
                        max_j = j
        return s[max_i:(max_j + 1)]

class SolutionDP(object):
    def longestPalindrome(self, s):
        """
        :type s: str
        :rtype: str
        """
        if not len(s):
            return ''

        n = len(s)
        M = [[False] * n for _ in range(n)]

        for i in range(n):
            M[i][i] = True
            max_len = 1
            pal_s = s[i]

        for i in range(n - 1):
            if s[i] == s[i + 1]:
                M[i][i + 1] = True
                max_len = 2
                pal_s = s[i:(i + 2)]

        for j in range(n):
            for i in range(0, j - 1):
                if s[i] == s[j] and M[i + 1][j - 1]:
                    M[i][j] = True
                    if max_len < j - i + 1:
                        max_len = j - i + 1
                        pal_s = s[i:(j + 1)]
        return pal_s

def main():
    import time

    s = 'babad'    # Ans: bab.

    start_time = time.time()
    print('By naive: {}'
          .format(SolutionNaive().longestPalindrome(s)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By DP: {}'.format(SolutionDP().longestPalindrome(s)))
    print('Time: {}'.format(time.time() - start_time))

    s = 'cbbd'     # Ans: bb.

    start_time = time.time()
    print('By naive: {}'
          .format(SolutionNaive().longestPalindrome(s)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By DP: {}'.format(SolutionDP().longestPalindrome(s)))
    print('Time: {}'.format(time.time() - start_time))

    s = 'abcba'    # Ans: abcba

    start_time = time.time()
    print('By naive: {}'
          .format(SolutionNaive().longestPalindrome(s)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By DP: {}'.format(SolutionDP().longestPalindrome(s)))
    print('Time: {}'.format(time.time() - start_time))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import sys
import os
from wufoo_model import WufooSurveyModel, WufooFieldModel
from flask.ext.restful import Resource
from flask import Flask, request, make_response, jsonify
from flask.ext.sqlalchemy import SQLAlchemy
from database import db
from json import dumps, loads
from flask.ext.security import current_user
import traceback

"""
class WufooDAO(object):
    def __init__(self, username):
        self.username = username
"""

class WufooResource(Resource):

    def get(self, **kwargs):
        print 'inside wufoo  get'
        try:
            print request.args
            print request.data
            print request.json
        except:
            print traceback.print_exc
        return jsonify(status=200)

    def post(self):
        """
        Get wufoo data from webhook
        """
        data = request.form
        print data
        create = data.get("create")
        # if creating/registering survey to user
        if create:
            if current_user.is_anonymous():
                return dumps([{"status":400}])
            else:
                url = data.get("url")
                if not url:
                    return jsonify(emsg="No url given")
                else:
                    print 'attempting to add survey'
                    try:
                        new_survey = WufooSurveyModel(username=current_user.email, url=url)
                        db.session.add(new_survey)
                        db.session.commit()
                        return make_response(dumps({"status":200, "msg":"Survey successfully added"}))
                    except:
                        traceback.print_exc()
                        return jsonify(status=500)
        # if webhook and not the user registering the survey for the first time
        else:
            print 'not create'
            # parse json load
            created_by = data.get("CreatedBy")
            entry_id = data.get("EntryId")
            form_structure = data.get("FormStructure")
            form_structure_dict =  loads(form_structure)
            url = form_structure_dict.get("Url")
            field_structure = data.get("FieldStructure")
            field_structure_dict = loads(field_structure)
            fields = field_structure_dict.get("Fields")
            form_url = "%s.wufoo.com/forms/%surl" %(created_by, url)
            print 'attempt to query for survey'
            # get survey
            survey = WufooSurveyModel.query.filter_by(url=form_url).last()
            if not survey:
                print 'survey does not exist yet'
                # if survey doesn't exist yet, pass
                return jsonify(status="This survey does not exist yet.")
            # create new entry
            new_entry = WufooEntryModel(entry_id=entry_id)
            survey_values = survey.values

            if survey_values:
                values = True
            else:
                values = False

            survey_values = []
            new_fields = []
            field_ids = [field.id for field in survey.fields]
            for field in fields:
                # keep track if any modifications
                modified_field = True
                new_field = True
                field_id = field.get("ID")
                title = field.get("Title")
                field_type = field.get("Type")
                subfields = field.get("Subfields")
                field_value = data.get("field_id")

                # check if field_id, field_type, and field_title are still the same
                if field_id in field_ids:
                    new_field = False
                    old_field_index = field_ids.index('field_id')
                    old_field = survey.fields[old_field_index]
                    if old_field.field_id == field_id and old_field.title == title and old_field.field_type == field_type:
                        modified_field = False

                    if new_field or modified_field:
                        new_field = WufooFieldModel(field_id=field_id, title = title, field_type = field_type)
                        survey.fields.append(new_field)
                # make new value
                new_value = WufooValue(field_value)
                db.session.add(new_value)
                # add new value to new entry
                new_entry.values.append(new_value)

        db.session.add(new_entry)
        # add new entries to survey
        survey.entries.append(new_entry)
        db.session.add(survey)
        db.session.commit()
<commit_msg>add new wufoo survey works<commit_after>import sys
import os
from wufoo_model import WufooSurveyModel, WufooFieldModel
from flask.ext.restful import Resource
from flask import Flask, request, make_response, jsonify
from flask.ext.sqlalchemy import SQLAlchemy
from database import db
from json import dumps, loads
from flask.ext.security import current_user
import traceback

"""
class WufooDAO(object):
    def __init__(self, username):
        self.username = username
"""

class WufooResource(Resource):

    def get(self, **kwargs):
        print 'inside wufoo  get'
        try:
            print request.args
            print request.data
            print request.json
        except:
            print traceback.print_exc
        return jsonify(status=200)

    def post(self):
        """
        Get wufoo data from webhook
        """
        # potential data coming in from wufoo.com
        data = request.form
        # potential data coming in from leanworkbench.com
        lwb_data = loads(request.data)
        create = lwb_data.get("create")
        # if creating/registering survey to user
        if create:
            if current_user.is_anonymous():
                return dumps([{"status":400}])
            else:
                url = lwb_data.get("url")
                if not url:
                    return jsonify(emsg="No url given")
                else:
                    print 'attempting to add survey'
                    try:
                        new_survey = WufooSurveyModel(username=current_user.email, url=url)
                        db.session.add(new_survey)
                        db.session.commit()
                        return make_response(dumps({"status":200, "msg":"Survey successfully added"}))
                    except:
                        traceback.print_exc()
                        return jsonify(status=500)
        # if webhook and not the user registering the survey for the first time
        else:
            print 'not create'
            # parse json load
            created_by = data.get("CreatedBy")
            entry_id = data.get("EntryId")
            form_structure = data.get("FormStructure")
            form_structure_dict =  loads(form_structure)
            url = form_structure_dict.get("Url")
            field_structure = data.get("FieldStructure")
            field_structure_dict = loads(field_structure)
            fields = field_structure_dict.get("Fields")
            form_url = "%s.wufoo.com/forms/%surl" %(created_by, url)
            print 'attempt to query for survey'
            # get survey
            survey = WufooSurveyModel.query.filter_by(url=form_url).last()
            if not survey:
                print 'survey does not exist yet'
                # if survey doesn't exist yet, pass
                return jsonify(status="This survey does not exist yet.")
            # create new entry
            print 'create new entry for survey'
            new_entry = WufooEntryModel(entry_id=entry_id)
            survey_values = survey.values

            if survey_values:
                values = True
            else:
                values = False

            survey_values = []
            new_fields = []
            field_ids = [field.id for field in survey.fields]
            for field in fields:
                # keep track if any modifications
                modified_field = True
                new_field = True
                field_id = field.get("ID")
                title = field.get("Title")
                field_type = field.get("Type")
                subfields = field.get("Subfields")
                field_value = data.get("field_id")

                # check if field_id, field_type, and field_title are still the same
                if field_id in field_ids:
                    new_field = False
                    old_field_index = field_ids.index('field_id')
                    old_field = survey.fields[old_field_index]
                    if old_field.field_id == field_id and old_field.title == title and old_field.field_type == field_type:
                        modified_field = False

                    if new_field or modified_field:
                        new_field = WufooFieldModel(field_id=field_id, title = title, field_type = field_type)
                        survey.fields.append(new_field)
                # make new value
                new_value = WufooValue(field_value)
                db.session.add(new_value)
                # add new value to new entry
                new_entry.values.append(new_value)

        db.session.add(new_entry)
        # add new entries to survey
        survey.entries.append(new_entry)
        db.session.add(survey)
        db.session.commit()
<|endoftext|>


<commit_before>import numpy as np
import matplotlib.pyplot as plt
from PIL import Image

from learntools.core import *

class SetTraininableAndNumClasses(CodingProblem):
    _vars = ['num_classes', 'my_new_model']
    _hint = \
"""num_classes is the number of categories your model chooses between for each prediction.
Think about the problem to determine the answer. " + my_new_model[0].layers.trainable is
set to either True or False"""

    _solution = CS(
"""
num_classes = 2    # First line that was changed
resnet_weights_path = '../input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'

my_new_model = Sequential()
my_new_model.add(ResNet50(include_top=False, pooling='avg', weights=resnet_weights_path))
my_new_model.add(Dense(num_classes, activation='softmax'))

# Indicate whether the first layer should be trained/changed or not.
my_new_model.layers[0].trainable = False   # Other line that was changed
"""
)

    def check(self, num_classes, my_new_model):
        assert (num_classes == 2), ("num_classes should be 2.  You set it to %d".format(num_classes))
        assert (not my_new_model.layers[0].trainable), \
        ("Layer 0 of your model should not be trainable. It was trained on another dataset, and will be hard to improve.")

class CompileTransferLearningModel(ThoughtExperiment):
    _solution = \
"""
The compile model doesn't change the values in any convolutions.  In fact, your model has not even
received an argument with data yet.  Compile specifies how your model will make updates a later
`fit` step where it receives data.  That is the part that will take longer.
"""

class WhatCompileArgsAffectModel(ThoughtExperiment):
    _solution = \
    """
- **optimizer** determines how we determine the numerical values that make up the model. So it can affect the resulting model and predictions
- **loss** determines what goal we optimize when determining numerical values in the model. So it can affect the resulting model and predictions
- **metrics** determines only what we print out while the model is being built, but it doesn't affect the model itself.

You may not understand all of this yet. That's totally fine for now.  It will become
clearer in an upcoming lesson (called A Deeper Understanding of Deep Learning).
"""

class FitTransferModel(CodingProblem):
    _vars = ['fit_stats']
    _hint = "To get steps_per_epoch, divide the number of images by the batch size."
    def check(self, fit_stats):
        their_val_dir = fit_stats.validation_data.directory
        their_val_loss = fit_stats.history['val_loss'][0]
        their_num_steps = fit_stats.params['steps']
        assert (their_val_dir == '../input/dogs-gone-sideways/val'),\
               ("The validation directory should be `../input/dogs-gone-sideways/val`. Yours was {}".format(their_val_dir))
        assert (their_num_steps == 22), ("Should have 22 steps per epoch. You had {}".format(their_num_steps))
        assert (their_val_loss < 0.6), \
                ("Your validation loss is {}. It can vary from one run to the next, but it should be lower then that. Something isn't right".format(their_val_loss))
    _solution = CS(
"""
image_size = 224
data_generator = ImageDataGenerator(preprocess_input)

train_generator = data_generator.flow_from_directory(
                                        directory=____,
                                        target_size=(image_size, image_size),
                                        batch_size=10,
                                        class_mode='categorical')

validation_generator = data_generator.flow_from_directory(
                                        directory=____,
                                        target_size=(image_size, image_size),
                                        class_mode='categorical')

# fit_stats below saves some statistics describing how model fitting went
# the key role of the following line is how it changes my_new_model by fitting to data
fit_stats = my_new_model.fit_generator(train_generator,
                                       steps_per_epoch=____,
                                       validation_data=____,
                                       validation_steps=1)
"""
    )

qvars = bind_exercises(globals(), [
    SetTraininableAndNumClasses,
    CompileTransferLearningModel,
    WhatCompileArgsAffectModel,
    FitTransferModel,
    ],
    tutorial_id=76,
    var_format='step_{n}',
    )

__all__ = list(qvars)
<commit_msg>remove flaky requirement that DL model has val score in certain range<commit_after>import numpy as np
import matplotlib.pyplot as plt
from PIL import Image

from learntools.core import *

class SetTraininableAndNumClasses(CodingProblem):
    _vars = ['num_classes', 'my_new_model']
    _hint = \
"""num_classes is the number of categories your model chooses between for each prediction.
Think about the problem to determine the answer. " + my_new_model[0].layers.trainable is
set to either True or False"""

    _solution = CS(
"""
num_classes = 2    # First line that was changed
resnet_weights_path = '../input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'

my_new_model = Sequential()
my_new_model.add(ResNet50(include_top=False, pooling='avg', weights=resnet_weights_path))
my_new_model.add(Dense(num_classes, activation='softmax'))

# Indicate whether the first layer should be trained/changed or not.
my_new_model.layers[0].trainable = False   # Other line that was changed
"""
)

    def check(self, num_classes, my_new_model):
        assert (num_classes == 2), ("num_classes should be 2.  You set it to %d".format(num_classes))
        assert (not my_new_model.layers[0].trainable), \
        ("Layer 0 of your model should not be trainable. It was trained on another dataset, and will be hard to improve.")

class CompileTransferLearningModel(ThoughtExperiment):
    _solution = \
"""
The compile model doesn't change the values in any convolutions.  In fact, your model has not even
received an argument with data yet.  Compile specifies how your model will make updates a later
`fit` step where it receives data.  That is the part that will take longer.
"""

class WhatCompileArgsAffectModel(ThoughtExperiment):
    _solution = \
    """
- **optimizer** determines how we determine the numerical values that make up the model. So it can affect the resulting model and predictions
- **loss** determines what goal we optimize when determining numerical values in the model. So it can affect the resulting model and predictions
- **metrics** determines only what we print out while the model is being built, but it doesn't affect the model itself.

You may not understand all of this yet. That's totally fine for now.  It will become
clearer in an upcoming lesson (called A Deeper Understanding of Deep Learning).
"""

class FitTransferModel(CodingProblem):
    _vars = ['fit_stats']
    _hint = "To get steps_per_epoch, divide the number of images by the batch size."
    def check(self, fit_stats):
        their_val_dir = fit_stats.validation_data.directory
        their_val_loss = fit_stats.history['val_loss'][0]
        their_num_steps = fit_stats.params['steps']
        assert (their_val_dir == '../input/dogs-gone-sideways/val'),\
               ("The validation directory should be `../input/dogs-gone-sideways/val`. Yours was {}".format(their_val_dir))
        assert (their_num_steps == 22), ("Should have 22 steps per epoch. You had {}".format(their_num_steps))
    _solution = CS(
"""
image_size = 224
data_generator = ImageDataGenerator(preprocess_input)

train_generator = data_generator.flow_from_directory(
                                        directory=____,
                                        target_size=(image_size, image_size),
                                        batch_size=10,
                                        class_mode='categorical')

validation_generator = data_generator.flow_from_directory(
                                        directory=____,
                                        target_size=(image_size, image_size),
                                        class_mode='categorical')

# fit_stats below saves some statistics describing how model fitting went
# the key role of the following line is how it changes my_new_model by fitting to data
fit_stats = my_new_model.fit_generator(train_generator,
                                       steps_per_epoch=____,
                                       validation_data=____,
                                       validation_steps=1)
"""
    )

qvars = bind_exercises(globals(), [
    SetTraininableAndNumClasses,
    CompileTransferLearningModel,
    WhatCompileArgsAffectModel,
    FitTransferModel,
    ],
    tutorial_id=76,
    var_format='step_{n}',
    )

__all__ = list(qvars)
<|endoftext|>