<commit_before>from contextlib import contextmanager
from unittest import TestCase

from mock import patch
from pyelasticsearch.exceptions import ElasticHttpNotFoundError

from regcore.db.es import (
    ESDiffs, ESLayers, ESNotices, ESRegulations)

class ESBase(object):
    """Mixin methods for boiler plate around mocking out Elastic Search. Each
    method sets the passed arguments as self.call_args"""
    @contextmanager
    def expect_get(self, doc_type, id, doc=None):
        """Expect an attempt to find a single document
           :param doc: document to return or None to test no document"""
        with patch('regcore.db.es.ElasticSearch') as es:
            if doc is None:
                es.return_value.get.side_effect = ElasticHttpNotFoundError
            else:
                es.return_value.get.return_value = {'_source': doc}
            yield
            self.call_args = es.return_value.get.call_args
        self.assertEqual((doc_type, id), self.call_args[0][1:])

    @contextmanager
    def expect_put(self, doc_type, id):
        """Expect a document to be written. Set the arguments passed to
        self.call_args"""
        with patch('regcore.db.es.ElasticSearch') as es:
            yield
            self.call_args = es.return_value.index.call_args
            self.assertEqual(doc_type, self.call_args[0][1])
            self.assertEqual(id, self.call_args[1].get('id'))

    @contextmanager
    def expect_bulk_put(self, doc_type, num_docs):
        """Expect multiple documents to be written.."""
        with patch('regcore.db.es.ElasticSearch') as es:
            yield
            self.call_args = es.return_value.bulk_index.call_args
            self.assertEqual(doc_type, self.call_args[0][1])
            self.assertEqual(num_docs, len(self.call_args[0][2]))

    @contextmanager
    def expect_search(self, doc_type, query, results):
        """Expect a search to be performed and respond with these results"""
        with patch('regcore.db.es.ElasticSearch') as es:
            es.return_value.search.return_value = {'hits': {'hits': results}}
            yield
            self.call_args = es.return_value.search.call_args
            self.assertEqual(self.call_args[1]['doc_type'], doc_type)
            self.assertEqual(query, self.call_args[0][0]['query'])

class ESRegulationsTest(TestCase, ESBase):
    def test_get_404(self):
        with self.expect_get('reg_tree', 'verver/lablab'):
            self.assertIsNone(ESRegulations().get('lablab', 'verver'))

    def test_get_success(self):
        return_value = {'first': 0, 'version': 'remove', 'id': 'also',
                        'label_string': 'a', 'regulation': '100'}
        with self.expect_get('reg_tree', 'verver/lablab', return_value):
            self.assertEqual(ESRegulations().get('lablab', 'verver'),
                             {"first": 0})

    def test_bulk_put(self):
        n2 = {'text': 'some text', 'label': ['111', '2'], 'children': []}
        n3 = {'text': 'other', 'label': ['111', '3'], 'children': []}
        # Use a copy of the children
        root = {'text': 'root', 'label': ['111'], 'children': [dict(n2),
                                                               dict(n3)]}
        nodes = [root, n2, n3]

        with self.expect_bulk_put('reg_tree', 3):
            ESRegulations().bulk_put(nodes, 'verver', '111')

        root.update({'version': 'verver', 'regulation': '111',
                     'label_string': '111', 'id': 'verver/111', 'root': True})
        n2.update({'version': 'verver', 'regulation': '111',
                   'label_string': '111-2', 'id': 'verver/111-2',
                   'root': False})
        n3.update({'version': 'verver', 'regulation': '111',
                   'label_string': '111-3', 'id': 'verver/111-3',
                   'root': False})
        self.assertEqual(nodes, self.call_args[0][2])

    def test_listing(self):
        query = {'match': {'label_string': 'lll'}}
        results = [{'fields': {'version': 'ver1', 'label_string': 'lll'}},
                   {'fields': {'version': 'aaa', 'label_string': 'lll'}},
                   {'fields': {'version': '333', 'label_string': 'lll'}},
                   {'fields': {'version': 'four', 'label_string': 'lll'}}]
        with self.expect_search('reg_tree', query, results):
            entries = ESRegulations().listing('lll')

        self.assertEqual([('333', 'lll'), ('aaa', 'lll'), ('four', 'lll'),
                          ('ver1', 'lll')], entries)

        query = {'match': {'root': True}}
        with self.expect_search('reg_tree', query, results):
            ESRegulations().listing()

class ESLayersTest(TestCase, ESBase):
    def test_get_404(self):
        with self.expect_get('layers', 'verver/namnam/lablab'):
            self.assertIsNone(ESLayers().get('namnam', 'lablab', 'verver'))

    def test_get_success(self):
        return_value = {'layer': {'some': 'body'}}
        with self.expect_get('layers', 'verver/namnam/lablab', return_value):
            self.assertEqual(ESLayers().get('namnam', 'lablab', 'verver'),
                             {"some": "body"})

    def test_bulk_put(self):
        layers = [{'111-22': [], '111-22-a': [], 'label': '111-22'},
                  {'111-23': [], 'label': '111-23'}]
        with self.expect_bulk_put('layers', 2):
            ESLayers().bulk_put(layers, 'verver', 'name', '111')

        del layers[0]['label']
        del layers[1]['label']
        transformed = [
            {'id': 'verver/name/111-22', 'version': 'verver',
             'name': 'name', 'label': '111-22', 'layer': layers[0]},
            {'id': 'verver/name/111-23', 'version': 'verver',
             'name': 'name', 'label': '111-23', 'layer': layers[1]}]
        self.assertEqual(transformed, self.call_args[0][2])

class ESNoticesTest(TestCase, ESBase):
    def test_get_404(self):
        with self.expect_get('notice', 'docdoc'):
            self.assertIsNone(ESNotices().get('docdoc'))

    def test_get_success(self):
        with self.expect_get('notice', 'docdoc', {'some': 'body'}):
            self.assertEqual(ESNotices().get('docdoc'),
                             {"some": 'body'})

    def test_put(self):
        with self.expect_put('notice', 'docdoc'):
            ESNotices().put('docdoc', {"some": "structure"})
        self.assertEqual(self.call_args[0][2], {"some": "structure"})

    def test_listing(self):
        query = {'match_all': {}}
        results = [{'_id': 22, '_somethingelse': 5, 'fields': {
                        'effective_on': '2005-05-05'}},
                   {'_id': 9, '_somethingelse': 'blue', 'fields': {}}]
        with self.expect_search('notice', query, results):
            entries = ESNotices().listing()

        self.assertEqual([{'document_number': 22,
                           'effective_on': '2005-05-05'},
                          {'document_number': 9}], entries)

        query = {'match': {'cfr_parts': '876'}}
        with self.expect_search('notice', query, results):
            ESNotices().listing('876')

class ESDiffTest(TestCase, ESBase):
    def test_get_404(self):
        with self.expect_get('diff', 'lablab/oldold/newnew'):
            self.assertIsNone(ESDiffs().get('lablab', 'oldold', 'newnew'))

    def test_get_success(self):
        return_value = {'label': 'lablab',
                        'old_version': 'oldold',
                        'new_version': 'newnew',
                        'diff': {'some': 'body'}}
        with self.expect_get('diff', 'lablab/oldold/newnew', return_value):
            self.assertEqual(ESDiffs().get('lablab', 'oldold', 'newnew'),
                             {"some": 'body'})

    def test_put(self):
        with self.expect_put('diff', 'lablab/oldold/newnew'):
            ESDiffs().put('lablab', 'oldold', 'newnew', {"some": "structure"})
        self.assertEqual(self.call_args[0][2],
                         {'label': 'lablab',
                          'old_version': 'oldold',
                          'new_version': 'newnew',
                          'diff': {'some': 'structure'}})
<commit_msg>Used wrong doc_type for layers in tests<commit_after>from contextlib import contextmanager
from unittest import TestCase

from mock import patch
from pyelasticsearch.exceptions import ElasticHttpNotFoundError

from regcore.db.es import ESDiffs, ESLayers, ESNotices, ESRegulations

class ESBase(object):
    """Mixin methods for boiler plate around mocking out Elastic Search. Each
    method sets the passed arguments as self.call_args"""
    @contextmanager
    def expect_get(self, doc_type, id, doc=None):
        """Expect an attempt to find a single document
           :param doc: document to return or None to test no document"""
        with patch('regcore.db.es.ElasticSearch') as es:
            if doc is None:
                es.return_value.get.side_effect = ElasticHttpNotFoundError
            else:
                es.return_value.get.return_value = {'_source': doc}
            yield
            self.call_args = es.return_value.get.call_args
        self.assertEqual((doc_type, id), self.call_args[0][1:])

    @contextmanager
    def expect_put(self, doc_type, id):
        """Expect a document to be written. Set the arguments passed to
        self.call_args"""
        with patch('regcore.db.es.ElasticSearch') as es:
            yield
            self.call_args = es.return_value.index.call_args
            self.assertEqual(doc_type, self.call_args[0][1])
            self.assertEqual(id, self.call_args[1].get('id'))

    @contextmanager
    def expect_bulk_put(self, doc_type, num_docs):
        """Expect multiple documents to be written.."""
        with patch('regcore.db.es.ElasticSearch') as es:
            yield
            self.call_args = es.return_value.bulk_index.call_args
            self.assertEqual(doc_type, self.call_args[0][1])
            self.assertEqual(num_docs, len(self.call_args[0][2]))

    @contextmanager
    def expect_search(self, doc_type, query, results):
        """Expect a search to be performed and respond with these results"""
        with patch('regcore.db.es.ElasticSearch') as es:
            es.return_value.search.return_value = {'hits': {'hits': results}}
            yield
            self.call_args = es.return_value.search.call_args
            self.assertEqual(self.call_args[1]['doc_type'], doc_type)
            self.assertEqual(query, self.call_args[0][0]['query'])

class ESRegulationsTest(TestCase, ESBase):
    def test_get_404(self):
        with self.expect_get('reg_tree', 'verver/lablab'):
            self.assertIsNone(ESRegulations().get('lablab', 'verver'))

    def test_get_success(self):
        return_value = {'first': 0, 'version': 'remove', 'id': 'also',
                        'label_string': 'a', 'regulation': '100'}
        with self.expect_get('reg_tree', 'verver/lablab', return_value):
            self.assertEqual(ESRegulations().get('lablab', 'verver'),
                             {"first": 0})

    def test_bulk_put(self):
        n2 = {'text': 'some text', 'label': ['111', '2'], 'children': []}
        n3 = {'text': 'other', 'label': ['111', '3'], 'children': []}
        # Use a copy of the children
        root = {'text': 'root', 'label': ['111'], 'children': [dict(n2),
                                                               dict(n3)]}
        nodes = [root, n2, n3]

        with self.expect_bulk_put('reg_tree', 3):
            ESRegulations().bulk_put(nodes, 'verver', '111')

        root.update({'version': 'verver', 'regulation': '111',
                     'label_string': '111', 'id': 'verver/111', 'root': True})
        n2.update({'version': 'verver', 'regulation': '111',
                   'label_string': '111-2', 'id': 'verver/111-2',
                   'root': False})
        n3.update({'version': 'verver', 'regulation': '111',
                   'label_string': '111-3', 'id': 'verver/111-3',
                   'root': False})
        self.assertEqual(nodes, self.call_args[0][2])

    def test_listing(self):
        query = {'match': {'label_string': 'lll'}}
        results = [{'fields': {'version': 'ver1', 'label_string': 'lll'}},
                   {'fields': {'version': 'aaa', 'label_string': 'lll'}},
                   {'fields': {'version': '333', 'label_string': 'lll'}},
                   {'fields': {'version': 'four', 'label_string': 'lll'}}]
        with self.expect_search('reg_tree', query, results):
            entries = ESRegulations().listing('lll')

        self.assertEqual([('333', 'lll'), ('aaa', 'lll'), ('four', 'lll'),
                          ('ver1', 'lll')], entries)

        query = {'match': {'root': True}}
        with self.expect_search('reg_tree', query, results):
            ESRegulations().listing()

class ESLayersTest(TestCase, ESBase):
    def test_get_404(self):
        with self.expect_get('layer', 'verver/namnam/lablab'):
            self.assertIsNone(ESLayers().get('namnam', 'lablab', 'verver'))

    def test_get_success(self):
        return_value = {'layer': {'some': 'body'}}
        with self.expect_get('layer', 'verver/namnam/lablab', return_value):
            self.assertEqual(ESLayers().get('namnam', 'lablab', 'verver'),
                             {"some": "body"})

    def test_bulk_put(self):
        layers = [{'111-22': [], '111-22-a': [], 'label': '111-22'},
                  {'111-23': [], 'label': '111-23'}]
        with self.expect_bulk_put('layer', 2):
            ESLayers().bulk_put(layers, 'verver', 'name', '111')

        del layers[0]['label']
        del layers[1]['label']
        transformed = [
            {'id': 'verver/name/111-22', 'version': 'verver',
             'name': 'name', 'label': '111-22', 'layer': layers[0]},
            {'id': 'verver/name/111-23', 'version': 'verver',
             'name': 'name', 'label': '111-23', 'layer': layers[1]}]
        self.assertEqual(transformed, self.call_args[0][2])

class ESNoticesTest(TestCase, ESBase):
    def test_get_404(self):
        with self.expect_get('notice', 'docdoc'):
            self.assertIsNone(ESNotices().get('docdoc'))

    def test_get_success(self):
        with self.expect_get('notice', 'docdoc', {'some': 'body'}):
            self.assertEqual(ESNotices().get('docdoc'),
                             {"some": 'body'})

    def test_put(self):
        with self.expect_put('notice', 'docdoc'):
            ESNotices().put('docdoc', {"some": "structure"})
        self.assertEqual(self.call_args[0][2], {"some": "structure"})

    def test_listing(self):
        query = {'match_all': {}}
        results = [{'_id': 22, '_somethingelse': 5, 'fields': {
                        'effective_on': '2005-05-05'}},
                   {'_id': 9, '_somethingelse': 'blue', 'fields': {}}]
        with self.expect_search('notice', query, results):
            entries = ESNotices().listing()

        self.assertEqual([{'document_number': 22,
                           'effective_on': '2005-05-05'},
                          {'document_number': 9}], entries)

        query = {'match': {'cfr_parts': '876'}}
        with self.expect_search('notice', query, results):
            ESNotices().listing('876')

class ESDiffTest(TestCase, ESBase):
    def test_get_404(self):
        with self.expect_get('diff', 'lablab/oldold/newnew'):
            self.assertIsNone(ESDiffs().get('lablab', 'oldold', 'newnew'))

    def test_get_success(self):
        return_value = {'label': 'lablab',
                        'old_version': 'oldold',
                        'new_version': 'newnew',
                        'diff': {'some': 'body'}}
        with self.expect_get('diff', 'lablab/oldold/newnew', return_value):
            self.assertEqual(ESDiffs().get('lablab', 'oldold', 'newnew'),
                             {"some": 'body'})

    def test_put(self):
        with self.expect_put('diff', 'lablab/oldold/newnew'):
            ESDiffs().put('lablab', 'oldold', 'newnew', {"some": "structure"})
        self.assertEqual(self.call_args[0][2],
                         {'label': 'lablab',
                          'old_version': 'oldold',
                          'new_version': 'newnew',
                          'diff': {'some': 'structure'}})
<|endoftext|>


<commit_before>import logging

from django import forms
from django.contrib import admin
from django.core.urlresolvers import reverse
from django.utils.html import escape
from django.utils.translation import ugettext_lazy as _
from django.db.models import Count

from sorl.thumbnail.admin import AdminImageMixin

from bluebottle.bb_projects.models import ProjectTheme, ProjectPhase
from bluebottle.tasks.admin import TaskAdminInline
from bluebottle.common.admin_utils import ImprovedModelForm
from bluebottle.geo.admin import LocationFilter
from bluebottle.geo.models import Location
from bluebottle.utils.model_dispatcher import (get_project_model,
                                               get_project_phaselog_model,
                                               get_project_document_model)
from bluebottle.utils.admin import export_as_csv_action
from bluebottle.votes.models import Vote

from .forms import ProjectDocumentForm
from .models import (PartnerOrganization, ProjectBudgetLine, Project)

logger = logging.getLogger(__name__)

PROJECT_MODEL = get_project_model()
PROJECT_PHASELOG_MODEL = get_project_phaselog_model()
PROJECT_DOCUMENT_MODEL = get_project_document_model()

class ProjectThemeAdmin(admin.ModelAdmin):
    list_display = admin.ModelAdmin.list_display + ('slug', 'disabled',)

admin.site.register(ProjectTheme, ProjectThemeAdmin)

class ProjectThemeFilter(admin.SimpleListFilter):
    title = _('Theme')
    parameter_name = 'theme'

    def lookups(self, request, model_admin):
        themes = [obj.theme for obj in
                  model_admin.model.objects.order_by('theme__name').distinct(
                      'theme__name').exclude(theme__isnull=True).all()]
        return [(theme.id, _(theme.name)) for theme in themes]

    def queryset(self, request, queryset):
        if self.value():
            return queryset.filter(theme__id__exact=self.value())
        else:
            return queryset

class ProjectDocumentInline(admin.StackedInline):
    model = PROJECT_DOCUMENT_MODEL
    form = ProjectDocumentForm
    extra = 0
    raw_id_fields = ('author',)
    readonly_fields = ('download_url',)
    fields = readonly_fields + ('file', 'author')

    def download_url(self, obj):
        url = obj.document_url

        if url is not None:
            return "{1}".format(str(url), 'Download')
        return '(None)'

    download_url.allow_tags = True

class ProjectPhaseLogInline(admin.TabularInline):
    model = PROJECT_PHASELOG_MODEL
    can_delete = False
    ordering = ('-start',)
    extra = 0

    def has_add_permission(self, request):
        return False

    readonly_fields = ('status', 'start')
    fields = readonly_fields

class FundingFilter(admin.SimpleListFilter):
    title = _('Funding')
    parameter_name = 'funding'

    def lookups(self, request, model_admin):
        return (
            ('yes', _('Funding')),
            ('no', _('Not funding')),
        )

    def queryset(self, request, queryset):
        if self.value() == 'yes':
            return queryset.filter(amount_asked__gt=0)
        return queryset

class PartnerOrganizationAdmin(AdminImageMixin, admin.ModelAdmin):
    prepopulated_fields = {"slug": ("name",)}

admin.site.register(PartnerOrganization, PartnerOrganizationAdmin)

class ProjectBudgetLineInline(admin.TabularInline):
    model = ProjectBudgetLine
    extra = 0

class ProjectAdminForm(forms.ModelForm):
    theme = forms.ModelChoiceField(
        queryset=ProjectTheme.objects.all().filter(disabled=False))

class ProjectAdmin(AdminImageMixin, ImprovedModelForm):
    form = ProjectAdminForm
    date_hierarchy = 'created'
    ordering = ('-created',)
    save_on_top = True

    search_fields = ('title', 'owner__first_name', 'owner__last_name',
                     'organization__name')

    raw_id_fields = ('owner', 'organization',)

    prepopulated_fields = {'slug': ('title',)}

    inlines = (ProjectBudgetLineInline, TaskAdminInline, ProjectDocumentInline,
               ProjectPhaseLogInline)

    list_filter = ('country__subregion__region',)

    def get_list_filter(self, request):
        filters = ('status', 'is_campaign', ProjectThemeFilter,
                   'country__subregion__region', 'partner_organization',
                   FundingFilter)

        # Only show Location column if there are any
        if Location.objects.count():
            filters +=  (LocationFilter, )
        return filters

    def get_list_display(self, request):
        fields = ('get_title_display', 'get_owner_display', 'created',
                  'status', 'is_campaign', 'deadline', 'donated_percentage')
        # Only show Location column if there are any
        if Location.objects.count():
            fields +=  ('location', )
        # Only show Vote_count column if there are any votes
        if Vote.objects.count():
            fields +=  ('vote_count', )
        return fields

    def get_list_editable(self, request):
        return ('is_campaign', )

    readonly_fields = ('vote_count', 'amount_donated',
                       'amount_needed', 'popularity')

    export_fields = ['title', 'owner', 'created', 'status',
                     'deadline', 'amount_asked', 'amount_donated']

    actions = (export_as_csv_action(fields=export_fields), )

    fieldsets = (
        (_('Main'), {'fields': ('owner', 'organization', 'partner_organization',
                                'status', 'title', 'slug', 'project_type',
                                'is_campaign')}),

        (_('Story'), {'fields': ('pitch', 'story', 'reach')}),

        (_('Details'), {'fields': ('language', 'theme', 'image',
                                   'video_url', 'country',
                                   'latitude', 'longitude',
                                   'location', 'place', 'tags')}),

        (_('Goal'), {'fields': ('amount_asked', 'amount_extra',
                                'amount_donated', 'amount_needed',
                                'popularity', 'vote_count')}),

        (_('Dates'), {'fields': ('voting_deadline', 'deadline',
                                 'date_submitted', 'campaign_started',
                                 'campaign_ended', 'campaign_funded')}),

        (_('Bank details'), {'fields': ('account_holder_name',
                                        'account_holder_address',
                                        'account_holder_postal_code',
                                        'account_holder_city',
                                        'account_holder_country',
                                        'account_number',
                                        'account_bic',
                                        'account_bank_country')})
    )

    def vote_count(self, obj):
        return obj.vote_set.count()

    def donated_percentage(self, obj):
        if not obj.amount_asked:
            return "-"
        percentage = "%.2f" % (100 * obj.amount_donated / obj.amount_asked)
        return "{0} %".format(percentage)

    def queryset(self, request):
        # Optimization: Select related fields that are used in admin specific
        # display fields.
        queryset = super(ProjectAdmin, self).queryset(request)
        queryset = queryset.select_related(
            'projectpitch', 'projectplan', 'projectcampaign', 'owner',
            'organization'
        ).extra(
            {'admin_vote_count': 'SELECT COUNT(*) from votes_vote where "votes_vote"."project_id" = "projects_project"."id"'}
        )

        return queryset

    def num_votes(self, obj):
        self.queryset(None)
        return obj.admin_vote_count

    num_votes.short_description = _('Vote Count')
    num_votes.admin_order_field = 'admin_vote_count'

    def get_title_display(self, obj):
        if len(obj.title) > 35:
            return u'{short_title} …' \
                .format(title=escape(obj.title), short_title=obj.title[:45])
        return obj.title

    get_title_display.allow_tags = True
    get_title_display.admin_order_field = 'title'
    get_title_display.short_description = _('title')

    def get_owner_display(self, obj):
        owner_name = obj.owner.get_full_name()
        if owner_name:
            owner_name = u' ({name})'.format(name=owner_name)
        return u'{email}{name}'.format(name=owner_name, email=obj.owner.email)

    get_owner_display.admin_order_field = 'owner__last_name'
    get_owner_display.short_description = _('owner')

    def project_owner(self, obj):
        object = obj.owner
        url = reverse('admin:{0}_{1}_change'.format(
            object._meta.app_label, object._meta.module_name), args=[object.id])
        return "{1}".format(
            str(url), object.first_name + ' ' + object.last_name)

    project_owner.allow_tags = True

admin.site.register(Project, ProjectAdmin)

admin.site.register(ProjectPhase)
<commit_msg>Order export projects above delete projects<commit_after>from collections import OrderedDict
import logging

from django import forms
from django.contrib import admin
from django.core.urlresolvers import reverse
from django.utils.html import escape
from django.utils.translation import ugettext_lazy as _
from django.db.models import Count

from sorl.thumbnail.admin import AdminImageMixin

from bluebottle.bb_projects.models import ProjectTheme, ProjectPhase
from bluebottle.tasks.admin import TaskAdminInline
from bluebottle.common.admin_utils import ImprovedModelForm
from bluebottle.geo.admin import LocationFilter
from bluebottle.geo.models import Location
from bluebottle.utils.model_dispatcher import (get_project_model,
                                               get_project_phaselog_model,
                                               get_project_document_model)
from bluebottle.utils.admin import export_as_csv_action
from bluebottle.votes.models import Vote

from .forms import ProjectDocumentForm
from .models import (PartnerOrganization, ProjectBudgetLine, Project)

logger = logging.getLogger(__name__)

PROJECT_MODEL = get_project_model()
PROJECT_PHASELOG_MODEL = get_project_phaselog_model()
PROJECT_DOCUMENT_MODEL = get_project_document_model()

class ProjectThemeAdmin(admin.ModelAdmin):
    list_display = admin.ModelAdmin.list_display + ('slug', 'disabled',)

admin.site.register(ProjectTheme, ProjectThemeAdmin)

class ProjectThemeFilter(admin.SimpleListFilter):
    title = _('Theme')
    parameter_name = 'theme'

    def lookups(self, request, model_admin):
        themes = [obj.theme for obj in
                  model_admin.model.objects.order_by('theme__name').distinct(
                      'theme__name').exclude(theme__isnull=True).all()]
        return [(theme.id, _(theme.name)) for theme in themes]

    def queryset(self, request, queryset):
        if self.value():
            return queryset.filter(theme__id__exact=self.value())
        else:
            return queryset

class ProjectDocumentInline(admin.StackedInline):
    model = PROJECT_DOCUMENT_MODEL
    form = ProjectDocumentForm
    extra = 0
    raw_id_fields = ('author',)
    readonly_fields = ('download_url',)
    fields = readonly_fields + ('file', 'author')

    def download_url(self, obj):
        url = obj.document_url

        if url is not None:
            return "{1}".format(str(url), 'Download')
        return '(None)'

    download_url.allow_tags = True

class ProjectPhaseLogInline(admin.TabularInline):
    model = PROJECT_PHASELOG_MODEL
    can_delete = False
    ordering = ('-start',)
    extra = 0

    def has_add_permission(self, request):
        return False

    readonly_fields = ('status', 'start')
    fields = readonly_fields

class FundingFilter(admin.SimpleListFilter):
    title = _('Funding')
    parameter_name = 'funding'

    def lookups(self, request, model_admin):
        return (
            ('yes', _('Funding')),
            ('no', _('Not funding')),
        )

    def queryset(self, request, queryset):
        if self.value() == 'yes':
            return queryset.filter(amount_asked__gt=0)
        return queryset

class PartnerOrganizationAdmin(AdminImageMixin, admin.ModelAdmin):
    prepopulated_fields = {"slug": ("name",)}

admin.site.register(PartnerOrganization, PartnerOrganizationAdmin)

class ProjectBudgetLineInline(admin.TabularInline):
    model = ProjectBudgetLine
    extra = 0

class ProjectAdminForm(forms.ModelForm):
    theme = forms.ModelChoiceField(
        queryset=ProjectTheme.objects.all().filter(disabled=False))

class ProjectAdmin(AdminImageMixin, ImprovedModelForm):
    form = ProjectAdminForm
    date_hierarchy = 'created'
    ordering = ('-created',)
    save_on_top = True

    search_fields = ('title', 'owner__first_name', 'owner__last_name',
                     'organization__name')

    raw_id_fields = ('owner', 'organization',)

    prepopulated_fields = {'slug': ('title',)}

    inlines = (ProjectBudgetLineInline, TaskAdminInline, ProjectDocumentInline,
               ProjectPhaseLogInline)

    list_filter = ('country__subregion__region',)

    def get_list_filter(self, request):
        filters = ('status', 'is_campaign', ProjectThemeFilter,
                   'country__subregion__region', 'partner_organization',
                   FundingFilter)

        # Only show Location column if there are any
        if Location.objects.count():
            filters +=  (LocationFilter, )
        return filters

    def get_list_display(self, request):
        fields = ('get_title_display', 'get_owner_display', 'created',
                  'status', 'is_campaign', 'deadline', 'donated_percentage')
        # Only show Location column if there are any
        if Location.objects.count():
            fields +=  ('location', )
        # Only show Vote_count column if there are any votes
        if Vote.objects.count():
            fields +=  ('vote_count', )
        return fields

    def get_list_editable(self, request):
        return ('is_campaign', )

    readonly_fields = ('vote_count', 'amount_donated',
                       'amount_needed', 'popularity')

    export_fields = ['title', 'owner', 'created', 'status',
                     'deadline', 'amount_asked', 'amount_donated']

    actions = (export_as_csv_action(fields=export_fields), )

    def get_actions(self, request):
        "Order the action in reverse (delete at the bottom)."
        actions = super(ProjectAdmin, self).get_actions(request)
        return OrderedDict(reversed(actions.items()))

    fieldsets = (
        (_('Main'), {'fields': ('owner', 'organization', 'partner_organization',
                                'status', 'title', 'slug', 'project_type',
                                'is_campaign')}),

        (_('Story'), {'fields': ('pitch', 'story', 'reach')}),

        (_('Details'), {'fields': ('language', 'theme', 'image',
                                   'video_url', 'country',
                                   'latitude', 'longitude',
                                   'location', 'place', 'tags')}),

        (_('Goal'), {'fields': ('amount_asked', 'amount_extra',
                                'amount_donated', 'amount_needed',
                                'popularity', 'vote_count')}),

        (_('Dates'), {'fields': ('voting_deadline', 'deadline',
                                 'date_submitted', 'campaign_started',
                                 'campaign_ended', 'campaign_funded')}),

        (_('Bank details'), {'fields': ('account_holder_name',
                                        'account_holder_address',
                                        'account_holder_postal_code',
                                        'account_holder_city',
                                        'account_holder_country',
                                        'account_number',
                                        'account_bic',
                                        'account_bank_country')})
    )

    def vote_count(self, obj):
        return obj.vote_set.count()

    def donated_percentage(self, obj):
        if not obj.amount_asked:
            return "-"
        percentage = "%.2f" % (100 * obj.amount_donated / obj.amount_asked)
        return "{0} %".format(percentage)

    def queryset(self, request):
        # Optimization: Select related fields that are used in admin specific
        # display fields.
        queryset = super(ProjectAdmin, self).queryset(request)
        queryset = queryset.select_related(
            'projectpitch', 'projectplan', 'projectcampaign', 'owner',
            'organization'
        ).extra(
            {'admin_vote_count': 'SELECT COUNT(*) from votes_vote where "votes_vote"."project_id" = "projects_project"."id"'}
        )

        return queryset

    def num_votes(self, obj):
        self.queryset(None)
        return obj.admin_vote_count

    num_votes.short_description = _('Vote Count')
    num_votes.admin_order_field = 'admin_vote_count'

    def get_title_display(self, obj):
        if len(obj.title) > 35:
            return u'{short_title} …' \
                .format(title=escape(obj.title), short_title=obj.title[:45])
        return obj.title

    get_title_display.allow_tags = True
    get_title_display.admin_order_field = 'title'
    get_title_display.short_description = _('title')

    def get_owner_display(self, obj):
        owner_name = obj.owner.get_full_name()
        if owner_name:
            owner_name = u' ({name})'.format(name=owner_name)
        return u'{email}{name}'.format(name=owner_name, email=obj.owner.email)

    get_owner_display.admin_order_field = 'owner__last_name'
    get_owner_display.short_description = _('owner')

    def project_owner(self, obj):
        object = obj.owner
        url = reverse('admin:{0}_{1}_change'.format(
            object._meta.app_label, object._meta.module_name), args=[object.id])
        return "{1}".format(
            str(url), object.first_name + ' ' + object.last_name)

    project_owner.allow_tags = True

admin.site.register(Project, ProjectAdmin)

admin.site.register(ProjectPhase)
<|endoftext|>


<commit_before>import xml.etree.ElementTree as ElementTree
import util
import copy

def remove_outlier_points(file_name):
    # Todo
    """
    Finds and removes the outlier points from the stroke in the given file.
    :param file_name: The file that will be scanned for outliers.
    :return:
    """
    strokes, xml = build_structure(file_name, get_xml=True)

    distance_data = []

    # Creating a list that contains the speed of the pen between two points.
    for index in range(len(strokes)):
        distance_data.append([])
        for point_index in range(len(strokes[index][:-1])):
            distance_data[index].append((util.point_2_point(util.Point(strokes[index][point_index][0].x,
                                                                               strokes[index][point_index][0].y),
                                                                    util.Point(strokes[index][point_index+1][0].x,
                                                                               strokes[index][point_index+1][0].y)),
                                        (index, point_index),
                                        (index, point_index+1)))
            # If the delta time is 0 then it is set to 0.01 (the shortest time spam that can be recorded)

    length_data = []

    for stroke in distance_data:
        for point in stroke:
            length_data .append(point[0])

    # Finding the quartiles for outlier detection
    q1, q2, q3 = get_quartiles(length_data)

    length_threshold = q3 + 1.5*(q3-q1)

    tree = ElementTree.parse(file_name)
    root = tree.getroot()

    outliers = get_outlier_points(length_threshold, distance_data)

    i = 0
    while i < len(outliers):
        print(str(outliers[i][0]) + " " + str(outliers[i][1]))
        root.find('StrokeSet')[outliers[i][0]].remove(root.find('StrokeSet')[outliers[i][0]][outliers[i][1]])
        for j in range(len(outliers)):
            if outliers[i][0] == outliers[j][0]:
                outliers[j] = (outliers[j][0], outliers[j][1]-1)
        del outliers[i]
        i += 1

    tree.write(file_name)

def get_quartiles(data):
    """
    Finds the first, second and third quartiles for a given data set.
    :param data: Data that will be analyzed.
    :return: First, second and third quartiles.
    """
    ordered_data = copy.copy(data)
    ordered_data.sort()

    # Separating the odd and the even length cases.
    if len(ordered_data) % 2 == 1:
        q2 = ordered_data[int(len(ordered_data) / 2)]
    else:
        q2 = (ordered_data[int(len(ordered_data) / 2) - 1] + ordered_data[int(len(ordered_data) / 2)]) / 2

    if len(ordered_data[int(len(ordered_data) / 2) + len(ordered_data) % 2:]) % 2 == 1:
        q1 = ordered_data[int(len(ordered_data) / 2 - 1 - int(len(ordered_data) / 2) / 2)]
        q3 = ordered_data[int(len(ordered_data) / 2 + int(len(ordered_data) / 2) / 2)]

    else:
        q1 = (ordered_data[int(len(ordered_data) / 2 - 1 - int(len(ordered_data) / 2) / 2)] +
              ordered_data[int(len(ordered_data) / 2 - int(len(ordered_data) / 2) / 2)]) / 2
        q3 = (ordered_data[int(len(ordered_data) / 2 - 1 + len(ordered_data) % 2 + int(len(ordered_data) / 2) / 2)] +
              ordered_data[int(len(ordered_data) / 2 + len(ordered_data) % 2 + int(len(ordered_data) / 2) / 2)]) / 2

    return q1, q2, q3

def get_outlier_points(length_threshold, distance_data):
    # Todo
    suspects = {}
    default = 0
    for i, stroke in enumerate(distance_data):
        for j, point_distance in enumerate(stroke):
            if point_distance[0] > length_threshold:
                suspects[point_distance[1]] = 1 if suspects.get(point_distance[1], default) == 0 else 2
                suspects[point_distance[2]] = 1 if suspects.get(point_distance[2], default) == 0 else 2
            elif j == 1 and suspects.get(point_distance[1], default) != 0:
                suspects[distance_data[i][0][1]] = 2
            if j == len(stroke)-1 and suspects.get(point_distance[1], default) != 0\
                    and suspects[point_distance[1]] == 1 and suspects.get(point_distance[2], default) != 0 and\
                            suspects[point_distance[2]] == 1:
                suspects[point_distance[1]] = 2
                suspects[point_distance[2]] = 2

    outliers = []
    for key, value in suspects.items():
        if value == 2:
            outliers.append(key)

    return outliers

def mark_horizontal(file_name, indexes):
    """
    Creates an attribute in the given file, for every stroke,
     and gives it value, based on if it is horizontal or not.
    :param file_name: Name of the xml.
    :param indexes: Horizontal indexes, that will be marked as "Yes".
    :return:
    """
    tree = ElementTree.parse(file_name)
    root = tree.getroot()

    for i in range(len(root.find('StrokeSet'))):
        if i in indexes:
            root.find('StrokeSet')[i].attrib['Horizontal'] = "Yes"
        else:
            root.find('StrokeSet')[i].attrib['Horizontal'] = "No"

    tree.write(file_name)

def build_structure(file_name, get_xml):
    """
    Creates a multi layer list structure of the given xml.
    :param file_name: Name of the XML.
    :param get_xml: Boolean value determining, whether the xml object structure is required.
    :return: The structured list of the XML.
    """
    with open(file_name, 'r') as file:
        tree = ElementTree.parse(file)
        root = tree.getroot()

        # text_lines = [text_line.attrib['text'] for text_line in root[1][1:]]

        strokes = []
        xml = []
        # StrokeSet tag stores the strokes in the xml
        for index in range(len(root.find('StrokeSet'))):
            strokes.append([(util.Point(float(point.attrib['x']), float(point.attrib['y'])),
                             float(point.attrib['time'])) for point in root.find('StrokeSet')[index][:]])
            xml.append([point for point in root.find('StrokeSet')[index]])

    if get_xml:
        return strokes, xml
    else:
        return strokes

def main():
    remove_outlier_points('/home/patrik/Desktop/TestStrokes/corrected/d10-718.xml')

if __name__ == "__main__":
    main()
<commit_msg>xml error cleaner remake v2<commit_after>import xml.etree.ElementTree as ElementTree
import util
import copy

def remove_outlier_points(file_name):
    """
    Finds and removes the outlier points from the stroke in the given file.
    :param file_name: The file that will be scanned for outliers.
    :return:
    """
    strokes, xml = build_structure(file_name, get_xml=True)

    distance_data = []

    # Creating a list that contains the speed of the pen between two points.
    for index in range(len(strokes)):
        distance_data.append([])
        for point_index in range(len(strokes[index][:-1])):
            distance_data[index].append((util.point_2_point(util.Point(strokes[index][point_index][0].x,
                                                                               strokes[index][point_index][0].y),
                                                                    util.Point(strokes[index][point_index+1][0].x,
                                                                               strokes[index][point_index+1][0].y)),
                                        (index, point_index),
                                        (index, point_index+1)))
            # If the delta time is 0 then it is set to 0.01 (the shortest time spam that can be recorded)

    length_data = []

    for stroke in distance_data:
        for point in stroke:
            length_data .append(point[0])

    # Finding the quartiles for outlier detection
    q1, q2, q3 = get_quartiles(length_data)

    length_threshold = q3 + 1.5*(q3-q1)

    tree = ElementTree.parse(file_name)
    root = tree.getroot()

    outliers = get_outlier_points(length_threshold, distance_data)

    i = 0
    while i < len(outliers):
        print(str(outliers[i][0]) + " " + str(outliers[i][1]))
        root.find('StrokeSet')[outliers[i][0]].remove(root.find('StrokeSet')[outliers[i][0]][outliers[i][1]])
        for j in range(len(outliers)):
            if outliers[i][0] == outliers[j][0]:
                outliers[j] = (outliers[j][0], outliers[j][1]-1)
        del outliers[i]
        i += 1

    tree.write(file_name)

def get_quartiles(data):
    """
    Finds the first, second and third quartiles for a given data set.
    :param data: Data that will be analyzed.
    :return: First, second and third quartiles.
    """
    ordered_data = copy.copy(data)
    ordered_data.sort()

    # Separating the odd and the even length cases.
    if len(ordered_data) % 2 == 1:
        q2 = ordered_data[int(len(ordered_data) / 2)]
    else:
        q2 = (ordered_data[int(len(ordered_data) / 2) - 1] + ordered_data[int(len(ordered_data) / 2)]) / 2

    if len(ordered_data[int(len(ordered_data) / 2) + len(ordered_data) % 2:]) % 2 == 1:
        q1 = ordered_data[int(len(ordered_data) / 2 - 1 - int(len(ordered_data) / 2) / 2)]
        q3 = ordered_data[int(len(ordered_data) / 2 + int(len(ordered_data) / 2) / 2)]

    else:
        q1 = (ordered_data[int(len(ordered_data) / 2 - 1 - int(len(ordered_data) / 2) / 2)] +
              ordered_data[int(len(ordered_data) / 2 - int(len(ordered_data) / 2) / 2)]) / 2
        q3 = (ordered_data[int(len(ordered_data) / 2 - 1 + len(ordered_data) % 2 + int(len(ordered_data) / 2) / 2)] +
              ordered_data[int(len(ordered_data) / 2 + len(ordered_data) % 2 + int(len(ordered_data) / 2) / 2)]) / 2

    return q1, q2, q3

def get_outlier_points(length_threshold, distance_data):
    suspects = {}
    default = 0
    for i, stroke in enumerate(distance_data):
        for j, point_pair in enumerate(stroke):
            if point_pair[0] > length_threshold:
                suspects[point_pair[1]] = 1 if suspects.get(point_pair[1], default) == 0 else 2
                suspects[point_pair[2]] = 1 if suspects.get(point_pair[2], default) == 0 else 2

            # Todo
            # elif j == 1 and suspects.get(point_pair[1], default) != 0:
            #     suspects[distance_data[i][0][1]] = 2
            #     suspects[distance_data[i][0][2]] = 2
            # if j == len(stroke)-1 and suspects.get(point_pair[1], default) != 0\
            #         and suspects[point_pair[1]] == 1 and suspects.get(point_pair[2], default) != 0 and\
            #                 suspects[point_pair[2]] == 1:
            #     print("asd")
            #     suspects[point_pair[1]] = 2
            #     suspects[point_pair[2]] = 2

    outliers = []
    for key, value in suspects.items():
        if value == 2:
            outliers.append(key)

    return outliers

def mark_horizontal(file_name, indexes):
    """
    Creates an attribute in the given file, for every stroke,
     and gives it value, based on if it is horizontal or not.
    :param file_name: Name of the xml.
    :param indexes: Horizontal indexes, that will be marked as "Yes".
    :return:
    """
    tree = ElementTree.parse(file_name)
    root = tree.getroot()

    for i in range(len(root.find('StrokeSet'))):
        if i in indexes:
            root.find('StrokeSet')[i].attrib['Horizontal'] = "Yes"
        else:
            root.find('StrokeSet')[i].attrib['Horizontal'] = "No"

    tree.write(file_name)

def build_structure(file_name, get_xml):
    """
    Creates a multi layer list structure of the given xml.
    :param file_name: Name of the XML.
    :param get_xml: Boolean value determining, whether the xml object structure is required.
    :return: The structured list of the XML.
    """
    with open(file_name, 'r') as file:
        tree = ElementTree.parse(file)
        root = tree.getroot()

        # text_lines = [text_line.attrib['text'] for text_line in root[1][1:]]

        strokes = []
        xml = []
        # StrokeSet tag stores the strokes in the xml
        for index in range(len(root.find('StrokeSet'))):
            strokes.append([(util.Point(float(point.attrib['x']), float(point.attrib['y'])),
                             float(point.attrib['time'])) for point in root.find('StrokeSet')[index][:]])
            xml.append([point for point in root.find('StrokeSet')[index]])

    if get_xml:
        return strokes, xml
    else:
        return strokes

def main():
    # remove_outlier_points('/home/patrik/Desktop/TestStrokes/corrected/d10-718.xml')
    pass

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from BeautifulSoup import BeautifulSoup
from HTMLParser import HTMLParseError
from urllib2 import urlopen

class PDGARanking:
    def __init__(self, number):
        self.number = number
        self.url = 'http://www.pdga.com/player-stats?PDGANum=%s' % number
        self.response = None
        self.load()
        self.parse()

    def load(self):
        self.response = urlopen(
            self.url).read()

    def parse(self):
        try:
            soup = BeautifulSoup(self.response)

            # Get current rating
            p = soup.find('p', attrs={'class': 'current-rating'})
            rating = p.contents[1].strip()

            # Get the name also
            h1 = soup.find('h1', attrs={'class': 'page-title'})
            name = h1.contents[0].split('#')[0].strip()

            # And location
            location = soup.find('p', attrs={'class':
                                 'location'}).contents[1]

        except HTMLParseError:
            self.rating = None
            self.name = None
        except AttributeError:
            self.rating = None
            self.name = None
        else:
            self.rating = int(rating)
            self.name = name
            self.location = location

if __name__ == '__main__':
    import sys
    num = sys.argv[1]
    rank = PDGARanking(num)
    print rank.rating
    print rank.name
<commit_msg>Updated PDGA util to reflect markup changes<commit_after>from BeautifulSoup import BeautifulSoup
from HTMLParser import HTMLParseError
from urllib2 import urlopen

class PDGARanking:
    def __init__(self, number):
        self.number = number
        self.url = 'http://www.pdga.com/player-stats?PDGANum=%s' % number
        self.response = None
        self.load()
        self.parse()

    def load(self):
        self.response = urlopen(
            self.url).read()

    def parse(self):
        try:
            soup = BeautifulSoup(self.response)

            # Get current rating
            p = soup.find('li', attrs={'class': 'current-rating'})
            rating = p.contents[1].strip()

            # Get the name also
            h1 = soup.find('h1', attrs={'id': 'page-title'})
            name = h1.contents[0].split('#')[0].strip()
            name = name.replace('\t', '')

            # And location
            location = soup.find('li', attrs={'class':
                                 'location'}).contents[1]

        except HTMLParseError, e:
            raise Exception(e)
            self.rating = None
            self.name = None
        except AttributeError:
            self.rating = None
            self.name = None
        else:
            self.rating = int(rating)
            self.name = name
            self.location = location

if __name__ == '__main__':
    import sys
    num = sys.argv[1]
    rank = PDGARanking(num)
    print rank.rating
    print rank.name
<|endoftext|>


<commit_before>from pynrfjprog import API, Hex
import unittest
import os

DIR = os.path.dirname(__file__)
FILENAME = os.path.join(DIR, '/armgcc/nvmc.hex')

class NVMCTests(unittest.TestCase):

    @classmethod
    def setUpClass(self):
        self.device_family = API.DeviceFamily.NRF51  # Start out with nRF51 as expected target device, will be checked and changed if target is an nRF52 series device.
        self.api = API.API(self.device_family)

        self._connect_to_target()

        self.device_version = self.api.read_device_version() # Check our assumption that the target was an nRF51 series device.
        if 'NRF52' in self.device_version: # If target is an nRF52 series device reconnect w/ correct device parameters.
            self.device_family = API.DeviceFamily.NRF52
            self.api.close()
            self._connect_to_target()

        self.api.recover() # Erase all Flash, RAM and disable any protection before running test suite.

    @classmethod
    def tearDownClass(self):
        self.api.recover()
        self.api.disconnect_from_emu()
        self.api.close()

    def setUp(self):
        self.api.erase_all()

    def tearDown(self):
        pass

    def test_erase_all(self):
        print(DIR)
        print(FILENAME)
        self.api.write_u32(0x20000, 0xDEADBEEF, 1)
        self._program_hex_file(FILENAME)
        self.api.reset()
        self.api.go()

        expectedData = 0xDEADBEEF
        readData = self.api.read_u32(0x20000)
        self.assertEqual(expectedData, readData)

    @classmethod
    def _connect_to_target(self):
        """
        Initialzes and connects self.api to target. self.api can now perform operations on connected device.
        """
        self.api.open()
        self.api.connect_to_emu_without_snr()
        self.api.connect_to_device()

    @classmethod
    def _program_hex_file(self, hex_file_path):
        """
        Programs hex_file to the target device.
        """
        data = Hex.Hex(hex_file_path)
        for segment in data:
            api.write(segment.address, segment.data, True)

unittest.main(verbosity = 2) # Call the unit tests with desired verbosity.
<commit_msg>test.py working. needs simplifying<commit_after>from pynrfjprog import API, Hex
import unittest
import os
import sys

DIR = os.path.dirname(os.path.realpath(sys.argv[0]))
FILENAME = os.path.join(DIR, 'armgcc/nvmc.hex')

class NVMCTests(unittest.TestCase):

    @classmethod
    def setUpClass(self):
        self.device_family = API.DeviceFamily.NRF51  # Start out with nRF51 as expected target device, will be checked and changed if target is an nRF52 series device.
        self.api = API.API(self.device_family)

        self._connect_to_target()

        self.device_version = self.api.read_device_version() # Check our assumption that the target was an nRF51 series device.
        if 'NRF52' in self.device_version: # If target is an nRF52 series device reconnect w/ correct device parameters.
            self.device_family = API.DeviceFamily.NRF52
            self.api.close()
            self._connect_to_target()

        self.api.recover() # Erase all Flash, RAM and disable any protection before running test suite.

    @classmethod
    def tearDownClass(self):
        self.api.recover()
        self.api.disconnect_from_emu()
        self.api.close()

    def setUp(self):
        self.api.erase_all()

    def tearDown(self):
        pass

    def test_erase_all(self):
        print(DIR)
        print(FILENAME)
        self.api.write_u32(0x20000, 0xDEADBEEF, 1)
        self._program_hex_file(FILENAME)
        self.api.sys_reset()
        self.api.go()

        expectedData = 0xFFFFFFFF
        readData = self.api.read_u32(0x20000)
        self.assertEqual(expectedData, readData)

    @classmethod
    def _connect_to_target(self):
        """
        Initialzes and connects self.api to target. self.api can now perform operations on connected device.
        """
        self.api.open()
        self.api.connect_to_emu_without_snr()
        self.api.connect_to_device()

    @classmethod
    def _program_hex_file(self, hex_file_path):
        """
        Programs hex_file to the target device.
        """
        data = Hex.Hex(hex_file_path)
        for segment in data:
            self.api.write(segment.address, segment.data, True)

unittest.main(verbosity = 2) # Call the unit tests with desired verbosity.
<|endoftext|>


<commit_before>"""Using a list of GitHub user dictionaries, get further details e.g.
followers, repos, organisations"""

import argparse
import json
import logging
import os
import ratelim
import requests
import sys
import time

from datetime import datetime

def details_url(login, detail_type):
    """Construct a GitHub API compliant API for a user and
    detail type"""
    url = ('https://api.github.com/users/' +
           '{}/'.format(login) +
           '{}'.format(detail_type))
    return url

@ratelim.greedy(5000, 3600)
def request_details(login, detail_type='repos', auth=None):
    """GET request to api.github for user and detail type. Optionally
    authenticate for higher rate limits"""
    url = details_url(login, detail_type)
    if not auth:
        req = requests.get(url)
    else:
        req = requests.get(url,
                           auth=auth)
    return req

def get_file_path():
    """Get the path to the current file"""
    return os.path.dirname(os.path.realpath(sys.argv[0]))

def out_file_name(out_path, detail_type):
    """Formatted file name"""
    file_name = '{}_github_uk_user_{}.json'.format(
        datetime.now().strftime("%Y%m%d%H"), detail_type)
    return os.path.join(out_path, file_name)

def username_passw():
    """Get GitHub username and password from the user's environment"""
    try:
        auth = (os.environ['GH_USERN'], os.environ['GH_PASSW'])
    except KeyError:
        raise KeyError("No username or password found in the environment. Please set " +
                       "these and try again")
        auth = None
    return auth

@ratelim.patient(1, 2)
def request_rate_limit_remaining(auth_details=None):
    """Check the remaining GitHub API calls remaining, returning the
    response object. Optionally authenticate with a username and password tuple
    to check against you login's remaining rate limit"""
    try:
        if not auth_details:
            r = requests.get('https://api.github.com/rate_limit').json()
        else:
            r = requests.get('https://api.github.com/rate_limit',
                             auth=(auth_details[0],
                             auth_details[1])).json()
        return r
    except requests.exceptions.ConnectionError as e:
        return {}

def returned_rate_limit_remaining(returned_request):
    """the remaining number of calls from a GitHub response object
    using the response's headers. Returns an integer value"""
    return int(returned_request.headers.get('x-ratelimit-remaining', 0))

def rate_limit_ok(auth_details=None):
    """Check whether any calls are remaining. If there are, return True, else
    wait until the limit is refreshed and then return True."""
    if not auth_details:
        r = request_rate_limit_remaining()
    else:
        r = request_rate_limit_remaining(auth_details)

    remaining = r.get('resources', {}).get('core', {}).get('remaining', 0)
    if remaining != 0:
        return True
    else:
        time_till_renewal = r.get(
            'resources',
            {}).get('core', {}).get('reset', None)
    try:
        delta = datetime.fromtimestamp(time_till_renewal) - datetime.now()
        time.sleep(delta.seconds)
    except TypeError as e:
        t = 60
        print('Waiting for {} seconds.'.format(t))
        time.sleep(t)
        rate_limit_ok(auth_details)

def details(data, detail_type, auth_details=None):
    """Get the detail_type details for all users in data and save them to outfile"""
    user_dict = {}
    if rate_limit_ok(auth_details):
        for user in data:
            login = user.get('user')
            r = request_details(login, detail_type=detail_type, auth=auth_details)
            login_names = [{key: x.get(key, {})
                            for key in ['id', 'login', 'name', ]} for x in r.json() if type(x) is dict]
            user_dict[login] = login_names
            rate_remaining = returned_rate_limit_remaining(r)
            if rate_remaining > 0:
                continue
            else:
                rate_limit_ok(auth_details)
    return user_dict

def main():
    """Main function"""
    logging.basicConfig(level=logging.DEBUG,
                        filename='/tmp/github.user_details.log')

    parser = argparse.ArgumentParser(description="Get details on GitHub Users")

    parser.add_argument(dest='datafile',
                        action='store',
                        help='path to data file')

    parser.add_argument(dest='outpath',
                        action='store',
                        help='path to out directory')

    args = parser.parse_args()

    auth_details = (username_passw())

    # make the outpath if it doesn't exist
    if not os.path.exists(args.outpath):
        os.mkdir(args.outpath)

    # open the data file contianing login names
    with open(args.datafile, 'r') as fp:
        data = json.load(fp)

    for detail_type in ['repos']:
        result = details(data, detail_type, auth_details)

    with open(out_file_name(args.outpath, detail_type), 'w') as fp:
        json.dump(result, fp)

if __name__ == "__main__":
    main()
<commit_msg>Refactor seconds delay code<commit_after>"""Using a list of GitHub user dictionaries, get further details e.g.
followers, repos, organisations"""

import argparse
import json
import logging
import os
import ratelim
import requests
import sys
import time

from datetime import datetime

def details_url(login, detail_type):
    """Construct a GitHub API compliant API for a user and
    detail type"""
    url = ('https://api.github.com/users/' +
           '{}/'.format(login) +
           '{}'.format(detail_type))
    return url

@ratelim.greedy(5000, 3600)
def request_details(login, detail_type='repos', auth=None):
    """GET request to api.github for user and detail type. Optionally
    authenticate for higher rate limits"""
    url = details_url(login, detail_type)
    if not auth:
        req = requests.get(url)
    else:
        req = requests.get(url,
                           auth=auth)
    return req

def get_file_path():
    """Get the path to the current file"""
    return os.path.dirname(os.path.realpath(sys.argv[0]))

def out_file_name(out_path, detail_type):
    """Formatted file name"""
    file_name = '{}_github_uk_user_{}.json'.format(
        datetime.now().strftime("%Y%m%d%H"), detail_type)
    return os.path.join(out_path, file_name)

def username_passw():
    """Get GitHub username and password from the user's environment"""
    try:
        auth = (os.environ['GH_USERN'], os.environ['GH_PASSW'])
    except KeyError:
        raise KeyError("No username or password found in the environment. Please set " +
                       "these and try again")
        auth = None
    return auth

@ratelim.patient(1, 2)
def request_rate_limit_remaining(auth_details=None):
    """Check the remaining GitHub API calls remaining, returning the
    response object. Optionally authenticate with a username and password tuple
    to check against you login's remaining rate limit"""
    try:
        if not auth_details:
            r = requests.get('https://api.github.com/rate_limit').json()
        else:
            r = requests.get('https://api.github.com/rate_limit',
                             auth=(auth_details[0],
                             auth_details[1])).json()
        return r
    except requests.exceptions.ConnectionError as e:
        return {}

def returned_rate_limit_remaining(returned_request):
    """the remaining number of calls from a GitHub response object
    using the response's headers. Returns an integer value"""
    return int(returned_request.headers.get('x-ratelimit-remaining', 0))

def rate_limit_ok(auth_details=None):
    """Check whether any calls are remaining. If there are, return True, else
    wait until the limit is refreshed and then return True."""
    if not auth_details:
        r = request_rate_limit_remaining()
    else:
        r = request_rate_limit_remaining(auth_details)

    remaining = r.get('resources', {}).get('core', {}).get('remaining', 0)
    if remaining != 0:
        return True
    else:
        time_till_renewal = r.get(
            'resources',
            {}).get('core', {}).get('reset', None)
    try:
        delta = datetime.now() - datetime.utcfromtimestamp(time_till_renewal)
        print('sleeping for {} seconds'.format(delta.total_seconds()))
        time.sleep(delta.total_seconds())
    except TypeError as e:
        t = 60
        print('Waiting for {} seconds.'.format(t))
        time.sleep(t)
        rate_limit_ok(auth_details)

def details(data, detail_type, auth_details=None):
    """Get the detail_type details for all users in data and save them to outfile"""
    user_dict = {}
    if rate_limit_ok(auth_details):
        for user in data:
            login = user.get('user')
            r = request_details(login, detail_type=detail_type, auth=auth_details)
            login_names = [{key: x.get(key, {})
                            for key in ['id', 'login', 'name', ]} for x in r.json() if type(x) is dict]
            user_dict[login] = login_names
            rate_remaining = returned_rate_limit_remaining(r)
            if rate_remaining > 0:
                continue
            else:
                rate_limit_ok(auth_details)
    return user_dict

def main():
    """Main function"""
    logging.basicConfig(level=logging.DEBUG,
                        filename='/tmp/github.user_details.log')

    parser = argparse.ArgumentParser(description="Get details on GitHub Users")

    parser.add_argument(dest='datafile',
                        action='store',
                        help='path to data file')

    parser.add_argument(dest='outpath',
                        action='store',
                        help='path to out directory')

    args = parser.parse_args()

    auth_details = (username_passw())

    # make the outpath if it doesn't exist
    if not os.path.exists(args.outpath):
        os.mkdir(args.outpath)

    # open the data file contianing login names
    with open(args.datafile, 'r') as fp:
        data = json.load(fp)

    for detail_type in ['repos']:
        result = details(data, detail_type, auth_details)

    with open(out_file_name(args.outpath, detail_type), 'w') as fp:
        json.dump(result, fp)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import

import os
import logging

from celery import shared_task

LOG = logging.getLogger(__name__)

@shared_task
def clone_database(database, credential):

    task_name = "clone_database"
    LOG.debug("task name: %s" % clone_database.name)

    return<commit_msg>celery<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import

import os
import logging

from dbaas.celery import app

LOG = logging.getLogger(__name__)

@app.task
def clone_database(self, database, credential):

    task_name = "clone_database"
    LOG.debug("task name: %s" % clone_database.name)

    return<|endoftext|>


<commit_before>from functools import reduce

from django.conf import settings
from django.core.management import BaseCommand
from django.core.management import CommandError
from django.utils.translation import activate, deactivate, ugettext
from parler.models import TranslatableModel

from nextcloudappstore.core.models import Category

class Command(BaseCommand):
    help = 'Imports all database translations from their respective .po files'
    translated_fields = (
        (Category, ('name', 'description')),
    )
    source_lang = 'en'

    def handle(self, *args, **options):
        is_verbose = options['verbosity'] >= 2
        langs = [lang[0] for lang in settings.LANGUAGES if
                 lang[0] != self.source_lang]
        for code in langs:
            activate(code)
            for model, attrs in self.translated_fields:
                if issubclass(model, TranslatableModel):
                    self._import_translations(model, attrs, code, is_verbose)
                else:
                    msg = 'Model "%s" is not translatable' % model.__name__
                    raise CommandError(msg)
            deactivate()

        msg = 'Imported db translations'
        self.stdout.write(self.style.SUCCESS(msg))

    def _import_translations(self, model, attrs, code, is_verbose=False):
        """
        Import translations for fields on a model for a language code
        :param model: the translated model
        :param attrs: the attributes to translate
        :param code: the language code
        :return:
        """
        for obj in model.objects.all():
            obj.set_current_language(self.source_lang, True)
            attrs_src = list(map(lambda f: getattr(obj, f), attrs))
            attrs_trans = list(map(lambda f: ugettext(f), attrs_src))
            obj.set_current_language(code, True)

            if self._has_translations(obj, attrs, attrs_trans):
                for attr, src, trans in zip(attrs, attrs_src, attrs_trans):
                    self._update_trans(obj, attr, code, src, trans, is_verbose)
                obj.save()

    def _has_translations(self, obj, attrs, attrs_trans):
        """
        Checks if any of the attributes has a translation
        :param obj: translatable model instance
        :param attrs: model attributes to check
        :param attrs_trans: translated fields for the current attribute values
        :return: True if the model has translations for the current code
        """
        translations_present = [self._has_translation(*(obj, attr, trans))
                                for attr, trans
                                in zip(attrs, attrs_trans)]
        return reduce(lambda a, b: a or b, translations_present, False)

    def _has_translation(self, obj, attr, trans):
        """
        Checks if an attribute has a translation
        :param obj: translatable model instance
        :param attrs: model attributes to check
        :param trans: translated field for the current attribute value
        :return: True if the attribute has translations for the current code
        """
        return trans.strip() != '' and getattr(obj, attr) != trans

    def _update_trans(self, obj, attr, code, src, trans, is_verbose=False):
        """
        Updates a translation for an attribute on the object
        :param obj: object which has the translatable attribute
        :param attr: attribute name
        :param code: language code
        :param src: attribute value in the source language
        :param trans: attribute value in the translated language
        :return:
        """
        if getattr(obj, attr) != trans:
            if is_verbose:
                msg = ('Update %s.%s for %s: "%s" -> "%s"'
                       % (
                           obj.__class__.__name__, attr, code,
                           getattr(obj, attr),
                           trans))
                self._print(msg)
            setattr(obj, attr, trans)
        else:
            setattr(obj, attr, src)

    def _print(self, msg):
        self.stdout.write(self.style.SUCCESS(msg))
<commit_msg>simplification<commit_after>from functools import reduce

from django.conf import settings
from django.core.management import BaseCommand
from django.core.management import CommandError
from django.utils.translation import activate, ugettext
from parler.models import TranslatableModel

from nextcloudappstore.core.models import Category

class Command(BaseCommand):
    help = 'Imports all database translations from their respective .po files'
    translated_fields = (
        (Category, ('name', 'description')),
    )
    source_lang = 'en'

    def handle(self, *args, **options):
        is_verbose = options['verbosity'] >= 2
        language_codes = [language[0] for language in settings.LANGUAGES if
                          language[0] != self.source_lang]
        for language_code in language_codes:
            for model, fields in self.translated_fields:
                if issubclass(model, TranslatableModel):
                    self._import_translations(model, fields, language_code,
                                              is_verbose)
                else:
                    msg = 'Model "%s" is not translatable' % model.__name__
                    raise CommandError(msg)

        msg = 'Imported db translations'
        self.stdout.write(self.style.SUCCESS(msg))

    def _import_translations(self, model, fields, language_code, is_verbose):
        """
        Import translations for fields on a model for a language code
        :param model: the translated model
        :param fields: the attributes to translate
        :param language_code: the language code
        :return:
        """
        for obj in model.objects.all():
            obj.set_current_language(self.source_lang, True)
            english_values = [getattr(obj, field) for field in fields]

            activate(language_code)
            translated_values = [ugettext(value) for value in english_values]
            obj.set_current_language(language_code, True)
            if self._has_translations(obj, fields, translated_values):
                for field, english, translation in zip(fields, english_values,
                                                       translated_values):
                    self._update_trans(obj, field, language_code, english,
                                       translation, is_verbose)
                obj.save()

    def _has_translations(self, obj, fields, translations):
        """
        Checks if any of the attributes has a translation
        :param obj: translatable model instance
        :param fields: model attributes to check
        :param translations: translated fields for the current attribute values
        :return: True if the model has translations for the current code
        """
        translations_present = [self._has_translation(obj, attr, trans)
                                for attr, trans
                                in zip(fields, translations)]
        return reduce(lambda a, b: a or b, translations_present, False)

    def _has_translation(self, obj, attr, trans):
        """
        Checks if an attribute has a translation
        :param obj: translatable model instance
        :param attrs: model attributes to check
        :param trans: translated field for the current attribute value
        :return: True if the attribute has translations for the current code
        """
        return trans.strip() != '' and getattr(obj, attr) != trans

    def _update_trans(self, obj, attr, code, src, trans, is_verbose=False):
        """
        Updates a translation for an attribute on the object
        :param obj: object which has the translatable attribute
        :param attr: attribute name
        :param code: language code
        :param src: attribute value in the source language
        :param trans: attribute value in the translated language
        :return:
        """
        if getattr(obj, attr) != trans:
            if is_verbose:
                msg = ('Update %s.%s for %s: "%s" -> "%s"'
                       % (
                           obj.__class__.__name__, attr, code,
                           getattr(obj, attr),
                           trans))
                self._print(msg)
            setattr(obj, attr, trans)
        else:
            setattr(obj, attr, src)

    def _print(self, msg):
        self.stdout.write(self.style.SUCCESS(msg))
<|endoftext|>


<commit_before>import logging
from pylons import request, response, session, tmpl_context as c, url
from pylons.controllers.util import abort, redirect
from pylons.decorators import rest
from lr.lib.base import BaseController, render
import lr.model as m
from lr.lib.harvest import harvest
import json
from lr.model import LRNode as sourceLRNode
log = logging.getLogger(__name__)

class SwordError(Exception):
    def __init__(self,value):
      self.value = value
    def __str__(self):
      return repr(self.value)
class SwordserviceController(BaseController):
    def __init__(self):
        self.h = harvest()
    def __before__(self):
        response.headers['content-type'] = 'application/atom+xml;charset=utf-8'
        self.parse_params()
    def index(self):
        c.collection_url = 'http://' + request.host
        if sourceLRNode.nodeDescription.node_name is not None:
            c.node_name = sourceLRNode.nodeDescription.node_name
        else:
            c.node_name = sourceLRNode.nodeDescription.node_id
        c.node_description = sourceLRNode.nodeDescription.node_description
        if sourceLRNode.communityDescription.community_name is not None:
            c.community_name = sourceLRNode.communityDescription.community_name
        else:
            c.community_name = sourceLRNode.communityDescription.community_id
        return render('sword.mako')
    def parse_params(self):
        c.user_agent = request.headers['user-agent']
        c.no_op = False
        c.verbose = False
        if request.headers.has_key('X-On-Behalf-Of'):
            c.on_behalf_of = request.headers['X-On-Behalf-Of']
        if request.headers.has_key('X-Verbose'):
            c.verbose = request.headers['X-Verbose']
        if request.headers.has_key('X-No-Op'):
            c.no_op = request.headers['X-No-Op']
        c.tos_url = m.base_model.appConfig['tos.url']
        c.generator_url = 'http://' + request.host + '/sword'
    def create(self):
        if c.no_op:
            result = {'OK':True}
            c.doc = {'doc_ID':12345}
        else:
            result = m.publish(json.loads(request.body))
        if result['OK']:
            c.content_url = 'http://' + request.host + '/obtain/'+result['doc_ID']
            c.doc = self.h.get_record(result['doc_ID'])
            return render('sword-publish.mako')
        else:
            raise SwordError(result['error'])
<commit_msg>collection url was reporting http vs https incorrectly<commit_after>import logging
from pylons import request, response, session, tmpl_context as c, url
from pylons.controllers.util import abort, redirect
from pylons.decorators import rest
from lr.lib.base import BaseController, render
import lr.model as m
from lr.lib.harvest import harvest
import json
from lr.model import LRNode as sourceLRNode
log = logging.getLogger(__name__)

class SwordError(Exception):
    def __init__(self,value):
      self.value = value
    def __str__(self):
      return repr(self.value)
class SwordserviceController(BaseController):
    def __init__(self):
        self.h = harvest()
    def __before__(self):
        response.headers['content-type'] = 'application/atom+xml;charset=utf-8'
        self.parse_params()
    def index(self):
        c.collection_url = request.url
        if sourceLRNode.nodeDescription.node_name is not None:
            c.node_name = sourceLRNode.nodeDescription.node_name
        else:
            c.node_name = sourceLRNode.nodeDescription.node_id
        c.node_description = sourceLRNode.nodeDescription.node_description
        if sourceLRNode.communityDescription.community_name is not None:
            c.community_name = sourceLRNode.communityDescription.community_name
        else:
            c.community_name = sourceLRNode.communityDescription.community_id
        return render('sword.mako')
    def parse_params(self):
        c.user_agent = request.headers['user-agent']
        c.no_op = False
        c.verbose = False
        if request.headers.has_key('X-On-Behalf-Of'):
            c.on_behalf_of = request.headers['X-On-Behalf-Of']
        if request.headers.has_key('X-Verbose'):
            c.verbose = request.headers['X-Verbose']
        if request.headers.has_key('X-No-Op'):
            c.no_op = request.headers['X-No-Op']
        c.tos_url = m.base_model.appConfig['tos.url']
        c.generator_url = 'http://' + request.host + '/sword'
    def create(self):
        if c.no_op:
            result = {'OK':True}
            c.doc = {'doc_ID':12345}
        else:
            result = m.publish(json.loads(request.body))
        if result['OK']:
            c.content_url = 'http://' + request.host + '/obtain/'+result['doc_ID']
            c.doc = self.h.get_record(result['doc_ID'])
            return render('sword-publish.mako')
        else:
            raise SwordError(result['error'])
<|endoftext|>


<commit_before>import tensorflow as tf
from tensorflow.python.training import moving_averages

FLAGS = tf.app.flags.FLAGS

def _variable_on_cpu(name, shape, initializer, trainable = True):
    """Helper to create a Variable stored on CPU memory.

    Args:
            name: name of the variable
            shape: list of ints
            initializer: initializer for Variable

    Returns:
            Variable Tensor
    """
    with tf.device('/cpu:0'):
        var = tf.get_variable(name, shape, initializer=initializer, trainable = trainable)
    return var

def _variable_with_weight_decay(name, shape, wd = 0.0):
    """Helper to create an initialized Variable with weight decay.

    Note that the Variable is initialized with a xavier initialization.
    A weight decay is added only if one is specified.

    Args:
            name: name of the variable
            shape: list of ints
            wd: add L2Loss weight decay multiplied by this float. If None, weight
                    decay is not added for this Variable.

    Returns:
            Variable Tensor
    """
    var = _variable_on_cpu(name, shape, tf.contrib.layers.xavier_initializer())
    # print("change var")
    # var = tf.Variable(tf.truncated_normal(shape, mean= 0.0, stddev = 1.0), name = name)
    if wd != 0.0:
        weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
        tf.add_to_collection('losses', weight_decay)
    return var

def _conv2d(x, w, b, strides = [1,1,1,1]):
    return tf.nn.bias_add(tf.nn.conv2d(x, w,strides=strides, padding = 'SAME'), b)

def _conv3d(x, w, b, strides = [1,1,1,1,1], padding = 'SAME'):
    return tf.nn.bias_add(tf.nn.conv3d(x, w,strides=strides, padding = padding), b)

def add_leaky_relu(hl_tensor, leaky_param):
    return tf.maximum(hl_tensor, tf.mul(leaky_param, hl_tensor))

def _deconv2d(x, w, b, output_shape, strides = [1,1,1,1]):
    return tf.nn.bias_add(tf.nn.conv2d_transpose(x, w, output_shape, strides), b)

def _unpooling(x, output_size):
    """ NEAREST_NEIGHBOR resize """
    return tf.image.resize_images(x, output_size[1], output_size[2],1)

def _add_leaky_relu(hl_tensor, leaky_param):
    """ add leaky relu layer
        Args:
            leaky_params should be from 0.01 to 0.1
    """
    return tf.maximum(hl_tensor, tf.mul(leaky_param, hl_tensor))

def _max_pool(x, ksize, strides):
    """ 2d pool layer"""
    pool = tf.nn.max_pool(x, ksize=ksize, strides= strides,
            padding='VALID')
    return pool

def _max_pool3(x, ksize, strides, name):
    """ 3d pool layer"""
    pool = tf.nn.max_pool3d(x, ksize=ksize, strides= strides,
        padding='VALID', name = name)
    return pool

def _avg_pool3(x, ksize, strides, name):
    """ 3d average pool layer """
    pool = tf.nn.avg_pool3d(x, ksize = ksize, strides = strides,
            padding = 'VALID', name = name)
    return pool

def _batch_norm(inputs, decay = 0.999, center = True, scale = False, epsilon = 0.001,
				moving_vars = 'moving_vars', activation = None, is_training = None,
				trainable = True, restore = True, scope = None, reuse = None):
    """ Copied from slim/ops.py
        Adds a Batch Normalization layer.
        Args:

            inputs: a tensor of size [batch_size, height, width, channels]
                    or [batch_size, channels].
            decay: decay for the moving average.
            center: If True, subtract beta. If False, beta is not created and ignored.
            scale: If True, multiply by gamma. If False, gamma is
                    not used. When the next layer is linear (also e.g. ReLU), this can be
                    disabled since the scaling can be done by the next layer.
            epsilon: small float added to variance to avoid dividing by zero.
            moving_vars: collection to store the moving_mean and moving_variance.
            activation: activation function.
            is_training: a placeholder whether or not the model is in training mode.
            trainable: whether or not the variables should be trainable or not.
            restore: whether or not the variables should be marked for restore.
            scope: Optional scope for variable_op_scope.
            reuse: whether or not the layer and its variables should be reused. To be
                            able to reuse the layer scope must be given.

        Returns:
            a tensor representing the output of the operation.
    """
    inputs_shape = inputs.get_shape()
    with tf.variable_op_scope([inputs], scope, 'BatchNorm', reuse = reuse):
        axis = list(range(len(inputs_shape) - 1))
        params_shape = inputs_shape[-1:]
        beta, gamma = None, None

        if center:
                beta = _variable_on_cpu('beta', params_shape, tf.zeros_initializer)
        if scale:
                gamma = _variable_on_cpu('gamma', params_shape, tf.ones_initializer)

        # moving_collections = [moving_vars, tf.GraphKeys.MOVING_AVERAGE_VARIABLES]
        moving_mean = _variable_on_cpu('moving_mean', params_shape,tf.zeros_initializer, trainable = False)
        # tf.add_to_collection(tf.GraphKeys.MOVING_AVERAGE_VARIABLES, moving_mean)
        moving_variance = _variable_on_cpu('moving_variance', params_shape, tf.ones_initializer, trainable = False)
        # tf.add_to_collection(tf.GraphKeys.MOVING_AVERAGE_VARIABLES, moving_variance)

        def train_phase():
            mean, variance = tf.nn.moments(inputs, axis)
            update_moving_mean = moving_averages.assign_moving_average(moving_mean, mean, decay)
            update_moving_variance = moving_averages.assign_moving_average(moving_variance,
                                                            variance, decay)
            with tf.control_dependencies([update_moving_mean, update_moving_variance]):
                return tf.identity(mean), tf.identity(variance)

        def test_phase():
            return moving_mean, moving_variance

        mean, variance = tf.cond(is_training, train_phase, test_phase)
        outputs = tf.nn.batch_normalization(inputs, mean, variance, beta, gamma, epsilon)
        outputs.set_shape(inputs.get_shape())

        if activation:
            outputs = activation(outputs)

        return outputs

def triplet_loss(infer, labels, radius = 2.0):
    """
    Args:
        infer: inference concatenate together with 2 * batch_size
        labels: 0 or 1 with batch_size
        radius:
    Return:
        loss: triplet loss
    """

    feature_1, feature_2 = tf.split(0,2,infer)

    feature_diff = tf.reduce_sum(tf.square(feature_1 - feature_2), 1)
    feature_list = tf.dynamic_partition(feature_diff, labels, 2)

    pos_list = feature_list[1]
    neg_list  = (tf.maximum(0.0, radius * radius - feature_list[0]))
    full_list = tf.concat(0,[pos_list, neg_list])
    loss = tf.reduce_mean(full_list)

    return loss

def l2_loss(infer, label, layer_name):
    with tf.variable_scope(layer_name):
        loss = tf.reduce_mean(tf.square(infer - label))
    return loss

def convolution_2d_layer(x, kernel_shape, kernel_stride, wd, layer_name):
    with tf.variable_scope(layer_name):
        weights = _variable_with_weight_decay('weights', kernel_shape, wd)
        biases = _variable_on_cpu('biases', [kernel_shape[-1]], tf.constant_initializer(0.0))
        conv = _conv2d(x, weights, biases, strides =  kernel_stride)
    return conv

def deconvolution_2d_layer(x, kernel_shape, kernel_stride, output_shape, wd, layer_name):
    with tf.variable_scope(layer_name):
        weights = _variable_with_weight_decay('weights', kernel_shape, wd)
        biases = _variable_on_cpu('biases', [kernel_shape[-2]], tf.constant_initializer(0.0))
        deconv = _deconv2d(x, weights, biases, output_shape, kernel_stride)
    return deconv

def maxpool_2d_layer(x, kernel_shape, kernel_stride, layer_name):
    with tf.variable_scope(layer_name):
        max_pool = _max_pool(x, kernel_shape, kernel_stride, name)
    return max_pool
<commit_msg>change deconv<commit_after>import tensorflow as tf
from tensorflow.python.training import moving_averages

FLAGS = tf.app.flags.FLAGS

def _variable_on_cpu(name, shape, initializer, trainable = True):
    """Helper to create a Variable stored on CPU memory.

    Args:
            name: name of the variable
            shape: list of ints
            initializer: initializer for Variable

    Returns:
            Variable Tensor
    """
    with tf.device('/cpu:0'):
        var = tf.get_variable(name, shape, initializer=initializer, trainable = trainable)
    return var

def _variable_with_weight_decay(name, shape, wd = 0.0):
    """Helper to create an initialized Variable with weight decay.

    Note that the Variable is initialized with a xavier initialization.
    A weight decay is added only if one is specified.

    Args:
            name: name of the variable
            shape: list of ints
            wd: add L2Loss weight decay multiplied by this float. If None, weight
                    decay is not added for this Variable.

    Returns:
            Variable Tensor
    """
    var = _variable_on_cpu(name, shape, tf.contrib.layers.xavier_initializer())
    # print("change var")
    # var = tf.Variable(tf.truncated_normal(shape, mean= 0.0, stddev = 1.0), name = name)
    if wd != 0.0:
        weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
        tf.add_to_collection('losses', weight_decay)
    return var

def _conv2d(x, w, b, strides = [1,1,1,1]):
    return tf.nn.bias_add(tf.nn.conv2d(x, w,strides=strides, padding = 'SAME'), b)

def _conv3d(x, w, b, strides = [1,1,1,1,1], padding = 'SAME'):
    return tf.nn.bias_add(tf.nn.conv3d(x, w,strides=strides, padding = padding), b)

def add_leaky_relu(hl_tensor, leaky_param):
    return tf.maximum(hl_tensor, tf.mul(leaky_param, hl_tensor))

def _deconv2d(x, w, b, output_shape, strides = [1,1,1,1]):
    return tf.nn.bias_add(tf.nn.conv2d_transpose(x, w, output_shape, strides), b)

def _unpooling(x, output_size):
    """ NEAREST_NEIGHBOR resize """
    return tf.image.resize_images(x, output_size[1], output_size[2],1)

def _add_leaky_relu(hl_tensor, leaky_param):
    """ add leaky relu layer
        Args:
            leaky_params should be from 0.01 to 0.1
    """
    return tf.maximum(hl_tensor, tf.mul(leaky_param, hl_tensor))

def _max_pool(x, ksize, strides):
    """ 2d pool layer"""
    pool = tf.nn.max_pool(x, ksize=ksize, strides= strides,
            padding='VALID')
    return pool

def _max_pool3(x, ksize, strides, name):
    """ 3d pool layer"""
    pool = tf.nn.max_pool3d(x, ksize=ksize, strides= strides,
        padding='VALID', name = name)
    return pool

def _avg_pool3(x, ksize, strides, name):
    """ 3d average pool layer """
    pool = tf.nn.avg_pool3d(x, ksize = ksize, strides = strides,
            padding = 'VALID', name = name)
    return pool

def _batch_norm(inputs, decay = 0.999, center = True, scale = False, epsilon = 0.001,
				moving_vars = 'moving_vars', activation = None, is_training = None,
				trainable = True, restore = True, scope = None, reuse = None):
    """ Copied from slim/ops.py
        Adds a Batch Normalization layer.
        Args:

            inputs: a tensor of size [batch_size, height, width, channels]
                    or [batch_size, channels].
            decay: decay for the moving average.
            center: If True, subtract beta. If False, beta is not created and ignored.
            scale: If True, multiply by gamma. If False, gamma is
                    not used. When the next layer is linear (also e.g. ReLU), this can be
                    disabled since the scaling can be done by the next layer.
            epsilon: small float added to variance to avoid dividing by zero.
            moving_vars: collection to store the moving_mean and moving_variance.
            activation: activation function.
            is_training: a placeholder whether or not the model is in training mode.
            trainable: whether or not the variables should be trainable or not.
            restore: whether or not the variables should be marked for restore.
            scope: Optional scope for variable_op_scope.
            reuse: whether or not the layer and its variables should be reused. To be
                            able to reuse the layer scope must be given.

        Returns:
            a tensor representing the output of the operation.
    """
    inputs_shape = inputs.get_shape()
    with tf.variable_op_scope([inputs], scope, 'BatchNorm', reuse = reuse):
        axis = list(range(len(inputs_shape) - 1))
        params_shape = inputs_shape[-1:]
        beta, gamma = None, None

        if center:
                beta = _variable_on_cpu('beta', params_shape, tf.zeros_initializer)
        if scale:
                gamma = _variable_on_cpu('gamma', params_shape, tf.ones_initializer)

        # moving_collections = [moving_vars, tf.GraphKeys.MOVING_AVERAGE_VARIABLES]
        moving_mean = _variable_on_cpu('moving_mean', params_shape,tf.zeros_initializer, trainable = False)
        # tf.add_to_collection(tf.GraphKeys.MOVING_AVERAGE_VARIABLES, moving_mean)
        moving_variance = _variable_on_cpu('moving_variance', params_shape, tf.ones_initializer, trainable = False)
        # tf.add_to_collection(tf.GraphKeys.MOVING_AVERAGE_VARIABLES, moving_variance)

        def train_phase():
            mean, variance = tf.nn.moments(inputs, axis)
            update_moving_mean = moving_averages.assign_moving_average(moving_mean, mean, decay)
            update_moving_variance = moving_averages.assign_moving_average(moving_variance,
                                                            variance, decay)
            with tf.control_dependencies([update_moving_mean, update_moving_variance]):
                return tf.identity(mean), tf.identity(variance)

        def test_phase():
            return moving_mean, moving_variance

        mean, variance = tf.cond(is_training, train_phase, test_phase)
        outputs = tf.nn.batch_normalization(inputs, mean, variance, beta, gamma, epsilon)
        outputs.set_shape(inputs.get_shape())

        if activation:
            outputs = activation(outputs)

        return outputs

def triplet_loss(infer, labels, radius = 2.0):
    """
    Args:
        infer: inference concatenate together with 2 * batch_size
        labels: 0 or 1 with batch_size
        radius:
    Return:
        loss: triplet loss
    """

    feature_1, feature_2 = tf.split(0,2,infer)

    feature_diff = tf.reduce_sum(tf.square(feature_1 - feature_2), 1)
    feature_list = tf.dynamic_partition(feature_diff, labels, 2)

    pos_list = feature_list[1]
    neg_list  = (tf.maximum(0.0, radius * radius - feature_list[0]))
    full_list = tf.concat(0,[pos_list, neg_list])
    loss = tf.reduce_mean(full_list)

    return loss

def l2_loss(infer, label, layer_name):
    with tf.variable_scope(layer_name):
        loss = tf.reduce_mean(tf.square(infer - label))
    return loss

def convolution_2d_layer(x, kernel_shape, kernel_stride, wd, layer_name):
    with tf.variable_scope(layer_name):
        weights = _variable_with_weight_decay('weights', kernel_shape, wd)
        biases = _variable_on_cpu('biases', [kernel_shape[-1]], tf.constant_initializer(0.0))
        conv = _conv2d(x, weights, biases, strides =  kernel_stride)
    return conv

def deconvolution_2d_layer(x, kernel_shape, kernel_stride, output_shape, wd, padding, layer_name):
    with tf.variable_scope(layer_name):
        weights = _variable_with_weight_decay('weights', kernel_shape, wd)
        biases = _variable_on_cpu('biases', [kernel_shape[-2]], tf.constant_initializer(0.0))
        deconv = _deconv2d(x, weights, biases, output_shape, kernel_stride, padding)
    return deconv

def maxpool_2d_layer(x, kernel_shape, kernel_stride, layer_name):
    with tf.variable_scope(layer_name):
        max_pool = _max_pool(x, kernel_shape, kernel_stride, name)
    return max_pool
<|endoftext|>