<commit_before>import logging

from django.core.management.base import BaseCommand

import django.conf
django.conf.settings.CELERY_ALWAYS_EAGER = True

import mysite.search.tasks
import mysite.customs.bugtrackers.roundup_general
import mysite.customs.bugtrackers
import mysite.search.tasks.launchpad_tasks

import mysite.customs.miro
import mysite.customs.bugtrackers.trac
import mysite.customs.bugtrackers.gnome_love
import mysite.customs.bugtrackers.fedora_fitfinish
import mysite.customs.bugtrackers.mozilla
import mysite.customs.bugtrackers.wikimedia
import mysite.customs.bugtrackers.kde

### All this code runs synchronously once a day.
### For now, we can crawl all the bug trackers in serial.

### One day, though, we will crawl so many bug trackers that it will take
### too long.

### I suggest that, at that point, we fork a number of worker processes, and
### turn these lists into a queue or something. That should be easy with the
### multiprocessing module. Then we can do N at once.

### We could do something smart with statistics, detecting the average
### refresh time within the bug tracker of a bug, then polling at some
### approximation of that rate. (But, really, why bother?)

### Since most of the time we're waiting on the network, we could also use
### Twisted or Stackless Python or something. That would be super cool, too.
### If somone can show me how to migrate this codebase to it, that'd be neat.
### I'm unlikely to look into it myself, though.

### (If at some point we start indexing absolutely every bug in free and open
### source software, then the above ideas will actually become meaningful.)

### -- New Age Asheesh, 2010-05-31.

class Command(BaseCommand):
    help = "Call this once a day to make sure we run Bug search-related nightly jobs."

    def update_bugzilla_trackers(self):
        bugzilla_trackers = {
            'Miro':
                mysite.customs.miro.grab_miro_bugs,
            'KDE junior jobs':
                mysite.customs.bugtrackers.kde.grab,
            'Wikimedia easy bugs':
                mysite.customs.bugtrackers.wikimedia.grab,
            'GNOME Love':
                mysite.customs.bugtrackers.gnome_love.grab,
            'Mozilla "good first bug"s':
                mysite.customs.bugtrackers.mozilla.grab,

            # FIXME
            # Really, the Bugzilla import code should be reworked to be as
            # clean and tidy as the Mercurial/Python/Roundup stuff, with
            # an abstract class with a .update() method.
            # Then simple sub-classes can handle each of the different projects'
            # details.

            # What the heck is up with the Fedora code being
            # special-cased like this? Well, I'll clean it up another day,
            # so long as it seems to work right now.
            'Fedora "fit and finish" new bugs':
                mysite.search.tasks.bugzilla_instances.LearnAboutNewFedoraFitAndFinishBugs.apply,

            'Fedora "fit and finish" refreshing old bugs':
                mysite.search.tasks.bugzilla_instances.RefreshAllFedoraFitAndFinishBugs.apply,
            }

        for bugzilla_tracker in bugzilla_trackers:
            logging.info("Refreshing bugs from %s." % bugzilla_tracker)
            callable = bugzilla_trackers[bugzilla_tracker]

    def find_and_update_enabled_roundup_trackers(self):
        enabled_roundup_trackers = []

        ### First, the "find" step
        for thing_name in dir(mysite.customs.bugtrackers.roundup_general):
            thing = getattr(mysite.customs.bugtrackers.roundup_general,
                            thing_name)
            if hasattr(thing, 'enabled'):
                if getattr(thing, 'enabled'):
                    enabled_roundup_trackers.append(thing)

        ### Okay, now update!
        for thing in enabled_roundup_trackers:
            thing.update()

    def handle(self, *args, **options):
        # Make celery always eager, baby

        # A bunch of classes whose .run() we want to .delay()
        dot_run_these = [
            # Twisted
            mysite.search.tasks.trac_instances.LearnAboutNewEasyTwistedBugs,
            mysite.search.tasks.trac_instances.RefreshAllTwistedEasyBugs,
        ]
        for thing in dot_run_these:
            thing().run()

        # Just functions
        run_these = [
            # various projects hosted on Launchpad
            mysite.search.tasks.launchpad_tasks.refresh_bugs_from_all_indexed_launchpad_projects,
            mysite.search.tasks.launchpad_tasks.refresh_all_launchpad_bugs,

            # sweet sweet sugar
            mysite.search.tasks.trac_instances.learn_about_new_sugar_easy_bugs,
            mysite.search.tasks.trac_instances.refresh_all_sugar_easy_bugs,

            # miro, whee
            mysite.customs.miro.grab_miro_bugs,

            # KDE
            mysite.customs.bugtrackers.kde.grab,

            # Wikimedia
            mysite.customs.bugtrackers.wikimedia.grab,

            # GNOME is in Love
            mysite.customs.bugtrackers.gnome_love.grab,

            # Mozira
            mysite.customs.bugtrackers.mozilla.grab,

            # Fedora:
            mysite.search.tasks.bugzilla_instances.LearnAboutNewFedoraFitAndFinishBugs.apply,
            mysite.search.tasks.bugzilla_instances.RefreshAllFedoraFitAndFinishBugs.apply,
        ]
        for callable in run_these:
            logging.info("About to run %s" % callable)
            callable()

        # And for Roundup bug trackers, use our special handling
        self.find_and_update_enabled_roundup_trackers()

        # And for Bugzilla bug trackers, use our special handling!
        self.update_bugzilla_trackers()

<commit_msg>Instantiate each RoundupTracker<commit_after>import logging

from django.core.management.base import BaseCommand

import django.conf
django.conf.settings.CELERY_ALWAYS_EAGER = True

import mysite.search.tasks
import mysite.customs.bugtrackers.roundup_general
import mysite.customs.bugtrackers
import mysite.search.tasks.launchpad_tasks

import mysite.customs.miro
import mysite.customs.bugtrackers.trac
import mysite.customs.bugtrackers.gnome_love
import mysite.customs.bugtrackers.fedora_fitfinish
import mysite.customs.bugtrackers.mozilla
import mysite.customs.bugtrackers.wikimedia
import mysite.customs.bugtrackers.kde

### All this code runs synchronously once a day.
### For now, we can crawl all the bug trackers in serial.

### One day, though, we will crawl so many bug trackers that it will take
### too long.

### I suggest that, at that point, we fork a number of worker processes, and
### turn these lists into a queue or something. That should be easy with the
### multiprocessing module. Then we can do N at once.

### We could do something smart with statistics, detecting the average
### refresh time within the bug tracker of a bug, then polling at some
### approximation of that rate. (But, really, why bother?)

### Since most of the time we're waiting on the network, we could also use
### Twisted or Stackless Python or something. That would be super cool, too.
### If somone can show me how to migrate this codebase to it, that'd be neat.
### I'm unlikely to look into it myself, though.

### (If at some point we start indexing absolutely every bug in free and open
### source software, then the above ideas will actually become meaningful.)

### -- New Age Asheesh, 2010-05-31.

class Command(BaseCommand):
    help = "Call this once a day to make sure we run Bug search-related nightly jobs."

    def update_bugzilla_trackers(self):
        bugzilla_trackers = {
            'Miro':
                mysite.customs.miro.grab_miro_bugs,
            'KDE junior jobs':
                mysite.customs.bugtrackers.kde.grab,
            'Wikimedia easy bugs':
                mysite.customs.bugtrackers.wikimedia.grab,
            'GNOME Love':
                mysite.customs.bugtrackers.gnome_love.grab,
            'Mozilla "good first bug"s':
                mysite.customs.bugtrackers.mozilla.grab,

            # FIXME
            # Really, the Bugzilla import code should be reworked to be as
            # clean and tidy as the Mercurial/Python/Roundup stuff, with
            # an abstract class with a .update() method.
            # Then simple sub-classes can handle each of the different projects'
            # details.

            # What the heck is up with the Fedora code being
            # special-cased like this? Well, I'll clean it up another day,
            # so long as it seems to work right now.
            'Fedora "fit and finish" new bugs':
                mysite.search.tasks.bugzilla_instances.LearnAboutNewFedoraFitAndFinishBugs.apply,

            'Fedora "fit and finish" refreshing old bugs':
                mysite.search.tasks.bugzilla_instances.RefreshAllFedoraFitAndFinishBugs.apply,
            }

        for bugzilla_tracker in bugzilla_trackers:
            logging.info("Refreshing bugs from %s." % bugzilla_tracker)
            callable = bugzilla_trackers[bugzilla_tracker]

    def find_and_update_enabled_roundup_trackers(self):
        enabled_roundup_trackers = []

        ### First, the "find" step
        for thing_name in dir(mysite.customs.bugtrackers.roundup_general):
            thing = getattr(mysite.customs.bugtrackers.roundup_general,
                            thing_name)
            if hasattr(thing, 'enabled'):
                if getattr(thing, 'enabled'):
                    enabled_roundup_trackers.append(thing)

        ### Okay, now update!
        for thing in enabled_roundup_trackers:
            instantiated = thing()
            instantiated.update()

    def handle(self, *args, **options):
        # Make celery always eager, baby

        # A bunch of classes whose .run() we want to .delay()
        dot_run_these = [
            # Twisted
            mysite.search.tasks.trac_instances.LearnAboutNewEasyTwistedBugs,
            mysite.search.tasks.trac_instances.RefreshAllTwistedEasyBugs,
        ]
        for thing in dot_run_these:
            thing().run()

        # Just functions
        run_these = [
            # various projects hosted on Launchpad
            mysite.search.tasks.launchpad_tasks.refresh_bugs_from_all_indexed_launchpad_projects,
            mysite.search.tasks.launchpad_tasks.refresh_all_launchpad_bugs,

            # sweet sweet sugar
            mysite.search.tasks.trac_instances.learn_about_new_sugar_easy_bugs,
            mysite.search.tasks.trac_instances.refresh_all_sugar_easy_bugs,

            # miro, whee
            mysite.customs.miro.grab_miro_bugs,

            # KDE
            mysite.customs.bugtrackers.kde.grab,

            # Wikimedia
            mysite.customs.bugtrackers.wikimedia.grab,

            # GNOME is in Love
            mysite.customs.bugtrackers.gnome_love.grab,

            # Mozira
            mysite.customs.bugtrackers.mozilla.grab,

            # Fedora:
            mysite.search.tasks.bugzilla_instances.LearnAboutNewFedoraFitAndFinishBugs.apply,
            mysite.search.tasks.bugzilla_instances.RefreshAllFedoraFitAndFinishBugs.apply,
        ]
        for callable in run_these:
            logging.info("About to run %s" % callable)
            callable()

        # And for Roundup bug trackers, use our special handling
        self.find_and_update_enabled_roundup_trackers()

        # And for Bugzilla bug trackers, use our special handling!
        self.update_bugzilla_trackers()

<|endoftext|>


<commit_before># -*- coding: utf8 -*-
# Copyright (c) 2014 by Ecreall under licence AGPL terms
# avalaible on http://www.gnu.org/licenses/agpl.html

# licence: AGPL
# author: Amen Souissi

from pyramid.view import view_config

from substanced.util import get_oid

from dace.processinstance.core import DEFAULTMAPPING_ACTIONS_VIEWS

from novaideo.content.processes.novaideo_view_manager.behaviors import (
    SeeMySupports)
from novaideo.content.novaideo_application import NovaIdeoApplication
from novaideo import _
from novaideo.views.filter import (
    FILTER_SOURCES)
from .see_my_contents import SeeMyContentsView

CONTENTS_MESSAGES = {
    '0': _(u"""Aucune propositions appréciées """
          u"""et ${tokens} jeton(s) d'appréciation restant(s)"""),
    '1': _(u"""Une proposition appréciée """
            u"""et ${tokens} jeton(s) d'appréciation restant(s)"""),
    '*': _(u"""${nember} propositions appréciées """
            u"""et ${tokens} jeton(s) d'appréciation restant(s)""")
}

@view_config(
    name='seemysupports',
    context=NovaIdeoApplication,
    renderer='pontus:templates/views_templates/grid.pt',
    )
class SeeMySupportsView(SeeMyContentsView):
    title = _('My supports')
    name = 'seemysupports'
    behaviors = [SeeMySupports]
    template = 'novaideo:views/novaideo_view_manager/templates/search_result.pt'
    viewid = 'seemysupports'
    contents_messages = CONTENTS_MESSAGES
    selected_filter = [('metadata_filter', ['negation', 'keywords', 'states']),
                       ('temporal_filter', ['negation', 'created_date']),
                       'text_filter', 'other_filter']
    include_archived = False

    def _get_title(self, **args):
        return _(self.contents_messages[args.get('index')],
            mapping={'nember': args.get('len_result'),
                     'tokens': len(getattr(args.get('user'), 'tokens', []))})

    def _get_content_ids(self, user):
        return [get_oid(o) for o in getattr(user, 'supports', [])]

DEFAULTMAPPING_ACTIONS_VIEWS.update(
    {SeeMySupports: SeeMySupportsView})

FILTER_SOURCES.update(
    {SeeMySupportsView.name: SeeMySupportsView})
<commit_msg>fix view title<commit_after># -*- coding: utf8 -*-
# Copyright (c) 2014 by Ecreall under licence AGPL terms
# avalaible on http://www.gnu.org/licenses/agpl.html

# licence: AGPL
# author: Amen Souissi

from pyramid.view import view_config

from substanced.util import get_oid

from dace.processinstance.core import DEFAULTMAPPING_ACTIONS_VIEWS

from novaideo.content.processes.novaideo_view_manager.behaviors import (
    SeeMySupports)
from novaideo.content.novaideo_application import NovaIdeoApplication
from novaideo import _
from novaideo.views.filter import (
    FILTER_SOURCES)
from .see_my_contents import SeeMyContentsView

CONTENTS_MESSAGES = {
    '0': _(u"""Aucun contenus appréciées """
          u"""et ${tokens} jeton(s) d'appréciation restant(s)"""),
    '1': _(u"""Un contenu appréciée """
            u"""et ${tokens} jeton(s) d'appréciation restant(s)"""),
    '*': _(u"""${nember} contenus appréciées """
            u"""et ${tokens} jeton(s) d'appréciation restant(s)""")
}

@view_config(
    name='seemysupports',
    context=NovaIdeoApplication,
    renderer='pontus:templates/views_templates/grid.pt',
    )
class SeeMySupportsView(SeeMyContentsView):
    title = _('My supports')
    name = 'seemysupports'
    behaviors = [SeeMySupports]
    template = 'novaideo:views/novaideo_view_manager/templates/search_result.pt'
    viewid = 'seemysupports'
    contents_messages = CONTENTS_MESSAGES
    selected_filter = [('metadata_filter', ['negation', 'keywords', 'states']),
                       ('temporal_filter', ['negation', 'created_date']),
                       'text_filter', 'other_filter']
    include_archived = False

    def _get_title(self, **args):
        return _(self.contents_messages[args.get('index')],
            mapping={'nember': args.get('len_result'),
                     'tokens': len(getattr(args.get('user'), 'tokens', []))})

    def _get_content_ids(self, user):
        return [get_oid(o) for o in getattr(user, 'supports', [])]

DEFAULTMAPPING_ACTIONS_VIEWS.update(
    {SeeMySupports: SeeMySupportsView})

FILTER_SOURCES.update(
    {SeeMySupportsView.name: SeeMySupportsView})
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenUpgrade module for Odoo
#    @copyright 2014-Today: Odoo Community Association
#    @author: Sylvain LE GAL <https://twitter.com/legalsylvain>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.openupgrade import openupgrade
from openerp.addons.openupgrade_records.lib import apriori

@openupgrade.migrate()
def migrate(cr, version):
    openupgrade.update_module_names(
        cr, apriori.renamed_modules.iteritems()
    )
    openupgrade.check_values_selection_field(
        cr, 'ir_act_report_xml', 'report_type',
        ['controller', 'pdf', 'qweb-html', 'qweb-pdf', 'sxw', 'webkit'])
    openupgrade.check_values_selection_field(
        cr, 'ir_ui_view', 'type', [
            'calendar', 'diagram', 'form', 'gantt', 'graph', 'kanban',
            'qweb', 'search', 'tree'])
<commit_msg>drop report_document_user view in base to alter table columns<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenUpgrade module for Odoo
#    @copyright 2014-Today: Odoo Community Association
#    @author: Sylvain LE GAL <https://twitter.com/legalsylvain>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.openupgrade import openupgrade
from openerp.addons.openupgrade_records.lib import apriori

@openupgrade.migrate()
def migrate(cr, version):
    # Drop view that inhibits changing field types. It will be recreated BTW
    cr.execute('drop view if exists report_document_user cascade')

    openupgrade.update_module_names(
        cr, apriori.renamed_modules.iteritems()
    )
    openupgrade.check_values_selection_field(
        cr, 'ir_act_report_xml', 'report_type',
        ['controller', 'pdf', 'qweb-html', 'qweb-pdf', 'sxw', 'webkit'])
    openupgrade.check_values_selection_field(
        cr, 'ir_ui_view', 'type', [
            'calendar', 'diagram', 'form', 'gantt', 'graph', 'kanban',
            'qweb', 'search', 'tree'])
<|endoftext|>


<commit_before>import pygit2
from datetime import datetime

from django.contrib import admin
from django.contrib.admin import SimpleListFilter
from django.shortcuts import render
from django.conf import settings
from django.contrib.admin.util import unquote
from django.contrib.admin import helpers

from cms.models import Post, Category
from cms.git import repo

class CategoriesListFilter(SimpleListFilter):
    title = "categories"
    parameter_name = "category_slug"

    def lookups(self, request, model_admin):
        """
        Returns a list of tuples. The first element in each
        tuple is the coded value for the option that will
        appear in the URL query. The second element is the
        human-readable name for the option that will appear
        in the right sidebar.
        """
        return Category.objects.values_list('slug', 'title')

    def queryset(self, request, queryset):
        """
        Returns queryset filtered on categories and primary_category.
        """
        if self.value():
            category = Category.objects.get(slug=self.value())
            return queryset.filter(primary_category=category)

class PostAdmin(admin.ModelAdmin):
    list_display = (
        'title', 'subtitle', 'primary_category', 'created_at', 'language',
        'source', '_derivatives', 'uuid')

    list_filter = ('created_at', CategoriesListFilter,)
    search_fields = ('title', 'description', 'content')
    raw_id_fields = ('source', 'owner')
    fieldsets = (
        (None, {'fields': ('title', 'subtitle', 'description', 'content', )}),
        ('Meta',
            {'fields': (
                'primary_category', 'owner', 'created_at', 'source',
                'language')})
    )

    def _derivatives(self, post):
        return len(post.post_set.all())
    _derivatives.short_description = 'Derivatives'
    _derivatives.allow_tags = True

    def save_model(self, request, obj, form, change):
        if not obj.owner:
            obj.owner = request.user
        obj.last_author = request.user

        super(PostAdmin, self).save_model(
            request,
            obj,
            form,
            change
        )

    def add_view(self, request, form_url='', extra_context=None):
        object_id = request.GET.get('source', '')
        extra_context = extra_context or {}
        obj = self.get_object(request, unquote(object_id))

        if obj is not None:
            ModelForm = self.get_form(request, obj)
            form = ModelForm(instance=obj)
            extra_context['sourceform'] = helpers.AdminForm(
                form, self.get_fieldsets(request, obj),
                self.get_prepopulated_fields(request, obj),
                self.get_readonly_fields(request, obj),
                model_admin=self)
        return super(PostAdmin, self).add_view(request, form_url, extra_context)

    def change_view(self, request, object_id, form_url='', extra_context=None):
        extra_context = extra_context or {}
        obj = self.get_object(request, unquote(object_id))

        if not self.has_change_permission(request, obj):
            raise PermissionDenied

        if obj is None:
            raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {'name': force_text(opts.verbose_name), 'key': escape(object_id)})

        if obj.source:
            ModelForm = self.get_form(request, obj.source)
            form = ModelForm(instance=obj.source)
            extra_context['sourceform'] = helpers.AdminForm(
                form, self.get_fieldsets(request, obj.source),
                self.get_prepopulated_fields(request, obj.source),
                self.get_readonly_fields(request, obj.source),
                model_admin=self)
        return super(PostAdmin, self).change_view(
            request, object_id, form_url, extra_context)

class CategoryAdmin(admin.ModelAdmin):
    prepopulated_fields = {"slug": ("title",)}
    list_display = ('title', 'subtitle', 'uuid')

    def save_model(self, request, obj, form, change):
        obj.last_author = request.user
        super(CategoryAdmin, self).save_model(request, obj, form, change)

@admin.site.register_view('github/', 'Github Configuration')
def my_view(request, *args, **kwargs):
    branch = repo.lookup_branch(repo.head.shorthand)
    last = repo[branch.target]
    commits = []
    for commit in repo.walk(last.id, pygit2.GIT_SORT_TIME):
        commits.append(commit)

    context = {
        'github_url': settings.GIT_REPO_URL,
        'repo': repo,
        'commits': [
            {
                'message': c.message,
                'author': c.author.name,
                'commit_time': datetime.fromtimestamp(c.commit_time)
            }
            for c in commits
        ]
    }
    return render(request, 'cms/admin/github.html', context)

admin.site.register(Post, PostAdmin)
admin.site.register(Category, CategoryAdmin)
<commit_msg>fix missing imports<commit_after>import pygit2
from datetime import datetime

from django.contrib import admin
from django.contrib.admin import SimpleListFilter
from django.shortcuts import render
from django.conf import settings
from django.contrib.admin.util import unquote
from django.contrib.admin import helpers
from django.http import Http404
from django.utils.html import escape
from django.core.exceptions import PermissionDenied

from cms.models import Post, Category
from cms.git import repo

class CategoriesListFilter(SimpleListFilter):
    title = "categories"
    parameter_name = "category_slug"

    def lookups(self, request, model_admin):
        """
        Returns a list of tuples. The first element in each
        tuple is the coded value for the option that will
        appear in the URL query. The second element is the
        human-readable name for the option that will appear
        in the right sidebar.
        """
        return Category.objects.values_list('slug', 'title')

    def queryset(self, request, queryset):
        """
        Returns queryset filtered on categories and primary_category.
        """
        if self.value():
            category = Category.objects.get(slug=self.value())
            return queryset.filter(primary_category=category)

class PostAdmin(admin.ModelAdmin):
    list_display = (
        'title', 'subtitle', 'primary_category', 'created_at', 'language',
        'source', '_derivatives', 'uuid')

    list_filter = ('created_at', CategoriesListFilter,)
    search_fields = ('title', 'description', 'content')
    raw_id_fields = ('source', 'owner')
    fieldsets = (
        (None, {'fields': ('title', 'subtitle', 'description', 'content', )}),
        ('Meta',
            {'fields': (
                'primary_category', 'owner', 'created_at', 'source',
                'language')})
    )

    def _derivatives(self, post):
        return len(post.post_set.all())
    _derivatives.short_description = 'Derivatives'
    _derivatives.allow_tags = True

    def save_model(self, request, obj, form, change):
        if not obj.owner:
            obj.owner = request.user
        obj.last_author = request.user

        super(PostAdmin, self).save_model(
            request,
            obj,
            form,
            change
        )

    def add_view(self, request, form_url='', extra_context=None):
        object_id = request.GET.get('source', '')
        extra_context = extra_context or {}
        obj = self.get_object(request, unquote(object_id))

        if obj is not None:
            ModelForm = self.get_form(request, obj)
            form = ModelForm(instance=obj)
            extra_context['sourceform'] = helpers.AdminForm(
                form, self.get_fieldsets(request, obj),
                self.get_prepopulated_fields(request, obj),
                self.get_readonly_fields(request, obj),
                model_admin=self)
        return super(PostAdmin, self).add_view(
            request, form_url, extra_context)

    def change_view(self, request, object_id, form_url='', extra_context=None):
        extra_context = extra_context or {}
        obj = self.get_object(request, unquote(object_id))

        if not self.has_change_permission(request, obj):
            raise PermissionDenied

        if obj is None:
            raise Http404(
                'Post object with primary key %(key)r does not exist.' %
                {'key': escape(object_id)})

        if obj.source:
            ModelForm = self.get_form(request, obj.source)
            form = ModelForm(instance=obj.source)
            extra_context['sourceform'] = helpers.AdminForm(
                form, self.get_fieldsets(request, obj.source),
                self.get_prepopulated_fields(request, obj.source),
                self.get_readonly_fields(request, obj.source),
                model_admin=self)
        return super(PostAdmin, self).change_view(
            request, object_id, form_url, extra_context)

class CategoryAdmin(admin.ModelAdmin):
    prepopulated_fields = {"slug": ("title",)}
    list_display = ('title', 'subtitle', 'uuid')

    def save_model(self, request, obj, form, change):
        obj.last_author = request.user
        super(CategoryAdmin, self).save_model(request, obj, form, change)

@admin.site.register_view('github/', 'Github Configuration')
def my_view(request, *args, **kwargs):
    branch = repo.lookup_branch(repo.head.shorthand)
    last = repo[branch.target]
    commits = []
    for commit in repo.walk(last.id, pygit2.GIT_SORT_TIME):
        commits.append(commit)

    context = {
        'github_url': settings.GIT_REPO_URL,
        'repo': repo,
        'commits': [
            {
                'message': c.message,
                'author': c.author.name,
                'commit_time': datetime.fromtimestamp(c.commit_time)
            }
            for c in commits
        ]
    }
    return render(request, 'cms/admin/github.html', context)

admin.site.register(Post, PostAdmin)
admin.site.register(Category, CategoryAdmin)
<|endoftext|>


<commit_before># This program handles the communication over I2C between a Raspberry Pi and an ADXL345 Accelerometer
# Made by: MrTijn/Tijndagamer
# Copyright 2015

import smbus

class MrTijnADXL345:

    # Global Variables
    earthGravityMS2               = 9.80665
    moonGravityMS2                = 1.62243
    marsGravityMS2                = 3.71
    gravityMS2                    = 0
    scaleMultiplier               = 0.004
    bus                           = smbus.SMBus(1)
    address                       = None

    # ADXL345 Registers
    dataFormat                    = 0x31
    bandwithRate                  = 0x2C
    POWER_CTL                     = 0x2D

    bandwithRate1600HZ            = 0x0F
    bandwithRate800HZ             = 0x0E
    bandwithRate400HZ             = 0x0D
    bandwithRate200HZ             = 0x0C
    bandwithRate100HZ             = 0x0B
    bandwithRate50HZ              = 0x0A
    bandwithRate25HZ              = 0x09

    range2G                       = 0x00
    range4G                       = 0x01
    range8G                       = 0x02
    range16G                      = 0x03

    Measure                       = 0x08

    DATAX0                        = 0x32
    DATAX1                        = 0x33
    DATAY0                        = 0x34
    DATAY1                        = 0x35
    DATAZ0                        = 0x36
    DATAZ1                        = 0x37

    def __init__(self, baseRange = range2G, address = 0x52, celestialBody = "earth"):
        if celestialBody == "earth":
            self.gravityMS2 = self.earthGravityMS2
        elif celestialBody == "moon":
            self.gravityMS2 = self.moonGravityMS2
        elif celestialBody == "mars":
            self.gravityMS2 = self.marsGravityMS2
        else:
            raise InputError("celestial body", celestialBody)

        self.address = address
        self.SetBandwithRate(self.bandwithRate100HZ)
        self.SetRange(baseRange)
        self.EnableMeasurement()

    # Enables measurement by writing 0x08 to POWER_CTL, register 0x27
    def EnableMeasurement(self):
        try:
            self.bus.write_byte_data(self.address, self.POWER_CTL, self.Measure)
        except:
            print("Error in EnableMeasurement(), are you sure that the ADXL345 is plugged in and wired correctly?")

    # Disables measurement by writing 0x00 to POWER_CTL, register 0x27
    def DisableMeasurement(self):
        self.bus.write_byte_data(self.address, self.POWER_CTL, 0x00)

    # Reads POWER_CTL, register 0x27
    def ReadMeasurementMode(self):
        return self.bus.read_byte_data(self.address, self.POWER_CTL)

    # Changes the bandwithRate by writing rate to bandwithRate, register 0x2C
    def SetBandwithRate(self, rate):
        self.bus.write_byte_data(self.address, self.bandwithRate, rate)

    # Reads bandwithRate, register 0x2C
    def ReadBandwithRate(self):
        rawBandwithRate = self.bus.read_byte_data(self.address, self.bandwithRate)
        return rawBandwithRate & 0x0F

    # Changes the range of the ADXL345. Available ranges are 2G, 4G, 8G and 16G.
    def SetRange(self, range):
        value = self.bus.read_byte_data(self.address, self.dataFormat)

        value &= ~0x0F;
        value |= range;
        value |= 0x08;

        self.bus.write_byte_data(self.address, self.dataFormat, value)

    # Reads the range the ADXL345 is set to.
    def ReadRange(self):
        #todo
        pass

    # Gets all the axes and returns them in a dictionary
    def GetAllAxes(self, round = False):
        # Read the raw bytes from the ADXL345
        bytes = self.bus.read_i2c_block_data(self.address, self.DATAX0, 6)

        x = bytes[0] | (bytes[1] << 8)
        if(x & (1 << 16 - 1)):
            x = x - (1<<16)

        y = bytes[2] | (bytes[3] << 8)
        if(y & (1 << 16 - 1)):
            y = y - (1<<16)

        z = bytes[4] | (bytes[5] << 8)
        if(z & (1 << 16 - 1)):
            z = z - (1<<16)

        x = x * self.scaleMultiplier
        y = y * self.scaleMultiplier
        z = z * self.scaleMultiplier

        x = x * self.gravityMS2
        y = y * self.gravityMS2
        z = z * self.gravityMS2

        if round == True:
            x = round(x, 4)
            y = round(y, 4)
            z = round(z, 4)

        return {"x": x, "y": y, "z": z}

    def GetOneValue(self, value, round = False):
        readRegister = 0x00

        if value == "x":
            readRegister = self.DATAX0
        elif value == "y":
            readRegister = self.DATAY0
        elif value == "z":
            readRegister = self.DATAZ0

        #Read the raw bytes from the ADXL345
        bytes = self.bus.read_i2c_block_data(self.address, readRegister, 2)

        x = bytes[0] | (bytes[1] << 8)
        if(x & (1 << 16 - 1)):
            x = x - (1<<16)

        x = x * self.scaleMultiplier

        x = x * self.gravityMS2

        if round == True:
            x = round(x, 4)

        return x

if __name__ == "__main__":
    accelerometer = MrTijnADXL345()
    axes = accelerometer.GetAllAxes()
    print("x: %.3f" % ( axes['x'] ))
    print("y: %.3f" % ( axes['y'] ))
    print("z: %.3f" % ( axes['z'] ))

class InputError(Exception):

    def __init__(self, errorMessage, inputExpression):
        self.errorMessage = errorMessage
        self.inputExpression = inputExpression
        print(inputExpression + " is an invalid " + errorMessage)
<commit_msg>ADXL345 - Update 1.2.1.1, Changed class name<commit_after># This program handles the communication over I2C between a Raspberry Pi and an ADXL345 Accelerometer
# Made by: MrTijn/Tijndagamer
# Copyright 2015

import smbus

class ADXL345:

    # Global Variables
    earthGravityMS2               = 9.80665
    moonGravityMS2                = 1.62243
    marsGravityMS2                = 3.71
    gravityMS2                    = 0
    scaleMultiplier               = 0.004
    bus                           = smbus.SMBus(1)
    address                       = None

    # ADXL345 Registers
    dataFormat                    = 0x31
    bandwithRate                  = 0x2C
    POWER_CTL                     = 0x2D
    Measure                       = 0x08

    bandwithRate1600HZ            = 0x0F
    bandwithRate800HZ             = 0x0E
    bandwithRate400HZ             = 0x0D
    bandwithRate200HZ             = 0x0C
    bandwithRate100HZ             = 0x0B
    bandwithRate50HZ              = 0x0A
    bandwithRate25HZ              = 0x09

    range2G                       = 0x00
    range4G                       = 0x01
    range8G                       = 0x02
    range16G                      = 0x03

    DATAX0                        = 0x32
    DATAX1                        = 0x33
    DATAY0                        = 0x34
    DATAY1                        = 0x35
    DATAZ0                        = 0x36
    DATAZ1                        = 0x37

    def __init__(self, baseRange = range2G, address = 0x52, celestialBody = "earth"):
        if celestialBody == "earth":
            self.gravityMS2 = self.earthGravityMS2
        elif celestialBody == "moon":
            self.gravityMS2 = self.moonGravityMS2
        elif celestialBody == "mars":
            self.gravityMS2 = self.marsGravityMS2
        else:
            raise InputError("celestial body", celestialBody)

        self.address = address
        self.SetBandwithRate(self.bandwithRate100HZ)
        self.SetRange(baseRange)
        self.EnableMeasurement()

    # Enables measurement by writing 0x08 to POWER_CTL, register 0x27
    def EnableMeasurement(self):
        try:
            self.bus.write_byte_data(self.address, self.POWER_CTL, self.Measure)
        except:
            print("Error in EnableMeasurement(), are you sure that the ADXL345 is plugged in and wired correctly?")

    # Disables measurement by writing 0x00 to POWER_CTL, register 0x27
    def DisableMeasurement(self):
        self.bus.write_byte_data(self.address, self.POWER_CTL, 0x00)

    # Reads POWER_CTL, register 0x27
    def ReadMeasurementMode(self):
        return self.bus.read_byte_data(self.address, self.POWER_CTL)

    # Changes the bandwithRate by writing rate to bandwithRate, register 0x2C
    def SetBandwithRate(self, rate):
        self.bus.write_byte_data(self.address, self.bandwithRate, rate)

    # Reads bandwithRate, register 0x2C
    def ReadBandwithRate(self):
        rawBandwithRate = self.bus.read_byte_data(self.address, self.bandwithRate)
        return rawBandwithRate & 0x0F

    # Changes the range of the ADXL345. Available ranges are 2G, 4G, 8G and 16G.
    def SetRange(self, range):
        value = self.bus.read_byte_data(self.address, self.dataFormat)

        value &= ~0x0F;
        value |= range;
        value |= 0x08;

        self.bus.write_byte_data(self.address, self.dataFormat, value)

    # Reads the range the ADXL345 is set to.
    def ReadRange(self):
        #todo
        pass

    # Gets all the axes and returns them in a dictionary
    def GetAllAxes(self, round = False):
        # Read the raw bytes from the ADXL345
        bytes = self.bus.read_i2c_block_data(self.address, self.DATAX0, 6)

        x = bytes[0] | (bytes[1] << 8)
        if(x & (1 << 16 - 1)):
            x = x - (1<<16)

        y = bytes[2] | (bytes[3] << 8)
        if(y & (1 << 16 - 1)):
            y = y - (1<<16)

        z = bytes[4] | (bytes[5] << 8)
        if(z & (1 << 16 - 1)):
            z = z - (1<<16)

        x = x * self.scaleMultiplier
        y = y * self.scaleMultiplier
        z = z * self.scaleMultiplier

        x = x * self.gravityMS2
        y = y * self.gravityMS2
        z = z * self.gravityMS2

        if round == True:
            x = round(x, 4)
            y = round(y, 4)
            z = round(z, 4)

        return {"x": x, "y": y, "z": z}

    #Gets one specific value and returns it
    def GetOneValue(self, value, round = False):
        readRegister = 0x00

        if value == "x":
            readRegister = self.DATAX0
        elif value == "y":
            readRegister = self.DATAY0
        elif value == "z":
            readRegister = self.DATAZ0

        #Read the raw bytes from the ADXL345
        bytes = self.bus.read_i2c_block_data(self.address, readRegister, 2)

        x = bytes[0] | (bytes[1] << 8)
        if(x & (1 << 16 - 1)):
            x = x - (1<<16)

        x = x * self.scaleMultiplier

        x = x * self.gravityMS2

        if round == True:
            x = round(x, 4)

        return x

if __name__ == "__main__":
    accelerometer = MrTijnADXL345()
    axes = accelerometer.GetAllAxes()
    print("x: %.3f" % ( axes['x'] ))
    print("y: %.3f" % ( axes['y'] ))
    print("z: %.3f" % ( axes['z'] ))

class InputError(Exception):

    def __init__(self, errorMessage, inputExpression):
        self.errorMessage = errorMessage
        self.inputExpression = inputExpression
        print(inputExpression + " is an invalid " + errorMessage)
<|endoftext|>


<commit_before><commit_msg>tests(base): Fix indentation<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    livereload.watcher
    ~~~~~~~~~~~~~~~~~~

    A file watch management for LiveReload Server.

    :copyright: (c) 2013 by Hsiaoming Yang
"""

import os
import glob

class Watcher(object):
    """A file watcher registery."""
    def __init__(self):
        self._tasks = {}
        self._mtimes = {}

        # filepath that is changed
        self.filepath = None

    def ignore(self, filename):
        """Ignore a given filename or not."""
        _, ext = os.path.splitext(filename)
        return ext in ['.pyc', '.pyo', '.o', '.swp']

    def watch(self, path, func=None):
        """Add a task to watcher."""
        self._tasks[path] = func

    def examine(self):
        """Check if there are changes, if true, run the given task."""
        # clean filepath
        self.filepath = None
        for path in self._tasks:
            if self.is_changed(path):
                func = self._tasks[path]
                # run function
                func and func()
        return self.filepath

    def is_changed(self, path):
        if os.path.isfile(path):
            return self.is_file_changed(path)
        elif os.path.isdir(path):
            return self.is_folder_changed(path)
        return self.is_glob_changed(path)

    def is_file_changed(self, path):
        if not os.path.isfile(path):
            return False

        if self.ignore(path):
            return False

        mtime = os.path.getmtime(path)

        if path not in self._mtimes:
            self._mtimes[path] = mtime
            self.filepath = path
            return True

        if self._mtimes[path] != mtime:
            self._mtimes[path] = mtime
            self.filepath = path
            return True

        self._mtimes[path] = mtime
        return False

    def is_folder_changed(self, path):
        for root, dirs, files in os.walk(path, followlinks=True):
            if '.git' in dirs:
                dirs.remove('.git')
            if '.hg' in dirs:
                dirs.remove('.hg')
            if '.svn' in dirs:
                dirs.remove('.svn')
            if '.cvs' in dirs:
                dirs.remove('.cvs')

            for f in files:
                if self.is_file_changed(os.path.join(root, f)):
                    return True
        return False

    def is_glob_changed(self, path):
        for f in glob.glob(path):
            if self.is_file_changed(f):
                return True
        return False
<commit_msg>Detect changed on file added.<commit_after># -*- coding: utf-8 -*-
"""
    livereload.watcher
    ~~~~~~~~~~~~~~~~~~

    A file watch management for LiveReload Server.

    :copyright: (c) 2013 by Hsiaoming Yang
"""

import os
import glob
import time

class Watcher(object):
    """A file watcher registery."""
    def __init__(self):
        self._tasks = {}
        self._mtimes = {}

        # filepath that is changed
        self.filepath = None
        self._start = time.time()

    def ignore(self, filename):
        """Ignore a given filename or not."""
        _, ext = os.path.splitext(filename)
        return ext in ['.pyc', '.pyo', '.o', '.swp']

    def watch(self, path, func=None):
        """Add a task to watcher."""
        self._tasks[path] = func

    def examine(self):
        """Check if there are changes, if true, run the given task."""
        # clean filepath
        self.filepath = None
        for path in self._tasks:
            if self.is_changed(path):
                func = self._tasks[path]
                # run function
                func and func()
        return self.filepath

    def is_changed(self, path):
        if os.path.isfile(path):
            return self.is_file_changed(path)
        elif os.path.isdir(path):
            return self.is_folder_changed(path)
        return self.is_glob_changed(path)

    def is_file_changed(self, path):
        if not os.path.isfile(path):
            return False

        if self.ignore(path):
            return False

        mtime = os.path.getmtime(path)

        if path not in self._mtimes:
            self._mtimes[path] = mtime
            self.filepath = path
            return mtime > self._start

        if self._mtimes[path] != mtime:
            self._mtimes[path] = mtime
            self.filepath = path
            return True

        self._mtimes[path] = mtime
        return False

    def is_folder_changed(self, path):
        for root, dirs, files in os.walk(path, followlinks=True):
            if '.git' in dirs:
                dirs.remove('.git')
            if '.hg' in dirs:
                dirs.remove('.hg')
            if '.svn' in dirs:
                dirs.remove('.svn')
            if '.cvs' in dirs:
                dirs.remove('.cvs')

            for f in files:
                if self.is_file_changed(os.path.join(root, f)):
                    return True
        return False

    def is_glob_changed(self, path):
        for f in glob.glob(path):
            if self.is_file_changed(f):
                return True
        return False
<|endoftext|>


<commit_before>#!/usr/bin/env python
# xia2find.py
#   Copyright (C) 2006 CCLRC, Graeme Winter
#
#   This code is distributed under the terms and conditions of the
#   CCP4 Program Suite Licence Agreement as a CCP4 Library.
#   A copy of the CCP4 licence can be obtained by writing to the
#   CCP4 Secretary, Daresbury Laboratory, Warrington WA4 4AD, UK.
#
# 12th July 2006
#
# A small program to find diffraction data sets as "sweeps" and print
# out the results.
#
#

import sys
import os

if not os.environ.has_key('DPA_ROOT'):
    raise RuntimeError, 'DPA_ROOT not defined'
if not os.environ.has_key('XIA2CORE_ROOT'):
    raise RuntimeError, 'XIA2CORE_ROOT not defined'

sys.path.append(os.path.join(os.environ['DPA_ROOT']))

from Schema.Sweep import SweepFactory
from Experts.FindImages import image2template_directory

known_image_extensions = ['img', 'mccd', 'mar2300', 'osc', 'cbf']
known_sweeps = { }

def is_image_name(file):

    global known_image_extensions

    if os.path.isfile(file):
        if os.path.split(file)[-1].split('.')[-1] in known_image_extensions:
            return True

    return False

def get_sweep(file):

    global known_sweeps

    if not is_image_name(file):
        return

    try:
        template, directory = image2template_directory(file)
        key = (directory, template)
        if not known_sweeps.has_key(key):
            sweeplist = SweepFactory(template, directory)
            for sweep in sweeplist:
                key = (sweep.getDirectory(), sweep.getTemplate())
                if not known_sweeps.has_key(key):
                    known_sweeps[key] = sweep

    except:
        pass

    return

def visit(root, directory, files):
    for f in files:
        get_sweep(os.path.join(directory, f))

def print_sweeps():

    global known_sweeps

    sweeps = known_sweeps.keys()
    sweeps.sort()

    for sweep in sweeps:
        s = known_sweeps[sweep]
        print 'Sweep: %s' % os.path.join(s.getDirectory(), s.getTemplate())
        print 'Images: %d to %d' % (min(s.getImages()), max(s.getImages()))
        print 'Collected from/to: %f %f' % s.getCollect()
        print 'Wavelength: %f' % s.getWavelength()
        print 'Distance: %f' % s.getDistance()
        print 'Beam: %f %f' % tuple(s.getBeam())
        print 'Oscillations: %f to %f (%f)' % tuple(s.getPhi())
        print ''

if __name__ == '__main__':

    if len(sys.argv) < 2:
        os.path.walk(os.getcwd(), visit, os.getcwd())
    else:
        os.path.walk(sys.argv[1], visit, os.getcwd())

    print_sweeps()
<commit_msg>Now can handle multiple sweeps with matching template etc; print in exposure order; nice ;o)<commit_after>#!/usr/bin/env python
# xia2find.py
#   Copyright (C) 2006 CCLRC, Graeme Winter
#
#   This code is distributed under the terms and conditions of the
#   CCP4 Program Suite Licence Agreement as a CCP4 Library.
#   A copy of the CCP4 licence can be obtained by writing to the
#   CCP4 Secretary, Daresbury Laboratory, Warrington WA4 4AD, UK.
#
# 12th July 2006
#
# A small program to find diffraction data sets as "sweeps" and print
# out the results.
#
#

import sys
import os

if not os.environ.has_key('DPA_ROOT'):
    raise RuntimeError, 'DPA_ROOT not defined'
if not os.environ.has_key('XIA2CORE_ROOT'):
    raise RuntimeError, 'XIA2CORE_ROOT not defined'

sys.path.append(os.path.join(os.environ['DPA_ROOT']))

from Schema.Sweep import SweepFactory
from Experts.FindImages import image2template_directory

known_image_extensions = ['img', 'mccd', 'mar2300', 'osc', 'cbf']
known_sweeps = { }

def is_image_name(file):

    global known_image_extensions

    if os.path.isfile(file):
        if os.path.split(file)[-1].split('.')[-1] in known_image_extensions:
            return True

    return False

def get_sweep(file):

    global known_sweeps

    if not is_image_name(file):
        return

    try:
        template, directory = image2template_directory(file)
        key = (directory, template)
        if not known_sweeps.has_key(key):
            sweeplist = SweepFactory(template, directory)
            known_sweeps[key] = sweeplist

    except:
        pass

    return

def visit(root, directory, files):
    for f in files:
        get_sweep(os.path.join(directory, f))

def print_sweeps():

    global known_sweeps

    sweeplists = known_sweeps.keys()
    sweeplists.sort()

    for sweep in sweeplists:
        sweeps = known_sweeps[sweep]
        # this should sort on exposure epoch ...?
        sweeps.sort()
        for s in sweeps:
            print 'Sweep: %s' % os.path.join(s.getDirectory(), s.getTemplate())
            print 'Images: %d to %d' % (min(s.getImages()), max(s.getImages()))
            print 'Collected from/to: %f %f' % s.getCollect()
            print 'Wavelength: %f' % s.getWavelength()
            print 'Distance: %f' % s.getDistance()
            print 'Beam: %f %f' % tuple(s.getBeam())
            print 'Oscillations: %f to %f (%f)' % tuple(s.getPhi())
            print ''

if __name__ == '__main__':

    if len(sys.argv) < 2:
        os.path.walk(os.getcwd(), visit, os.getcwd())
    else:
        os.path.walk(sys.argv[1], visit, os.getcwd())

    print_sweeps()
<|endoftext|>


<commit_before>import linecache, sys, random
from unidecode import unidecode

def clCoeff(oNode, finallist):
	if type(finallist[oNode]) != type(None):
		dNode = finallist[oNode].split(' ')
	else:
		return 0

	# Gather the same info for each of the neighbors
	# Make this a function, can do recursive calls to check multiple depths?
	twoSteps = ''
	for neighbor in dNode:
		if len(neighbor) > 0 and type(finallist[int(neighbor)]) != type(None):
			twoSteps += finallist[int(neighbor)]
	twoSteps = twoSteps.split(' ')

	# Have to find the number of overlaps
	oneANDtwo = 0
	for firstStep in dNode:
		oneANDtwo += twoSteps.count(firstStep)

	if len(dNode) > 1:
		# CLUSTERING COEFFICIENT
		ai = float(len(dNode))
		Ei = float(oneANDtwo)
		# Calculate the coefficient
		# Links are often (not always) counted twice...
		clCoeff = (2*Ei)/(ai*(ai-1))
		return clCoeff
	else:
		return 0

def getName(oNode):
	oname = linecache.getline('titles-sorted.txt',int(oNode))[:-1]
	return oname

def getOutlinks(oNode, finallist):
	dNode = finallist[oNode].split(' ')
	namesList = []
	for indexNo in dNode:
		namesList += [linecache.getline('titles-sorted.txt',int(indexNo))[:-1]]<commit_msg>Lol now getOutlinks returns<commit_after>import linecache, sys, random
from unidecode import unidecode

def clCoeff(oNode, finallist):
	if type(finallist[oNode]) != type(None):
		dNode = finallist[oNode].split(' ')
	else:
		return 0

	# Gather the same info for each of the neighbors
	# Make this a function, can do recursive calls to check multiple depths?
	twoSteps = ''
	for neighbor in dNode:
		if len(neighbor) > 0 and type(finallist[int(neighbor)]) != type(None):
			twoSteps += finallist[int(neighbor)]
	twoSteps = twoSteps.split(' ')

	# Have to find the number of overlaps
	oneANDtwo = 0
	for firstStep in dNode:
		oneANDtwo += twoSteps.count(firstStep)

	if len(dNode) > 1:
		# CLUSTERING COEFFICIENT
		ai = float(len(dNode))
		Ei = float(oneANDtwo)
		# Calculate the coefficient
		# Links are often (not always) counted twice...
		clCoeff = (2*Ei)/(ai*(ai-1))
		return clCoeff
	else:
		return 0

def getName(oNode):
	oname = linecache.getline('titles-sorted.txt',int(oNode))[:-1]
	return oname

def getOutlinks(oNode, finallist):
	dNode = finallist[oNode].split(' ')
	namesList = []
	for indexNo in dNode:
		namesList += [linecache.getline('titles-sorted.txt',int(indexNo))[:-1]]
	return [dNode, namesList]<|endoftext|>


<commit_before>"""Neural Image Style Transfer utils."""

import tensorflow as tf

def gram_style(a):
  with tf.name_scope('gram'):
    chn = int(a.shape[-1])
    a = tf.reshape(a, [-1, chn])
    n = tf.shape(a)[0]
    gram = tf.matmul(a, a, transpose_a=True)
    return gram / tf.cast(n, tf.float32)

def mean_l1_loss(g1, g2):
  with tf.name_scope('mean_l1_loss'):
    return tf.reduce_mean(tf.abs(g1-g2))

class StyleLoss(object):
  """Image Style Loss.

  A variant of style component of Artistic Image Style Transfer loss,
  mainly inspired by [1].

  [1] Leon A. Gatys et al. "A Neural Algorithm of Artistic Style"
      https://arxiv.org/abs/1508.06576
  """

  def __init__(self, style_layers, ema_decay=None,
               style_func=gram_style,
               loss_func=mean_l1_loss):
    """Initilize style loss.

    Args:
      style_layers: List of tensors that are used to compute statistics that
        define a style.
      ema_decay: Number in range [0.0 .. 1.0] or None. Loss function is computed against
        moving averaged versions of style statistics if ema_decay is not None.
        This is useful when each optimisation step only covers some part of
        the full output image.
      style_func: Function that is used to compute layer statistics.
      loss_func: Function that is used to compute difference between two
        outputs of 'style_func'.
    """
    self.input_grams = [style_func(s) for s in style_layers]
    self.ema = None
    if ema_decay is not None:
      self.ema = tf.train.ExponentialMovingAverage(decay=ema_decay)
      update_ema_op = self.ema.apply(self.input_grams)
      with tf.control_dependencies([update_ema_op]):
        self.effective_grams = [g + tf.stop_gradient(self.ema.average(g)-g)
                                for g in self.input_grams]
    else:
      self.effective_grams = self.input_grams

    self.target_vars = [tf.Variable(tf.zeros_like(g), trainable=False)
                        for g in self.input_grams]
    self.style_losses = [loss_func(g, gv)
                         for g, gv in zip(self.effective_grams, self.target_vars)]
    self.style_loss = tf.add_n(self.style_losses)

  def set_style(self, input_feeds):
    """Set target style variables.

    Expected usage:
      style_loss = StyleLoss(style_layers)
      ...
      init_op = tf.global_variables_initializer()
      init_op.run()

      feeds = {... session.run() 'feeds' argument that will make 'style_layers'
               tensors evaluate to activation values of style image...}
      style_loss.set_style(feeds)  # this must be called after 'init_op.run()'
    """
    sess = tf.get_default_session()
    computed = sess.run(self.input_grams, input_feeds)
    for v, g in zip(self.target_vars, computed):
      v.load(g)
<commit_msg>mean_l2_loss func<commit_after>"""Neural Image Style Transfer utils."""

import tensorflow as tf

def gram_style(a):
  with tf.name_scope('gram'):
    chn = int(a.shape[-1])
    a = tf.reshape(a, [-1, chn])
    n = tf.shape(a)[0]
    gram = tf.matmul(a, a, transpose_a=True)
    return gram / tf.cast(n, tf.float32)

def mean_l1_loss(g1, g2):
  with tf.name_scope('mean_l1_loss'):
    return tf.reduce_mean(tf.abs(g1-g2))

def mean_l2_loss(g1, g2):
  with tf.name_scope('mean_l2_loss'):
    return tf.sqrt(tf.reduce_mean(tf.square(g1 - g2)))

class StyleLoss(object):
  """Image Style Loss.

  A variant of style component of Artistic Image Style Transfer loss,
  mainly inspired by [1].

  [1] Leon A. Gatys et al. "A Neural Algorithm of Artistic Style"
      https://arxiv.org/abs/1508.06576
  """

  def __init__(self, style_layers, ema_decay=None,
               style_func=gram_style,
               loss_func=mean_l1_loss):
    """Initilize style loss.

    Args:
      style_layers: List of tensors that are used to compute statistics that
        define a style.
      ema_decay: Number in range [0.0 .. 1.0] or None. Loss function is computed against
        moving averaged versions of style statistics if ema_decay is not None.
        This is useful when each optimisation step only covers some part of
        the full output image.
      style_func: Function that is used to compute layer statistics.
      loss_func: Function that is used to compute difference between two
        outputs of 'style_func'.
    """
    self.input_grams = [style_func(s) for s in style_layers]
    self.ema = None
    if ema_decay is not None:
      self.ema = tf.train.ExponentialMovingAverage(decay=ema_decay)
      update_ema_op = self.ema.apply(self.input_grams)
      with tf.control_dependencies([update_ema_op]):
        self.effective_grams = [g + tf.stop_gradient(self.ema.average(g)-g)
                                for g in self.input_grams]
    else:
      self.effective_grams = self.input_grams

    self.target_vars = [tf.Variable(tf.zeros_like(g), trainable=False)
                        for g in self.input_grams]
    self.style_losses = [loss_func(g, gv)
                         for g, gv in zip(self.effective_grams, self.target_vars)]
    self.style_loss = tf.add_n(self.style_losses)

  def set_style(self, input_feeds):
    """Set target style variables.

    Expected usage:
      style_loss = StyleLoss(style_layers)
      ...
      init_op = tf.global_variables_initializer()
      init_op.run()

      feeds = {... session.run() 'feeds' argument that will make 'style_layers'
               tensors evaluate to activation values of style image...}
      style_loss.set_style(feeds)  # this must be called after 'init_op.run()'
    """
    sess = tf.get_default_session()
    computed = sess.run(self.input_grams, input_feeds)
    for v, g in zip(self.target_vars, computed):
      v.load(g)
<|endoftext|>


<commit_before>"""
Our experiment environment.
"""
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.animation as animation

class Environment:

    def __init__(self, ball, target, robot=None, dimensions=None):
        """
        Our room goes from [0, 0, 0] to [x, y, z]
        :param dimensions: 1x3 array of the dimensions of the room.
        """
        self.ball = ball
        self.target = target
        self.robot = robot
        if dimensions:
            self.dimensions = np.array(dimensions)
        else:
            self.dimensions = np.array([10.0, 10.0, 100.0])

    def run(self, duration):
        """Run for a certain duration

        :param duration: duration to run environment in seconds
        :return score: The value showing how close the ball is to the target
        """
        duration_ms = int(duration * 1000)

        for _ in xrange(duration_ms):
            self.ball.step()
            if self._collision():
                self.ball.attach()
                break
        return self.target.get_score(self.ball.leading_point())

    def _collision(self):
        """Check if the object collides with the walls
        :return: Boolean, whether there was a collision
        """
        if self.ball.attached:
            return False

        if self.target.is_hit(self.ball.leading_point()):
            return True

        for i in range(len(self.ball.position)):
            in_negative_space = self.ball.position[i] <= 0
            past_boundary = self.ball.position[i] >= self.dimensions[i]
            if in_negative_space or past_boundary:
                return True

        return False

    def plot(self, ax=None, show=True):
        """Plot throw trajectory and ball

        :param ax: Current axis if a figure already exists
        :param show: (Default: True) Whether to show the figure
        """
        if ax is None:
            fig = plt.figure(figsize=(12, 12))
            ax = Axes3D(fig)

        # Set the limits to be environment ranges
        ax.set_xlim([0, self.dimensions[0]])
        ax.set_ylim([0, self.dimensions[1]])
        ax.set_zlim([0, max(10, self.ball.positions[:, 2].max())])

        # Plot Trajectory
        ax.plot(self.ball.positions[:, 0], self.ball.positions[:, 1],
                self.ball.positions[:, 2], 'r--', label='Trajectory')

        # Plot objects
        self.ball.plot(ax)
        self.target.plot(ax)

        if show:
            plt.show()

    def animate(self, duration):
        """Animates the running of the program

        :param duration: Duration of animation in seconds
        """
        # We want it at 30 fps
        frames = int(30 * duration)
        iter_per_frame = int(1000 / 30)

        def update(i):
            ax.clear()
            for _ in xrange(iter_per_frame):
                self.ball.step()
                if self._collision():
                    self.ball.attach()
            self.plot(ax=ax, show=False)

        fig = plt.figure(figsize=(8, 8))
        ax = Axes3D(fig)
        self.plot(ax=ax, show=False)

        # If we don't assign its return to something, it doesn't run.
        # Seems like really weird behavior..
        _ = animation.FuncAnimation(fig, update, frames=frames, blit=False)
        plt.show()
<commit_msg>Added axis labels to env<commit_after>"""
Our experiment environment.
"""
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.animation as animation

class Environment:

    def __init__(self, ball, target, robot=None, dimensions=None):
        """
        Our room goes from [0, 0, 0] to [x, y, z]
        :param dimensions: 1x3 array of the dimensions of the room.
        """
        self.ball = ball
        self.target = target
        self.robot = robot
        if dimensions:
            self.dimensions = np.array(dimensions)
        else:
            self.dimensions = np.array([10.0, 10.0, 100.0])

    def run(self, duration):
        """Run for a certain duration

        :param duration: duration to run environment in seconds
        :return score: The value showing how close the ball is to the target
        """
        duration_ms = int(duration * 1000)

        for _ in xrange(duration_ms):
            self.ball.step()
            if self._collision():
                self.ball.attach()
                break
        return self.target.get_score(self.ball.leading_point())

    def _collision(self):
        """Check if the object collides with the walls
        :return: Boolean, whether there was a collision
        """
        if self.ball.attached:
            return False

        if self.target.is_hit(self.ball.leading_point()):
            return True

        for i in range(len(self.ball.position)):
            in_negative_space = self.ball.position[i] <= 0
            past_boundary = self.ball.position[i] >= self.dimensions[i]
            if in_negative_space or past_boundary:
                return True

        return False

    def plot(self, ax=None, show=True):
        """Plot throw trajectory and ball

        :param ax: Current axis if a figure already exists
        :param show: (Default: True) Whether to show the figure
        """
        if ax is None:
            fig = plt.figure(figsize=(12, 12))
            ax = Axes3D(fig)

        # Set the limits to be environment ranges
        ax.set_xlim([0, self.dimensions[0]])
        ax.set_ylim([0, self.dimensions[1]])
        ax.set_zlim([0, max(10, self.ball.positions[:, 2].max())])

        # And set our labels
        ax.set_xlabel('X')
        ax.set_ylabel('Y')
        ax.set_zlabel('Z')

        # Plot Trajectory
        ax.plot(self.ball.positions[:, 0], self.ball.positions[:, 1],
                self.ball.positions[:, 2], 'r--', label='Trajectory')

        # Plot objects
        self.ball.plot(ax)
        self.target.plot(ax)

        if show:
            plt.show()

    def animate(self, duration):
        """Animates the running of the program

        :param duration: Duration of animation in seconds
        """
        # We want it at 30 fps
        frames = int(30 * duration)
        iter_per_frame = int(1000 / 30)

        def update(i):
            ax.clear()
            for _ in xrange(iter_per_frame):
                self.ball.step()
                if self._collision():
                    self.ball.attach()
            self.plot(ax=ax, show=False)

        fig = plt.figure(figsize=(8, 8))
        ax = Axes3D(fig)
        self.plot(ax=ax, show=False)

        # If we don't assign its return to something, it doesn't run.
        # Seems like really weird behavior..
        _ = animation.FuncAnimation(fig, update, frames=frames, blit=False)
        plt.show()
<|endoftext|>


<commit_before># Copyright (c) 2015 Matthew Earl
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
#     The above copyright notice and this permission notice shall be included
#     in all copies or substantial portions of the Software.
#
#     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
#     OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#     MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
#     NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#     DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#     OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
#     USE OR OTHER DEALINGS IN THE SOFTWARE.

"""
Component and board class definitions.

"""

__all__ = (
    'Board',
    'Component',
    'LeadedComponent',
    'StripBoard',
    'Terminal',
)

import abc

class Board():
    """
    Base class for a grid-based prototyping board.

    Attributes:
        size: (width, height) pair giving the width and height of the board in
            terms of the number of holes in either dimension.
        holes: Set of (x, y) pairs indicating the coordinates of holes in the
            board: If a pair (x, y) is in the set then the board has a hole at
            this position.
        strips: A list of lists of holes that are conductively connected. The
            ordering of each list indicates connectedness, in that if the kth
            element of a strip is drilled out then elements i and j are
            considered disconnected for all i < k, j > k. As such a given hole
            can only be connected to up to two other holes, so some board
            topologies (eg. T junctions) cannot be represented.

    """
    def __init__(self, size, holes, strips):
        """
        Initialize the object.

        Arguments initialize the object's attributes.

        """
        self.size = tuple(size)
        self.holes = set(holes)
        self.strips = set(tuple(strip) for strip in strips)

        # Check each hole is within the bounds of the board.
        if not all(0 <= x < size[0] and 0 <= y < size[1] for (x, y) in
                   self.holes):
            raise ValueError

        # Check all the strip elements are located on holes.
        if not all(h in self.holes for strip in self.strips for h in strip):
            raise ValueError

class StripBoard(Board):
    """
    A rectangular board in which holes are connected if and only if they are in
    the same row.

    """
    def __init__(self, size):
        holes = [(x, y) for y in range(size[1]) for x in range(size[0])]
        strips = [[(x, y) for x in range(size[0])] for y in range(size[1])]
        super().__init__(size, holes, strips)

class Terminal():
    """
    Object to represent a terminal of given component object.

    """
    pass

class Component(metaclass=abc.ABCMeta):
    """
    Object to represent a component.

    A component has a number of terminals, and a component can be placed in
    numerous positions on a given board.

    A position is defined by a mapping of the component's terminals to (x, y)
    coordinates. See methods for more information.

    """
    def __init__(self, terminals):
        self.terminals = list(terminals)

    @abc.abstractmethod
    def get_relative_positions(self):
        """
        Get relative positions for this component.

        Each position is a dict mapping the component's terminals to (x, y)
        positions, and the first terminal always maps to (0, 0).

        This function assumes an infinite grid of holes is available.

        """
        raise NotImplemented

    def get_positions(self, board):
        """
        Get absolute positions for this component, given a board.

        Each position is a dict mapping the component's terminals to (x, y)
        positions. Each (x, y) position is a hole in the board.

        """
        for hx, hy in board.holes:
            for rel_pos in self.get_relative_positions():
                abs_pos = {terminal: (hx + x, hy + y)
                            for terminal, (x, y) in rel_pos.items()}

                if all(v in board.holes for v in abs_pos.values()):
                    yield abs_pos

class LeadedComponent(Component):
    """
    Class for leaded components.

    For example, through-hole resistors, diodes, capacitors.

    """

    def __init__(self, max_length, *,
                 allow_vertical=True,
                 allow_horizontal=True):
        terminals = (Terminal(), Terminal())

        self._max_length = max_length
        self._allow_horizontal = allow_horizontal
        self._allow_vertical = allow_vertical

        super().__init__(terminals)

    def get_relative_positions(self):
        for length in range(1, self._max_length + 1):
            if self._allow_vertical:
                yield {self_terminals[0]: (0, 0),
                       self.terminals[1]: (length, 0)}
                yield {self.terminals[1]: (0, 0),
                       self.terminals[0]: (length, 0)}
            if self._allow_horizontal:
                yield {self.terminals[0]: (0, 0),
                       self.terminals[1]: (0, length)}
                yield {self.terminals[1]: (0, 0),
                       self.terminals[0]: (0, length)}

<commit_msg>Update component.Board to have a traces attribute instead of a strip attribute. traces attribute describes interhole connections pairwise, which is much more useful than the strip model.<commit_after># Copyright (c) 2015 Matthew Earl
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
#     The above copyright notice and this permission notice shall be included
#     in all copies or substantial portions of the Software.
#
#     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
#     OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#     MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
#     NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#     DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#     OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
#     USE OR OTHER DEALINGS IN THE SOFTWARE.

"""
Component and board class definitions.

"""

__all__ = (
    'Board',
    'Component',
    'LeadedComponent',
    'StripBoard',
    'Terminal',
)

import abc

class Board():
    """
    Base class for a grid-based prototyping board.

    Attributes:
        size: (width, height) pair giving the width and height of the board in
            terms of the number of holes in either dimension.
        holes: Set of (x, y) pairs indicating the coordinates of holes in the
            board: If a pair (x, y) is in the set then the board has a hole at
            this position.
        traces: A list of pairs of holes that are conductively connected.

    """
    def __init__(self, size, holes, traces):
        """
        Initialize the object.

        Arguments initialize the object's attributes.

        """
        self.size = tuple(size)
        self.holes = set(holes)
        self.traces = set(tuple(sorted(hole_pair)) for hole_pair in traces)

        # Check each hole is within the bounds of the board.
        if not all(0 <= x < size[0] and 0 <= y < size[1] for (x, y) in
                   self.holes):
            raise ValueError

        # Check all holes in the traces are holes in the board.
        if not {h for t for t in self.traces for h in t} <= self.holes:
            raise ValueError

class StripBoard(Board):
    """
    A rectangular board in which holes are connected if and only if they are in
    the same row.

    """
    def __init__(self, size):
        holes = [(x, y) for y in range(size[1]) for x in range(size[0])]
        traces = [((x, y), (x + 1, y))
                    for y in range(size[1]) for x in range(size[0] - 1)]
        super().__init__(size, holes, traces)

class Terminal():
    """
    Object to represent a terminal of given component object.

    """
    pass

class Component(metaclass=abc.ABCMeta):
    """
    Object to represent a component.

    A component has a number of terminals, and a component can be placed in
    numerous positions on a given board.

    A position is defined by a mapping of the component's terminals to (x, y)
    coordinates. See methods for more information.

    """
    def __init__(self, terminals):
        self.terminals = list(terminals)

    @abc.abstractmethod
    def get_relative_positions(self):
        """
        Get relative positions for this component.

        Each position is a dict mapping the component's terminals to (x, y)
        positions, and the first terminal always maps to (0, 0).

        This function assumes an infinite grid of holes is available.

        """
        raise NotImplemented

    def get_positions(self, board):
        """
        Get absolute positions for this component, given a board.

        Each position is a dict mapping the component's terminals to (x, y)
        positions. Each (x, y) position is a hole in the board.

        """
        for hx, hy in board.holes:
            for rel_pos in self.get_relative_positions():
                abs_pos = {terminal: (hx + x, hy + y)
                            for terminal, (x, y) in rel_pos.items()}

                if all(v in board.holes for v in abs_pos.values()):
                    yield abs_pos

class LeadedComponent(Component):
    """
    Class for leaded components.

    For example, through-hole resistors, diodes, capacitors.

    """

    def __init__(self, max_length, *,
                 allow_vertical=True,
                 allow_horizontal=True):
        terminals = (Terminal(), Terminal())

        self._max_length = max_length
        self._allow_horizontal = allow_horizontal
        self._allow_vertical = allow_vertical

        super().__init__(terminals)

    def get_relative_positions(self):
        for length in range(1, self._max_length + 1):
            if self._allow_vertical:
                yield {self_terminals[0]: (0, 0),
                       self.terminals[1]: (length, 0)}
                yield {self.terminals[1]: (0, 0),
                       self.terminals[0]: (length, 0)}
            if self._allow_horizontal:
                yield {self.terminals[0]: (0, 0),
                       self.terminals[1]: (0, length)}
                yield {self.terminals[1]: (0, 0),
                       self.terminals[0]: (0, length)}

<|endoftext|>


<commit_before>"""
Command line interface implementation.

Andrew DeOrio <awdeorio@umich.edu>
"""
from __future__ import print_function
import sys
import socket
import configparser
import smtplib
import textwrap
import jinja2
import click
from .template_message import TemplateMessage
from .sendmail_client import SendmailClient
from .utils import MailmergeError

# Python 2 pathlib support requires backport
try:
    from pathlib2 import Path
except ImportError:
    from pathlib import Path

# Python 2 UTF8 support requires csv backport
try:
    from backports import csv
except ImportError:
    import csv

@click.command(context_settings={"help_option_names": ['-h', '--help']})
@click.version_option()  # Auto detect version from setup.py
@click.option(
    "--sample", is_flag=True, default=False,
    help="Create sample database, template email, and config",
)
@click.option(
    "--dry-run/--no-dry-run", default=True,
    help="Don't send email, just print (True)",
)
@click.option(
    "--limit", is_flag=False, default=1,
    type=click.IntRange(0, None),
    help="Limit the number of messages (1)",
)
@click.option(
    "--no-limit", is_flag=True, default=False,
    help="Do not limit the number of messages",
)
@click.option(
    "--template", "template_path",
    default="mailmerge_template.txt",
    type=click.Path(),
    help="template email file name (mailmerge_template.txt)"
)
@click.option(
    "--database", "database_path",
    default="mailmerge_database.csv",
    type=click.Path(),
    help="database CSV file name (mailmerge_database.csv)",
)
@click.option(
    "--config", "config_path",
    default="mailmerge_server.conf",
    type=click.Path(),
    help="configuration file name (mailmerge_server.conf)",
)
def main(sample, dry_run, limit, no_limit,
         database_path, template_path, config_path):
    """
    Mailmerge is a simple, command line mail merge tool.

    For examples and formatting features, see:
    https://github.com/awdeorio/mailmerge
    """
    # We need an argument for each command line option.  That also means a lot
    # of local variables.
    # pylint: disable=too-many-arguments, too-many-locals

    # Convert paths from string to Path objects
    # https://github.com/pallets/click/issues/405
    template_path = Path(template_path)
    database_path = Path(database_path)
    config_path = Path(config_path)

    check_input_files(template_path, database_path, config_path, sample)

    # No limit is an alias for limit=-1
    if no_limit:
        limit = -1

    try:
        template_message = TemplateMessage(template_path)
        csv_database = read_csv_database(database_path)
        sendmail_client = SendmailClient(config_path, dry_run)
        for i, row in enumerate_limit(csv_database, limit):
            sender, recipients, message = template_message.render(row)
            sendmail_client.sendmail(sender, recipients, message)
            print(">>> message {}".format(i))
            print(message.as_string())
            for filename in get_attachment_filenames(message):
                print(">>> attached {}".format(filename))
            print(">>> sent message {}".format(i))
    except jinja2.exceptions.TemplateError as err:
        sys.exit(">>> Error in Jinja2 template: {}".format(err))
    except csv.Error as err:
        sys.exit(">>> Error reading CSV file: {}".format(err))
    except smtplib.SMTPAuthenticationError as err:
        sys.exit(">>> Authentication error: {}".format(err))
    except configparser.Error as err:
        sys.exit(
            ">>> Error reading config file {filename}: {message}"
            .format(filename=config_path, message=err)
        )
    except smtplib.SMTPException as err:
        sys.exit(">>> Error sending message", err, sep=' ', file=sys.stderr)
    except socket.error:
        sys.exit(">>> Error connecting to server")
    except MailmergeError as err:
        sys.exit(">>> {}".format(err))

    # Hints for user
    if not no_limit:
        print(
            ">>> Limit was {limit} message{pluralizer}.  "
            "To remove the limit, use the --no-limit option."
            .format(limit=limit, pluralizer=("" if limit == 1 else "s"))
        )
    if dry_run:
        print(
            ">>> This was a dry run.  "
            "To send messages, use the --no-dry-run option."
        )

if __name__ == "__main__":
    # No value for parameter, that's how click works
    # pylint: disable=no-value-for-parameter
    main()

def check_input_files(template_path, database_path, config_path, sample):
    """Check if input files are present and hint the user."""
    if sample:
        create_sample_input_files(template_path, database_path, config_path)
        sys.exit(0)

    if not template_path.exists():
        sys.exit(textwrap.dedent(u"""
            Error: can't find template {template_path}

            Create a sample (--sample) or specify a file (--template)
            See https://github.com/awdeorio/mailmerge for examples.\
        """.format(template_path=template_path)))

    if not database_path.exists():
        sys.exit(textwrap.dedent(u"""
            Error: can't find database {database_path}

            Create a sample (--sample) or specify a file (--database)
            See https://github.com/awdeorio/mailmerge for examples.\
        """.format(database_path=database_path)))

    if not config_path.exists():
        sys.exit(textwrap.dedent(u"""
            Error: can't find config {config_path}
            Create a sample (--sample) or specify a file (--config)
            See https://github.com/awdeorio/mailmerge for examples.\
        """.format(config_path=config_path)))

def create_sample_input_files(template_path, database_path, config_path):
    """Create sample template, database and server config."""
    for path in [template_path, database_path, config_path]:
        if path.exists():
            sys.exit("Error: file exists: {}".format(path))
    with template_path.open("w") as template_file:
        template_file.write(textwrap.dedent(u"""\
            TO: {{email}}
            SUBJECT: Testing mailmerge
            FROM: My Self <myself@mydomain.com>

            Hi, {{name}},

            Your number is {{number}}.
        """))
    with database_path.open("w") as database_file:
        database_file.write(textwrap.dedent(u"""\
            email,name,number
            myself@mydomain.com,"Myself",17
            bob@bobdomain.com,"Bob",42
        """))
    with config_path.open("w") as config_file:
        config_file.write(textwrap.dedent(u"""\
            # Example: GMail, with SSL/TLS security
            [smtp_server]
            host = smtp.gmail.com
            port = 465
            security = SSL/TLS
            username = YOUR_USERNAME_HERE

            # Example: University of Michigan, with SSL/TLS security
            # [smtp_server]
            # host = smtp.mail.umich.edu
            # port = 465
            # security = SSL/TLS
            # username = YOUR_USERNAME_HERE

            # Example: University of Michigan EECS Dept., with STARTTLS security
            # [smtp_server]
            # host = newman.eecs.umich.edu
            # port = 25
            # security = STARTTLS
            # username = YOUR_USERNAME_HERE

            # Example: University of Michigan EECS Dept., with no security
            # [smtp_server]
            # host = newman.eecs.umich.edu
            # port = 25
        """))
    print(textwrap.dedent("""\
        Created sample template email "{template_path}"
        Created sample database "{database_path}"
        Created sample config file "{config_path}"
        Edit these files, then run mailmerge again.
    """.format(
        template_path=template_path,
        database_path=database_path,
        config_path=config_path,
    )))

def read_csv_database(database_path):
    """Read database CSV file, providing one line at a time."""
    # Modify the default dialect to be strict.  This will trigger errors for
    # things like unclosed quotes.
    class StrictExcel(csv.excel):
        """Strict version of default dialect."""

        # pylint: disable=too-few-public-methods
        strict = True

    # Open file and read using strict dialect
    with database_path.open("r") as database_file:
        reader = csv.DictReader(database_file, dialect=StrictExcel)
        for row in reader:
            yield row

def enumerate_limit(iterable, limit):
    """Enumerate iterable, stopping after limit iterations.

    When limit == -1, enumerate entire iterable.
    """
    for i, value in enumerate(iterable):
        if limit != -1 and i >= limit:
            return
        yield i, value

def get_attachment_filenames(message):
    """Return a list of attachment filenames."""
    if message.get_content_maintype() != "multipart":
        return []

    filenames = []
    for part in message.walk():
        if part.get_content_maintype() == "multipart":
            continue
        if part.get_content_maintype() == "text":
            continue
        if part.get("Content-Disposition") == "inline":
            continue
        if part.get("Content-Disposition") is None:
            continue
        filenames.append(part.get_filename())
    return filenames
<commit_msg>simplify read_csv_database<commit_after>"""
Command line interface implementation.

Andrew DeOrio <awdeorio@umich.edu>
"""
from __future__ import print_function
import sys
import socket
import configparser
import smtplib
import textwrap
import jinja2
import click
from .template_message import TemplateMessage
from .sendmail_client import SendmailClient
from .utils import MailmergeError

# Python 2 pathlib support requires backport
try:
    from pathlib2 import Path
except ImportError:
    from pathlib import Path

# Python 2 UTF8 support requires csv backport
try:
    from backports import csv
except ImportError:
    import csv

@click.command(context_settings={"help_option_names": ['-h', '--help']})
@click.version_option()  # Auto detect version from setup.py
@click.option(
    "--sample", is_flag=True, default=False,
    help="Create sample database, template email, and config",
)
@click.option(
    "--dry-run/--no-dry-run", default=True,
    help="Don't send email, just print (True)",
)
@click.option(
    "--limit", is_flag=False, default=1,
    type=click.IntRange(0, None),
    help="Limit the number of messages (1)",
)
@click.option(
    "--no-limit", is_flag=True, default=False,
    help="Do not limit the number of messages",
)
@click.option(
    "--template", "template_path",
    default="mailmerge_template.txt",
    type=click.Path(),
    help="template email file name (mailmerge_template.txt)"
)
@click.option(
    "--database", "database_path",
    default="mailmerge_database.csv",
    type=click.Path(),
    help="database CSV file name (mailmerge_database.csv)",
)
@click.option(
    "--config", "config_path",
    default="mailmerge_server.conf",
    type=click.Path(),
    help="configuration file name (mailmerge_server.conf)",
)
def main(sample, dry_run, limit, no_limit,
         database_path, template_path, config_path):
    """
    Mailmerge is a simple, command line mail merge tool.

    For examples and formatting features, see:
    https://github.com/awdeorio/mailmerge
    """
    # We need an argument for each command line option.  That also means a lot
    # of local variables.
    # pylint: disable=too-many-arguments, too-many-locals

    # Convert paths from string to Path objects
    # https://github.com/pallets/click/issues/405
    template_path = Path(template_path)
    database_path = Path(database_path)
    config_path = Path(config_path)

    check_input_files(template_path, database_path, config_path, sample)

    # No limit is an alias for limit=-1
    if no_limit:
        limit = -1

    try:
        template_message = TemplateMessage(template_path)
        csv_database = read_csv_database(database_path)
        sendmail_client = SendmailClient(config_path, dry_run)
        for i, row in enumerate_limit(csv_database, limit):
            sender, recipients, message = template_message.render(row)
            sendmail_client.sendmail(sender, recipients, message)
            print(">>> message {}".format(i))
            print(message.as_string())
            for filename in get_attachment_filenames(message):
                print(">>> attached {}".format(filename))
            print(">>> sent message {}".format(i))
    except jinja2.exceptions.TemplateError as err:
        sys.exit(">>> Error in Jinja2 template: {}".format(err))
    except csv.Error as err:
        sys.exit(">>> Error reading CSV file: {}".format(err))
    except smtplib.SMTPAuthenticationError as err:
        sys.exit(">>> Authentication error: {}".format(err))
    except configparser.Error as err:
        sys.exit(
            ">>> Error reading config file {filename}: {message}"
            .format(filename=config_path, message=err)
        )
    except smtplib.SMTPException as err:
        sys.exit(">>> Error sending message", err, sep=' ', file=sys.stderr)
    except socket.error:
        sys.exit(">>> Error connecting to server")
    except MailmergeError as err:
        sys.exit(">>> {}".format(err))

    # Hints for user
    if not no_limit:
        print(
            ">>> Limit was {limit} message{pluralizer}.  "
            "To remove the limit, use the --no-limit option."
            .format(limit=limit, pluralizer=("" if limit == 1 else "s"))
        )
    if dry_run:
        print(
            ">>> This was a dry run.  "
            "To send messages, use the --no-dry-run option."
        )

if __name__ == "__main__":
    # No value for parameter, that's how click works
    # pylint: disable=no-value-for-parameter
    main()

def check_input_files(template_path, database_path, config_path, sample):
    """Check if input files are present and hint the user."""
    if sample:
        create_sample_input_files(template_path, database_path, config_path)
        sys.exit(0)

    if not template_path.exists():
        sys.exit(textwrap.dedent(u"""
            Error: can't find template {template_path}

            Create a sample (--sample) or specify a file (--template)
            See https://github.com/awdeorio/mailmerge for examples.\
        """.format(template_path=template_path)))

    if not database_path.exists():
        sys.exit(textwrap.dedent(u"""
            Error: can't find database {database_path}

            Create a sample (--sample) or specify a file (--database)
            See https://github.com/awdeorio/mailmerge for examples.\
        """.format(database_path=database_path)))

    if not config_path.exists():
        sys.exit(textwrap.dedent(u"""
            Error: can't find config {config_path}
            Create a sample (--sample) or specify a file (--config)
            See https://github.com/awdeorio/mailmerge for examples.\
        """.format(config_path=config_path)))

def create_sample_input_files(template_path, database_path, config_path):
    """Create sample template, database and server config."""
    for path in [template_path, database_path, config_path]:
        if path.exists():
            sys.exit("Error: file exists: {}".format(path))
    with template_path.open("w") as template_file:
        template_file.write(textwrap.dedent(u"""\
            TO: {{email}}
            SUBJECT: Testing mailmerge
            FROM: My Self <myself@mydomain.com>

            Hi, {{name}},

            Your number is {{number}}.
        """))
    with database_path.open("w") as database_file:
        database_file.write(textwrap.dedent(u"""\
            email,name,number
            myself@mydomain.com,"Myself",17
            bob@bobdomain.com,"Bob",42
        """))
    with config_path.open("w") as config_file:
        config_file.write(textwrap.dedent(u"""\
            # Example: GMail, with SSL/TLS security
            [smtp_server]
            host = smtp.gmail.com
            port = 465
            security = SSL/TLS
            username = YOUR_USERNAME_HERE

            # Example: University of Michigan, with SSL/TLS security
            # [smtp_server]
            # host = smtp.mail.umich.edu
            # port = 465
            # security = SSL/TLS
            # username = YOUR_USERNAME_HERE

            # Example: University of Michigan EECS Dept., with STARTTLS security
            # [smtp_server]
            # host = newman.eecs.umich.edu
            # port = 25
            # security = STARTTLS
            # username = YOUR_USERNAME_HERE

            # Example: University of Michigan EECS Dept., with no security
            # [smtp_server]
            # host = newman.eecs.umich.edu
            # port = 25
        """))
    print(textwrap.dedent("""\
        Created sample template email "{template_path}"
        Created sample database "{database_path}"
        Created sample config file "{config_path}"
        Edit these files, then run mailmerge again.
    """.format(
        template_path=template_path,
        database_path=database_path,
        config_path=config_path,
    )))

def read_csv_database(database_path):
    """Read database CSV file, providing one line at a time.

    We'll use a class to modify the csv library's default dialect ('excel') to
    enable strict syntax checking.  This will trigger errors for things like
    unclosed quotes.
    """
    # Our helper class is really simple
    # pylint: disable=too-few-public-methods, missing-class-docstring

    class StrictExcel(csv.excel):
        strict = True

    with database_path.open("r") as database_file:
        reader = csv.DictReader(database_file, dialect=StrictExcel)
        for row in reader:
            yield row

def enumerate_limit(iterable, limit):
    """Enumerate iterable, stopping after limit iterations.

    When limit == -1, enumerate entire iterable.
    """
    for i, value in enumerate(iterable):
        if limit != -1 and i >= limit:
            return
        yield i, value

def get_attachment_filenames(message):
    """Return a list of attachment filenames."""
    if message.get_content_maintype() != "multipart":
        return []

    filenames = []
    for part in message.walk():
        if part.get_content_maintype() == "multipart":
            continue
        if part.get_content_maintype() == "text":
            continue
        if part.get("Content-Disposition") == "inline":
            continue
        if part.get("Content-Disposition") is None:
            continue
        filenames.append(part.get_filename())
    return filenames
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Copyright 2016 Fedele Mantuano (https://twitter.com/fedelemantuano)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

__version__ = "3.9.1"

if __name__ == "__main__":
    print(__version__)
<commit_msg>Updated version<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Copyright 2016 Fedele Mantuano (https://twitter.com/fedelemantuano)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

__version__ = "3.9.2"

if __name__ == "__main__":
    print(__version__)
<|endoftext|>