<commit_before># -*- coding: utf-8 -*-
##
## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""WebSession Regression Test Suite."""

__revision__ = "$Id$"

import unittest

from invenio.config import sweburl
from invenio.testutils import make_test_suite, warn_user_about_tests_and_run, \
                              test_web_page_content, merge_error_messages

class WebSessionWebPagesAvailabilityTest(unittest.TestCase):
    """Check WebSession web pages whether they are up or not."""

    def test_your_account_pages_availability(self):
        """websession - availability of Your Account pages"""

        baseurl = sweburl + '/youraccount/'

        _exports = ['', 'edit', 'change', 'lost', 'display',
                    'send_email', 'youradminactivities',
                    'delete', 'logout', 'login', 'register']

        error_messages = []
        for url in [baseurl + page for page in _exports]:
            error_messages.extend(test_web_page_content(url))
        if error_messages:
            self.fail(merge_error_messages(error_messages))
        return

    def test_your_groups_pages_availability(self):
        """websession - availability of Your Groups pages"""

        baseurl = sweburl + '/yourgroups/'

        _exports = ['', 'display', 'create', 'join', 'leave', 'edit', 'members']

        error_messages = []
        for url in [baseurl + page for page in _exports]:
            error_messages.extend(test_web_page_content(url))
        if error_messages:
            self.fail(merge_error_messages(error_messages))
        return

test_suite = make_test_suite(WebSessionWebPagesAvailabilityTest)

if __name__ == "__main__":
    warn_user_about_tests_and_run(test_suite)
<commit_msg>Added test for "Lost Your Password" functionality for internal accounts.<commit_after># -*- coding: utf-8 -*-
##
## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""WebSession Regression Test Suite."""

__revision__ = \
    "$Id$"

import unittest

from mechanize import Browser

from invenio.config import sweburl, adminemail
from invenio.testutils import make_test_suite, warn_user_about_tests_and_run, \
                              test_web_page_content, merge_error_messages

class WebSessionWebPagesAvailabilityTest(unittest.TestCase):
    """Check WebSession web pages whether they are up or not."""

    def test_your_account_pages_availability(self):
        """websession - availability of Your Account pages"""

        baseurl = sweburl + '/youraccount/'

        _exports = ['', 'edit', 'change', 'lost', 'display',
                    'send_email', 'youradminactivities',
                    'delete', 'logout', 'login', 'register']

        error_messages = []
        for url in [baseurl + page for page in _exports]:
            error_messages.extend(test_web_page_content(url))
        if error_messages:
            self.fail(merge_error_messages(error_messages))
        return

    def test_your_groups_pages_availability(self):
        """websession - availability of Your Groups pages"""

        baseurl = sweburl + '/yourgroups/'

        _exports = ['', 'display', 'create', 'join', 'leave', 'edit', 'members']

        error_messages = []
        for url in [baseurl + page for page in _exports]:
            error_messages.extend(test_web_page_content(url))
        if error_messages:
            self.fail(merge_error_messages(error_messages))
        return

class WebSessionLostYourPasswordTest(unittest.TestCase):
    """Test Lost Your Passwords functionality."""

    def test_lost_your_password_for_internal_accounts(self):
        """websession - sending lost password for internal admin account"""

        try_with_account = adminemail

        # click on "send lost password" for adminemail internal account
        browser = Browser()
        browser.open(sweburl + "/youraccount/lost")
        browser.select_form(nr=0)
        browser['p_email'] = try_with_account
        browser.submit()

        # verify the response:
        expected_response = "Okay, password has been emailed to " + \
                            try_with_account
        lost_password_response_body = browser.response().read()
        try:
            lost_password_response_body.index(expected_response)
        except ValueError:
            self.fail("Expected to see %s, got %s." % \
                      (expected_response, lost_password_response_body))

test_suite = make_test_suite(WebSessionWebPagesAvailabilityTest,
                             WebSessionLostYourPasswordTest)

if __name__ == "__main__":
    warn_user_about_tests_and_run(test_suite)
<|endoftext|>


<commit_before>"""
==================
Breadth-first search
==================

Basic algorithms for breadth-first searching.
"""
__author__ = """\n""".join(['Aric Hagberg <hagberg@lanl.gov>'])

__all__ = ['bfs_edges', 'bfs_tree',
           'bfs_predecessors', 'bfs_successors']

import networkx as nx
from collections import defaultdict

def bfs_edges(G,source):
    """Produce edges in a breadth-first-search starting at source."""
    # Based on http://www.ics.uci.edu/~eppstein/PADS/BFS.py
    # by D. Eppstein, July 2004.
    visited=set([source])
    stack = [(source,iter(G[source]))]
    while stack:
        parent,children = stack[0]
        try:
            child = children.next()
            if child not in visited:
                yield parent,child
                visited.add(child)
                stack.append((child,iter(G[child])))
        except StopIteration:
            stack.pop(0)

def bfs_tree(G, source):
    """Return directed tree of breadth-first-search from source."""
    return nx.DiGraph(bfs_edges(G,source))

def bfs_predecessors(G, source):
    """Return dictionary of predecessors in breadth-first-search from source."""
    return dict((t,s) for s,t in bfs_edges(G,source))

def bfs_successors(G, source):
    """Return dictionary of successors in breadth-first-search from source."""
    d=defaultdict(list)
    for k, v in bfs_edges(G,source):
        d[k].append(v)
    return dict(d)

<commit_msg>Update bfs documentation heading<commit_after>"""
====================
Breadth-first search
====================

Basic algorithms for breadth-first searching.
"""
__author__ = """\n""".join(['Aric Hagberg <hagberg@lanl.gov>'])

__all__ = ['bfs_edges', 'bfs_tree',
           'bfs_predecessors', 'bfs_successors']

import networkx as nx
from collections import defaultdict

def bfs_edges(G,source):
    """Produce edges in a breadth-first-search starting at source."""
    # Based on http://www.ics.uci.edu/~eppstein/PADS/BFS.py
    # by D. Eppstein, July 2004.
    visited=set([source])
    stack = [(source,iter(G[source]))]
    while stack:
        parent,children = stack[0]
        try:
            child = children.next()
            if child not in visited:
                yield parent,child
                visited.add(child)
                stack.append((child,iter(G[child])))
        except StopIteration:
            stack.pop(0)

def bfs_tree(G, source):
    """Return directed tree of breadth-first-search from source."""
    return nx.DiGraph(bfs_edges(G,source))

def bfs_predecessors(G, source):
    """Return dictionary of predecessors in breadth-first-search from source."""
    return dict((t,s) for s,t in bfs_edges(G,source))

def bfs_successors(G, source):
    """Return dictionary of successors in breadth-first-search from source."""
    d=defaultdict(list)
    for k, v in bfs_edges(G,source):
        d[k].append(v)
    return dict(d)

<|endoftext|>


<commit_before>import lldb
from lldbsuite.test.decorators import *
from lldbsuite.test.lldbtest import *
from lldbsuite.test import lldbutil

class TestOrderedSet(TestBase):
  mydir = TestBase.compute_mydir(__file__)

  def test_ordered_set(self):
    self.build()
    src_file = "main.m"
    src_file_spec = lldb.SBFileSpec(src_file)
    (target, process, thread, main_breakpoint) = lldbutil.run_to_source_breakpoint(self,
          "break here", src_file_spec, exe_name = "a.out")
    frame = thread.GetSelectedFrame()
    self.expect("expr -d run -- orderedSet", substrs=["3 elements"])
    self.expect("expr -d run -- *orderedSet", substrs=["(int)1", "(int)2", "(int)3"])
<commit_msg>@skipUnlessDarwin TestOrderedSet<commit_after>import lldb
from lldbsuite.test.decorators import *
from lldbsuite.test.lldbtest import *
from lldbsuite.test import lldbutil

class TestOrderedSet(TestBase):
  mydir = TestBase.compute_mydir(__file__)

  @skipUnlessDarwin
  def test_ordered_set(self):
    self.build()
    src_file = "main.m"
    src_file_spec = lldb.SBFileSpec(src_file)
    (target, process, thread, main_breakpoint) = lldbutil.run_to_source_breakpoint(self,
          "break here", src_file_spec, exe_name = "a.out")
    frame = thread.GetSelectedFrame()
    self.expect("expr -d run -- orderedSet", substrs=["3 elements"])
    self.expect("expr -d run -- *orderedSet", substrs=["(int)1", "(int)2", "(int)3"])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.10.4 on 2016-12-17 19:12
from __future__ import unicode_literals

from django.db import migrations, models
import django.db.models.deletion
import uuid

class Migration(migrations.Migration):

    initial = True

    dependencies = [
        ('competences', '0001_initial'),
        ('disciplines', '0001_initial'),
        ('results', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='ChoiceModulesPool',
            fields=[
                ('archived', models.BooleanField(default=False, verbose_name='В архиве')),
                ('created', models.DateTimeField(auto_now_add=True, verbose_name='Дата создания')),
                ('updated', models.DateTimeField(auto_now=True, verbose_name='Обновлен')),
                ('status', models.CharField(choices=[('h', 'Скрыт'), ('p', 'Опубликован')], default='h', max_length=1, verbose_name='Статус публикации')),
                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
                ('title', models.CharField(default='Название пула', max_length=256, verbose_name='Название пула модулей по выбору')),
                ('description', models.TextField(blank=True, default='', max_length=16384, verbose_name='Описание')),
            ],
            options={
                'verbose_name': 'пул модулей по выбору',
                'verbose_name_plural': 'пулы модулей по выбору',
            },
        ),
        migrations.CreateModel(
            name='EducationalProgramTrajectoriesPool',
            fields=[
                ('archived', models.BooleanField(default=False, verbose_name='В архиве')),
                ('created', models.DateTimeField(auto_now_add=True, verbose_name='Дата создания')),
                ('updated', models.DateTimeField(auto_now=True, verbose_name='Обновлен')),
                ('status', models.CharField(choices=[('h', 'Скрыт'), ('p', 'Опубликован')], default='h', max_length=1, verbose_name='Статус публикации')),
                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
                ('title', models.CharField(default='Название пула', max_length=256, verbose_name='Название пула модулей траектории обр. программ')),
                ('description', models.TextField(blank=True, default='', max_length=16384, verbose_name='Описание')),
            ],
            options={
                'verbose_name': 'пул модулей траектории обр. программ',
                'verbose_name_plural': 'пулы модулей траектории обр. программ',
            },
        ),
        migrations.CreateModel(
            name='GeneralBaseModulesPool',
            fields=[
                ('archived', models.BooleanField(default=False, verbose_name='В архиве')),
                ('created', models.DateTimeField(auto_now_add=True, verbose_name='Дата создания')),
                ('updated', models.DateTimeField(auto_now=True, verbose_name='Обновлен')),
                ('status', models.CharField(choices=[('h', 'Скрыт'), ('p', 'Опубликован')], default='h', max_length=1, verbose_name='Статус публикации')),
                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
                ('title', models.CharField(default='Название пула', max_length=256, verbose_name='Название базового модуля программы')),
                ('description', models.TextField(blank=True, default='', max_length=16384, verbose_name='Описание')),
            ],
            options={
                'verbose_name': 'базовый модуль программы',
                'verbose_name_plural': 'базовые модули программы',
            },
        ),
        migrations.CreateModel(
            name='Module',
            fields=[
                ('archived', models.BooleanField(default=False, verbose_name='В архиве')),
                ('created', models.DateTimeField(auto_now_add=True, verbose_name='Дата создания')),
                ('updated', models.DateTimeField(auto_now=True, verbose_name='Обновлен')),
                ('status', models.CharField(choices=[('h', 'Скрыт'), ('p', 'Опубликован')], default='h', max_length=1, verbose_name='Статус публикации')),
                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
                ('title', models.CharField(max_length=256, verbose_name='Название модуля')),
                ('description', models.TextField(blank=True, default='', max_length=16384, verbose_name='Описание модуля')),
                ('results_text', models.TextField(blank=True, default='', max_length=16384, verbose_name='Результаты обучения')),
                ('competences', models.ManyToManyField(blank=True, to='competences.Competence', verbose_name='Компетенции')),
                ('disciplines', models.ManyToManyField(blank=True, to='disciplines.Discipline', verbose_name='Дисциплины')),
                ('results', models.ManyToManyField(blank=True, to='results.Result', verbose_name='Результаты обучения')),
            ],
            options={
                'verbose_name': 'модуль',
                'verbose_name_plural': 'модули',
            },
        ),
        migrations.CreateModel(
            name='Type',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('archived', models.BooleanField(default=False, verbose_name='В архиве')),
                ('created', models.DateTimeField(auto_now_add=True, verbose_name='Дата создания')),
                ('updated', models.DateTimeField(auto_now=True, verbose_name='Обновлен')),
                ('status', models.CharField(choices=[('h', 'Скрыт'), ('p', 'Опубликован')], default='h', max_length=1, verbose_name='Статус публикации')),
                ('title', models.CharField(default='Название типа модуля', max_length=256, verbose_name='Название типа модуля')),
                ('description', models.TextField(blank=True, default='', max_length=16384, verbose_name='Описание')),
            ],
            options={
                'verbose_name': 'тип модуля',
                'verbose_name_plural': 'типы модулей',
            },
        ),
        migrations.AddField(
            model_name='module',
            name='type',
            field=models.ForeignKey(default=0, null=True, on_delete=django.db.models.deletion.CASCADE, to='modules.Type', verbose_name='Тип модуля'),
        ),
        migrations.AddField(
            model_name='generalbasemodulespool',
            name='modules',
            field=models.ManyToManyField(to='modules.Module', verbose_name='модули'),
        ),
        migrations.AddField(
            model_name='educationalprogramtrajectoriespool',
            name='modules',
            field=models.ManyToManyField(to='modules.Module', verbose_name='модули'),
        ),
        migrations.AddField(
            model_name='choicemodulespool',
            name='modules',
            field=models.ManyToManyField(to='modules.Module', verbose_name='модули'),
        ),
    ]
<commit_msg>default=1<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.10.4 on 2016-12-17 19:12
from __future__ import unicode_literals

from django.db import migrations, models
import django.db.models.deletion
import uuid

class Migration(migrations.Migration):

    initial = True

    dependencies = [
        ('competences', '0001_initial'),
        ('disciplines', '0001_initial'),
        ('results', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='ChoiceModulesPool',
            fields=[
                ('archived', models.BooleanField(default=False, verbose_name='В архиве')),
                ('created', models.DateTimeField(auto_now_add=True, verbose_name='Дата создания')),
                ('updated', models.DateTimeField(auto_now=True, verbose_name='Обновлен')),
                ('status', models.CharField(choices=[('h', 'Скрыт'), ('p', 'Опубликован')], default='h', max_length=1, verbose_name='Статус публикации')),
                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
                ('title', models.CharField(default='Название пула', max_length=256, verbose_name='Название пула модулей по выбору')),
                ('description', models.TextField(blank=True, default='', max_length=16384, verbose_name='Описание')),
            ],
            options={
                'verbose_name': 'пул модулей по выбору',
                'verbose_name_plural': 'пулы модулей по выбору',
            },
        ),
        migrations.CreateModel(
            name='EducationalProgramTrajectoriesPool',
            fields=[
                ('archived', models.BooleanField(default=False, verbose_name='В архиве')),
                ('created', models.DateTimeField(auto_now_add=True, verbose_name='Дата создания')),
                ('updated', models.DateTimeField(auto_now=True, verbose_name='Обновлен')),
                ('status', models.CharField(choices=[('h', 'Скрыт'), ('p', 'Опубликован')], default='h', max_length=1, verbose_name='Статус публикации')),
                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
                ('title', models.CharField(default='Название пула', max_length=256, verbose_name='Название пула модулей траектории обр. программ')),
                ('description', models.TextField(blank=True, default='', max_length=16384, verbose_name='Описание')),
            ],
            options={
                'verbose_name': 'пул модулей траектории обр. программ',
                'verbose_name_plural': 'пулы модулей траектории обр. программ',
            },
        ),
        migrations.CreateModel(
            name='GeneralBaseModulesPool',
            fields=[
                ('archived', models.BooleanField(default=False, verbose_name='В архиве')),
                ('created', models.DateTimeField(auto_now_add=True, verbose_name='Дата создания')),
                ('updated', models.DateTimeField(auto_now=True, verbose_name='Обновлен')),
                ('status', models.CharField(choices=[('h', 'Скрыт'), ('p', 'Опубликован')], default='h', max_length=1, verbose_name='Статус публикации')),
                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
                ('title', models.CharField(default='Название пула', max_length=256, verbose_name='Название базового модуля программы')),
                ('description', models.TextField(blank=True, default='', max_length=16384, verbose_name='Описание')),
            ],
            options={
                'verbose_name': 'базовый модуль программы',
                'verbose_name_plural': 'базовые модули программы',
            },
        ),
        migrations.CreateModel(
            name='Module',
            fields=[
                ('archived', models.BooleanField(default=False, verbose_name='В архиве')),
                ('created', models.DateTimeField(auto_now_add=True, verbose_name='Дата создания')),
                ('updated', models.DateTimeField(auto_now=True, verbose_name='Обновлен')),
                ('status', models.CharField(choices=[('h', 'Скрыт'), ('p', 'Опубликован')], default='h', max_length=1, verbose_name='Статус публикации')),
                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
                ('title', models.CharField(max_length=256, verbose_name='Название модуля')),
                ('description', models.TextField(blank=True, default='', max_length=16384, verbose_name='Описание модуля')),
                ('results_text', models.TextField(blank=True, default='', max_length=16384, verbose_name='Результаты обучения')),
                ('competences', models.ManyToManyField(blank=True, to='competences.Competence', verbose_name='Компетенции')),
                ('disciplines', models.ManyToManyField(blank=True, to='disciplines.Discipline', verbose_name='Дисциплины')),
                ('results', models.ManyToManyField(blank=True, to='results.Result', verbose_name='Результаты обучения')),
            ],
            options={
                'verbose_name': 'модуль',
                'verbose_name_plural': 'модули',
            },
        ),
        migrations.CreateModel(
            name='Type',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('archived', models.BooleanField(default=False, verbose_name='В архиве')),
                ('created', models.DateTimeField(auto_now_add=True, verbose_name='Дата создания')),
                ('updated', models.DateTimeField(auto_now=True, verbose_name='Обновлен')),
                ('status', models.CharField(choices=[('h', 'Скрыт'), ('p', 'Опубликован')], default='h', max_length=1, verbose_name='Статус публикации')),
                ('title', models.CharField(default='Название типа модуля', max_length=256, verbose_name='Название типа модуля')),
                ('description', models.TextField(blank=True, default='', max_length=16384, verbose_name='Описание')),
            ],
            options={
                'verbose_name': 'тип модуля',
                'verbose_name_plural': 'типы модулей',
            },
        ),
        migrations.AddField(
            model_name='module',
            name='type',
            field=models.ForeignKey(default=1, null=True, on_delete=django.db.models.deletion.CASCADE, to='modules.Type', verbose_name='Тип модуля'),
        ),
        migrations.AddField(
            model_name='generalbasemodulespool',
            name='modules',
            field=models.ManyToManyField(to='modules.Module', verbose_name='модули'),
        ),
        migrations.AddField(
            model_name='educationalprogramtrajectoriespool',
            name='modules',
            field=models.ManyToManyField(to='modules.Module', verbose_name='модули'),
        ),
        migrations.AddField(
            model_name='choicemodulespool',
            name='modules',
            field=models.ManyToManyField(to='modules.Module', verbose_name='модули'),
        ),
    ]
<|endoftext|>


<commit_before>"""
Tests for the Credit xBlock service
"""

from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory

from openedx.core.djangoapps.credit.services import CreditService
from openedx.core.djangoapps.credit.models import CreditCourse
from openedx.core.djangoapps.credit.api.eligibility import set_credit_requirements

from student.models import CourseEnrollment, UserProfile

class CreditServiceTests(ModuleStoreTestCase):
    """
    Tests for the Credit xBlock service
    """

    def setUp(self, **kwargs):
        super(CreditServiceTests, self).setUp()

        self.service = CreditService()
        self.course = CourseFactory.create(org='edX', number='DemoX', display_name='Demo_Course')
        self.credit_course = CreditCourse.objects.create(course_key=self.course.id, enabled=True)
        self.profile = UserProfile.objects.create(user_id=self.user.id, name='Foo Bar')

    def test_user_not_found(self):
        """
        Makes sure that get_credit_state returns None if user_id cannot be found
        """

        self.assertIsNone(self.service.get_credit_state(0, self.course.id))

    def test_user_not_enrolled(self):
        """
        Makes sure that get_credit_state returns None if user_id is not enrolled
        in the test course
        """

        self.assertIsNone(self.service.get_credit_state(self.user.id, self.course.id))

    def test_inactive_enrollment(self):
        """
        Makes sure that get_credit_state returns None if the user's enrollment is
        inactive
        """

        enrollment = CourseEnrollment.enroll(self.user, self.course.id)
        enrollment.is_active = False
        enrollment.save()

        self.assertIsNone(self.service.get_credit_state(self.user.id, self.course.id))

    def test_not_credit_course(self):
        """
        Makes sure that get_credit_state returns None if the test course is not
        Credit eligible
        """

        CourseEnrollment.enroll(self.user, self.course.id)

        self.credit_course.enabled = False
        self.credit_course.save()

        self.assertIsNone(self.service.get_credit_state(self.user.id, self.course.id))

    def test_no_profile_name(self):
        """
        Makes sure that get_credit_state returns None if the user does not
        have a corresponding UserProfile. This shouldn't happen in
        real environments
        """

        profile = UserProfile.objects.get(user_id=self.user.id)
        profile.delete()

        self.assertIsNone(self.service.get_credit_state(self.user.id, self.course.id))

    def test_get_and_set_credit_state(self):
        """
        Happy path through the service
        """

        CourseEnrollment.enroll(self.user, self.course.id)

        # set course requirements
        set_credit_requirements(
            self.course.id,
            [
                {
                    "namespace": "grade",
                    "name": "grade",
                    "display_name": "Grade",
                    "criteria": {
                        "min_grade": 0.8
                    },
                },
            ]
        )

        # mark the grade as satisfied
        self.service.set_credit_requirement_status(
            self.user.id,
            self.course.id,
            'grade',
            'grade'
        )

        credit_state = self.service.get_credit_state(self.user.id, self.course.id)

        self.assertIsNotNone(credit_state)
        self.assertEqual(credit_state['enrollment_mode'], 'honor')
        self.assertEqual(credit_state['profile_fullname'], 'Foo Bar')
        self.assertEqual(len(credit_state['credit_requirement_status']), 1)
        self.assertEqual(credit_state['credit_requirement_status'][0]['name'], 'grade')
        self.assertEqual(credit_state['credit_requirement_status'][0]['status'], 'satisfied')

    def test_bad_user(self):
        """
        Try setting requirements status with a bad user_id
        """

        # set course requirements
        set_credit_requirements(
            self.course.id,
            [
                {
                    "namespace": "grade",
                    "name": "grade",
                    "display_name": "Grade",
                    "criteria": {
                        "min_grade": 0.8
                    },
                },
            ]
        )

        # mark the grade as satisfied
        retval = self.service.set_credit_requirement_status(
            0,
            self.course.id,
            'grade',
            'grade'
        )
        self.assertIsNone(retval)

    def test_course_id_string(self):
        """
        Make sure we can pass a course_id (string) and get back correct results as well
        """

        CourseEnrollment.enroll(self.user, self.course.id)

        # set course requirements
        set_credit_requirements(
            self.course.id,
            [
                {
                    "namespace": "grade",
                    "name": "grade",
                    "display_name": "Grade",
                    "criteria": {
                        "min_grade": 0.8
                    },
                },
            ]
        )

        # mark the grade as satisfied
        self.service.set_credit_requirement_status(
            self.user.id,
            unicode(self.course.id),
            'grade',
            'grade'
        )

        credit_state = self.service.get_credit_state(self.user.id, unicode(self.course.id))

        self.assertIsNotNone(credit_state)
        self.assertEqual(credit_state['enrollment_mode'], 'honor')
        self.assertEqual(credit_state['profile_fullname'], 'Foo Bar')
        self.assertEqual(len(credit_state['credit_requirement_status']), 1)
        self.assertEqual(credit_state['credit_requirement_status'][0]['name'], 'grade')
        self.assertEqual(credit_state['credit_requirement_status'][0]['status'], 'satisfied')
<commit_msg>Skip contrived CME test<commit_after>"""
Tests for the Credit xBlock service
"""
import unittest

from django.conf import settings

from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory

from openedx.core.djangoapps.credit.services import CreditService
from openedx.core.djangoapps.credit.models import CreditCourse
from openedx.core.djangoapps.credit.api.eligibility import set_credit_requirements

from student.models import CourseEnrollment, UserProfile

class CreditServiceTests(ModuleStoreTestCase):
    """
    Tests for the Credit xBlock service
    """

    def setUp(self, **kwargs):
        super(CreditServiceTests, self).setUp()

        self.service = CreditService()
        self.course = CourseFactory.create(org='edX', number='DemoX', display_name='Demo_Course')
        self.credit_course = CreditCourse.objects.create(course_key=self.course.id, enabled=True)
        self.profile = UserProfile.objects.create(user_id=self.user.id, name='Foo Bar')

    def test_user_not_found(self):
        """
        Makes sure that get_credit_state returns None if user_id cannot be found
        """

        self.assertIsNone(self.service.get_credit_state(0, self.course.id))

    def test_user_not_enrolled(self):
        """
        Makes sure that get_credit_state returns None if user_id is not enrolled
        in the test course
        """

        self.assertIsNone(self.service.get_credit_state(self.user.id, self.course.id))

    def test_inactive_enrollment(self):
        """
        Makes sure that get_credit_state returns None if the user's enrollment is
        inactive
        """

        enrollment = CourseEnrollment.enroll(self.user, self.course.id)
        enrollment.is_active = False
        enrollment.save()

        self.assertIsNone(self.service.get_credit_state(self.user.id, self.course.id))

    def test_not_credit_course(self):
        """
        Makes sure that get_credit_state returns None if the test course is not
        Credit eligible
        """

        CourseEnrollment.enroll(self.user, self.course.id)

        self.credit_course.enabled = False
        self.credit_course.save()

        self.assertIsNone(self.service.get_credit_state(self.user.id, self.course.id))

    @unittest.skipIf(settings.FEATURES.get('DISABLE_CME_REGISTRATION_TESTS', False), 'This is contrived and does not work for CME')
    def test_no_profile_name(self):
        """
        Makes sure that get_credit_state returns None if the user does not
        have a corresponding UserProfile. This shouldn't happen in
        real environments
        """

        profile = UserProfile.objects.get(user_id=self.user.id)
        profile.delete()

        self.assertIsNone(self.service.get_credit_state(self.user.id, self.course.id))

    def test_get_and_set_credit_state(self):
        """
        Happy path through the service
        """

        CourseEnrollment.enroll(self.user, self.course.id)

        # set course requirements
        set_credit_requirements(
            self.course.id,
            [
                {
                    "namespace": "grade",
                    "name": "grade",
                    "display_name": "Grade",
                    "criteria": {
                        "min_grade": 0.8
                    },
                },
            ]
        )

        # mark the grade as satisfied
        self.service.set_credit_requirement_status(
            self.user.id,
            self.course.id,
            'grade',
            'grade'
        )

        credit_state = self.service.get_credit_state(self.user.id, self.course.id)

        self.assertIsNotNone(credit_state)
        self.assertEqual(credit_state['enrollment_mode'], 'honor')
        self.assertEqual(credit_state['profile_fullname'], 'Foo Bar')
        self.assertEqual(len(credit_state['credit_requirement_status']), 1)
        self.assertEqual(credit_state['credit_requirement_status'][0]['name'], 'grade')
        self.assertEqual(credit_state['credit_requirement_status'][0]['status'], 'satisfied')

    def test_bad_user(self):
        """
        Try setting requirements status with a bad user_id
        """

        # set course requirements
        set_credit_requirements(
            self.course.id,
            [
                {
                    "namespace": "grade",
                    "name": "grade",
                    "display_name": "Grade",
                    "criteria": {
                        "min_grade": 0.8
                    },
                },
            ]
        )

        # mark the grade as satisfied
        retval = self.service.set_credit_requirement_status(
            0,
            self.course.id,
            'grade',
            'grade'
        )
        self.assertIsNone(retval)

    def test_course_id_string(self):
        """
        Make sure we can pass a course_id (string) and get back correct results as well
        """

        CourseEnrollment.enroll(self.user, self.course.id)

        # set course requirements
        set_credit_requirements(
            self.course.id,
            [
                {
                    "namespace": "grade",
                    "name": "grade",
                    "display_name": "Grade",
                    "criteria": {
                        "min_grade": 0.8
                    },
                },
            ]
        )

        # mark the grade as satisfied
        self.service.set_credit_requirement_status(
            self.user.id,
            unicode(self.course.id),
            'grade',
            'grade'
        )

        credit_state = self.service.get_credit_state(self.user.id, unicode(self.course.id))

        self.assertIsNotNone(credit_state)
        self.assertEqual(credit_state['enrollment_mode'], 'honor')
        self.assertEqual(credit_state['profile_fullname'], 'Foo Bar')
        self.assertEqual(len(credit_state['credit_requirement_status']), 1)
        self.assertEqual(credit_state['credit_requirement_status'][0]['name'], 'grade')
        self.assertEqual(credit_state['credit_requirement_status'][0]['status'], 'satisfied')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import os
import glob
import imp
import zipfile

from openerp import tools
from openerp.osv import osv

class base_module_scan(osv.osv_memory):
    """ scan module """

    _name = "base.module.scan"
    _description = "scan module"

    def watch_dir(self, cr, uid, ids, context):
        mod_obj = self.pool.get('ir.module.module')
        all_mods = mod_obj.read(cr, uid, mod_obj.search(cr, uid, []), ['name', 'state'])
        known_modules = [x['name'] for x in all_mods]
        ls_ad = glob.glob(os.path.join(tools.config['addons_path'], '*', '__terp__.py'))
        modules = [module_name_re.match(name).group(1) for name in ls_ad]
        for fname in os.listdir(tools.config['addons_path']):
            if zipfile.is_zipfile(fname):
                modules.append( fname.split('.')[0])
        for module in modules:
            if module in known_modules:
                continue
            terp = mod_obj.get_module_info(module)
            if not terp.get('installable', True):
                continue

            # XXX check if this code is correct...
            fm = imp.find_module(module)
            try:
                imp.load_module(module, *fm)
            finally:
                if fm[0]:
                    fm[0].close()

            values = mod_obj.get_values_from_terp(terp)
            mod_id = mod_obj.create(cr, uid, dict(name=module, state='uninstalled', **values))
            dependencies = terp.get('depends', [])
            for d in dependencies:
                cr.execute('insert into ir_module_module_dependency (module_id,name) values (%s, %s)', (mod_id, d))
        for module in known_modules:
            terp = mod_obj.get_module_info(module)
            if terp.get('installable', True):
                for mod in all_mods:
                    if mod['name'] == module and mod['state'] == 'uninstallable':
                        mod_obj.write(cr, uid, [mod['id']], {'state': 'uninstalled'})
        return {}

base_module_scan()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[REM] Removed unused base_module_scan wizard.<commit_after><|endoftext|>


<commit_before>import queue
import threading
from time import sleep

# Pre-load the model graphs so it doesn't have to be done for each job
from django.core.management.base import BaseCommand
from photonix.classifiers.color import ColorModel, run_on_photo
from photonix.photos.utils.tasks import requeue_stuck_tasks

print('Loading style color model')
model = ColorModel()
q = queue.Queue()

def worker():
    while True:
        task = q.get()

        if task is None:
            break

        task.start()
        run_on_photo(task.subject_id)
        task.complete()

        q.task_done()

class Command(BaseCommand):
    help = 'Runs the workers with the color classification model.'

    def run_processors(self):
        num_workers = 4
        threads = []

        print('Starting {} color classification workers\n'.format(num_workers))

        for i in range(num_workers):
            t = threading.Thread(target=worker)
            t.start()
            threads.append(t)

        try:
            while True:
                requeue_stuck_tasks('classify.color')

                # Load 'Pending' tasks onto worker threads
                for task in Task.objects.filter(type='classify.color', status='P')[:64]:
                    q.put(task)

                # Wait until all threads have finished
                q.join()
                sleep(1)

        except KeyboardInterrupt:
            # Shut down threads cleanly
            for i in range(num_workers):
                q.put(None)
            for t in threads:
                t.join()

    def handle(self, *args, **options):
        self.run_processors()
<commit_msg>Fixed color classifier that had a missing import<commit_after>import queue
import threading
from time import sleep

from django.core.management.base import BaseCommand
# Pre-load the model graphs so it doesn't have to be done for each job
from photonix.classifiers.color import ColorModel, run_on_photo
from photonix.photos.models import Task
from photonix.photos.utils.tasks import requeue_stuck_tasks

print('Loading style color model')
model = ColorModel()
q = queue.Queue()

def worker():
    while True:
        task = q.get()

        if task is None:
            break

        task.start()
        run_on_photo(task.subject_id)
        task.complete()

        q.task_done()

class Command(BaseCommand):
    help = 'Runs the workers with the color classification model.'

    def run_processors(self):
        num_workers = 4
        threads = []

        print('Starting {} color classification workers\n'.format(num_workers))

        for i in range(num_workers):
            t = threading.Thread(target=worker)
            t.start()
            threads.append(t)

        try:
            while True:
                requeue_stuck_tasks('classify.color')

                # Load 'Pending' tasks onto worker threads
                for task in Task.objects.filter(type='classify.color', status='P')[:64]:
                    q.put(task)

                # Wait until all threads have finished
                q.join()
                sleep(1)

        except KeyboardInterrupt:
            # Shut down threads cleanly
            for i in range(num_workers):
                q.put(None)
            for t in threads:
                t.join()

    def handle(self, *args, **options):
        self.run_processors()
<|endoftext|>


<commit_before># -------------------------------------------------------------------------------
# Copyright IBM Corp. 2017
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------------

from pixiedust.display.chart.renderers import PixiedustRenderer
from pixiedust.display.chart.renderers.baseChartDisplay import commonChartOptions
from pixiedust.utils import Logger
from ..baseChartDisplay import BaseChartDisplay
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from six import with_metaclass
from abc import abstractmethod, ABCMeta
import numpy as np
import math
import matplotlib.ticker as ticker

try:
    import mpld3
    import mpld3.plugins as plugins
    from pixiedust.display.chart.plugins.chart import ChartPlugin
    from pixiedust.display.chart.plugins.dialog import DialogPlugin
    from pixiedust.display.chart.plugins.elementInfo import ElementInfoPlugin
    mpld3Available = True
except ImportError:
    mpld3Available = False

@PixiedustRenderer(rendererId="matplotlib")
@Logger()
class MatplotlibBaseDisplay(with_metaclass(ABCMeta, BaseChartDisplay)):

    def __init__(self, options, entity, dataHandler=None):
        super(MatplotlibBaseDisplay,self).__init__(options,entity,dataHandler)
        self.needsStretching = False

    @abstractmethod
    def matplotlibRender(self):
        pass

    @property
    def useMpld3(self):
        return mpld3Available and self.options.get("mpld3", "false") == "true"

    def canStretch(self):
        return True

    def isStretchingOn(self):
        return self.getBooleanOption('stretch', False) and not self.useMpld3

    @commonChartOptions
    def getChartOptions(self):
        options = []
        if mpld3Available:
            options.append({
                'name': 'mpld3',
                'refresh': True,
                'description': 'D3 Rendering (mpld3)',
                'metadata': {
                    'type': 'checkbox',
                    'default': "false"
                }
            })
        if self.needsStretching:
            options.append({
                'name': 'stretch',
                'description': 'Stretch image',
                'metadata':{
                    'type': 'checkbox',
                    'default': 'false'
                }
            })
        return options

    def setTicks(self, fig, ax):
        if self.handlerId=="lineChart" and self.getBooleanOption("logy",False):
            start, end = ax.get_ylim()
            ax.yaxis.set_minor_locator(ticker.MultipleLocator((end - start) / 4))
            ax.yaxis.set_minor_formatter(ticker.LogFormatter(labelOnlyBase=False))
            ax.grid(True, which='both')

        labels = [s.get_text() for s in ax.get_xticklabels()]
        totalWidth = sum(len(s) for s in labels) * 5
        if totalWidth > self.getPreferredOutputWidth():
            self.needsStretching = self.canStretch()
            if self.isStretchingOn():
                #resize image width
                fig.set_size_inches( totalWidth/self.getDPI(),fig.get_figheight())
            else:
                self.addMessage("Some labels are not displayed because of a lack of space. Click on Stretch image to see them all")
                #filter down the list to max 20
                max = int(len(labels)/20)
                if max < 20:
                    xl = [(i,a[:10] + ".." if len(a) > 10 else a) for i,a in enumerate(labels)]
                else:
                    xl = [(i,a) for i,a in enumerate(labels) if (i % max == 0)]
                ax.set_xticks([x[0] for x in xl])
                ax.set_xticklabels([x[1] for x in xl])
                plt.xticks(rotation=30)
        else:
            plt.xticks(rotation=0)

    def setLegend(self, fig, ax):
        if ax.get_legend() is not None:
            numLabels = len(ax.get_legend_handles_labels()[1])
            nCol = int(min(max(math.sqrt( numLabels ), 3), 6))
            nRows = int(numLabels/nCol)
            bboxPos = max(1.15, 1.0 + ((float(nRows)/2)/10.0))
            ax.legend(loc='upper center', bbox_to_anchor=(0.5, bboxPos),ncol=nCol, fancybox=True, shadow=True)

    def getNumFigures(self):
        return 1    #default, subclasses can override

    def createFigure(self):
        numFigures = self.getNumFigures()
        if numFigures <= 1:
            return plt.subplots(figsize=( int(self.getPreferredOutputWidth()/ self.getDPI()), int(self.getPreferredOutputHeight() / self.getDPI()) ))

        numCols = 1 if self.isStretchingOn() else 2 #number of columns for a multiplots, TODO make the layout configurable
        numRows = int( numFigures/numCols ) + numFigures % numCols
        imageHeight =  ((self.getPreferredOutputWidth()/numCols) * self.getHeightWidthRatio()) * numRows
        fig,ax = plt.subplots(numRows, numCols, figsize=( int(self.getPreferredOutputWidth()/self.getDPI()), int(imageHeight/self.getDPI() )))
        if numFigures%numCols != 0:
            fig.delaxes(ax.item(numFigures))
            ax = np.delete(ax,numFigures)
        return (fig,ax)

    def getSubplotHSpace(self):
        return 0.5 if self.isStretchingOn() else 0.3

    def doRenderChart(self):
        self.colormap = cm.jet

        # set defaults for plot
        plt.rcParams['savefig.dpi'] = 96
        plt.rcParams['font.family'] = "serif"
        plt.rcParams['font.serif'] = "cm"

        fig = None
        try:
            # go
            fig, ax = self.createFigure()

            if self.useMpld3:
                #TODO: rework this piece
                #keyFieldLabels = self.getKeyFieldLabels()
                #if (len(keyFieldLabels) > 0 and self.supportsKeyFieldLabels(self.handlerId) and self.supportsAggregation(self.handlerId)):
                #    plugins.connect(fig, ChartPlugin(self, keyFieldLabels))
                plugins.connect(fig, DialogPlugin(self, self.handlerId, self.dialogBody))

            #let subclass do the actual rendering
            newAx = self.matplotlibRender(fig, ax)
            if newAx is not None:
                ax = newAx

            axes = ax
            if not isinstance(axes, (list,np.ndarray)):
                axes = np.asarray([axes])

            #finalize the chart
            for i, a in np.ndenumerate(axes):
                if a.title is None or not a.title.get_visible() or a.title.get_text() == '':
                    self.setLegend(fig, a)
                self.setTicks(fig, a)
            sharex = True if len(axes) <=1 else len([a for a in axes if a._sharex is not None]) > 0
            if len(axes)>1 and not sharex:
                #adjust the height between subplots
                plt.subplots_adjust(hspace=self.getSubplotHSpace())

            #mpld3 has a bug when autolocator are used. Change to FixedLocators
            if self.useMpld3:
                self.addMessage("Warning: While great, D3 rendering is using MPLD3 library which has limitations that have not yet been fixed")
                from matplotlib import ticker
                self.debug("Converting to FixedLocator for mpld3")
                for a in axes:
                    locator = a.xaxis.get_major_locator()
                    if not isinstance(locator, ticker.FixedLocator):
                        vmin,vmax = a.xaxis.get_data_interval()
                        a.xaxis.set_major_locator(ticker.FixedLocator(locator.tick_values(vmin, vmax)))

            #Render the figure
            return self.renderFigure(fig)
        finally:
            if fig is not None:
                plt.close(fig)

    def renderFigure(self, fig):
        def genMarkup(chartFigure):
            return self.env.from_string("""
                    {0}
                    {{%for message in messages%}}

{{{{message}}}}

                    {{%endfor%}}
                """.format(chartFigure)
            ).render(messages=self.messages)

        if not self.useMpld3:
            import base64
            try:
                from io import BytesIO as pngIO
            except ImportError:
                from StringIO import StringIO as pngIO
            png=pngIO()
            plt.savefig(png, pad_inches=0.05, bbox_inches='tight', dpi=self.getDPI())
            try:
                return(
                    genMarkup("""

                        """.format(base64.b64encode(png.getvalue()).decode("ascii"))
                    )
                )
            finally:
                png.close()
        else:
            mpld3.enable_notebook()
            try:
                return genMarkup(mpld3.fig_to_html(fig))
            finally:
                mpld3.disable_notebook()

    def connectElementInfo(self, element, data):
        if not hasattr(element, "get_figure") and hasattr(element,"get_children"):
            for i,child in enumerate(element.get_children()):
                plugins.connect(child.get_figure(), ElementInfoPlugin(child,data[i]))
        elif hasattr(element, "get_figure"):
            plugins.connect(element.get_figure(), ElementInfoPlugin(element, data ))

    """
    Helper to safely access i position of an ax object
    """
    def getAxItem(self, ax, pos):
        if isinstance(ax, np.ndarray):
            return ax.item(pos)
        elif pos == 0:
            return ax
        raise ValueError("Trying to access pos {} from a single ax".format(pos))
<commit_msg>Issue #276: large image in matplotlib bar chart<commit_after># -------------------------------------------------------------------------------
# Copyright IBM Corp. 2017
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------------

from pixiedust.display.chart.renderers import PixiedustRenderer
from pixiedust.display.chart.renderers.baseChartDisplay import commonChartOptions
from pixiedust.utils import Logger
from ..baseChartDisplay import BaseChartDisplay
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from six import with_metaclass
from abc import abstractmethod, ABCMeta
import numpy as np
import math
import matplotlib.ticker as ticker

try:
    import mpld3
    import mpld3.plugins as plugins
    from pixiedust.display.chart.plugins.chart import ChartPlugin
    from pixiedust.display.chart.plugins.dialog import DialogPlugin
    from pixiedust.display.chart.plugins.elementInfo import ElementInfoPlugin
    mpld3Available = True
except ImportError:
    mpld3Available = False

@PixiedustRenderer(rendererId="matplotlib")
@Logger()
class MatplotlibBaseDisplay(with_metaclass(ABCMeta, BaseChartDisplay)):

    def __init__(self, options, entity, dataHandler=None):
        super(MatplotlibBaseDisplay,self).__init__(options,entity,dataHandler)
        self.needsStretching = False

    @abstractmethod
    def matplotlibRender(self):
        pass

    @property
    def useMpld3(self):
        return mpld3Available and self.options.get("mpld3", "false") == "true"

    def canStretch(self):
        return True

    def isStretchingOn(self):
        return self.getBooleanOption('stretch', False) and not self.useMpld3

    @commonChartOptions
    def getChartOptions(self):
        options = []
        if mpld3Available:
            options.append({
                'name': 'mpld3',
                'refresh': True,
                'description': 'D3 Rendering (mpld3)',
                'metadata': {
                    'type': 'checkbox',
                    'default': "false"
                }
            })
        if self.needsStretching:
            options.append({
                'name': 'stretch',
                'description': 'Stretch image',
                'metadata':{
                    'type': 'checkbox',
                    'default': 'false'
                }
            })
        return options

    def setTicks(self, fig, ax):
        if self.handlerId=="lineChart" and self.getBooleanOption("logy",False):
            start, end = ax.get_ylim()
            ax.yaxis.set_minor_locator(ticker.MultipleLocator((end - start) / 4))
            ax.yaxis.set_minor_formatter(ticker.LogFormatter(labelOnlyBase=False))
            ax.grid(True, which='both')

        labels = [s.get_text() for s in ax.get_xticklabels()]
        totalWidth = sum(len(s) for s in labels) * 5
        if totalWidth > self.getPreferredOutputWidth():
            self.needsStretching = self.canStretch()
            if self.isStretchingOn():
                #resize image width
                fig.set_size_inches(min(totalWidth/self.getDPI(), 32768/self.getDPI()),fig.get_figheight())
            else:
                self.addMessage("Some labels are not displayed because of a lack of space. Click on Stretch image to see them all")
                #filter down the list to max 20
                max = int(len(labels)/20)
                if max < 20:
                    xl = [(i,a[:10] + ".." if len(a) > 10 else a) for i,a in enumerate(labels)]
                else:
                    xl = [(i,a) for i,a in enumerate(labels) if (i % max == 0)]
                ax.set_xticks([x[0] for x in xl])
                ax.set_xticklabels([x[1] for x in xl])
                plt.xticks(rotation=30)
        else:
            plt.xticks(rotation=0)

    def setLegend(self, fig, ax):
        if ax.get_legend() is not None:
            numLabels = len(ax.get_legend_handles_labels()[1])
            nCol = int(min(max(math.sqrt( numLabels ), 3), 6))
            nRows = int(numLabels/nCol)
            bboxPos = max(1.15, 1.0 + ((float(nRows)/2)/10.0))
            ax.legend(loc='upper center', bbox_to_anchor=(0.5, bboxPos),ncol=nCol, fancybox=True, shadow=True)

    def getNumFigures(self):
        return 1    #default, subclasses can override

    def createFigure(self):
        numFigures = self.getNumFigures()
        if numFigures <= 1:
            return plt.subplots(figsize=( int(self.getPreferredOutputWidth()/ self.getDPI()), int(self.getPreferredOutputHeight() / self.getDPI()) ))

        numCols = 1 if self.isStretchingOn() else 2 #number of columns for a multiplots, TODO make the layout configurable
        numRows = int( numFigures/numCols ) + numFigures % numCols
        imageHeight =  ((self.getPreferredOutputWidth()/numCols) * self.getHeightWidthRatio()) * numRows
        fig,ax = plt.subplots(numRows, numCols, figsize=( int(self.getPreferredOutputWidth()/self.getDPI()), int(imageHeight/self.getDPI() )))
        if numFigures%numCols != 0:
            fig.delaxes(ax.item(numFigures))
            ax = np.delete(ax,numFigures)
        return (fig,ax)

    def getSubplotHSpace(self):
        return 0.5 if self.isStretchingOn() else 0.3

    def doRenderChart(self):
        self.colormap = cm.jet

        # set defaults for plot
        plt.rcParams['savefig.dpi'] = 96
        plt.rcParams['font.family'] = "serif"
        plt.rcParams['font.serif'] = "cm"

        fig = None
        try:
            # go
            fig, ax = self.createFigure()

            if self.useMpld3:
                #TODO: rework this piece
                #keyFieldLabels = self.getKeyFieldLabels()
                #if (len(keyFieldLabels) > 0 and self.supportsKeyFieldLabels(self.handlerId) and self.supportsAggregation(self.handlerId)):
                #    plugins.connect(fig, ChartPlugin(self, keyFieldLabels))
                plugins.connect(fig, DialogPlugin(self, self.handlerId, self.dialogBody))

            #let subclass do the actual rendering
            newAx = self.matplotlibRender(fig, ax)
            if newAx is not None:
                ax = newAx

            axes = ax
            if not isinstance(axes, (list,np.ndarray)):
                axes = np.asarray([axes])

            #finalize the chart
            for i, a in np.ndenumerate(axes):
                if a.title is None or not a.title.get_visible() or a.title.get_text() == '':
                    self.setLegend(fig, a)
                self.setTicks(fig, a)
            sharex = True if len(axes) <=1 else len([a for a in axes if a._sharex is not None]) > 0
            if len(axes)>1 and not sharex:
                #adjust the height between subplots
                plt.subplots_adjust(hspace=self.getSubplotHSpace())

            #mpld3 has a bug when autolocator are used. Change to FixedLocators
            if self.useMpld3:
                self.addMessage("Warning: While great, D3 rendering is using MPLD3 library which has limitations that have not yet been fixed")
                from matplotlib import ticker
                self.debug("Converting to FixedLocator for mpld3")
                for a in axes:
                    locator = a.xaxis.get_major_locator()
                    if not isinstance(locator, ticker.FixedLocator):
                        vmin,vmax = a.xaxis.get_data_interval()
                        a.xaxis.set_major_locator(ticker.FixedLocator(locator.tick_values(vmin, vmax)))

            #Render the figure
            return self.renderFigure(fig)
        finally:
            if fig is not None:
                plt.close(fig)

    def renderFigure(self, fig):
        def genMarkup(chartFigure):
            return self.env.from_string("""
                    {0}
                    {{%for message in messages%}}

{{{{message}}}}

                    {{%endfor%}}
                """.format(chartFigure)
            ).render(messages=self.messages)

        if not self.useMpld3:
            import base64
            try:
                from io import BytesIO as pngIO
            except ImportError:
                from StringIO import StringIO as pngIO
            png=pngIO()
            plt.savefig(png, pad_inches=0.05, bbox_inches='tight', dpi=self.getDPI())
            try:
                return(
                    genMarkup("""

                        """.format(base64.b64encode(png.getvalue()).decode("ascii"))
                    )
                )
            finally:
                png.close()
        else:
            mpld3.enable_notebook()
            try:
                return genMarkup(mpld3.fig_to_html(fig))
            finally:
                mpld3.disable_notebook()

    def connectElementInfo(self, element, data):
        if not hasattr(element, "get_figure") and hasattr(element,"get_children"):
            for i,child in enumerate(element.get_children()):
                plugins.connect(child.get_figure(), ElementInfoPlugin(child,data[i]))
        elif hasattr(element, "get_figure"):
            plugins.connect(element.get_figure(), ElementInfoPlugin(element, data ))

    """
    Helper to safely access i position of an ax object
    """
    def getAxItem(self, ax, pos):
        if isinstance(ax, np.ndarray):
            return ax.item(pos)
        elif pos == 0:
            return ax
        raise ValueError("Trying to access pos {} from a single ax".format(pos))
<|endoftext|>


<commit_before>import json
import logging

from django.core.management.base import BaseCommand
from django.db import transaction

from osf.models import PreprintProvider, PreprintService, Subject
from scripts import utils as script_utils
from osf.models.validators import validate_subject_hierarchy
from website.preprints.tasks import on_preprint_updated

logger = logging.getLogger(__name__)

BEPRESS_PROVIDER = PreprintProvider.objects.filter(_id='osf').first()

def validate_input(custom_provider, data):
    logger.info('Validating data')
    includes = data.get('include', [])
    excludes = data.get('exclude', [])
    customs = data.get('custom', {})
    merges = data.get('merge', {})
    assert not set(includes) & set(excludes), 'There must be no overlap between includes and excludes'
    for text in includes:
        assert Subject.objects.filter(provider=BEPRESS_PROVIDER, text=text).exists(), 'Unable to find included subject with text {}'.format(text)
    included_subjects = Subject.objects.filter(provider=BEPRESS_PROVIDER, text__in=includes).include_children()
    logger.info('Successfully validated `include`')
    for text in excludes:
        try:
            Subject.objects.get(provider=BEPRESS_PROVIDER, text=text)
        except Subject.DoesNotExist:
            raise RuntimeError('Unable to find excluded subject with text {}'.format(text))
        assert included_subjects.filter(text=text).exists(), 'Excluded subject with text {} was not included'.format(text)
    included_subjects.exclude(text__in=excludes)
    logger.info('Successfully validated `exclude`')
    for cust_name, map_dict in customs.iteritems():
        assert not included_subjects.filter(text=cust_name).exists(), 'Custom text {} already exists in mapped set'.format(cust_name)
        assert Subject.objects.filter(provider=BEPRESS_PROVIDER, text=map_dict.get('bepress')).exists(), 'Unable to find specified BePress subject with text {}'.format(map_dict.get('bepress'))
        if map_dict.get('parent'):  # Null parent possible
            assert map_dict['parent'] in set(customs.keys()) | set(included_subjects.values_list('text', flat=True)), 'Unable to find specified parent with text {} in mapped set'.format(map_dict['parent'])
            # TODO: hierarchy length validation? Probably more trouble than worth here, done on .save
    logger.info('Successfully validated `custom`')
    included_subjects = included_subjects | Subject.objects.filter(text__in=[map_dict['bepress'] for map_dict in customs.values()])
    for merged_from, merged_into in merges.iteritems():
        assert not included_subjects.filter(text=merged_from).exists(), 'Cannot merge subject "{}" that will be included'.format(merged_from)
        assert merged_into in set(included_subjects.values_list('text', flat=True)) | set(customs.keys()), 'Unable to determine merge target for "{}"'.format(merged_into)
    included_subjects = included_subjects | Subject.objects.filter(text__in=merges.keys())
    missing_subjects = Subject.objects.filter(id__in=set([hier[-1].id for ps in PreprintService.objects.filter(provider=custom_provider) for hier in ps.subject_hierarchy])).exclude(id__in=included_subjects.values_list('id', flat=True))
    assert not missing_subjects.exists(), 'Incomplete mapping -- following subjects in use but not included:\n{}'.format(missing_subjects.all())
    logger.info('Successfully validated mapping completeness')

def create_subjects_recursive(custom_provider, root_text, exclude_texts, parent=None):
    logger.info('Duplicating BePress subject {} on {}'.format(root_text, custom_provider._id))
    bepress_subj = Subject.objects.get(provider=BEPRESS_PROVIDER, text=root_text)
    custom_subj = Subject(text=root_text, parent=parent, bepress_subject=bepress_subj, provider=custom_provider)
    custom_subj.save()
    # This is not a problem now, as all excluded subjects are leafs, but it could be problematic if non-leafs had their children excluded.
    # It could also be problematic if they didn't, if any of those children are used by existing preprints.
    # TODO: Determine correct resolution
    for child_text in bepress_subj.children.exclude(text__in=exclude_texts).values_list('text', flat=True):
        create_subjects_recursive(custom_provider, child_text, exclude_texts, parent=custom_subj)

def do_create_subjects(custom_provider, includes, excludes):
    for root_text in includes:
        create_subjects_recursive(custom_provider, root_text, excludes)

def map_custom_subject(custom_provider, name, parent, mapping):
    logger.info('Attempting to create subject {} on {} from {} with {}'.format(name, custom_provider._id, mapping, 'parent {}'.format(parent) if parent else 'no parent'))
    if parent:
        parent_subject = Subject.objects.filter(provider=custom_provider, text=parent).first()
    else:
        parent_subject = None
    bepress_subject = Subject.objects.get(provider=BEPRESS_PROVIDER, text=mapping)
    if parent and not parent_subject:
        return False
    custom_subject = Subject(provider=custom_provider, text=name, parent=parent_subject, bepress_subject=bepress_subject)
    custom_subject.save()
    return True

def do_custom_mapping(custom_provider, customs):
    tries = 0
    unmapped_customs = customs
    previous_len = len(unmapped_customs)
    while len(unmapped_customs):
        tries += 1
        if tries == 10:
            raise RuntimeError('Unable to map custom subjects with 10 iterations -- invalid input')
        successes = []
        for cust_name, map_dict in unmapped_customs.iteritems():
            if map_custom_subject(custom_provider, cust_name, map_dict.get('parent'), map_dict.get('bepress')):
                successes.append(cust_name)
            else:
                logger.warn('Failed. Retrying next iteration')
        [unmapped_customs.pop(key) for key in successes]
        new_len = len(unmapped_customs)
        if new_len == previous_len:
            raise RuntimeError('Unable to map any custom subjects on iteration -- invalid input')

def map_preprints_to_custom_subjects(custom_provider, merge_dict, dry_run=False):
    for preprint in PreprintService.objects.filter(provider=custom_provider):
        logger.info('Preparing to migrate preprint {}'.format(preprint.id))
        old_hier = preprint.subject_hierarchy
        subjects_to_map = [hier[-1] for hier in old_hier]
        merged_subject_ids = set(Subject.objects.filter(provider=custom_provider, text__in=[merge_dict[k] for k in set(merge_dict.keys()) & set([s.text for s in subjects_to_map])]).values_list('id', flat=True))
        subject_ids_to_map = set(s.id for s in subjects_to_map if s.text not in merge_dict.keys())
        aliased_subject_ids = set(Subject.objects.filter(bepress_subject__id__in=subject_ids_to_map, provider=custom_provider).values_list('id', flat=True)) | merged_subject_ids
        aliased_hiers = [s.object_hierarchy for s in Subject.objects.filter(id__in=aliased_subject_ids)]
        old_subjects = list(preprint.subjects.values_list('id', flat=True))
        preprint.subjects.clear()
        for hier in aliased_hiers:
            validate_subject_hierarchy([s._id for s in hier])
            for s in hier:
                preprint.subjects.add(s)
        # Update preprint in SHARE
        if not dry_run:
            on_preprint_updated(preprint._id, old_subjects=old_subjects, update_share=True)
        preprint.reload()
        new_hier = [s.object_hierarchy for s in preprint.subjects.exclude(children__in=preprint.subjects.all())]
        logger.info('Successfully migrated preprint {}.\n\tOld hierarchy:{}\n\tNew hierarchy:{}'.format(preprint.id, old_hier, new_hier))

def migrate(provider=None, data=None, dry_run=False):
    custom_provider = PreprintProvider.objects.filter(_id=provider).first()
    assert custom_provider, 'Unable to find specified provider: {}'.format(provider)
    assert custom_provider.id != BEPRESS_PROVIDER.id, 'Cannot add custom mapping to BePress provider'
    validate_input(custom_provider, data)
    do_create_subjects(custom_provider, data['include'], data.get('exclude', []))
    do_custom_mapping(custom_provider, data.get('custom', {}))
    map_preprints_to_custom_subjects(custom_provider, data.get('merge', {}), dry_run=dry_run)

class Command(BaseCommand):
    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)
        parser.add_argument(
            '--dry',
            action='store_true',
            dest='dry_run',
            help='Run migration and roll back changes to db',
        )
        parser.add_argument(
            '--data',
            action='store',
            dest='data',
            required=True,
            help='List of targets, of form {\n"include": [<list of subject texts to include at top level, children implicit>],'
            '\n"exclude": [<list of children to exclude from included trees>],'
            '\n"custom": [{"<Custom Name": {"parent": <Parent text>", "bepress": "<Bepress Name>"}}, ...]'
            '\n"merge": {"<Merged from (bepress)>": "<Merged into (custom)", ...}}',
        )
        parser.add_argument(
            '--provider',
            action='store',
            dest='provider',
            required=True,
            help='_id of the PreprintProvider object, e.g. "osf"'
        )

    def handle(self, *args, **options):
        dry_run = options.get('dry_run')
        provider = options['provider']
        data = options['data']
        if not dry_run:
            script_utils.add_file_logger(logger, __file__)
        with transaction.atomic():
            migrate(provider=provider, data=json.loads(data), dry_run=dry_run)
            if dry_run:
                raise RuntimeError('Dry run, transaction rolled back.')
<commit_msg>Allow easier complete BePress duplication<commit_after>import json
import logging

from django.core.management.base import BaseCommand
from django.db import transaction

from osf.models import PreprintProvider, PreprintService, Subject
from scripts import utils as script_utils
from osf.models.validators import validate_subject_hierarchy
from website.preprints.tasks import on_preprint_updated

logger = logging.getLogger(__name__)

BEPRESS_PROVIDER = PreprintProvider.objects.filter(_id='osf').first()

def validate_input(custom_provider, data, ignore_inex_collisions=False):
    logger.info('Validating data')
    includes = data.get('include', [])
    excludes = data.get('exclude', [])
    customs = data.get('custom', {})
    merges = data.get('merge', {})
    if not ignore_inex_collisions:
        assert not set(includes) & set(excludes), 'There must be no overlap between includes and excludes'
    for text in includes:
        assert Subject.objects.filter(provider=BEPRESS_PROVIDER, text=text).exists(), 'Unable to find included subject with text {}'.format(text)
    included_subjects = Subject.objects.filter(provider=BEPRESS_PROVIDER, text__in=includes).include_children()
    logger.info('Successfully validated `include`')
    for text in excludes:
        try:
            Subject.objects.get(provider=BEPRESS_PROVIDER, text=text)
        except Subject.DoesNotExist:
            raise RuntimeError('Unable to find excluded subject with text {}'.format(text))
        assert included_subjects.filter(text=text).exists(), 'Excluded subject with text {} was not included'.format(text)
    included_subjects.exclude(text__in=excludes)
    logger.info('Successfully validated `exclude`')
    for cust_name, map_dict in customs.iteritems():
        assert not included_subjects.filter(text=cust_name).exists(), 'Custom text {} already exists in mapped set'.format(cust_name)
        assert Subject.objects.filter(provider=BEPRESS_PROVIDER, text=map_dict.get('bepress')).exists(), 'Unable to find specified BePress subject with text {}'.format(map_dict.get('bepress'))
        if map_dict.get('parent'):  # Null parent possible
            assert map_dict['parent'] in set(customs.keys()) | set(included_subjects.values_list('text', flat=True)), 'Unable to find specified parent with text {} in mapped set'.format(map_dict['parent'])
            # TODO: hierarchy length validation? Probably more trouble than worth here, done on .save
    logger.info('Successfully validated `custom`')
    included_subjects = included_subjects | Subject.objects.filter(text__in=[map_dict['bepress'] for map_dict in customs.values()])
    for merged_from, merged_into in merges.iteritems():
        assert not included_subjects.filter(text=merged_from).exists(), 'Cannot merge subject "{}" that will be included'.format(merged_from)
        assert merged_into in set(included_subjects.values_list('text', flat=True)) | set(customs.keys()), 'Unable to determine merge target for "{}"'.format(merged_into)
    included_subjects = included_subjects | Subject.objects.filter(text__in=merges.keys())
    missing_subjects = Subject.objects.filter(id__in=set([hier[-1].id for ps in PreprintService.objects.filter(provider=custom_provider) for hier in ps.subject_hierarchy])).exclude(id__in=included_subjects.values_list('id', flat=True))
    assert not missing_subjects.exists(), 'Incomplete mapping -- following subjects in use but not included:\n{}'.format(missing_subjects.all())
    logger.info('Successfully validated mapping completeness')

def create_subjects_recursive(custom_provider, root_text, exclude_texts, parent=None):
    logger.info('Duplicating BePress subject {} on {}'.format(root_text, custom_provider._id))
    bepress_subj = Subject.objects.get(provider=BEPRESS_PROVIDER, text=root_text)
    custom_subj = Subject(text=root_text, parent=parent, bepress_subject=bepress_subj, provider=custom_provider)
    custom_subj.save()
    # This is not a problem now, as all excluded subjects are leafs, but it could be problematic if non-leafs had their children excluded.
    # It could also be problematic if they didn't, if any of those children are used by existing preprints.
    # TODO: Determine correct resolution
    for child_text in bepress_subj.children.exclude(text__in=exclude_texts).values_list('text', flat=True):
        create_subjects_recursive(custom_provider, child_text, exclude_texts, parent=custom_subj)

def do_create_subjects(custom_provider, includes, excludes):
    for root_text in includes:
        if root_text in excludes:
            # This should only be hit in the `--complete-bepress` case where a top-level is excluded
            continue
        create_subjects_recursive(custom_provider, root_text, excludes)

def map_custom_subject(custom_provider, name, parent, mapping):
    logger.info('Attempting to create subject {} on {} from {} with {}'.format(name, custom_provider._id, mapping, 'parent {}'.format(parent) if parent else 'no parent'))
    if parent:
        parent_subject = Subject.objects.filter(provider=custom_provider, text=parent).first()
    else:
        parent_subject = None
    bepress_subject = Subject.objects.get(provider=BEPRESS_PROVIDER, text=mapping)
    if parent and not parent_subject:
        return False
    custom_subject = Subject(provider=custom_provider, text=name, parent=parent_subject, bepress_subject=bepress_subject)
    custom_subject.save()
    return True

def do_custom_mapping(custom_provider, customs):
    tries = 0
    unmapped_customs = customs
    previous_len = len(unmapped_customs)
    while len(unmapped_customs):
        tries += 1
        if tries == 10:
            raise RuntimeError('Unable to map custom subjects with 10 iterations -- invalid input')
        successes = []
        for cust_name, map_dict in unmapped_customs.iteritems():
            if map_custom_subject(custom_provider, cust_name, map_dict.get('parent'), map_dict.get('bepress')):
                successes.append(cust_name)
            else:
                logger.warn('Failed. Retrying next iteration')
        [unmapped_customs.pop(key) for key in successes]
        new_len = len(unmapped_customs)
        if new_len == previous_len:
            raise RuntimeError('Unable to map any custom subjects on iteration -- invalid input')

def map_preprints_to_custom_subjects(custom_provider, merge_dict, dry_run=False):
    for preprint in PreprintService.objects.filter(provider=custom_provider):
        logger.info('Preparing to migrate preprint {}'.format(preprint.id))
        old_hier = preprint.subject_hierarchy
        subjects_to_map = [hier[-1] for hier in old_hier]
        merged_subject_ids = set(Subject.objects.filter(provider=custom_provider, text__in=[merge_dict[k] for k in set(merge_dict.keys()) & set([s.text for s in subjects_to_map])]).values_list('id', flat=True))
        subject_ids_to_map = set(s.id for s in subjects_to_map if s.text not in merge_dict.keys())
        aliased_subject_ids = set(Subject.objects.filter(bepress_subject__id__in=subject_ids_to_map, provider=custom_provider).values_list('id', flat=True)) | merged_subject_ids
        aliased_hiers = [s.object_hierarchy for s in Subject.objects.filter(id__in=aliased_subject_ids)]
        old_subjects = list(preprint.subjects.values_list('id', flat=True))
        preprint.subjects.clear()
        for hier in aliased_hiers:
            validate_subject_hierarchy([s._id for s in hier])
            for s in hier:
                preprint.subjects.add(s)
        # Update preprint in SHARE
        if not dry_run:
            on_preprint_updated(preprint._id, old_subjects=old_subjects, update_share=True)
        preprint.reload()
        new_hier = [s.object_hierarchy for s in preprint.subjects.exclude(children__in=preprint.subjects.all())]
        logger.info('Successfully migrated preprint {}.\n\tOld hierarchy:{}\n\tNew hierarchy:{}'.format(preprint.id, old_hier, new_hier))

def migrate(provider=None, data=None, dry_run=False, complete=False):
    custom_provider = PreprintProvider.objects.filter(_id=provider).first()
    assert custom_provider, 'Unable to find specified provider: {}'.format(provider)
    assert custom_provider.id != BEPRESS_PROVIDER.id, 'Cannot add custom mapping to BePress provider'
    validate_input(custom_provider, data, ignore_inex_collisions=complete)
    do_create_subjects(custom_provider, data['include'], data.get('exclude', []))
    do_custom_mapping(custom_provider, data.get('custom', {}))
    map_preprints_to_custom_subjects(custom_provider, data.get('merge', {}), dry_run=dry_run)

class Command(BaseCommand):
    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)
        parser.add_argument(
            '--dry',
            action='store_true',
            dest='dry_run',
            help='Run migration and roll back changes to db',
        )
        parser.add_argument(
            '--data',
            action='store',
            dest='data',
            required=True,
            help='List of targets, of form {\n"include": [<list of subject texts to include at top level, children implicit>],'
            '\n"exclude": [<list of children to exclude from included trees>],'
            '\n"custom": [{"<Custom Name": {"parent": <Parent text>", "bepress": "<Bepress Name>"}}, ...]'
            '\n"merge": {"<Merged from (bepress)>": "<Merged into (custom)", ...}}',
        )
        parser.add_argument(
            '--provider',
            action='store',
            dest='provider',
            required=True,
            help='_id of the PreprintProvider object, e.g. "osf"'
        )
        parser.add_argument(
            '--complete-bepress',
            action='store_true',
            dest='complete_bepress',
            help='Specifies that the entire BePress taxonomy should be copied. `data.include` is ignored, the other keys may still be used'
        )

    def handle(self, *args, **options):
        dry_run = options.get('dry_run')
        provider = options['provider']
        data = json.loads(options['data'])
        complete = options.get('complete_bepress')
        if complete:
            data['include'] = list(Subject.objects.filter(provider=BEPRESS_PROVIDER, parent__isnull=True).values_list('text', flat=True))
        if not dry_run:
            script_utils.add_file_logger(logger, __file__)
        with transaction.atomic():
            migrate(provider=provider, data=data, dry_run=dry_run, complete=complete)
            if dry_run:
                raise RuntimeError('Dry run, transaction rolled back.')
<|endoftext|>


<commit_before># import data from the new f1 filers csv. This replaces an older FEC one-off page of new filers.

import csv
from decimal import *
from dateutil.parser import parse as dateparse

from django.core.management.base import BaseCommand
from parsing.read_FEC_settings import FTP_DATA_DIR, CYCLE
from fec_alerts.models import f1filer
from summary_data.utils.overlay_utils import make_committee_from_f1filer

two_digit_cycle = "16"
#override
CYCLE = 2016

""" FEC 'update' means new file headers. Grr.  """
def transform_column_headers(new_row):
    return  {
        'cmte_id': new_row['COMMITTEE_ID'],
        'cmte_nm': new_row['COMMITTEE_NAME'],
        'cmte_st1': new_row['COMMITTEE_STREET_1'],
        'cmte_st2': new_row['COMMITTEE_STREET_2'],
        'cmte_city': new_row['COMMITTEE_CITY'],
        'cmte_st': new_row['COMMITTEE_STATE'],
        'cmte_zip': new_row['COMMITTEE_ZIP'],
        'affiliated_cmte_nm': new_row['AFFILIATED_COMMITTEE_NAME'],
        'filed_cmte_tp': new_row['FILED_COMMITTEE_TYPE'],
        'filed_cmte_dsgn': new_row['FILED_COMMITTEE_DESIGNATION'],
        'filing_freq': new_row['FILING_FREQUENCY'],
        'org_tp': new_row['ORGANIZATION_TYPE'],
        'tres_nm': new_row['TREASURER_NAME'],
        'receipt_dt': new_row['RECEIPT_DATE'],
        'cmte_email': new_row['COMMITTEE_EMAIL'],
        'cmte_web_url': new_row['COMMITTEE_WEB_URL'],
        'begin_image_num': new_row['BEGIN_IMAGE_NUMBER']
    }

def readfile(filelocation):
    fh = open(filelocation, 'r')
    reader = csv.DictReader(fh)
    count = 0
    for newstyle_row in reader:
        row = transform_column_headers(newstyle_row)
        try:
            thiscom = f1filer.objects.get(cmte_id=row['cmte_id'])
        except f1filer.DoesNotExist:
            print "Creating %s %s" % (row['cmte_id'], row['cmte_nm'])
            # first create the f1filer object:
            row['cycle'] = str(CYCLE)
            try:
                row['receipt_dt'] = dateparse(row['receipt_dt'])
            except:
                print "can't parse original receipt date='%s', skipping" % (row['receipt_dt'])
                continue

            try:
                del row[None]
            except KeyError:
                pass

            print row
            thisf1 = f1filer(**row)
            thisf1.save()

            ## if we are creating a new f1, check if it's a committee and if not, create one.
            make_committee_from_f1filer(row['cmte_id'], row['cycle'])

class Command(BaseCommand):
    def handle(self, *args, **options):
        print "load f1 filers"
        filename =  "/%s/Form1Filer_%s.csv" % (two_digit_cycle, two_digit_cycle)
        filelocation = FTP_DATA_DIR + filename
        readfile(filelocation)
<commit_msg>set update time on new committee page from new update script<commit_after># import data from the new f1 filers csv. This replaces an older FEC one-off page of new filers.

import csv
from decimal import *
from dateutil.parser import parse as dateparse

from django.core.management.base import BaseCommand
from parsing.read_FEC_settings import FTP_DATA_DIR, CYCLE
from fec_alerts.models import f1filer
from summary_data.utils.overlay_utils import make_committee_from_f1filer
from summary_data.utils.update_utils import set_update

from django.conf import settings

COMMITTEES_SCRAPE_KEY  = settings.COMMITTEES_SCRAPE_KEY

two_digit_cycle = "16"
#override
CYCLE = 2016

""" FEC 'update' means new file headers. Grr.  """
def transform_column_headers(new_row):
    return  {
        'cmte_id': new_row['COMMITTEE_ID'],
        'cmte_nm': new_row['COMMITTEE_NAME'],
        'cmte_st1': new_row['COMMITTEE_STREET_1'],
        'cmte_st2': new_row['COMMITTEE_STREET_2'],
        'cmte_city': new_row['COMMITTEE_CITY'],
        'cmte_st': new_row['COMMITTEE_STATE'],
        'cmte_zip': new_row['COMMITTEE_ZIP'],
        'affiliated_cmte_nm': new_row['AFFILIATED_COMMITTEE_NAME'],
        'filed_cmte_tp': new_row['FILED_COMMITTEE_TYPE'],
        'filed_cmte_dsgn': new_row['FILED_COMMITTEE_DESIGNATION'],
        'filing_freq': new_row['FILING_FREQUENCY'],
        'org_tp': new_row['ORGANIZATION_TYPE'],
        'tres_nm': new_row['TREASURER_NAME'],
        'receipt_dt': new_row['RECEIPT_DATE'],
        'cmte_email': new_row['COMMITTEE_EMAIL'],
        'cmte_web_url': new_row['COMMITTEE_WEB_URL'],
        'begin_image_num': new_row['BEGIN_IMAGE_NUMBER']
    }

def readfile(filelocation):
    fh = open(filelocation, 'r')
    reader = csv.DictReader(fh)
    count = 0
    for newstyle_row in reader:
        row = transform_column_headers(newstyle_row)
        try:
            thiscom = f1filer.objects.get(cmte_id=row['cmte_id'])
        except f1filer.DoesNotExist:
            print "Creating %s %s" % (row['cmte_id'], row['cmte_nm'])
            # first create the f1filer object:
            row['cycle'] = str(CYCLE)
            try:
                row['receipt_dt'] = dateparse(row['receipt_dt'])
            except:
                print "can't parse original receipt date='%s', skipping" % (row['receipt_dt'])
                continue

            try:
                del row[None]
            except KeyError:
                pass

            print row
            thisf1 = f1filer(**row)
            thisf1.save()

            ## if we are creating a new f1, check if it's a committee and if not, create one.
            make_committee_from_f1filer(row['cmte_id'], row['cycle'])

class Command(BaseCommand):
    def handle(self, *args, **options):
        print "load f1 filers"
        filename =  "/%s/Form1Filer_%s.csv" % (two_digit_cycle, two_digit_cycle)
        filelocation = FTP_DATA_DIR + filename
        readfile(filelocation)
        set_update(COMMITTEES_SCRAPE_KEY)
<|endoftext|>