<commit_before>#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from bigdl.nn.layer import *
from bigdl.nn.criterion import *
from bigdl.util.common import *
from bigdl.models.ml_pipeline.dl_classifier import *
from pyspark.sql.types import *
from pyspark.context import SparkContext
import numpy as np
import pytest
from numpy.testing import assert_allclose

class TestDLClassifer():
    def setup_method(self, method):
        """ setup any state tied to the execution of the given method in a
        class.  setup_method is invoked for every test method of a class.
        """
        sparkConf = create_spark_conf()
        self.sc = SparkContext(master="local[1]", appName="test model",
                               conf=sparkConf)
        self.sqlContext = SQLContext(self.sc)
        init_engine()

    def teardown_method(self, method):
        """ teardown any state that was previously setup with a setup_method
        call.
        """
        self.sc.stop()

    def test_all_set_get_methods(self):
        """ run tests for all the set and get methods involved in DLEstimator, DLModel,
            DLClassifier, DLClassifierModel
        """

        '''
        use linear model and MSE criterion to test the DLEstimator and DLModel
        '''
        linear_model = Sequential().add(Linear(2, 2))
        mse_criterion = MSECriterion()

        '''
        initialize a DLEstimator to test setter, getter for
        batchSize, maxEpoch, learningRate in DLEstimator
        '''
        estimator = DLEstimator(model=linear_model, criterion=mse_criterion,
                                feature_size=[2], label_size=[2])

        # check the set and get methods for batchSize
        assert estimator.setBatchSize(30).getBatchSize() == 30
        # check the set and get methods for maxEpoch
        assert estimator.setMaxEpoch(40).getMaxEpoch() == 40
        # check the set and get methods for learningRate
        assert estimator.setLearningRate(1e-4).getLearningRate() == 1e-4

        '''
        initialize a DLModel to test setter, getter for featureSize, batchSize in DLModel
        '''
        dl_model = DLClassifierModel(model=linear_model, featureSize=[1])

        # check the set and get methods for featureSize in DLModel
        assert dl_model.setFeatureSize([2]).getFeatureSize() == [2]
        # check the set and get methods for batchSize in DLModel
        assert dl_model.setBatchSize((20)).getBatchSize() == 20

        '''
        use linear model and ClassNLL criterion to test the DLClassifier and DLClassifierModel
        '''
        linear_model = Sequential().add(Linear(2, 2))
        classNLL_criterion = ClassNLLCriterion()

        '''
        initialize a DLClassifier to test setter, getter for
        batchSize, maxEpoch, learningRate in DLClassifier
        '''
        classifier = DLClassifier(model=linear_model, criterion=classNLL_criterion,
                                  feature_size=[2])

        # check the set and get methods for batchSize
        assert classifier.setBatchSize(20).getBatchSize() == 20
        # check the set and get methods for maxEpoch
        assert classifier.setMaxEpoch(50).getMaxEpoch() == 50
        # check the set and get methods for learningRate
        assert classifier.setLearningRate(1e-5).getLearningRate() == 1e-5
        '''
        initialize a DLClassifierModel to test setter, getter
        for featureSize, batchSize in DLClassifierModel
        '''
        dl_classifier_model = DLClassifierModel(model=linear_model, featureSize=[1])

        # check the set and get methods for featureSize
        assert dl_classifier_model.setFeatureSize([2]).getFeatureSize() == [2]

        # check the set and get methods for batchSize
        assert dl_classifier_model.setBatchSize((20)).getBatchSize() == 20

    def test_dlestimator_fit_dlmodel_transform(self):
        model = Sequential().add(Linear(2, 2))
        criterion = MSECriterion()
        estimator = DLEstimator(model, criterion, [2], [2]).setBatchSize(4)\
            .setLearningRate(0.01).setMaxEpoch(1000)

        data = self.sc.parallelize([
            ((2.0, 1.0), (1.0, 2.0)),
            ((1.0, 2.0), (2.0, 1.0)),
            ((2.0, 1.0), (1.0, 2.0)),
            ((1.0, 2.0), (2.0, 1.0))])

        schema = StructType([
            StructField("features", ArrayType(DoubleType(), False), False),
            StructField("label", ArrayType(DoubleType(), False), False)])
        df = self.sqlContext.createDataFrame(data, schema)
        dlModel = estimator.fit(df)

        dlModel.transform(df).registerTempTable("dlModelDF")  # Compatible with spark 1.6
        results = self.sqlContext.table("dlModelDF")

        count = results.rdd.count()
        data = results.rdd.collect()

        for i in range(count):
            row_label = data[i]['label']
            row_prediction = data[i]['prediction']
            assert_allclose(row_label[0], row_prediction[0], atol=0, rtol=1e-1)
            assert_allclose(row_label[1], row_prediction[1], atol=0, rtol=1e-1)

    def test_dlclassifier_fit_dlclassifiermodel_transform(self):
        model = Sequential().add(Linear(2, 2))
        criterion = ClassNLLCriterion()
        classifier = DLClassifier(model, criterion, [2]).setBatchSize(4)\
            .setLearningRate(0.01).setMaxEpoch(1000)
        data = self.sc.parallelize([
            ((2.0, 1.0), 1.0),
            ((1.0, 2.0), 2.0),
            ((2.0, 1.0), 1.0),
            ((1.0, 2.0), 2.0)])

        schema = StructType([
            StructField("features", ArrayType(DoubleType(), False), False),
            StructField("label", DoubleType(), False)])
        df = self.sqlContext.createDataFrame(data, schema)
        dlClassifierModel = classifier.fit(df)

        dlClassifierModel.transform(df).registerTempTable("dlClassifierModelDF")
        results = self.sqlContext.table("dlClassifierModelDF")

        count = results.rdd.count()
        data = results.rdd.collect()

        for i in range(count):
            row_label = data[i]['label']
            row_prediction = data[i]['prediction']
            assert row_label == row_prediction

if __name__ == "__main__":
    pytest.main()
<commit_msg>update unittest for dl_classifier (#1774)<commit_after>#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from bigdl.nn.layer import *
from bigdl.nn.criterion import *
from bigdl.util.common import *
from bigdl.models.ml_pipeline.dl_classifier import *
from pyspark.sql.types import *
from pyspark.context import SparkContext
import numpy as np
import pytest
from numpy.testing import assert_allclose

class TestDLClassifer():
    def setup_method(self, method):
        """ setup any state tied to the execution of the given method in a
        class.  setup_method is invoked for every test method of a class.
        """
        sparkConf = create_spark_conf()
        self.sc = SparkContext(master="local[1]", appName="test model",
                               conf=sparkConf)
        self.sqlContext = SQLContext(self.sc)
        init_engine()

    def teardown_method(self, method):
        """ teardown any state that was previously setup with a setup_method
        call.
        """
        self.sc.stop()

    def test_all_set_get_methods(self):
        """ run tests for all the set and get methods involved in DLEstimator, DLModel,
            DLClassifier, DLClassifierModel
        """

        '''
        use linear model and MSE criterion to test the DLEstimator and DLModel
        '''
        linear_model = Sequential().add(Linear(2, 2))
        mse_criterion = MSECriterion()

        '''
        initialize a DLEstimator to test setter, getter for
        batchSize, maxEpoch, learningRate in DLEstimator
        '''
        estimator = DLEstimator(model=linear_model, criterion=mse_criterion,
                                feature_size=[2], label_size=[2])

        # check the set and get methods for batchSize
        assert estimator.setBatchSize(30).getBatchSize() == 30
        # check the set and get methods for maxEpoch
        assert estimator.setMaxEpoch(40).getMaxEpoch() == 40
        # check the set and get methods for learningRate
        assert estimator.setLearningRate(1e-4).getLearningRate() == 1e-4

        '''
        initialize a DLModel to test setter, getter for featureSize, batchSize in DLModel
        '''
        dl_model = DLClassifierModel(model=linear_model, featureSize=[1])

        # check the set and get methods for featureSize in DLModel
        assert dl_model.setFeatureSize([2]).getFeatureSize() == [2]
        # check the set and get methods for batchSize in DLModel
        assert dl_model.setBatchSize((20)).getBatchSize() == 20

        '''
        use linear model and ClassNLL criterion to test the DLClassifier and DLClassifierModel
        '''
        linear_model = Sequential().add(Linear(2, 2))
        classNLL_criterion = ClassNLLCriterion()

        '''
        initialize a DLClassifier to test setter, getter for
        batchSize, maxEpoch, learningRate in DLClassifier
        '''
        classifier = DLClassifier(model=linear_model, criterion=classNLL_criterion,
                                  feature_size=[2])

        # check the set and get methods for batchSize
        assert classifier.setBatchSize(20).getBatchSize() == 20
        # check the set and get methods for maxEpoch
        assert classifier.setMaxEpoch(50).getMaxEpoch() == 50
        # check the set and get methods for learningRate
        assert classifier.setLearningRate(1e-5).getLearningRate() == 1e-5
        '''
        initialize a DLClassifierModel to test setter, getter
        for featureSize, batchSize in DLClassifierModel
        '''
        dl_classifier_model = DLClassifierModel(model=linear_model, featureSize=[1])

        # check the set and get methods for featureSize
        assert dl_classifier_model.setFeatureSize([2]).getFeatureSize() == [2]

        # check the set and get methods for batchSize
        assert dl_classifier_model.setBatchSize((20)).getBatchSize() == 20

    def test_dlestimator_fit_dlmodel_transform(self):
        model = Sequential().add(Linear(2, 2))
        criterion = MSECriterion()
        estimator = DLEstimator(model, criterion, [2], [2]).setBatchSize(4)\
            .setLearningRate(0.01).setMaxEpoch(1000)

        data = self.sc.parallelize([
            ((2.0, 1.0), (1.0, 2.0)),
            ((1.0, 2.0), (2.0, 1.0)),
            ((2.0, 1.0), (1.0, 2.0)),
            ((1.0, 2.0), (2.0, 1.0))])

        schema = StructType([
            StructField("features", ArrayType(DoubleType(), False), False),
            StructField("label", ArrayType(DoubleType(), False), False)])
        df = self.sqlContext.createDataFrame(data, schema)
        dlModel = estimator.fit(df)

        dlModel.transform(df).registerTempTable("dlModelDF")  # Compatible with spark 1.6
        results = self.sqlContext.table("dlModelDF")

        count = results.rdd.count()
        data = results.rdd.collect()

        for i in range(count):
            row_label = data[i][1]
            row_prediction = data[i][2]
            assert_allclose(row_label[0], row_prediction[0], atol=0, rtol=1e-1)
            assert_allclose(row_label[1], row_prediction[1], atol=0, rtol=1e-1)

    def test_dlclassifier_fit_dlclassifiermodel_transform(self):
        model = Sequential().add(Linear(2, 2))
        criterion = ClassNLLCriterion()
        classifier = DLClassifier(model, criterion, [2]).setBatchSize(4)\
            .setLearningRate(0.01).setMaxEpoch(1000)
        data = self.sc.parallelize([
            ((2.0, 1.0), 1.0),
            ((1.0, 2.0), 2.0),
            ((2.0, 1.0), 1.0),
            ((1.0, 2.0), 2.0)])

        schema = StructType([
            StructField("features", ArrayType(DoubleType(), False), False),
            StructField("label", DoubleType(), False)])
        df = self.sqlContext.createDataFrame(data, schema)
        dlClassifierModel = classifier.fit(df)

        dlClassifierModel.transform(df).registerTempTable("dlClassifierModelDF")
        results = self.sqlContext.table("dlClassifierModelDF")

        count = results.rdd.count()
        data = results.rdd.collect()

        for i in range(count):
            row_label = data[i][1]
            row_prediction = data[i][2]
            assert row_label == row_prediction

if __name__ == "__main__":
    pytest.main()
<|endoftext|>


<commit_before>"""
Views related to the LTI Passport feature.
"""
import json
import requests
import logging

from requests.exceptions import RequestException
from xmodule.modulestore.django import modulestore
from edxmako.shortcuts import render_to_response  # pylint: disable=import-error

from django.conf import settings
from django.contrib import messages
from django.utils.translation import ugettext as _
from django.core.urlresolvers import reverse
from django.shortcuts import redirect
from django.views.decorators.cache import cache_control
from django.views.decorators.csrf import ensure_csrf_cookie
from django.http import HttpResponseBadRequest

from labster_course_license.utils import LtiPassport, course_tree_info
from labster_course_license.models import CourseLicense
from ccx_keys.locator import CCXLocator
from ccx.views import coach_dashboard, get_ccx_for_coach
from ccx.overrides import get_override_for_ccx, override_field_for_ccx

log = logging.getLogger(__name__)

class LabsterApiError(Exception):
    """
    This exception is raised in the case where problems with Labster API appear.
    """
    pass

@ensure_csrf_cookie
@cache_control(no_cache=True, no_store=True, must_revalidate=True)
@coach_dashboard
def license_handler(request, course, ccx=None):
    """
    Labster License handler.
    """
    if request.method == 'GET':
        return dashboard(request, course, ccx)
    elif request.method == 'POST':
        return set_license(request, course, ccx)
    return HttpResponseBadRequest('Only GET and POST methods are supported.')

def dashboard(request, course, ccx):
    """
    Display the Course License Dashboard.
    """
    # right now, we can only have one ccx per user and course
    # so, if no ccx is passed in, we can sefely redirect to that
    if ccx is None:
        ccx = get_ccx_for_coach(course, request.user)

    context = {
        'course': course,
        'ccx': ccx,
    }

    if ccx:
        ccx_locator = CCXLocator.from_course_locator(course.id, ccx.id)
        passports = get_override_for_ccx(ccx, course, 'lti_passports', course.lti_passports)
        passports = map(lambda x: LtiPassport(x).as_dict(), passports)
        context['passports'] = passports
        context['license'] = CourseLicense.get_license(ccx_locator)
        context['labster_license_url'] = reverse('labster_license_handler', kwargs={'course_id': ccx_locator})
    else:
        context['ccx_coach_dashboard'] = reverse('ccx_coach_dashboard', kwargs={'course_id': course.id})
    return render_to_response('labster/course_license.html', context)

def update_course(ccx, course_info, descriptors):
    """
    Updates all the descriptors.
    """
    hidden = 'visible_to_staff_only'
    for descriptor in descriptors:
        info = course_info[descriptor]
        override_field_for_ccx(ccx, descriptor, hidden, info.is_hidden)
        update_course(ccx, course_info, info.children)

def _send_request(url, data):
    """
    Sends a request the Labster API.
    """
    headers = {
        "authorization": 'Token {}'.format(settings.LABSTER_API_AUTH_TOKEN),
        "content-type": 'application/json',
        "accept": 'application/json',
    }

    try:
        response = requests.post(url, headers=headers, data=json.dumps(data))
        response.raise_for_status()
        return response.json()
    except RequestException as ex:
        log.exception("Labster API is unavailable:\n%r", ex)
        raise LabsterApiError(_("Labster API is unavailable."))
    except ValueError as ex:
        log.error("Invalid JSON:\n%r", ex)
        raise LabsterApiError(_("Invalid JSON."))

def get_consumer_secret(user, license):
    """
    Return consumer and secret keys.
    Raises: LabsterApiError
    """
    data = {
        'user_id': user.id,
        'user_email': user.email,
        'license': license,
        'source': 'EDX',
    }
    url = settings.LABSTER_ENDPOINTS.get('consumer_secret')
    response = _send_request(url, data)
    return response['consumer_key'], response['secret_key']

def get_licensed_simulations(consumer_keys):
    """
    Return a list of available for the user simulation ids.
    Raises: LabsterApiError
    """
    data = {'consumer_keys': consumer_keys}
    url = settings.LABSTER_ENDPOINTS.get('available_simulations')
    response = _send_request(url, data)
    return set(response)

def passport_by_lti_id(passports, expected_lti_id):
    """
    Return an index of the passport with specified `lti_id(str)`.
    """
    for index, passport_str in enumerate(passports):
        passport = LtiPassport(passport_str)
        if passport.lti_id == expected_lti_id:
            return passport, index
    return None, None

def set_license(request, course, ccx):
    """
    Set lti passport for the CCX.
    """
    if not ccx:
        raise Http404

    course_key = course.location.course_key
    ccx_locator = CCXLocator.from_course_locator(course.id, ccx.id)
    url = reverse('labster_license_handler', kwargs={'course_id': ccx_locator})

    # Getting consumer and secret keys.
    license = request.POST.get('license', None)
    if not license:
        messages.error(request, _('Please set your license. The field cannot be empty.'))
        return redirect(url)

    try:
        consumer_key, secret_key = get_consumer_secret(request.user, license)
    except LabsterApiError:
        messages.error(
            request, _('There are some issues with applying your license. Please try again in a few minutes.')
        )
        return redirect(url)

    # Update passports
    passports = get_override_for_ccx(ccx, course, 'lti_passports', course.lti_passports)[:]
    passport, index = passport_by_lti_id(passports, settings.LABSTER_DEFAULT_LTI_ID)

    if passport:
        passport.consumer_key = consumer_key
        passport.secret_key = secret_key
        passports[index] = str(passport)
    else:
        passport = LtiPassport.construct(settings.LABSTER_DEFAULT_LTI_ID, consumer_key, secret_key)
        passports.append(str(passport))

    override_field_for_ccx(ccx, course, 'lti_passports', passports)
    CourseLicense.set_license(ccx_locator, license)

    update_course_structure = request.POST.get('update', None)
    if not update_course_structure:
        return redirect(url)

    # Getting a list of licensed simulations
    consumer_keys = [LtiPassport(passport_str).consumer_key for passport_str in passports]
    try:
        licensed_simulations = get_licensed_simulations(consumer_keys)
    except LabsterApiError:
        messages.error(
            request, _('Your license is successfully applied, but there was an error with updating your course.')
        )
        return redirect(url)

    # Updating the course: hides unlicensed simulations.
    store = modulestore()
    with store.bulk_operations(course_key):
        simulations = store.get_items(course_key, qualifiers={'category': 'lti'})
        course_info, chapters = course_tree_info(store, simulations, licensed_simulations)
        update_course(ccx, course_info, chapters)

    url = reverse('labster_license_handler', kwargs={'course_id': CCXLocator.from_course_locator(course.id, ccx.id)})
    return redirect(url)
<commit_msg>Remove user_id from the GetClientSecret keys request.<commit_after>"""
Views related to the LTI Passport feature.
"""
import json
import requests
import logging

from requests.exceptions import RequestException
from xmodule.modulestore.django import modulestore
from edxmako.shortcuts import render_to_response  # pylint: disable=import-error

from django.conf import settings
from django.contrib import messages
from django.utils.translation import ugettext as _
from django.core.urlresolvers import reverse
from django.shortcuts import redirect
from django.views.decorators.cache import cache_control
from django.views.decorators.csrf import ensure_csrf_cookie
from django.http import HttpResponseBadRequest

from labster_course_license.utils import LtiPassport, course_tree_info
from labster_course_license.models import CourseLicense
from ccx_keys.locator import CCXLocator
from ccx.views import coach_dashboard, get_ccx_for_coach
from ccx.overrides import get_override_for_ccx, override_field_for_ccx

log = logging.getLogger(__name__)

class LabsterApiError(Exception):
    """
    This exception is raised in the case where problems with Labster API appear.
    """
    pass

@ensure_csrf_cookie
@cache_control(no_cache=True, no_store=True, must_revalidate=True)
@coach_dashboard
def license_handler(request, course, ccx=None):
    """
    Labster License handler.
    """
    if request.method == 'GET':
        return dashboard(request, course, ccx)
    elif request.method == 'POST':
        return set_license(request, course, ccx)
    return HttpResponseBadRequest('Only GET and POST methods are supported.')

def dashboard(request, course, ccx):
    """
    Display the Course License Dashboard.
    """
    # right now, we can only have one ccx per user and course
    # so, if no ccx is passed in, we can sefely redirect to that
    if ccx is None:
        ccx = get_ccx_for_coach(course, request.user)

    context = {
        'course': course,
        'ccx': ccx,
    }

    if ccx:
        ccx_locator = CCXLocator.from_course_locator(course.id, ccx.id)
        passports = get_override_for_ccx(ccx, course, 'lti_passports', course.lti_passports)
        passports = map(lambda x: LtiPassport(x).as_dict(), passports)
        context['passports'] = passports
        context['license'] = CourseLicense.get_license(ccx_locator)
        context['labster_license_url'] = reverse('labster_license_handler', kwargs={'course_id': ccx_locator})
    else:
        context['ccx_coach_dashboard'] = reverse('ccx_coach_dashboard', kwargs={'course_id': course.id})
    return render_to_response('labster/course_license.html', context)

def update_course(ccx, course_info, descriptors):
    """
    Updates all the descriptors.
    """
    hidden = 'visible_to_staff_only'
    for descriptor in descriptors:
        info = course_info[descriptor]
        override_field_for_ccx(ccx, descriptor, hidden, info.is_hidden)
        update_course(ccx, course_info, info.children)

def _send_request(url, data):
    """
    Sends a request the Labster API.
    """
    headers = {
        "authorization": 'Token {}'.format(settings.LABSTER_API_AUTH_TOKEN),
        "content-type": 'application/json',
        "accept": 'application/json',
    }

    try:
        response = requests.post(url, headers=headers, data=json.dumps(data))
        response.raise_for_status()
        return response.json()
    except RequestException as ex:
        log.exception("Labster API is unavailable:\n%r", ex)
        raise LabsterApiError(_("Labster API is unavailable."))
    except ValueError as ex:
        log.error("Invalid JSON:\n%r", ex)
        raise LabsterApiError(_("Invalid JSON."))

def get_consumer_secret(user, license):
    """
    Return consumer and secret keys.
    Raises: LabsterApiError
    """
    data = {
        'user_email': user.email,
        'license': license,
        'source': 'EDX',
    }
    url = settings.LABSTER_ENDPOINTS.get('consumer_secret')
    response = _send_request(url, data)
    return response['consumer_key'], response['secret_key']

def get_licensed_simulations(consumer_keys):
    """
    Return a list of available for the user simulation ids.
    Raises: LabsterApiError
    """
    data = {'consumer_keys': consumer_keys}
    url = settings.LABSTER_ENDPOINTS.get('available_simulations')
    response = _send_request(url, data)
    return set(response)

def passport_by_lti_id(passports, expected_lti_id):
    """
    Return an index of the passport with specified `lti_id(str)`.
    """
    for index, passport_str in enumerate(passports):
        passport = LtiPassport(passport_str)
        if passport.lti_id == expected_lti_id:
            return passport, index
    return None, None

def set_license(request, course, ccx):
    """
    Set lti passport for the CCX.
    """
    if not ccx:
        raise Http404

    course_key = course.location.course_key
    ccx_locator = CCXLocator.from_course_locator(course.id, ccx.id)
    url = reverse('labster_license_handler', kwargs={'course_id': ccx_locator})

    # Getting consumer and secret keys.
    license = request.POST.get('license', None)
    if not license:
        messages.error(request, _('Please set your license. The field cannot be empty.'))
        return redirect(url)

    try:
        consumer_key, secret_key = get_consumer_secret(request.user, license)
    except LabsterApiError:
        messages.error(
            request, _('There are some issues with applying your license. Please try again in a few minutes.')
        )
        return redirect(url)

    # Update passports
    passports = get_override_for_ccx(ccx, course, 'lti_passports', course.lti_passports)[:]
    passport, index = passport_by_lti_id(passports, settings.LABSTER_DEFAULT_LTI_ID)

    if passport:
        passport.consumer_key = consumer_key
        passport.secret_key = secret_key
        passports[index] = str(passport)
    else:
        passport = LtiPassport.construct(settings.LABSTER_DEFAULT_LTI_ID, consumer_key, secret_key)
        passports.append(str(passport))

    override_field_for_ccx(ccx, course, 'lti_passports', passports)
    CourseLicense.set_license(ccx_locator, license)

    update_course_structure = request.POST.get('update', None)
    if not update_course_structure:
        return redirect(url)

    # Getting a list of licensed simulations
    consumer_keys = [LtiPassport(passport_str).consumer_key for passport_str in passports]
    try:
        licensed_simulations = get_licensed_simulations(consumer_keys)
    except LabsterApiError:
        messages.error(
            request, _('Your license is successfully applied, but there was an error with updating your course.')
        )
        return redirect(url)

    # Updating the course: hides unlicensed simulations.
    store = modulestore()
    with store.bulk_operations(course_key):
        simulations = store.get_items(course_key, qualifiers={'category': 'lti'})
        course_info, chapters = course_tree_info(store, simulations, licensed_simulations)
        update_course(ccx, course_info, chapters)

    url = reverse('labster_license_handler', kwargs={'course_id': CCXLocator.from_course_locator(course.id, ccx.id)})
    return redirect(url)
<|endoftext|>


<commit_before>#! /usr/bin/env python

# EPTOOLS Python Interface
# Example: Binary classification, coupled or factorized posterior
# mode.
# Can be configured with Gaussian or Laplace prior.
#
# Very similar to Matlab
#   testcode eptools/matlab/test/binclass/eptest_binclass.m
# Same data as glm-ie_v1.5/doc/classify_inference.m:
# Adult (a9a): Dimension 123, first 30000 for testing, last 2561
# for training.

import numpy as np
import scipy.sparse as ssp
import scipy.io  # DEBUG: Load Matlab files
import numbers
import time  # Profiling

import apbsint as abt

# Helper functions

# Main code

# Load dataset: CSV format. Inputs: Each row contains indices of nonzero
# features (value: 1).
# NOTE: Building a dense nested list is faster (by factor 10!) than
# building a ssp.lil_matrix row by row.
# NOTE: We remove 3 attributes (45, 116, 122), these are not touched by
# any patterns in the training set.
num_feat = n = 120  # After removing 3
tmat = []
fid = open('adult_a9a_inputs_comp.csv','r')
for line in fid:
    ind = [int(x) for x in line.split(',')]
    v = np.zeros(n+3,dtype=np.float64)
    v[ind] = 1.
    # Remove attributes 45, 116, 122
    tmat.append(list(np.hstack((v[:45], v[46:116], v[117:122]))))
fid.close()
num_cases = len(tmat)
print 'Dataset: Read %d cases.' % num_cases
inp_all = ssp.csr_matrix(tmat)
del tmat
num_test = 30000
num_train = num_cases-num_test
fid = open('adult_a9a_targets.csv','r')
targ_all = np.array([float(x) for x in fid.readline().split(',')],
                    dtype=np.float64)
fid.close()
if targ_all.size != num_cases:
    raise IndexError('Internal error: Wrong file size')

# Setup
#imode = 'CoupParallel'   # Coupled, parallel updating
#imode = 'CoupSequential' # Coupled, sequential updating
imode = 'Factorized'     # Factorized, sequential updating
is_fact = (imode == 'Factorized')
do_laplace = True        # Laplace prior
tau_lapl = 2./5.
#do_laplace = False       # Gaussian prior
#ssq_gauss = 25./4.
do_seldamp = True
#do_seldamp = False
seldamp_numk = 5
# DEBUG: Compare intermediate results of Matlab and Python code directly.
# We load Matlab variables from files written after init. (%d = 0) and after
# each sweep (%d = 1,2,...). Anything non-deterministic also loaded from there.
# NOTE: Implemented for 'Factorized' mode only.
#debug_matcomp_fname = '/home/seeger/exp/ept/eptb13_vars_it%d.mat'
try:
    debug_matcomp_fname
    do_debug_matcomp = True
except NameError:
    do_debug_matcomp = False

# Coupling factors
if not is_fact:
    bfct_test = abt.MatSparse(inp_all[:num_test,:].copy())
    bfct_train = abt.MatContainer([abt.MatEye(n),
                                   abt.MatSparse(inp_all[num_test:,:].copy())])
else:
    bfct_test = abt.MatFactorizedInf(inp_all[:num_test,:].copy())
    mx_tmp = ssp.vstack([ssp.eye(n,format='csr'), inp_all[num_test:,:]],
                        format='csr')
    bfct_train = abt.MatFactorizedInf(mx_tmp)
m = bfct_train.shape(0)
# Potential managers
if not do_laplace:
    pm_elem1 = abt.ElemPotManager('Gaussian',n,(0., ssq_gauss))
else:
    pm_elem1 = abt.ElemPotManager('Laplace',n,(0., tau_lapl))
pm_elem2 = abt.ElemPotManager('Probit',num_train,
                              (targ_all[num_test:].copy(), 0.))
pman_train = abt.PotManager((pm_elem1, pm_elem2))
pman_test = abt.PotManager(abt.ElemPotManager('Probit',num_test,
                                              (targ_all[:num_test].copy(), 0.)))
print 'Prior:      %s\nLikelihood: %s' % (pman_train.elem[0].name,
                                          pman_train.elem[1].name)
# Initialization of representation parameters
if not is_fact:
    model_train = abt.ModelCoupled(bfct_train,pman_train)
    model_test = abt.ModelCoupled(bfct_test,pman_test)
    repres = abt.RepresentationCoupled(bfct_train,
                                       keep_margs=(imode == 'CoupParallel'))
    if imode == 'CoupParallel':
        inf_driv = abt.EPCoupParallelInfDriver(model_train,repres)
    else:
        inf_driv = abt.EPCoupSequentialInfDriver(model_train,repres)
    if not do_laplace:
        inf_driv.init('ADF')
    else:
        # Same initialization as in glm-ie
        tvec = np.empty(m)
        tvec[:] = 1.
        repres.setpi(tvec)
        tvec[:n] = 0.
        tvec[n:] = 0.5*targ_all[num_test:]
        repres.setbeta(tvec)
        repres.refresh()
else:
    model_train = abt.ModelFactorized(bfct_train,pman_train)
    model_test = abt.ModelFactorized(bfct_test,pman_test)
    repres = abt.RepresentationFactorized(bfct_train)
    inf_driv = abt.EPFactorizedInfDriver(model_train,repres)
    if not do_laplace:
        # 'cav_var' argument does not matter, because coupling matrix for
        # Gaussian potentials is I
        inf_driv.init('ADF')
    else:
        # Below, we skip the prior potentials in the 1st sweep. We set their
        # pi's to 1, beta's to 0 (as in coupled mode and glm-ie)
        tvec = np.zeros(repres.size_pars())
        repres.setbeta(tvec)
        tvec[:n] = 1.
        repres.setpi(tvec)
        repres.refresh()
    if do_seldamp:
        if not do_laplace:
            # Exclude Gaussian potentials from selective damping mechanism
            # (they are not updated on anyway)
            sd_subind = pman_train.filterpots(set(['Gaussian']))
            repres.seldamp_reset(seldamp_numk,sd_subind,True)
        else:
            repres.seldamp_reset(seldamp_numk)
    # DEBUG
    if do_debug_matcomp:
        deb_mc = scipy.io.loadmat(debug_matcomp_fname % 0)
        deb_ep_pi = deb_mc['ep_pi'].ravel()
        deb_ep_beta = deb_mc['ep_beta'].ravel()
        deb_marg_pi = deb_mc['marg_pi'].ravel()
        deb_marg_beta = deb_mc['marg_beta'].ravel()
        print ('DEBUG[%d]: df(ep_pi)=%.4e, df(ep_beta)=%.4e, ' +
               'df(m_ep)=%.4e, df(m_beta)=%.4e') % \
               (0, abt.helpers.maxreldiff(repres.ep_pi,deb_ep_pi),
                abt.helpers.maxreldiff(repres.ep_beta,deb_ep_beta),
                abt.helpers.maxreldiff(repres.marg_pi,deb_marg_pi),
                abt.helpers.maxreldiff(repres.marg_beta,deb_marg_beta))
# Initial test set prediction
opts = abt.helpers.Struct()
opts.imode = imode
opts.ptype = 3
(h_q, rho_q, logz, h_p, rho_p) = inf_driv.predict(model_test,opts)
acc = 100.*float((np.sign(h_q)==targ_all[:num_test]).sum())/num_test
loglh = logz.sum()/num_test
print ('\nInitial predictions:\nAccuracy: %4.2f%%\n'
       'Test set log likelihood: %.6f') % (acc, loglh)

# EP inference
print '\nRunning EP inference (mode: %s)...' % imode
if imode == 'CoupParallel':
    opts = abt.helpers.Struct()
    opts.imode = imode
    opts.maxit = 20       # Max. number sweeps
    #opts.maxit = 40       # Max. number sweeps
    opts.deltaeps = 1e-4  # Convergence threshold
    opts.damp = 0.        # No damping
    opts.caveps = 1e-5
    opts.verbose = 1
    opts.res_det = True
    opts.bc_testmodel = model_test
    t_start = time.time()
    (res, res_det) = inf_driv.inference(opts)
    t_stop = time.time()
    print 'Time(inference): %.6fs' % (t_stop-t_start)
    if res.rstat==1:
        print '\nDone MAXIT iterations.'
    else:
        print '\nConverged to DELTAEPS accuracy.'
    print ('Number sweeps:   %d\n' +
           'Final delta:     %f\n' +
           'Number of skips: %d') % (res.nit,res.delta,res.nskip)
elif imode == 'CoupSequential':
    opts = abt.helpers.Struct()
    opts.imode = imode
    opts.maxit = 15       # Max. number sweeps
    opts.deltaeps = 1e-4  # Convergence threshold
    opts.damp = 0.        # No damping
    opts.caveps = 1e-5
    opts.skipeps = 1e-7
    opts.refresh = True   # Recompute repres. after each sweep
    opts.verbose = 1
    opts.res_det = True
    #opts.bc_testmodel = model_test
    t_start = time.time()
    (res, res_det) = inf_driv.inference(opts)
    t_stop = time.time()
    print 'Time(inference): %.6fs' % (t_stop-t_start)
    if res.rstat==1:
        print '\nDone MAXIT iterations.'
    else:
        print '\nConverged to DELTAEPS accuracy.'
    print ('Number sweeps:   %d\n' +
           'Final delta:     %f') % (res.nit,res.delta)
    print 'Skip histogram: ', res.nskip
elif imode == 'Factorized':
    opts = abt.helpers.Struct()
    opts.imode = imode
    opts.maxit = 100      # Max. number sweeps
    opts.deltaeps = 1e-4  # Convergence threshold
    opts.damp = 0.        # No damping
    opts.piminthres = 1e-7
    opts.refresh = True   # Recompute repres. after each sweep
    opts.verbose = 1
    opts.res_det = True
    if not do_laplace:
        # Gaussian potentials have coupling factor I, so EP updates do
        # not change anything. Skip them
        opts.skip_gauss = True
    else:
        # Skip updates for Laplace prior potentials in 1st sweep
        opts.upd_1stsweep = set(['Probit'])
    if do_debug_matcomp:
        opts.deb_matcomp_fname = debug_matcomp_fname
    opts.bc_testmodel = model_test
    t_start = time.time()
    (res, res_det) = inf_driv.inference(opts)
    t_stop = time.time()
    print 'Time(inference): %.6fs' % (t_stop-t_start)
    if res.rstat==1:
        print '\nDone MAXIT iterations.'
    else:
        print '\nConverged to DELTAEPS accuracy.'
    print ('Number sweeps:   %d\n' +
           'Final delta:     %f') % (res.nit,res.delta)
    print 'Skip histogram:  ', res.nskip
    if do_seldamp:
        print 'Select. damps:   %d' % res.nsdamp
else:
    raise ValueError("IMODE: Unknown value '" + imode + "'")

# Predictions
opts = abt.helpers.Struct()
opts.imode = imode
#opts.ptype = 0
opts.ptype = 3
#h_q = inf_driv.predict(model_test,opts)
(h_q, rho_q, logz, h_p, rho_p) = inf_driv.predict(model_test,opts)
acc = 100.*float((np.sign(h_q)==targ_all[:num_test]).sum())/num_test
loglh = logz.sum()/num_test
print ('\nFinal predictions:\nAccuracy: %4.2f%%\n'
       'Test set log likelihood: %.6f') % (acc, loglh)
<commit_msg>Added baseline results to test code as comments<commit_after>#! /usr/bin/env python

# EPTOOLS Python Interface
# Example: Binary classification, coupled or factorized posterior
# mode.
# Can be configured with Gaussian or Laplace prior.
#
# Very similar to Matlab
#   testcode eptools/matlab/test/binclass/eptest_binclass.m
# Same data as glm-ie_v1.5/doc/classify_inference.m:
# Adult (a9a): Dimension 123, first 30000 for testing, last 2561
# for training.

import numpy as np
import scipy.sparse as ssp
import scipy.io  # DEBUG: Load Matlab files
import numbers
import time  # Profiling

import apbsint as abt

# Helper functions

# Main code

# Load dataset: CSV format. Inputs: Each row contains indices of nonzero
# features (value: 1).
# NOTE: Building a dense nested list is faster (by factor 10!) than
# building a ssp.lil_matrix row by row.
# NOTE: We remove 3 attributes (45, 116, 122), these are not touched by
# any patterns in the training set.
num_feat = n = 120  # After removing 3
tmat = []
fid = open('adult_a9a_inputs_comp.csv','r')
for line in fid:
    ind = [int(x) for x in line.split(',')]
    v = np.zeros(n+3,dtype=np.float64)
    v[ind] = 1.
    # Remove attributes 45, 116, 122
    tmat.append(list(np.hstack((v[:45], v[46:116], v[117:122]))))
fid.close()
num_cases = len(tmat)
print 'Dataset: Read %d cases.' % num_cases
inp_all = ssp.csr_matrix(tmat)
del tmat
num_test = 30000
num_train = num_cases-num_test
fid = open('adult_a9a_targets.csv','r')
targ_all = np.array([float(x) for x in fid.readline().split(',')],
                    dtype=np.float64)
fid.close()
if targ_all.size != num_cases:
    raise IndexError('Internal error: Wrong file size')

# Setup
#imode = 'CoupParallel'   # Coupled, parallel updating
#imode = 'CoupSequential' # Coupled, sequential updating
imode = 'Factorized'     # Factorized, sequential updating
is_fact = (imode == 'Factorized')
do_laplace = True        # Laplace prior
tau_lapl = 2./5.
#do_laplace = False       # Gaussian prior
#ssq_gauss = 25./4.
do_seldamp = True
#do_seldamp = False
seldamp_numk = 5
# DEBUG: Compare intermediate results of Matlab and Python code directly.
# We load Matlab variables from files written after init. (%d = 0) and after
# each sweep (%d = 1,2,...). Anything non-deterministic also loaded from there.
# NOTE: Implemented for 'Factorized' mode only.
#debug_matcomp_fname = '/home/seeger/exp/ept/eptb13_vars_it%d.mat'
try:
    debug_matcomp_fname
    do_debug_matcomp = True
except NameError:
    do_debug_matcomp = False

# Coupling factors
if not is_fact:
    bfct_test = abt.MatSparse(inp_all[:num_test,:].copy())
    bfct_train = abt.MatContainer([abt.MatEye(n),
                                   abt.MatSparse(inp_all[num_test:,:].copy())])
else:
    bfct_test = abt.MatFactorizedInf(inp_all[:num_test,:].copy())
    mx_tmp = ssp.vstack([ssp.eye(n,format='csr'), inp_all[num_test:,:]],
                        format='csr')
    bfct_train = abt.MatFactorizedInf(mx_tmp)
m = bfct_train.shape(0)
# Potential managers
if not do_laplace:
    pm_elem1 = abt.ElemPotManager('Gaussian',n,(0., ssq_gauss))
else:
    pm_elem1 = abt.ElemPotManager('Laplace',n,(0., tau_lapl))
pm_elem2 = abt.ElemPotManager('Probit',num_train,
                              (targ_all[num_test:].copy(), 0.))
pman_train = abt.PotManager((pm_elem1, pm_elem2))
pman_test = abt.PotManager(abt.ElemPotManager('Probit',num_test,
                                              (targ_all[:num_test].copy(), 0.)))
print 'Prior:      %s\nLikelihood: %s' % (pman_train.elem[0].name,
                                          pman_train.elem[1].name)
# Initialization of representation parameters
if not is_fact:
    model_train = abt.ModelCoupled(bfct_train,pman_train)
    model_test = abt.ModelCoupled(bfct_test,pman_test)
    repres = abt.RepresentationCoupled(bfct_train,
                                       keep_margs=(imode == 'CoupParallel'))
    if imode == 'CoupParallel':
        inf_driv = abt.EPCoupParallelInfDriver(model_train,repres)
    else:
        inf_driv = abt.EPCoupSequentialInfDriver(model_train,repres)
    if not do_laplace:
        inf_driv.init('ADF')
    else:
        # Same initialization as in glm-ie
        tvec = np.empty(m)
        tvec[:] = 1.
        repres.setpi(tvec)
        tvec[:n] = 0.
        tvec[n:] = 0.5*targ_all[num_test:]
        repres.setbeta(tvec)
        repres.refresh()
else:
    model_train = abt.ModelFactorized(bfct_train,pman_train)
    model_test = abt.ModelFactorized(bfct_test,pman_test)
    repres = abt.RepresentationFactorized(bfct_train)
    inf_driv = abt.EPFactorizedInfDriver(model_train,repres)
    if not do_laplace:
        # 'cav_var' argument does not matter, because coupling matrix for
        # Gaussian potentials is I
        inf_driv.init('ADF')
    else:
        # Below, we skip the prior potentials in the 1st sweep. We set their
        # pi's to 1, beta's to 0 (as in coupled mode and glm-ie)
        tvec = np.zeros(repres.size_pars())
        repres.setbeta(tvec)
        tvec[:n] = 1.
        repres.setpi(tvec)
        repres.refresh()
    if do_seldamp:
        if not do_laplace:
            # Exclude Gaussian potentials from selective damping mechanism
            # (they are not updated on anyway)
            sd_subind = pman_train.filterpots(set(['Gaussian']))
            repres.seldamp_reset(seldamp_numk,sd_subind,True)
        else:
            repres.seldamp_reset(seldamp_numk)
    # DEBUG
    if do_debug_matcomp:
        deb_mc = scipy.io.loadmat(debug_matcomp_fname % 0)
        deb_ep_pi = deb_mc['ep_pi'].ravel()
        deb_ep_beta = deb_mc['ep_beta'].ravel()
        deb_marg_pi = deb_mc['marg_pi'].ravel()
        deb_marg_beta = deb_mc['marg_beta'].ravel()
        print ('DEBUG[%d]: df(ep_pi)=%.4e, df(ep_beta)=%.4e, ' +
               'df(m_ep)=%.4e, df(m_beta)=%.4e') % \
               (0, abt.helpers.maxreldiff(repres.ep_pi,deb_ep_pi),
                abt.helpers.maxreldiff(repres.ep_beta,deb_ep_beta),
                abt.helpers.maxreldiff(repres.marg_pi,deb_marg_pi),
                abt.helpers.maxreldiff(repres.marg_beta,deb_marg_beta))
# Initial test set prediction
opts = abt.helpers.Struct()
opts.imode = imode
opts.ptype = 3
(h_q, rho_q, logz, h_p, rho_p) = inf_driv.predict(model_test,opts)
acc = 100.*float((np.sign(h_q)==targ_all[:num_test]).sum())/num_test
loglh = logz.sum()/num_test
print ('\nInitial predictions:\nAccuracy: %4.2f%%\n'
       'Test set log likelihood: %.6f') % (acc, loglh)

# EP inference
print '\nRunning EP inference (mode: %s)...' % imode
if imode == 'CoupParallel':
    opts = abt.helpers.Struct()
    opts.imode = imode
    opts.maxit = 20       # Max. number sweeps
    #opts.maxit = 40       # Max. number sweeps
    opts.deltaeps = 1e-4  # Convergence threshold
    opts.damp = 0.        # No damping
    opts.caveps = 1e-5
    opts.verbose = 1
    opts.res_det = True
    opts.bc_testmodel = model_test
    t_start = time.time()
    (res, res_det) = inf_driv.inference(opts)
    t_stop = time.time()
    print 'Time(inference): %.6fs' % (t_stop-t_start)
    if res.rstat==1:
        print '\nDone MAXIT iterations.'
    else:
        print '\nConverged to DELTAEPS accuracy.'
    print ('Number sweeps:   %d\n' +
           'Final delta:     %f\n' +
           'Number of skips: %d') % (res.nit,res.delta,res.nskip)
elif imode == 'CoupSequential':
    opts = abt.helpers.Struct()
    opts.imode = imode
    opts.maxit = 15       # Max. number sweeps
    opts.deltaeps = 1e-4  # Convergence threshold
    opts.damp = 0.        # No damping
    opts.caveps = 1e-5
    opts.skipeps = 1e-7
    opts.refresh = True   # Recompute repres. after each sweep
    opts.verbose = 1
    opts.res_det = True
    #opts.bc_testmodel = model_test
    t_start = time.time()
    (res, res_det) = inf_driv.inference(opts)
    t_stop = time.time()
    print 'Time(inference): %.6fs' % (t_stop-t_start)
    if res.rstat==1:
        print '\nDone MAXIT iterations.'
    else:
        print '\nConverged to DELTAEPS accuracy.'
    print ('Number sweeps:   %d\n' +
           'Final delta:     %f') % (res.nit,res.delta)
    print 'Skip histogram: ', res.nskip
elif imode == 'Factorized':
    opts = abt.helpers.Struct()
    opts.imode = imode
    opts.maxit = 100      # Max. number sweeps
    opts.deltaeps = 1e-4  # Convergence threshold
    opts.damp = 0.        # No damping
    opts.piminthres = 1e-7
    opts.refresh = True   # Recompute repres. after each sweep
    opts.verbose = 1
    opts.res_det = True
    if not do_laplace:
        # Gaussian potentials have coupling factor I, so EP updates do
        # not change anything. Skip them
        opts.skip_gauss = True
    else:
        # Skip updates for Laplace prior potentials in 1st sweep
        opts.upd_1stsweep = set(['Probit'])
    if do_debug_matcomp:
        opts.deb_matcomp_fname = debug_matcomp_fname
    opts.bc_testmodel = model_test
    t_start = time.time()
    (res, res_det) = inf_driv.inference(opts)
    t_stop = time.time()
    print 'Time(inference): %.6fs' % (t_stop-t_start)
    if res.rstat==1:
        print '\nDone MAXIT iterations.'
    else:
        print '\nConverged to DELTAEPS accuracy.'
    print ('Number sweeps:   %d\n' +
           'Final delta:     %f') % (res.nit,res.delta)
    print 'Skip histogram:  ', res.nskip
    if do_seldamp:
        print 'Select. damps:   %d' % res.nsdamp
else:
    raise ValueError("IMODE: Unknown value '" + imode + "'")

# Predictions
opts = abt.helpers.Struct()
opts.imode = imode
#opts.ptype = 0
opts.ptype = 3
#h_q = inf_driv.predict(model_test,opts)
(h_q, rho_q, logz, h_p, rho_p) = inf_driv.predict(model_test,opts)
acc = 100.*float((np.sign(h_q)==targ_all[:num_test]).sum())/num_test
loglh = logz.sum()/num_test
print ('\nFinal predictions:\nAccuracy: %4.2f%%\n'
       'Test set log likelihood: %.6f') % (acc, loglh)

# Baseline results (23/3/2014, commit 8fd5e9d3c9c60bbaee0c4cad050a66ec656e7f10)
#
#--- [Factorized, Laplace (tau=2/5), sel. damp] ---
#Time(inference): 2.925671s
#
#Done MAXIT iterations.
#Number sweeps:   100
#Final delta:     0.005832
#Skip histogram:   [267879      0      0      0    101]
#Select. damps:   112
#
#Final predictions:
#Accuracy: 83.92%
#Test set log likelihood: -0.346497
#---
#--- [Factorized, Laplace (tau=2/5), no sel. damp] ---
#Time(inference): 2.690239s
#
#Done MAXIT iterations.
#Number sweeps:   100
#Final delta:     0.004497
#Skip histogram:   [267980      0      0      0      0]
#
#Final predictions:
#Accuracy: 83.91%
#Test set log likelihood: -0.346438
#---
#--- [Factorized, Gaussian (ssq=25/4), sel. damp] ---
#Time(inference): 2.900740s
#
#Done MAXIT iterations.
#Number sweeps:   100
#Final delta:     0.009161
#Skip histogram:   [256100      0      0      0      0]
#Select. damps:   0
#
#Final predictions:
#Accuracy: 83.89%
#Test set log likelihood: -0.348321
#---
#--- [Factorized, Gaussian (ssq=25/4), no sel. damp] ---
#Time(inference): 2.652864s
#
#Done MAXIT iterations.
#Number sweeps:   100
#Final delta:     0.008146
#Skip histogram:   [256100      0      0      0      0]
#
#Final predictions:
#Accuracy: 83.87%
#Test set log likelihood: -0.347889
#---
#--- [CoupParallel, Laplace (tau=2/5)] ---
#Time(inference): 6.966938s
#
#Done MAXIT iterations.
#Number sweeps:   20
#Final delta:     0.001795
#Number of skips: 0
#
#Final predictions:
#Accuracy: 83.91%
#Test set log likelihood: -0.346969
#---
#--- [CoupParallel, Gaussian (ssq=25/4)] ---
#Time(inference): 7.035166s
#
#Done MAXIT iterations.
#Number sweeps:   20
#Final delta:     0.026932
#Number of skips: 0
#
#Final predictions:
#Accuracy: 83.87%
#Test set log likelihood: -0.347569
#---
#==> Final delta much larger than for Laplace (same above). Why?
#    After 40 sweeps: delta=0.001204, 83.87%, llh=-0.347581
#==> This means that Laplace prior helps convergence of parallel updating
#--- [CoupSequential, Laplace (tau=2/5)] ---
#Time(inference): 44.869315s
#
#Done MAXIT iterations.
#Number sweeps:   15
#Final delta:     0.000117
#Skip histogram:  [40215     0     0     0]
#
#Final predictions:
#Accuracy: 83.91%
#Test set log likelihood: -0.346968
#---
#==> delta=0.002392 at iter. 10
#    Converges to high accuracy, but much slower than CoupParallel
#--- [CoupSequential, Gaussian (ssq=25/4)] ---
#Time(inference): 23.707730s
#
#Converged to DELTAEPS accuracy.
#Number sweeps:   7
#Final delta:     0.000080
#Skip histogram:  [17927     0     0     0]
#
#Final predictions:
#Accuracy: 83.87%
#Test set log likelihood: -0.347582
#---
#==> delta=0.001889 at iter. 5
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Partially based on AboutMessagePassing in the Ruby Koans
#

from runner.koan import *

class AboutAttributeAccess(Koan):

    class TypicalObject:
        pass

    def test_calling_undefined_functions_normally_results_in_errors(self):
        typical = self.TypicalObject()

        with self.assertRaises(___): typical.foobar()

    def test_calling_getattribute_causes_an_attribute_error(self):
        typical = self.TypicalObject()

        with self.assertRaises(___): typical.__getattribute__('foobar')

        # THINK ABOUT IT:
        #
        # If the method __getattribute__() causes the AttributeError, then
        # what would happen if we redefine __getattribute__()?

    # ------------------------------------------------------------------

    class CatchAllAttributeReads:
        def __getattribute__(self, attr_name):
            return "Someone called '" + attr_name + "' and it could not be found"

    def test_all_attribute_reads_are_caught(self):
        catcher = self.CatchAllAttributeReads()

        self.assertRegexpMatches(catcher.foobar, __)

    def test_intercepting_return_values_can_disrupt_the_call_chain(self):
        catcher = self.CatchAllAttributeReads()

        self.assertRegexpMatches(catcher.foobaz, __) # This is fine

        try:
            catcher.foobaz(1)
        except TypeError as ex:
            err_msg = ex.args[0]

        self.assertRegexpMatches(err_msg, __)

        # foobaz returns a string. What happens to the '(1)' part?
        # Try entering this into a python console to reproduce the issue:
        #
        #     "foobaz"(1)
        #

    def test_changes_to_the_getattribute_implementation_affects_getattr_function(self):
        catcher = self.CatchAllAttributeReads()

        self.assertRegexpMatches(getattr(catcher, 'any_attribute'), __)

    # ------------------------------------------------------------------

    class WellBehavedFooCatcher:
        def __getattribute__(self, attr_name):
            if attr_name[:3] == "foo":
                return "Foo to you too"
            else:
                return super().__getattribute__(attr_name)

    def test_foo_attributes_are_caught(self):
        catcher = self.WellBehavedFooCatcher()

        self.assertEqual(__, catcher.foo_bar)
        self.assertEqual(__, catcher.foo_baz)

    def test_non_foo_messages_are_treated_normally(self):
        catcher = self.WellBehavedFooCatcher()

        with self.assertRaises(___): catcher.normal_undefined_attribute

    # ------------------------------------------------------------------

    global stack_depth
    stack_depth = 0

    class RecursiveCatcher:
        def __init__(self):
            global stack_depth
            stack_depth = 0
            self.no_of_getattribute_calls = 0

        def __getattribute__(self, attr_name):
            global stack_depth # We need something that is outside the scope of this class
            stack_depth += 1

            if stack_depth<=10: # to prevent a stack overflow
                self.no_of_getattribute_calls += 1
                # Oops! We just accessed an attribute (no_of_getattribute_calls)
                # Guess what happens when self.no_of_getattribute_calls is
                # accessed?

            # Using 'object' directly because using super() here will also
            # trigger a __getattribute__() call.
            return object.__getattribute__(self, attr_name)

        def my_method(self):
            pass

    def test_getattribute_is_a_bit_overzealous_sometimes(self):
        catcher = self.RecursiveCatcher()
        catcher.my_method()
        global stack_depth
        self.assertEqual(__, stack_depth)

    # ------------------------------------------------------------------

    class MinimalCatcher:
        class DuffObject: pass

        def __init__(self):
            self.no_of_getattr_calls = 0

        def __getattr__(self, attr_name):
            self.no_of_getattr_calls += 1
            return self.DuffObject

        def my_method(self):
            pass

    def test_getattr_ignores_known_attributes(self):
        catcher = self.MinimalCatcher()
        catcher.my_method()

        self.assertEqual(__, catcher.no_of_getattr_calls)

    def test_getattr_only_catches_unknown_attributes(self):
        catcher = self.MinimalCatcher()
        catcher.purple_flamingos()
        catcher.free_pie()

        self.assertEqual(__,
            type(catcher.give_me_duff_or_give_me_death()).__name__)

        self.assertEqual(__, catcher.no_of_getattr_calls)

    # ------------------------------------------------------------------

    class PossessiveSetter(object):
        def __setattr__(self, attr_name, value):
            new_attr_name =  attr_name

            if attr_name[-5:] == 'comic':
                new_attr_name = "my_" + new_attr_name
            elif attr_name[-3:] == 'pie':
                new_attr_name = "a_" + new_attr_name

            object.__setattr__(self, new_attr_name, value)

    def test_setattr_intercepts_attribute_assignments(self):
        fanboy = self.PossessiveSetter()

        fanboy.comic = 'The Laminator, issue #1'
        fanboy.pie = 'blueberry'

        self.assertEqual(__, fanboy.a_pie)

        #
        # NOTE: Change the prefix to make this next assert pass
        #

        prefix = '__'
        self.assertEqual("The Laminator, issue #1", getattr(fanboy, prefix + '_comic'))

    # ------------------------------------------------------------------

    class ScarySetter:
        def __init__(self):
            self.num_of_coconuts = 9
            self._num_of_private_coconuts = 2

        def __setattr__(self, attr_name, value):
            new_attr_name =  attr_name

            if attr_name[0] != '_':
                new_attr_name = "altered_" + new_attr_name

            object.__setattr__(self, new_attr_name, value)

    def test_it_modifies_external_attribute_as_expected(self):
        setter = self.ScarySetter()
        setter.e = "mc hammer"

        self.assertEqual(__, setter.altered_e)

    def test_it_mangles_some_internal_attributes(self):
        setter = self.ScarySetter()

        try:
            coconuts = setter.num_of_coconuts
        except AttributeError:
            self.assertEqual(__, setter.altered_num_of_coconuts)

    def test_in_this_case_private_attributes_remain_unmangled(self):
        setter = self.ScarySetter()

        self.assertEqual(__, setter._num_of_private_coconuts)
<commit_msg>Koan done: attribute access<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Partially based on AboutMessagePassing in the Ruby Koans
#

from runner.koan import *

class AboutAttributeAccess(Koan):

    class TypicalObject:
        pass

    def test_calling_undefined_functions_normally_results_in_errors(self):
        typical = self.TypicalObject()

        with self.assertRaises(AttributeError): typical.foobar()

    def test_calling_getattribute_causes_an_attribute_error(self):
        typical = self.TypicalObject()

        with self.assertRaises(AttributeError): typical.__getattribute__('foobar')

        # THINK ABOUT IT:
        #
        # If the method __getattribute__() causes the AttributeError, then
        # what would happen if we redefine __getattribute__()?

    # ------------------------------------------------------------------

    class CatchAllAttributeReads:
        def __getattribute__(self, attr_name):
            return "Someone called '" + attr_name + "' and it could not be found"

    def test_all_attribute_reads_are_caught(self):
        catcher = self.CatchAllAttributeReads()

        self.assertRegexpMatches(catcher.foobar, "Someone called 'foobar' and it could not be found")

    def test_intercepting_return_values_can_disrupt_the_call_chain(self):
        catcher = self.CatchAllAttributeReads()

        # This is fine
        self.assertRegexpMatches(catcher.foobaz, "Someone called 'foobaz' and it could not be found")

        try:
            catcher.foobaz(1)
        except TypeError as ex:
            err_msg = ex.args[0]

        self.assertRegexpMatches(err_msg, "'str' object is not callable")

        # foobaz returns a string. What happens to the '(1)' part?
        # Try entering this into a python console to reproduce the issue:
        #
        #     "foobaz"(1)
        #

    def test_changes_to_the_getattribute_implementation_affects_getattr_function(self):
        catcher = self.CatchAllAttributeReads()

        self.assertRegexpMatches(getattr(catcher, 'any_attribute'), "Someone called 'any_attribute' and it could not be found")

    # ------------------------------------------------------------------

    class WellBehavedFooCatcher:
        def __getattribute__(self, attr_name):
            if attr_name[:3] == "foo":
                return "Foo to you too"
            else:
                return super().__getattribute__(attr_name)

    def test_foo_attributes_are_caught(self):
        catcher = self.WellBehavedFooCatcher()

        self.assertEqual("Foo to you too", catcher.foo_bar)
        self.assertEqual("Foo to you too", catcher.foo_baz)

    def test_non_foo_messages_are_treated_normally(self):
        catcher = self.WellBehavedFooCatcher()

        with self.assertRaises(AttributeError): catcher.normal_undefined_attribute

    # ------------------------------------------------------------------

    global stack_depth
    stack_depth = 0

    class RecursiveCatcher:
        def __init__(self):
            global stack_depth
            stack_depth = 0
            self.no_of_getattribute_calls = 0

        def __getattribute__(self, attr_name):
            global stack_depth # We need something that is outside the scope of this class
            stack_depth += 1

            if stack_depth<=10: # to prevent a stack overflow
                self.no_of_getattribute_calls += 1
                # Oops! We just accessed an attribute (no_of_getattribute_calls)
                # Guess what happens when self.no_of_getattribute_calls is
                # accessed?

            # Using 'object' directly because using super() here will also
            # trigger a __getattribute__() call.
            return object.__getattribute__(self, attr_name)

        def my_method(self):
            pass

    def test_getattribute_is_a_bit_overzealous_sometimes(self):
        catcher = self.RecursiveCatcher()
        catcher.my_method()
        global stack_depth
        self.assertEqual(11, stack_depth)

    # ------------------------------------------------------------------

    class MinimalCatcher:
        class DuffObject: pass

        def __init__(self):
            self.no_of_getattr_calls = 0

        def __getattr__(self, attr_name):
            self.no_of_getattr_calls += 1
            return self.DuffObject

        def my_method(self):
            pass

    def test_getattr_ignores_known_attributes(self):
        catcher = self.MinimalCatcher()
        catcher.my_method()

        self.assertEqual(0, catcher.no_of_getattr_calls)

    def test_getattr_only_catches_unknown_attributes(self):
        catcher = self.MinimalCatcher()
        catcher.purple_flamingos()
        catcher.free_pie()

        self.assertEqual("DuffObject",
            type(catcher.give_me_duff_or_give_me_death()).__name__)

        self.assertEqual(3, catcher.no_of_getattr_calls)

    # ------------------------------------------------------------------

    class PossessiveSetter(object):
        def __setattr__(self, attr_name, value):
            new_attr_name =  attr_name

            if attr_name[-5:] == 'comic':
                new_attr_name = "my_" + new_attr_name
            elif attr_name[-3:] == 'pie':
                new_attr_name = "a_" + new_attr_name

            object.__setattr__(self, new_attr_name, value)

    def test_setattr_intercepts_attribute_assignments(self):
        fanboy = self.PossessiveSetter()

        fanboy.comic = 'The Laminator, issue #1'
        fanboy.pie = 'blueberry'

        self.assertEqual("blueberry", fanboy.a_pie)

        #
        # NOTE: Change the prefix to make this next assert pass
        #

        prefix = "my"
        self.assertEqual("The Laminator, issue #1", getattr(fanboy, prefix + '_comic'))

    # ------------------------------------------------------------------

    class ScarySetter:
        def __init__(self):
            self.num_of_coconuts = 9
            self._num_of_private_coconuts = 2

        def __setattr__(self, attr_name, value):
            new_attr_name =  attr_name

            if attr_name[0] != '_':
                new_attr_name = "altered_" + new_attr_name

            object.__setattr__(self, new_attr_name, value)

    def test_it_modifies_external_attribute_as_expected(self):
        setter = self.ScarySetter()
        setter.e = "mc hammer"

        self.assertEqual("mc hammer", setter.altered_e)

    def test_it_mangles_some_internal_attributes(self):
        setter = self.ScarySetter()

        try:
            coconuts = setter.num_of_coconuts
        except AttributeError:
            self.assertEqual(9, setter.altered_num_of_coconuts)

    def test_in_this_case_private_attributes_remain_unmangled(self):
        setter = self.ScarySetter()

        self.assertEqual(2, setter._num_of_private_coconuts)
<|endoftext|>


<commit_before>
"""
test_exploreDA
--------------
The module which groups the main functions to explore a new data.

"""

import pandas as pd
import numpy as np
import datetime

from Plotting.contdistrib_plot import cont_distrib_plot
from Plotting.catdistrib_plot import barplot_plot
from Plotting.net_plotting import plot_net_distribution, plot_heat_net
from Plotting.geo_plotting import compute_spatial_density_sparse,\
    clean_coordinates, plot_in_map, plot_geo_heatmap
from Plotting.temp_plotting import temp_distrib

from Statistics.cont_statistics import quantile_compute, ranges_compute,\
    cont_count, log_cont_count
from Statistics.cat_statistics import cat_count
from Statistics.temp_statistics import count_temp_stats
from Statistics.coord_statistics import mean_coord_by_values

from SummaryStatistics.univariate_stats import compute_univariate_stats

def test():
    ## Parameters data
    contdata = pd.Series(np.random.random(100))
    catdata = pd.Series(np.random.randint(0, 10, 100))
    netdata = np.random.random((10, 10))
    xt = np.random.random(100)
    timedata = pd.Series(np.random.random(100),
                         index=[datetime.datetime.now() + datetime.timedelta(e)
                                for e in np.cumsum(xt)])

    df = pd.DataFrame([np.random.random(100), np.random.random(100), catdata])
    df = df.T
    df.columns = ['a', 'b', 'c']

    ### Statitics testing
    # Categorical variable
    cat_count(df, 'c')

    # Continious variable
    quantile_compute(contdata, 5)
    ranges_compute(contdata, 5)
    cont_count(df, 'a', 5)
    log_cont_count(df, 'a', 5)

    # Coordinate variables
    mean_coord_by_values(df, ['a', 'b'], 'c')

    # Temporal variable
    date_ranges = np.linspace(timedata.min(), timedata.max(), 5)[1:-1]
    count_temp_stats(timedata, date_ranges, tags=None)

    ### Plotting testing
    ## Testing univariate categorical variable plotting
    barplot_plot(catdata, logscale=False)
    barplot_plot(catdata, logscale=True)

    ## Testing univariate continious variable plotting
    cont_distrib_plot(contdata, n_bins=5, logscale=True)
    cont_distrib_plot(contdata, n_bins=5, logscale=False)

    ## Testing network plotting
    plot_net_distribution(netdata, 5)
    plot_heat_net(netdata, range(10))

    ## Testing geospatial plotting
    # Parameters
    longs, lats = np.random.random(100), np.random.random(100)
    n_x, n_y = 10, 10
    n_levs = 5
    sigma_smooth, order_smooth, null_lim = 5, 0, 0.1
    var0, var1 = None, np.random.random(100)
    coordinates = pd.DataFrame([longs, lats]).T

    # Auxiliar function
    clean_coordinates(coordinates)

    # Computing grid data
    compute_spatial_density_sparse(longs, lats, n_x, n_y, sigma_smooth,
                                   order_smooth, null_lim, var0)
    compute_spatial_density_sparse(longs, lats, n_x, n_y, sigma_smooth,
                                   order_smooth, null_lim, var1)

    ## Temporal plotting
    temp_distrib(pd.Series(timedata.index), 'day', logscale=False)
    temp_distrib(pd.Series(timedata.index), 'day', logscale=True)

    # Main plotters
    plot_in_map(coordinates, resolution='f', color_cont=None, marker_size=1)
    plot_in_map(coordinates, resolution='f', color_cont='b', marker_size=1)
    plot_geo_heatmap(coordinates, n_levs, n_x, n_y, var0)

    ### Summary testing
    # Parameters
    df = pd.DataFrame([np.random.random(100), np.random.randint(0, 1, 100),
                       np.random.random(100), np.random.random(100),
                       np.random.random(100).cumsum(),
                       [datetime.datetime(2005, 1, 1) + datetime.timedelta(e)
                        for e in xt*3500]])
    df = df.T
    df.columns = ['cont', 'cat', 'x', 'y', 't', 't_y']

    info_var_cat0 = {'type': 'categorical', 'variables': 'cat',
                     'ifplot': True, 'logscale': True}
    info_var_cat1 = {'type': 'categorical', 'variables': 'cat',
                     'ifplot': True, 'logscale': True}

    info_var_cont0 = {'type': 'continuous', 'n_bins': 10, 'ifplot': True,
                      'logscale': False, 'variables': 'cont'}
    info_var_cont1 = {'type': 'continuous', 'n_bins': 10, 'ifplot': True,
                      'logscale': True, 'variables': 'cont'}

    info_var_coord = {'type': 'coordinates', 'variables': ['x', 'y'],
                      'ifplot': True}

    info_var_temp = {'type': 'temporal', 'variables': 't', 'logscale': False,
                     'ifplot': True, 'agg_time': 'second',
                     'stats_pars': {'date_ranges': range(101)}}
    info_var_templog = {'type': 'temporal', 'variables': 't', 'logscale': True,
                        'ifplot': True, 'agg_time': 'second',
                        'stats_pars': {'date_ranges': range(101)}}

    info_var_ty = {'type': 'temporal', 'variables': 't_y', 'logscale': False,
                   'ifplot': True, 'agg_time': 'month',
                   'stats_pars': {'tags': ['pre', 'through', 'post'],
                                  'date_ranges': ['2006-01-01', '2012-12-31']
                                  }
                   }
    info_var_tylog = {'type': 'temporal', 'variables': 't_y', 'logscale': True,
                      'ifplot': True, 'agg_time': 'month',
                      'stats_pars': {'tags': None,
                                     'date_ranges': ['2006-01-01',
                                                     '2012-12-31']
                                     }
                      }

    info_var_tempdist = {'type': 'tmpdist', 'variables': ['x', 'y', 't'],
                         'logscale': False, 'ifplot': True}
    info_var_tempdistlog = {'type': 'tmpdist', 'variables': ['x', 'y', 't'],
                            'logscale': True, 'ifplot': True}

    # Test
    compute_univariate_stats(df, info_var_cat0)
    compute_univariate_stats(df, info_var_cat1)
    compute_univariate_stats(df, info_var_cont0)
    compute_univariate_stats(df, info_var_cont1)
    compute_univariate_stats(df, info_var_coord)
    compute_univariate_stats(df, info_var_ty)
    compute_univariate_stats(df, info_var_tylog)
#    compute_univariate_stats(df, info_var_temp)
#    compute_univariate_stats(df, info_var_templog)
#    compute_univariate_stats(df, info_var_tempdist)
#    compute_univariate_stats(df, info_var_tempdistlog)
<commit_msg>Studying test ExploreDA.<commit_after>
"""
test_exploreDA
--------------
The module which groups the main functions to explore a new data.

"""

import pandas as pd
import numpy as np
import datetime

from Plotting.contdistrib_plot import cont_distrib_plot
from Plotting.catdistrib_plot import barplot_plot
from Plotting.net_plotting import plot_net_distribution, plot_heat_net
from Plotting.geo_plotting import compute_spatial_density_sparse,\
    clean_coordinates, plot_in_map, plot_geo_heatmap
from Plotting.temp_plotting import temp_distrib

from Statistics.cont_statistics import quantile_compute, ranges_compute,\
    cont_count, log_cont_count
from Statistics.cat_statistics import cat_count
from Statistics.temp_statistics import count_temp_stats
from Statistics.coord_statistics import mean_coord_by_values

from SummaryStatistics.univariate_stats import compute_univariate_stats

def test():
    pass
#    ## Parameters data
#    contdata = pd.Series(np.random.random(100))
#    catdata = pd.Series(np.random.randint(0, 10, 100))
#    netdata = np.random.random((10, 10))
#    xt = np.random.random(100)
#    timedata = pd.Series(np.random.random(100),
#                         index=[datetime.datetime.now() + datetime.timedelta(e)
#                                for e in np.cumsum(xt)])
#
#    df = pd.DataFrame([np.random.random(100), np.random.random(100), catdata])
#    df = df.T
#    df.columns = ['a', 'b', 'c']
#
#    ### Statitics testing
#    # Categorical variable
#    cat_count(df, 'c')
#
#    # Continious variable
#    quantile_compute(contdata, 5)
#    ranges_compute(contdata, 5)
#    cont_count(df, 'a', 5)
#    log_cont_count(df, 'a', 5)
#
#    # Coordinate variables
#    mean_coord_by_values(df, ['a', 'b'], 'c')
#
#    # Temporal variable
#    date_ranges = np.linspace(timedata.min(), timedata.max(), 5)[1:-1]
#    count_temp_stats(timedata, date_ranges, tags=None)
#
#    ### Plotting testing
#    ## Testing univariate categorical variable plotting
#    barplot_plot(catdata, logscale=False)
#    barplot_plot(catdata, logscale=True)
#
#    ## Testing univariate continious variable plotting
#    cont_distrib_plot(contdata, n_bins=5, logscale=True)
#    cont_distrib_plot(contdata, n_bins=5, logscale=False)
#
#    ## Testing network plotting
#    plot_net_distribution(netdata, 5)
#    plot_heat_net(netdata, range(10))
#
#    ## Testing geospatial plotting
#    # Parameters
#    longs, lats = np.random.random(100), np.random.random(100)
#    n_x, n_y = 10, 10
#    n_levs = 5
#    sigma_smooth, order_smooth, null_lim = 5, 0, 0.1
#    var0, var1 = None, np.random.random(100)
#    coordinates = pd.DataFrame([longs, lats]).T
#
#    # Auxiliar function
#    clean_coordinates(coordinates)
#
#    # Computing grid data
#    compute_spatial_density_sparse(longs, lats, n_x, n_y, sigma_smooth,
#                                   order_smooth, null_lim, var0)
#    compute_spatial_density_sparse(longs, lats, n_x, n_y, sigma_smooth,
#                                   order_smooth, null_lim, var1)
#
#    ## Temporal plotting
#    temp_distrib(pd.Series(timedata.index), 'day', logscale=False)
#    temp_distrib(pd.Series(timedata.index), 'day', logscale=True)
#
#    # Main plotters
#    plot_in_map(coordinates, resolution='f', color_cont=None, marker_size=1)
#    plot_in_map(coordinates, resolution='f', color_cont='b', marker_size=1)
#    plot_geo_heatmap(coordinates, n_levs, n_x, n_y, var0)
#
#    ### Summary testing
#    # Parameters
#    df = pd.DataFrame([np.random.random(100), np.random.randint(0, 1, 100),
#                       np.random.random(100), np.random.random(100),
#                       np.random.random(100).cumsum(),
#                       [datetime.datetime(2005, 1, 1) + datetime.timedelta(e)
#                        for e in xt*3500]])
#    df = df.T
#    df.columns = ['cont', 'cat', 'x', 'y', 't', 't_y']
#
#    info_var_cat0 = {'type': 'categorical', 'variables': 'cat',
#                     'ifplot': True, 'logscale': True}
#    info_var_cat1 = {'type': 'categorical', 'variables': 'cat',
#                     'ifplot': True, 'logscale': True}
#
#    info_var_cont0 = {'type': 'continuous', 'n_bins': 10, 'ifplot': True,
#                      'logscale': False, 'variables': 'cont'}
#    info_var_cont1 = {'type': 'continuous', 'n_bins': 10, 'ifplot': True,
#                      'logscale': True, 'variables': 'cont'}
#
#    info_var_coord = {'type': 'coordinates', 'variables': ['x', 'y'],
#                      'ifplot': True}
#
#    info_var_temp = {'type': 'temporal', 'variables': 't', 'logscale': False,
#                     'ifplot': True, 'agg_time': 'second',
#                     'stats_pars': {'date_ranges': range(101)}}
#    info_var_templog = {'type': 'temporal', 'variables': 't', 'logscale': True,
#                        'ifplot': True, 'agg_time': 'second',
#                        'stats_pars': {'date_ranges': range(101)}}
#
#    info_var_ty = {'type': 'temporal', 'variables': 't_y', 'logscale': False,
#                   'ifplot': True, 'agg_time': 'month',
#                   'stats_pars': {'tags': ['pre', 'through', 'post'],
#                                  'date_ranges': ['2006-01-01', '2012-12-31']
#                                  }
#                   }
#    info_var_tylog = {'type': 'temporal', 'variables': 't_y', 'logscale': True,
#                      'ifplot': True, 'agg_time': 'month',
#                      'stats_pars': {'tags': None,
#                                     'date_ranges': ['2006-01-01',
#                                                     '2012-12-31']
#                                     }
#                      }
#
#    info_var_tempdist = {'type': 'tmpdist', 'variables': ['x', 'y', 't'],
#                         'logscale': False, 'ifplot': True}
#    info_var_tempdistlog = {'type': 'tmpdist', 'variables': ['x', 'y', 't'],
#                            'logscale': True, 'ifplot': True}
#
#    # Test
#    compute_univariate_stats(df, info_var_cat0)
#    compute_univariate_stats(df, info_var_cat1)
#    compute_univariate_stats(df, info_var_cont0)
#    compute_univariate_stats(df, info_var_cont1)
#    compute_univariate_stats(df, info_var_coord)
#    compute_univariate_stats(df, info_var_ty)
#    compute_univariate_stats(df, info_var_tylog)
##    compute_univariate_stats(df, info_var_temp)
##    compute_univariate_stats(df, info_var_templog)
##    compute_univariate_stats(df, info_var_tempdist)
##    compute_univariate_stats(df, info_var_tempdistlog)
<|endoftext|>


<commit_before># -*- coding: utf-8  -*-
from pywikibot import family

__version__ = '$Id$'

# The Wikimedia family that is known as Wikisource

class Family(family.Family):
    def __init__(self):
        family.Family.__init__(self)
        self.name = 'wikisource'

        self.languages_by_size = [
            'en', 'zh', 'ru', 'pt', 'fr', 'de', 'es', 'it', 'he', 'ar', 'hu',
            'fa', 'pl', 'th', 'cs', 'ro', 'hr', 'te', 'fi', 'bn', 'sv', 'tr',
            'nl', 'sl', 'vi', 'ko', 'uk', 'sr', 'ja', 'el', 'la', 'li', 'yi',
            'ml', 'az', 'is', 'bs', 'hy', 'ca', 'id', 'mk', 'ta', 'da', 'no',
            'et', 'bg', 'lt', 'gl', 'kn', 'cy', 'sk', 'zh-min-nan', 'fo',
        ]

        for lang in self.languages_by_size:
            self.langs[lang] = '%s.wikisource.org' % lang
            self.langs['-'] = 'wikisource.org'

        self.alphabetic = ['ang','ar','az','bg','bs','ca','cs','cy',
                      'da','de','el','en','es','et','fa','fi',
                      'fo','fr','gl','he','hr','ht','hu','id',
                      'is','it','ja', 'ko','la','lt','ml','nl',
                      'no','pl','pt','ro','ru','sk','sl','sr',
                      'sv','te','th','tr','uk','vi','yi','zh']

        self.obsolete = {
            'ang': None, # http://meta.wikimedia.org/wiki/Proposals_for_closing_projects/Closure_of_Old_English_Wikisource
            'dk': 'da',
            'ht': None, # http://meta.wikimedia.org/wiki/Proposals_for_closing_projects/Closure_of_Haitian_Creole_Wikisource
            'jp': 'ja',
            'minnan':'zh-min-nan',
            'nb': 'no',
            'tokipona': None,
            'zh-tw': 'zh',
            'zh-cn': 'zh'
        }

        self.interwiki_putfirst = {
            'en': self.alphabetic,
            'fi': self.alphabetic,
            'fr': self.alphabetic,
            'he': ['en'],
            'hu': ['en'],
            'pl': self.alphabetic,
            'simple': self.alphabetic
        }
        # Global bot allowed languages on http://meta.wikimedia.org/wiki/Bot_policy/Implementation#Current_implementation
        self.cross_allowed = [
            'el','fa','it','ko','no','vi','zh'
        ]
        # CentralAuth cross avaliable projects.
        self.cross_projects = [
            'wikipedia', 'wiktionary', 'wikibooks', 'wikiquote', 'wikinews', 'wikiversity',
            'meta', 'mediawiki', 'test', 'incubator', 'commons', 'species'
        ]

    def version(self, code):
        return '1.16wmf4'

    def shared_image_repository(self, code):
        return ('commons', 'commons')
<commit_msg>Changed handling of cross namespace interlanguage links for author namespace from trunk r8430<commit_after># -*- coding: utf-8  -*-
from pywikibot import family

__version__ = '$Id$'

# The Wikimedia family that is known as Wikisource

class Family(family.Family):
    def __init__(self):
        family.Family.__init__(self)
        self.name = 'wikisource'

        self.languages_by_size = [
            'en', 'zh', 'ru', 'pt', 'fr', 'de', 'es', 'it', 'he', 'ar', 'hu',
            'fa', 'pl', 'th', 'cs', 'ro', 'hr', 'te', 'fi', 'bn', 'sv', 'tr',
            'nl', 'sl', 'vi', 'ko', 'uk', 'sr', 'ja', 'el', 'la', 'li', 'yi',
            'ml', 'az', 'is', 'bs', 'hy', 'ca', 'id', 'mk', 'ta', 'da', 'no',
            'et', 'bg', 'lt', 'gl', 'kn', 'cy', 'sk', 'zh-min-nan', 'fo',
        ]

        for lang in self.languages_by_size:
            self.langs[lang] = '%s.wikisource.org' % lang
            self.langs['-'] = 'wikisource.org'

        self.alphabetic = ['ang','ar','az','bg','bs','ca','cs','cy',
                      'da','de','el','en','es','et','fa','fi',
                      'fo','fr','gl','he','hr','ht','hu','id',
                      'is','it','ja', 'ko','la','lt','ml','nl',
                      'no','pl','pt','ro','ru','sk','sl','sr',
                      'sv','te','th','tr','uk','vi','yi','zh']

        self.obsolete = {
            'ang': None, # http://meta.wikimedia.org/wiki/Proposals_for_closing_projects/Closure_of_Old_English_Wikisource
            'dk': 'da',
            'ht': None, # http://meta.wikimedia.org/wiki/Proposals_for_closing_projects/Closure_of_Haitian_Creole_Wikisource
            'jp': 'ja',
            'minnan':'zh-min-nan',
            'nb': 'no',
            'tokipona': None,
            'zh-tw': 'zh',
            'zh-cn': 'zh'
        }

        self.interwiki_putfirst = {
            'en': self.alphabetic,
            'fi': self.alphabetic,
            'fr': self.alphabetic,
            'he': ['en'],
            'hu': ['en'],
            'pl': self.alphabetic,
            'simple': self.alphabetic
        }
        # Global bot allowed languages on http://meta.wikimedia.org/wiki/Bot_policy/Implementation#Current_implementation
        self.cross_allowed = [
            'el','fa','it','ko','no','vi','zh'
        ]
        # CentralAuth cross avaliable projects.
        self.cross_projects = [
            'wikipedia', 'wiktionary', 'wikibooks', 'wikiquote', 'wikinews', 'wikiversity',
            'meta', 'mediawiki', 'test', 'incubator', 'commons', 'species'
        ]

        self.authornamespaces = {
            '_default': [0],
            'ar': [102],
            'bg': [100],
            'cs': [100],
            'da': [102],
            'en': [102],
            'fa': [102],
            'fr': [102],
            'hr': [100],
            'hu': [100],
            'hy': [100],
            'it': [102],
            'ko': [100],
            'la': [102],
            'nl': [102],
            'no': [102],
            'pl': [104],
            'pt': [102],
            'sv': [106],
            'tr': [100],
            'vi': [102],
            'zh': [102],
            }

        self.crossnamespace[0] = {
            '_default': self.authornamespaces,
        }
        self.crossnamespace[100] = {
            'bg': self.authornamespaces,
            'cs': self.authornamespaces,
            'hr': self.authornamespaces,
            'hu': self.authornamespaces,
            'hy': self.authornamespaces,
            'ko': self.authornamespaces,
            'tr': self.authornamespaces,
        }

        self.crossnamespace[102] = {
            'ar': self.authornamespaces,
            'da': self.authornamespaces,
            'en': self.authornamespaces,
            'fa': self.authornamespaces,
            'fr': self.authornamespaces,
            'it': self.authornamespaces,
            'la': self.authornamespaces,
            'nl': self.authornamespaces,
            'no': self.authornamespaces,
            'pt': self.authornamespaces,
            'vi': self.authornamespaces,
            'zh': self.authornamespaces,
        }

        self.crossnamespace[104] = {
            'pl': self.authornamespaces,
        }

        self.crossnamespace[106] = {
            'sv': self.authornamespaces,
        }

    def version(self, code):
        return '1.16wmf4'

    def shared_image_repository(self, code):
        return ('commons', 'commons')
<|endoftext|>