<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals

from django.conf import settings
from django.contrib.auth.models import User
from django.db import models
from django.utils.timezone import now
from django.utils.encoding import python_2_unicode_compatible

from core.db import LogicalDeleteMixin
from book.models import Book
from qq.models import CheckinRecord

@python_2_unicode_compatible
class Member(LogicalDeleteMixin):
    DEFAULT_AVATAR = settings.DEFAULT_MEMBER_AVATAR

    user = models.OneToOneField(User, related_name='member')

    sn = models.IntegerField('编号', db_index=True)
    qq = models.TextField('QQ', blank=True)
    nick_name = models.TextField('昵称', db_index=True, blank=True)
    avatar = models.URLField('头像', blank=True, default=DEFAULT_AVATAR)
    description = models.TextField('个人介绍', blank=True, default='个人介绍')
    books = models.ManyToManyField(Book, related_name='readers', blank=True)

    created_at = models.DateTimeField(default=now)
    updated_at = models.DateTimeField(default=now)
    last_read_at = models.DateTimeField(
        '最后一次读书时间', null=True, blank=True
    )

    class Meta:
        verbose_name = 'member'
        verbose_name_plural = 'members'

    def __str__(self):
        return '【{0}】{1}'.format(self.sn, self.nick_name)

    def save(self, *args, **kwargs):
        from qq.utils import update_member_info, update_member_books
        if not self.avatar:
            self.avatar = self.DEFAULT_AVATAR

        self.updated_at = now()
        if not (self.pk and self.user):
            user = User.objects.filter(username=self.qq)
            if not user.exists():
                user = User.objects.create_user(self.qq)
                user.set_password(settings.DEFAULT_PASSWORD)
                user.save()
            else:
                user = user[0]
            self.user = user

        new = True if not self.pk else False
        value = super(Member, self).save(*args, **kwargs)
        self.update_qq_record()
        if new:
            update_member_info.delay(self.pk)
            update_member_books.delay(self.pk)

        return value

    def update_qq_record(self):
        CheckinRecord.raw_objects.filter(qq=self.qq).update(
            sn=self.sn, nick_name=self.nick_name
        )

@python_2_unicode_compatible
class NewMember(LogicalDeleteMixin):
    status_need = 0
    status_approve = 1
    status_disappreove = 2
    status_choices = (
        (0, '待处理'),
        (1, '接受'),
        (2, '不接受'),
    )

    sn = models.IntegerField('编号')
    qq = models.TextField('QQ')
    nick_name = models.TextField('昵称')
    description = models.TextField('个人介绍', blank=True, default='个人介绍')
    status = models.SmallIntegerField(choices=status_choices,
                                      default=status_need)

    created_at = models.DateTimeField(default=now)
    updated_at = models.DateTimeField(default=now)
    last_read_at = models.DateTimeField(
        '最后一次读书时间', null=True, blank=True
    )

    class Meta:
        verbose_name = 'new_member'
        verbose_name_plural = 'new_members'

    def __str__(self):
        return 'New Member: 【{0}】{1}'.format(self.sn, self.nick_name)

    def approve(self):
        from qq.utils import update_member_info, update_member_books
        m = Member.raw_objects.filter(qq=self.qq).first()
        if m is None:
            m = Member()
        m.sn = self.sn
        m.qq = self.qq
        m.nick_name = self.nick_name
        m.description = self.description
        m.last_read_at = self.last_read_at
        m.save()
        update_member_info.delay(m.pk)
        update_member_books.delay(m.pk)

        self.status = self.status_approve
        self.save()
        return m

    def disapprove(self):
        self.status = self.status_disappreove
        self.save()
<commit_msg>small adjust member.models<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals

from django.conf import settings
from django.contrib.auth.models import User
from django.db import models
from django.utils.timezone import now
from django.utils.encoding import python_2_unicode_compatible

from core.db import LogicalDeleteMixin
from book.models import Book
from qq.models import CheckinRecord

@python_2_unicode_compatible
class Member(LogicalDeleteMixin):
    DEFAULT_AVATAR = settings.DEFAULT_MEMBER_AVATAR

    user = models.OneToOneField(User, related_name='member')

    sn = models.IntegerField('编号', db_index=True)
    qq = models.TextField('QQ', blank=True)
    nick_name = models.TextField('昵称', db_index=True, blank=True)
    avatar = models.URLField('头像', blank=True, default=DEFAULT_AVATAR)
    description = models.TextField('个人介绍', blank=True, default='个人介绍')
    books = models.ManyToManyField(Book, related_name='readers', blank=True)

    created_at = models.DateTimeField(default=now)
    updated_at = models.DateTimeField(default=now)
    last_read_at = models.DateTimeField(
        '最后一次读书时间', null=True, blank=True
    )

    class Meta:
        verbose_name = 'member'
        verbose_name_plural = 'members'

    def __str__(self):
        return '【{0}】{1}'.format(self.sn, self.nick_name)

    def save(self, *args, **kwargs):
        from qq.utils import update_member_info, update_member_books
        if not self.avatar:
            self.avatar = self.DEFAULT_AVATAR

        self.updated_at = now()
        if not (self.pk and self.user):
            user = User.objects.filter(username=self.qq)
            if not user.exists():
                user = User.objects.create_user(self.qq)
                user.set_password(settings.DEFAULT_PASSWORD)
                user.save()
            else:
                user = user[0]
            self.user = user

        new = (not self.pk)
        value = super(Member, self).save(*args, **kwargs)
        self.update_qq_record()
        if new:
            update_member_info.delay(self.pk)
            update_member_books.delay(self.pk)

        return value

    def update_qq_record(self):
        CheckinRecord.raw_objects.filter(qq=self.qq).update(
            sn=self.sn, nick_name=self.nick_name
        )

@python_2_unicode_compatible
class NewMember(LogicalDeleteMixin):
    status_need = 0
    status_approve = 1
    status_disappreove = 2
    status_choices = (
        (status_need, '待处理'),
        (status_approve, '接受'),
        (status_disappreove, '不接受'),
    )

    sn = models.IntegerField('编号')
    qq = models.TextField('QQ')
    nick_name = models.TextField('昵称')
    description = models.TextField('个人介绍', blank=True, default='个人介绍')
    status = models.SmallIntegerField(choices=status_choices,
                                      default=status_need)

    created_at = models.DateTimeField(default=now)
    updated_at = models.DateTimeField(default=now)
    last_read_at = models.DateTimeField(
        '最后一次读书时间', null=True, blank=True
    )

    class Meta:
        verbose_name = 'new_member'
        verbose_name_plural = 'new_members'

    def __str__(self):
        return 'New Member: 【{0}】{1}'.format(self.sn, self.nick_name)

    def approve(self):
        from qq.utils import update_member_info, update_member_books
        m = Member.raw_objects.filter(qq=self.qq).first()
        if m is None:
            m = Member()
        m.sn = self.sn
        m.qq = self.qq
        m.nick_name = self.nick_name
        m.description = self.description
        m.last_read_at = self.last_read_at
        m.save()
        update_member_info.delay(m.pk)
        update_member_books.delay(m.pk)

        self.status = self.status_approve
        self.save()
        return m

    def disapprove(self):
        self.status = self.status_disappreove
        self.save()
<|endoftext|>


<commit_before><commit_msg>Add parquet reader<commit_after><|endoftext|>


<commit_before>import numpy as np
import itertools as it

from models.model import AbstractModel

from distribution_estimation.approximator import LaplaceApproximation
from distribution_estimation.sampler import MonteCarlo
from distribution_estimation.kernel_optimiser import HyperParameterOptimiser

n_classes = 10

class GaussianProcess(AbstractModel):
    def __init__(self, kernel='sqr_exp', distribution_estimation='analytic', num_classes=2):
        super(GaussianProcess, self).__init__('GaussianProcesses')
        self.n_classes = num_classes
        self.hyper_params = None
        self.kernel = self.build_kernel(kernel)
        self.cov_function = None
        self.mean_function = None
        self.latent_function = None
        if distribution_estimation == 'analytic':
            self.estimator = LaplaceApproximation()
        else:
            # TODO: try with a sampling procedure
            raise NotImplementedError
        self.sampler = MonteCarlo(num_sampling_steps=100)
        self.optimiser = HyperParameterOptimiser()
        self.num_restarts_fitting = 1

    def build_kernel(self, kernel='sqr_exp'):
        def squared_exponential(x_i, x_j, cls=0):
            d = x_i - x_j
            res = (self.hyper_params['sigma'][cls] ** 2) * \
                  np.exp(-0.5 * np.dot(d.T, d) / self.hyper_params['lambda'][cls])
            return res

        def linear(x_i, x_j, cls=0):
            return self.hyper_params['sigma'][cls] ** 2 + np.dot(x_i.T, x_j)

        def periodic(x_i, x_j, cls=0):
            d = np.sin(0.5 * (x_i - x_j))
            return np.exp(-2.0 * np.dot(d.T, d) / (self.hyper_params['lambda'][cls]) ** 2)

        if kernel == 'sqr_exp': # best for multi: 1 - 1,1 / 16,3, 16.4
            self.hyper_params = {'sigma': np.random.uniform(2.0, 2.1, size=n_classes if self.n_classes > 2 else 1),
                                 'lambda': np.random.uniform(10.3, 10.4, size=n_classes if self.n_classes > 2 else 1)}
            return squared_exponential
        elif kernel == 'lin':
            self.hyper_params = {'sigma': np.random.uniform(0.7, 1.3, size=n_classes if self.n_classes > 2 else 1)}
            return linear
        elif kernel == 'periodic':
            self.hyper_params = {'lambda': np.random.uniform(0.7, 1.3, size=n_classes if self.n_classes > 2 else 1)}
            return periodic
        else:
            raise NotImplementedError

    def build_cov_function(self, data=None, mode='symm'):
        if mode == 'hetero':
            print("\t\tComputing the covariance between training and test samples")
            num_train_samples = self.data['x_train'].shape[0]
            num_samples_new = data.shape[0]
            if self.n_classes > 2:
                cov_function = [np.zeros((num_samples_new, num_train_samples)) for _ in xrange(self.n_classes)]
                for c in xrange(self.n_classes):
                    for i in xrange(num_samples_new):
                        for j in xrange(num_train_samples):
                            cov_function[c][i, j] = self.kernel(data[i], self.data['x_train'][j], cls=c)
            else:
                cov_function = np.zeros((num_samples_new, num_train_samples))
                for i in xrange(num_samples_new):
                    for j in xrange(num_train_samples):
                        cov_function[i, j] = self.kernel(data[i], self.data['x_train'][j])

        elif mode == 'auto':
            print("\t\tComputing the covariance between test samples")
            if self.n_classes > 2:
                cov_function = [np.array([self.kernel(data[i], data[i], cls=c) for i in xrange(data.shape[0])])
                                for c in xrange(n_classes)]
            else:
                cov_function = np.array([self.kernel(data[i], data[i]) for i in xrange(data.shape[0])])

        elif mode == 'symm':
            print("\t\tComputing the covariance between training samples")
            num_samples = data.shape[0]
            upper_tri_ids = it.combinations(np.arange(num_samples), 2)
            if self.n_classes > 2:
                cov_function = [np.zeros((num_samples, num_samples)) for _ in xrange(n_classes)]
                # compute upper triangular submatrix and then the lower using the symmetry property
                for c in xrange(n_classes):
                    for i, j in upper_tri_ids:
                        cov_function[c][i, j] = self.kernel(data[i], data[j], cls=c)
                    cov_function[c] += cov_function[c].T
                    # compute the diagonal
                    for i in xrange(num_samples):
                        cov_function[c][i, i] = self.kernel(data[i], data[i], cls=c)
            else:
                cov_function = np.zeros((num_samples, num_samples))
                for i, j in upper_tri_ids:
                    cov_function[i, j] = self.kernel(data[i], data[j])
                cov_function += cov_function.T
                for i in xrange(num_samples):
                    cov_function[i, i] = self.kernel(data[i], data[i])
        else:
            raise ValueError

        return cov_function

    def set_gp_functions(self, latent_init=None, cov_init=None, mean_init=None):
        if cov_init is None:
            self.cov_function = self.build_cov_function(self.data['x_train'], mode='symm')
        else:
            self.cov_function = cov_init
        if latent_init is None:
            if self.n_classes > 2:
                self.latent_function = np.zeros((n_classes, self.num_samples))
            else:
                self.latent_function = np.zeros(self.num_samples)
        else:
            self.latent_function = latent_init
        if mean_init is None:
            if self.n_classes > 2:
                self.mean_function = np.zeros((n_classes, self.num_samples))
            else:
                self.mean_function = np.zeros(self.num_samples)
        else:
            self.mean_function = mean_init

    def update_hypers_and_functions(self, new_hypers):
        self.hyper_params = new_hypers
        self.cov_function = self.build_cov_function(self.data['x_train'], mode='symm')

    def fit(self, train_data, **kwargs):
        pass

    def predict(self, new_data, **kwargs):
        pass

class MulticlassGaussianProcess(GaussianProcess):
    def __init__(self, kernel='sqr_exp', distribution_estimation='analytic', classification_mode='mixed_binary'):
        super(MulticlassGaussianProcess, self).__init__(kernel=kernel,
                                                        distribution_estimation=distribution_estimation,
                                                        num_classes=n_classes)
        self.classification_mode = classification_mode   # can be 'multi' but it is not finished

    def _stratify_training_data(self, c, size=100):
        ids_of_class_c = np.arange(self.data['y_train'].shape[0])[self.data['y_train'] == c]
        ids_the_rest = np.arange(self.data['y_train'].shape[0])[self.data['y_train'] != c]
        # pick a random subset 50% of size from them
        size_of_class_c = size // 2
        subset_ids_of_class_c = np.random.choice(ids_of_class_c, size=size_of_class_c, replace=False)
        subset_ids_rest = np.random.choice(ids_the_rest, size=size - size_of_class_c, replace=False)
        shuffled_order = np.random.permutation(subset_ids_of_class_c.shape[0] + subset_ids_rest.shape[0])
        data_dict = {'x_train': np.vstack((self.data['x_train'][subset_ids_of_class_c],
                                           self.data['x_train'][subset_ids_rest]))[shuffled_order],
                     'y_train': np.hstack((self.data['y_train'][subset_ids_of_class_c],
                                           self.data['y_train'][subset_ids_rest]))[shuffled_order]}
        return data_dict

    def fit(self, train_data, **kwargs):
        """
        Apply multi-class Laplace approximation with a squared expoenntial kernel covariance function.
        Use a shared signal amplitude and length-scale factors sigma and l for all 10 classes (i.e. latent functions)
        and all 28*28 input dimensions.
        :param train_data:
        :param kwargs:
        :return:
        """
        self.data = train_data

        self.num_samples, dim_x, dim_y = self.data['x_train'].shape
        self.data['x_train'] = self.data['x_train'].reshape(self.num_samples, dim_x * dim_y)

        if self.classification_mode == 'multi':
            self.set_gp_functions()
            f_posterior, approx_log_marg_likelihood = \
                self.estimator.approximate_multiclass(self.cov_function,
                                                      self.data['y_train'],
                                                      latent_init=self.latent_function)
            # TODO: persist the posterior latent function if an improvement is observed
            # better_hypers = self.optimiser.optimise_hyper_params_multiclass(f_posterior)
            # cov_posterior, mean_posterior = self._recompute_mean_cov(better_hypers)
            # self._set_gp_functions(latent_init=f_posterior, cov_init=cov_posterior, mean_init=mean_posterior)
            self.latent_function = f_posterior
        elif self.classification_mode == 'mixed_binary':
            self.binary_gps = list()
            for cls in range(1, 2):
                train_data_for_c = self._stratify_training_data(c=cls, size=200)
                gp = BinaryGaussianProcessClassifier(class_positive=cls)
                gp.fit(train_data_for_c)
                self.binary_gps.append((cls, gp))
        else:
            raise ValueError

    def predict(self, new_data, **kwargs):
        if len(new_data.shape) == 3:
            num_samples, dim_x, dim_y = new_data.shape
            new_data = new_data.reshape(num_samples, dim_x * dim_y)
        else:
            num_samples = new_data.shape[0]

        if self.classification_mode == 'multi':
            cov_function_test = {'auto': self.build_cov_function(new_data, mode='auto'),
                                 'hetero': self.build_cov_function(new_data, mode='hetero')}
            latent_mean, latent_cov = self.estimator.compute_latent_mean_cov_multiclass(cov_matrix_train=self.cov_function,
                                                                                cov_matrix_test=cov_function_test,
                                                                                f_posterior=self.latent_function)
            predicted_class_probs = self.sampler.sample(latent_mean=latent_mean, latent_cov=latent_cov)
            return np.argmax(predicted_class_probs, axis=1)

        elif self.classification_mode == 'mixed_binary':
            predictions = np.zeros((num_samples, n_classes))
            for cls, gp in self.binary_gps:
                predictions[:, cls] = gp.predict(new_data, return_probs=True)
            prediction_classes = np.argmax(np.vstack((predictions[:, cls], 1-predictions[:, cls])).T, axis=1)
        else:
            raise ValueError

        return prediction_classes

class BinaryGaussianProcessClassifier(GaussianProcess):
    def __init__(self, class_positive, kernel='sqr_exp', distribution_estimation='analytic'):
        super(BinaryGaussianProcessClassifier, self).__init__(kernel=kernel,
                                                              distribution_estimation=distribution_estimation,
                                                              num_classes=2)
        self.class_positive = class_positive

    def fit(self, train_data, **kwargs):
        self.data = train_data
        if len(self.data['x_train'].shape) == 3:
            self.num_samples, dim_x, dim_y = train_data.shape
            self.data = train_data.reshape(self.num_samples, dim_x * dim_y)
        self.num_samples = self.data['x_train'].shape[0]

        self.set_gp_functions()
        for i in range(1):
            f_posterior, a, approx_log_marg_likelihood = \
                self.estimator.approximate_binary(self.cov_function, self.data['y_train'],
                                                  latent_init=self.latent_function, cls=self.class_positive)
            # better_hypers = self.optimiser.optimise_hyper_params_binary(self.cov_function,
            #                                                             f_posterior,
            #                                                             a, self.data['y_train'],
            #                                                             self.hyper_params, self.class_positive)
            # self.hyper_params = better_hypers
            # self.set_gp_functions(latent_init=f_posterior)  # it also recomputes the new covariance
        self.latent_function = f_posterior

    def predict(self, new_data, **kwargs):
        # extend the covariance function
        cov_function_test = {'auto': self.build_cov_function(new_data, mode='auto'),
                             'hetero': self.build_cov_function(new_data, mode='hetero')}
        latent_mean, latent_cov = self.estimator.compute_latent_mean_cov_binary(cov_matrix_train=self.cov_function,
                                                                                cov_matrix_test=cov_function_test,
                                                                                f_posterior=self.latent_function)
        predict_probs = kwargs.get('return_probs', False)
        predicted_class_probs = self.estimator.approximate_posterior_integral(latent_mean=latent_mean,
                                                                              latent_cov=latent_cov)
        if predict_probs:
            return predicted_class_probs
        else:
            return (predicted_class_probs > 0.5).astype(np.int32)    # return +1 for class_one and 0 otherwise

if __name__ == "__main__":
    from utils.data_utils import load_MNIST

    data_train, data_test = load_MNIST(num_training=10000, num_validation=100)

    model = MulticlassGaussianProcess(classification_mode='mixed_binary')

    model.fit(data_train)

    predictions = model.predict(data_train['x_val'])
    print(predictions)
    print(data_train['y_val'])
    test_acc = np.sum(predictions == data_train['y_val']) / float(predictions.shape[0]) * 100.
    print("Validation accuracy: {0}"
          .format(test_acc))
<commit_msg>checkpoint: mixed binary gps ~92% valacc<commit_after>import numpy as np
import itertools as it

from models.model import AbstractModel

from distribution_estimation.approximator import LaplaceApproximation
from distribution_estimation.sampler import MonteCarlo
from distribution_estimation.kernel_optimiser import HyperParameterOptimiser

n_classes = 10

class GaussianProcess(AbstractModel):
    def __init__(self, kernel='sqr_exp', distribution_estimation='analytic', num_classes=2):
        super(GaussianProcess, self).__init__('GaussianProcesses')
        self.n_classes = num_classes
        self.hyper_params = None
        self.kernel = self.build_kernel(kernel)
        self.cov_function = None
        self.mean_function = None
        self.latent_function = None
        if distribution_estimation == 'analytic':
            self.estimator = LaplaceApproximation()
        else:
            # TODO: try with a sampling procedure
            raise NotImplementedError
        self.sampler = MonteCarlo(num_sampling_steps=100)
        self.optimiser = HyperParameterOptimiser()
        self.num_restarts_fitting = 1

    def build_kernel(self, kernel='sqr_exp'):
        def squared_exponential(x_i, x_j, cls=0):
            d = x_i - x_j
            res = (self.hyper_params['sigma'][cls] ** 2) * \
                  np.exp(-0.5 * np.dot(d.T, d) / self.hyper_params['lambda'][cls])
            return res

        def linear(x_i, x_j, cls=0):
            return self.hyper_params['sigma'][cls] ** 2 + np.dot(x_i.T, x_j)

        def periodic(x_i, x_j, cls=0):
            d = np.sin(0.5 * (x_i - x_j))
            return np.exp(-2.0 * np.dot(d.T, d) / (self.hyper_params['lambda'][cls]) ** 2)

        if kernel == 'sqr_exp': # best for multi: 1 - 1,1 / 16,3, 16.4
            self.hyper_params = {'sigma': np.random.uniform(2.0, 2.1, size=n_classes if self.n_classes > 2 else 1),
                                 'lambda': np.random.uniform(10.3, 10.4, size=n_classes if self.n_classes > 2 else 1)}
            return squared_exponential
        elif kernel == 'lin':
            self.hyper_params = {'sigma': np.random.uniform(0.7, 1.3, size=n_classes if self.n_classes > 2 else 1)}
            return linear
        elif kernel == 'periodic':
            self.hyper_params = {'lambda': np.random.uniform(0.7, 1.3, size=n_classes if self.n_classes > 2 else 1)}
            return periodic
        else:
            raise NotImplementedError

    def build_cov_function(self, data=None, mode='symm'):
        if mode == 'hetero':
            print("\t\tComputing the covariance between training and test samples")
            num_train_samples = self.data['x_train'].shape[0]
            num_samples_new = data.shape[0]
            if self.n_classes > 2:
                cov_function = [np.zeros((num_samples_new, num_train_samples)) for _ in xrange(self.n_classes)]
                for c in xrange(self.n_classes):
                    for i in xrange(num_samples_new):
                        for j in xrange(num_train_samples):
                            cov_function[c][i, j] = self.kernel(data[i], self.data['x_train'][j], cls=c)
            else:
                cov_function = np.zeros((num_samples_new, num_train_samples))
                for i in xrange(num_samples_new):
                    for j in xrange(num_train_samples):
                        cov_function[i, j] = self.kernel(data[i], self.data['x_train'][j])

        elif mode == 'auto':
            print("\t\tComputing the covariance between test samples")
            if self.n_classes > 2:
                cov_function = [np.array([self.kernel(data[i], data[i], cls=c) for i in xrange(data.shape[0])])
                                for c in xrange(n_classes)]
            else:
                cov_function = np.array([self.kernel(data[i], data[i]) for i in xrange(data.shape[0])])

        elif mode == 'symm':
            print("\t\tComputing the covariance between training samples")
            num_samples = data.shape[0]
            upper_tri_ids = it.combinations(np.arange(num_samples), 2)
            if self.n_classes > 2:
                cov_function = [np.zeros((num_samples, num_samples)) for _ in xrange(n_classes)]
                # compute upper triangular submatrix and then the lower using the symmetry property
                for c in xrange(n_classes):
                    for i, j in upper_tri_ids:
                        cov_function[c][i, j] = self.kernel(data[i], data[j], cls=c)
                    cov_function[c] += cov_function[c].T
                    # compute the diagonal
                    for i in xrange(num_samples):
                        cov_function[c][i, i] = self.kernel(data[i], data[i], cls=c)
            else:
                cov_function = np.zeros((num_samples, num_samples))
                for i, j in upper_tri_ids:
                    cov_function[i, j] = self.kernel(data[i], data[j])
                cov_function += cov_function.T
                for i in xrange(num_samples):
                    cov_function[i, i] = self.kernel(data[i], data[i])
        else:
            raise ValueError

        return cov_function

    def set_gp_functions(self, latent_init=None, cov_init=None, mean_init=None):
        if cov_init is None:
            self.cov_function = self.build_cov_function(self.data['x_train'], mode='symm')
        else:
            self.cov_function = cov_init
        if latent_init is None:
            if self.n_classes > 2:
                self.latent_function = np.zeros((n_classes, self.num_samples))
            else:
                self.latent_function = np.zeros(self.num_samples)
        else:
            self.latent_function = latent_init
        if mean_init is None:
            if self.n_classes > 2:
                self.mean_function = np.zeros((n_classes, self.num_samples))
            else:
                self.mean_function = np.zeros(self.num_samples)
        else:
            self.mean_function = mean_init

    def update_hypers_and_functions(self, new_hypers):
        self.hyper_params = new_hypers
        self.cov_function = self.build_cov_function(self.data['x_train'], mode='symm')

    def fit(self, train_data, **kwargs):
        pass

    def predict(self, new_data, **kwargs):
        pass

class MulticlassGaussianProcess(GaussianProcess):
    def __init__(self, kernel='sqr_exp', distribution_estimation='analytic', classification_mode='mixed_binary'):
        super(MulticlassGaussianProcess, self).__init__(kernel=kernel,
                                                        distribution_estimation=distribution_estimation,
                                                        num_classes=n_classes)
        self.classification_mode = classification_mode   # can be 'multi' but it is not finished

    def _stratify_training_data(self, c, size=100):
        ids_of_class_c = np.arange(self.data['y_train'].shape[0])[self.data['y_train'] == c]
        ids_the_rest = np.arange(self.data['y_train'].shape[0])[self.data['y_train'] != c]
        # pick a random subset 50% of size from them
        size_of_class_c = size // 2
        subset_ids_of_class_c = np.random.choice(ids_of_class_c, size=size_of_class_c, replace=False)
        subset_ids_rest = np.random.choice(ids_the_rest, size=size - size_of_class_c, replace=False)
        shuffled_order = np.random.permutation(subset_ids_of_class_c.shape[0] + subset_ids_rest.shape[0])
        data_dict = {'x_train': np.vstack((self.data['x_train'][subset_ids_of_class_c],
                                           self.data['x_train'][subset_ids_rest]))[shuffled_order],
                     'y_train': np.hstack((self.data['y_train'][subset_ids_of_class_c],
                                           self.data['y_train'][subset_ids_rest]))[shuffled_order]}
        return data_dict

    def fit(self, train_data, **kwargs):
        """
        Apply multi-class Laplace approximation with a squared expoenntial kernel covariance function.
        Use a shared signal amplitude and length-scale factors sigma and l for all 10 classes (i.e. latent functions)
        and all 28*28 input dimensions.
        :param train_data:
        :param kwargs:
        :return:
        """
        self.data = train_data

        self.num_samples, dim_x, dim_y = self.data['x_train'].shape
        self.data['x_train'] = self.data['x_train'].reshape(self.num_samples, dim_x * dim_y)

        if self.classification_mode == 'multi':
            self.set_gp_functions()
            f_posterior, approx_log_marg_likelihood = \
                self.estimator.approximate_multiclass(self.cov_function,
                                                      self.data['y_train'],
                                                      latent_init=self.latent_function)
            # TODO: persist the posterior latent function if an improvement is observed
            # better_hypers = self.optimiser.optimise_hyper_params_multiclass(f_posterior)
            # cov_posterior, mean_posterior = self._recompute_mean_cov(better_hypers)
            # self._set_gp_functions(latent_init=f_posterior, cov_init=cov_posterior, mean_init=mean_posterior)
            self.latent_function = f_posterior
        elif self.classification_mode == 'mixed_binary':
            self.binary_gps = list()
            for cls in range(n_classes):
                train_data_for_c = self._stratify_training_data(c=cls, size=200)
                gp = BinaryGaussianProcessClassifier(class_positive=cls)
                gp.fit(train_data_for_c)
                self.binary_gps.append((cls, gp))
        else:
            raise ValueError

    def predict(self, new_data, **kwargs):
        if len(new_data.shape) == 3:
            num_samples, dim_x, dim_y = new_data.shape
            new_data = new_data.reshape(num_samples, dim_x * dim_y)
        else:
            num_samples = new_data.shape[0]

        if self.classification_mode == 'multi':
            cov_function_test = {'auto': self.build_cov_function(new_data, mode='auto'),
                                 'hetero': self.build_cov_function(new_data, mode='hetero')}
            latent_mean, latent_cov = self.estimator.compute_latent_mean_cov_multiclass(cov_matrix_train=self.cov_function,
                                                                                cov_matrix_test=cov_function_test,
                                                                                f_posterior=self.latent_function)
            predicted_class_probs = self.sampler.sample(latent_mean=latent_mean, latent_cov=latent_cov)
            return np.argmax(predicted_class_probs, axis=1)

        elif self.classification_mode == 'mixed_binary':
            predictions = np.zeros((num_samples, n_classes))
            for cls, gp in self.binary_gps:
                predictions[:, cls] = gp.predict(new_data, return_probs=True)
            prediction_classes = np.argmax(predictions, axis=1)
        else:
            raise ValueError

        return prediction_classes

class BinaryGaussianProcessClassifier(GaussianProcess):
    def __init__(self, class_positive, kernel='sqr_exp', distribution_estimation='analytic'):
        super(BinaryGaussianProcessClassifier, self).__init__(kernel=kernel,
                                                              distribution_estimation=distribution_estimation,
                                                              num_classes=2)
        self.class_positive = class_positive

    def fit(self, train_data, **kwargs):
        self.data = train_data
        if len(self.data['x_train'].shape) == 3:
            self.num_samples, dim_x, dim_y = train_data.shape
            self.data = train_data.reshape(self.num_samples, dim_x * dim_y)
        self.num_samples = self.data['x_train'].shape[0]

        self.set_gp_functions()
        for i in range(1):
            f_posterior, a, approx_log_marg_likelihood = \
                self.estimator.approximate_binary(self.cov_function, self.data['y_train'],
                                                  latent_init=self.latent_function, cls=self.class_positive)
            # better_hypers = self.optimiser.optimise_hyper_params_binary(self.cov_function,
            #                                                             f_posterior,
            #                                                             a, self.data['y_train'],
            #                                                             self.hyper_params, self.class_positive)
            # self.hyper_params = better_hypers
            # self.set_gp_functions(latent_init=f_posterior)  # it also recomputes the new covariance
        self.latent_function = f_posterior

    def predict(self, new_data, **kwargs):
        # extend the covariance function
        cov_function_test = {'auto': self.build_cov_function(new_data, mode='auto'),
                             'hetero': self.build_cov_function(new_data, mode='hetero')}
        latent_mean, latent_cov = self.estimator.compute_latent_mean_cov_binary(cov_matrix_train=self.cov_function,
                                                                                cov_matrix_test=cov_function_test,
                                                                                f_posterior=self.latent_function)
        predict_probs = kwargs.get('return_probs', False)
        predicted_class_probs = self.estimator.approximate_posterior_integral(latent_mean=latent_mean,
                                                                              latent_cov=latent_cov)
        if predict_probs:
            return predicted_class_probs
        else:
            return (predicted_class_probs > 0.5).astype(np.int32)    # return +1 for class_one and 0 otherwise

if __name__ == "__main__":
    from utils.data_utils import load_MNIST

    data_train, data_test = load_MNIST(num_training=10000, num_validation=100)

    model = MulticlassGaussianProcess(classification_mode='mixed_binary')

    model.fit(data_train)

    predictions = model.predict(data_train['x_val'])
    print(predictions)
    print(data_train['y_val'])
    test_acc = np.sum(predictions == data_train['y_val']) / float(predictions.shape[0]) * 100.
    print("Validation accuracy: {0}"
          .format(test_acc))
<|endoftext|>


<commit_before># @ 2019 Akretion - www.akretion.com.br -
#   Magno Costa <magno.costa@akretion.com.br>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

from lxml import etree
from odoo.tests import SavepointCase

from odoo.addons.l10n_br_fiscal.constants.fiscal import (
    TAX_FRAMEWORK_SIMPLES,
    TAX_FRAMEWORK_SIMPLES_ALL,
    TAX_FRAMEWORK_NORMAL,
    CFOP_DESTINATION_INTERNAL,
    CFOP_DESTINATION_EXTERNAL,
    DOCUMENT_ISSUER_PARTNER,
)

class L10nBrPurchaseBaseTest(SavepointCase):

    @classmethod
    def setUpClass(self):
        super().setUpClass()
        self.main_company = self.env.ref('base.main_company')
        self.company = self.env.ref('base.main_company')
        self.po_products = self.env.ref(
            'l10n_br_purchase.main_po_only_products')
        # self.po_services = self.env.ref(
        #     'l10n_br_purchase.main_po_only_services')
        # self.po_prod_srv = self.env.ref(
        #     'l10n_br_purchase.main_po_product_service')
        self.fsc_op_purchase = self.env.ref('l10n_br_fiscal.fo_compras')
        self.fsc_op_line_purchase = self.env.ref(
            'l10n_br_fiscal.fo_compras_compras')
        self.fsc_op_line_purchase_resale = self.env.ref(
            'l10n_br_fiscal.fo_compras_compras_comercializacao')
        self.fsc_op_line_purchase_asset = self.env.ref(
            'l10n_br_fiscal.fo_compras_compras_ativo')
        self.fsc_op_line_purchase_use = self.env.ref(
            'l10n_br_fiscal.fo_compras_compras_uso_consumo')

        self.fsc_op_line_dist = self.env.ref(
            'l10n_br_fiscal.fo_compras_compras')
        self.fsc_op_line_serv = self.env.ref(
            'l10n_br_fiscal.fo_venda_servico')

        TAXES_NORMAL = {
            'icms': {
                'tax': self.env.ref('l10n_br_fiscal.tax_icms_12'),
                'cst': self.env.ref('l10n_br_fiscal.cst_icms_00'),
            },
            'issqn': {
                'tax': self.env.ref('l10n_br_fiscal.tax_issqn_5'),
            },
            'ipi': {
                'tax': self.env.ref('l10n_br_fiscal.tax_ipi_5'),
                'cst': self.env.ref('l10n_br_fiscal.cst_ipi_00'),
            },
            'pis': {
                'tax': self.env.ref('l10n_br_fiscal.tax_pis_0_65'),
                'cst': self.env.ref('l10n_br_fiscal.cst_pis_50'),
            },
            'cofins': {
                'tax': self.env.ref('l10n_br_fiscal.tax_cofins_3'),
                'cst': self.env.ref('l10n_br_fiscal.cst_pis_50'),
            },
            'icmsfcp': {
                'tax': False,
                'cst': False,
            },
        }

        TAXES_SIMPLES = {
            'icms': {
                'tax': self.env.ref('l10n_br_fiscal.tax_icms_sn_com_credito'),
                'cst': self.env.ref('l10n_br_fiscal.cst_icmssn_101'),
            },
            'issqn': {
                'tax': self.env.ref('l10n_br_fiscal.tax_issqn_5'),
            },
            'ipi': {
                'tax': self.env.ref('l10n_br_fiscal.tax_ipi_outros'),
                'cst': self.env.ref('l10n_br_fiscal.cst_ipi_49'),
            },
            'pis': {
                'tax': self.env.ref('l10n_br_fiscal.tax_pis_outros'),
                'cst': self.env.ref('l10n_br_fiscal.cst_pis_98'),
            },
            'cofins': {
                'tax': self.env.ref(
                    'l10n_br_fiscal.tax_cofins_outros'),
                'cst': self.env.ref('l10n_br_fiscal.cst_cofins_98'),
            },
            'icmsfcp': {
                'tax': False,
                'cst': False,
            },
        }

        self.FISCAL_DEFS = {
            CFOP_DESTINATION_INTERNAL: {
                self.fsc_op_line_purchase.name: {
                    'cfop': self.env.ref('l10n_br_fiscal.cfop_1101'),
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
                self.fsc_op_line_purchase_resale.name: {
                    'cfop': self.env.ref('l10n_br_fiscal.cfop_1102'),
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
                self.fsc_op_line_purchase_asset.name: {
                    'cfop': self.env.ref('l10n_br_fiscal.cfop_1551'),
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
                self.fsc_op_line_purchase_use.name: {
                    'cfop': self.env.ref('l10n_br_fiscal.cfop_1556'),
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
            },
            CFOP_DESTINATION_EXTERNAL: {
                self.fsc_op_line_purchase.name: {
                    'cfop': self.env.ref('l10n_br_fiscal.cfop_2101'),
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
                self.fsc_op_line_purchase_resale.name: {
                    'cfop': self.env.ref('l10n_br_fiscal.cfop_2102'),
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
                self.fsc_op_line_purchase_asset.name: {
                    'cfop': self.env.ref('l10n_br_fiscal.cfop_2551'),
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
                self.fsc_op_line_purchase_use.name: {
                    'cfop': self.env.ref('l10n_br_fiscal.cfop_2556'),
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
            },
            'service': {
                self.fsc_op_line_serv.name: {
                    'cfop': False,
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
            },
        }

    def _change_user_company(self, company):
        self.env.user.company_ids += company
        self.env.user.company_id = company

    def _run_purchase_order_onchanges(self, purchase_order):
        purchase_order.onchange_partner_id()
        purchase_order._onchange_fiscal_operation_id()

    def _run_purchase_line_onchanges(self, purchase_line):
        purchase_line._onchange_product_id_fiscal()
        purchase_line._onchange_fiscal_operation_id()
        purchase_line._onchange_fiscal_operation_line_id()
        purchase_line._onchange_fiscal_taxes()

    def _invoice_purchase_order(self, order):
        order.with_context(tracking_disable=True).button_confirm()

        invoice_values = {
            'partner_id': order.partner_id.id,
            'purchase_id': order.id,
            'account_id': order.partner_id.property_account_payable_id.id,
            'type': 'in_invoice',
        }

        invoice_values.update(order._prepare_br_fiscal_dict())
        document_type_id = order._context.get('document_type_id')

        if document_type_id:
            document_type = self.env['l10n_br_fiscal.document.type'].browse(
                document_type_id)
        else:
            document_type = order.company_id.document_type_id
            document_type_id = order.company_id.document_type_id.id

        document_serie = document_type.get_document_serie(
            order.company_id, order.fiscal_operation_id)

        invoice_values['document_serie_id'] = document_serie.id
        invoice_values['document_type_id'] = document_type_id
        invoice_values['issuer'] = DOCUMENT_ISSUER_PARTNER
        self.invoice = self.env['account.invoice'].with_context(
            tracking_disable=True).create(invoice_values)
        self.invoice.purchase_order_change()

        self.assertEqual(
            order.order_line.mapped('qty_invoiced'), [4.0, 2.0],
            'Purchase: all products should be invoiced"'
        )

        self.assertEquals(
            order.state, 'purchase', 'Error to confirm Purchase Order.'
        )

        for invoice in order.invoice_ids:
            self.assertTrue(
                invoice.fiscal_operation_id,
                "Error to included Operation on invoice "
                "dictionary from Purchase Order.",
            )

            self.assertTrue(
                invoice.fiscal_operation_type,
                "Error to included Operation Type on invoice"
                " dictionary from Purchase Order.",
            )

            for line in invoice.invoice_line_ids:
                self.assertTrue(
                    line.fiscal_operation_line_id,
                    "Error to included Operation "
                    "Line from Purchase Order Line.",
                )

        self.invoice_action = order.action_view_invoice()
        self.assertTrue(
            self.invoice_action,
            "Error opening invoice !",
        )

    def test_l10n_br_purchase_products(self):
        """Test brazilian Purchase Order with only Products."""
        self._change_user_company(self.company)
        self._run_purchase_order_onchanges(self.po_products)
        self.assertTrue(
            self.po_products.fiscal_operation_id,
            'Error to mapping Operation on Purchase Order.',
        )

        self.assertEquals(
            self.po_products.fiscal_operation_id.name,
            self.fsc_op_purchase.name,
            "Error to mapping correct Operation on Purchase Order "
            "after change fiscal category.",
        )

        for line in self.po_products.order_line:
            self._run_purchase_line_onchanges(line)

            self.assertTrue(
                line.fiscal_operation_id,
                "Error to mapping Fiscal Operation on Purchase Order Line.",
            )

            self.assertTrue(
                line.fiscal_operation_line_id,
                "Error to mapping Fiscal Operation"
                " Line on Purchase Order Line.",
            )

            cfop = self.FISCAL_DEFS[line.cfop_id.destination][
                line.fiscal_operation_line_id.name]['cfop']

            taxes = self.FISCAL_DEFS[line.cfop_id.destination][
                line.fiscal_operation_line_id.name][
                    line.company_id.tax_framework]

            self.assertEquals(
                line.cfop_id.code, cfop.code,
                "Error to mapping CFOP {} for {}.".format(cfop.code, cfop.name)
            )

            if line.company_id.tax_framework in TAX_FRAMEWORK_SIMPLES_ALL:
                icms_tax = line.icmssn_tax_id
            else:
                icms_tax = line.icms_tax_id

            # ICMS
            self.assertEquals(
                icms_tax.name, taxes['icms']['tax'].name,
                "Error to mapping Tax {} for {}.".format(
                    taxes['icms']['tax'].name,
                    line.fiscal_operation_line_id.name)
            )

            self.assertEquals(
                line.icms_cst_id.code, taxes['icms']['cst'].code,
                "Error to mapping CST {} from {} for {}.".format(
                    taxes['icms']['cst'].code,
                    taxes['icms']['tax'].name,
                    line.fiscal_operation_line_id.name)
                )

            # ICMS FCP
            self.assertFalse(
                line.icmsfcp_tax_id,
                "Error to mapping ICMS FCP 2%"
                " for Venda de Contribuinte Dentro do Estado.")

            # IPI
            self.assertEquals(
                line.ipi_tax_id.name, taxes['ipi']['tax'].name,
                "Error to mapping Tax {} for {}.".format(
                    taxes['ipi']['tax'].name,
                    line.fiscal_operation_line_id.name)
            )

            self.assertEquals(
                line.ipi_cst_id.code, taxes['ipi']['cst'].code,
                "Error to mapping CST {} from {} for {}.".format(
                    taxes['ipi']['cst'].code,
                    taxes['ipi']['tax'].name,
                    line.fiscal_operation_line_id.name)
            )

            # PIS
            self.assertEquals(
                line.pis_tax_id.name, taxes['pis']['tax'].name,
                "Error to mapping Tax {} for {}.".format(
                    taxes['pis']['tax'].name,
                    line.fiscal_operation_line_id.name)
            )

            self.assertEquals(
                line.pis_cst_id.code, taxes['pis']['cst'].code,
                "Error to mapping CST {} from {} for {}.".format(
                    taxes['pis']['cst'].code,
                    taxes['pis']['tax'].name,
                    line.fiscal_operation_line_id.name)
            )

            # COFINS
            self.assertEquals(
                line.cofins_tax_id.name, taxes['cofins']['tax'].name,
                "Error to mapping Tax {} for {}.".format(
                    taxes['cofins']['tax'].name,
                    line.fiscal_operation_line_id.name)
            )

            self.assertEquals(
                line.cofins_cst_id.code, taxes['cofins']['cst'].code,
                "Error to mapping CST {} from {} for {}.".format(
                    taxes['cofins']['cst'].code,
                    taxes['cofins']['tax'].name,
                    line.fiscal_operation_line_id.name)
            )

        self._invoice_purchase_order(self.po_products)
        self._change_user_company(self.main_company)

    def test_fields_view_get(self):
        """Test Purchase Order fields_view_get."""
        view_arch = etree.fromstring(
            self.po_products.fields_view_get()["arch"])

        self.assertTrue(
            view_arch.findall(".//field[@name='fiscal_operation_id']"),
            "Error to included Operation "
            "Line from Purchase Order Line.",
        )
<commit_msg>lint: sed -i 's/assertEquals/assertEqual/g'<commit_after># @ 2019 Akretion - www.akretion.com.br -
#   Magno Costa <magno.costa@akretion.com.br>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

from lxml import etree
from odoo.tests import SavepointCase

from odoo.addons.l10n_br_fiscal.constants.fiscal import (
    TAX_FRAMEWORK_SIMPLES,
    TAX_FRAMEWORK_SIMPLES_ALL,
    TAX_FRAMEWORK_NORMAL,
    CFOP_DESTINATION_INTERNAL,
    CFOP_DESTINATION_EXTERNAL,
    DOCUMENT_ISSUER_PARTNER,
)

class L10nBrPurchaseBaseTest(SavepointCase):

    @classmethod
    def setUpClass(self):
        super().setUpClass()
        self.main_company = self.env.ref('base.main_company')
        self.company = self.env.ref('base.main_company')
        self.po_products = self.env.ref(
            'l10n_br_purchase.main_po_only_products')
        # self.po_services = self.env.ref(
        #     'l10n_br_purchase.main_po_only_services')
        # self.po_prod_srv = self.env.ref(
        #     'l10n_br_purchase.main_po_product_service')
        self.fsc_op_purchase = self.env.ref('l10n_br_fiscal.fo_compras')
        self.fsc_op_line_purchase = self.env.ref(
            'l10n_br_fiscal.fo_compras_compras')
        self.fsc_op_line_purchase_resale = self.env.ref(
            'l10n_br_fiscal.fo_compras_compras_comercializacao')
        self.fsc_op_line_purchase_asset = self.env.ref(
            'l10n_br_fiscal.fo_compras_compras_ativo')
        self.fsc_op_line_purchase_use = self.env.ref(
            'l10n_br_fiscal.fo_compras_compras_uso_consumo')

        self.fsc_op_line_dist = self.env.ref(
            'l10n_br_fiscal.fo_compras_compras')
        self.fsc_op_line_serv = self.env.ref(
            'l10n_br_fiscal.fo_venda_servico')

        TAXES_NORMAL = {
            'icms': {
                'tax': self.env.ref('l10n_br_fiscal.tax_icms_12'),
                'cst': self.env.ref('l10n_br_fiscal.cst_icms_00'),
            },
            'issqn': {
                'tax': self.env.ref('l10n_br_fiscal.tax_issqn_5'),
            },
            'ipi': {
                'tax': self.env.ref('l10n_br_fiscal.tax_ipi_5'),
                'cst': self.env.ref('l10n_br_fiscal.cst_ipi_00'),
            },
            'pis': {
                'tax': self.env.ref('l10n_br_fiscal.tax_pis_0_65'),
                'cst': self.env.ref('l10n_br_fiscal.cst_pis_50'),
            },
            'cofins': {
                'tax': self.env.ref('l10n_br_fiscal.tax_cofins_3'),
                'cst': self.env.ref('l10n_br_fiscal.cst_pis_50'),
            },
            'icmsfcp': {
                'tax': False,
                'cst': False,
            },
        }

        TAXES_SIMPLES = {
            'icms': {
                'tax': self.env.ref('l10n_br_fiscal.tax_icms_sn_com_credito'),
                'cst': self.env.ref('l10n_br_fiscal.cst_icmssn_101'),
            },
            'issqn': {
                'tax': self.env.ref('l10n_br_fiscal.tax_issqn_5'),
            },
            'ipi': {
                'tax': self.env.ref('l10n_br_fiscal.tax_ipi_outros'),
                'cst': self.env.ref('l10n_br_fiscal.cst_ipi_49'),
            },
            'pis': {
                'tax': self.env.ref('l10n_br_fiscal.tax_pis_outros'),
                'cst': self.env.ref('l10n_br_fiscal.cst_pis_98'),
            },
            'cofins': {
                'tax': self.env.ref(
                    'l10n_br_fiscal.tax_cofins_outros'),
                'cst': self.env.ref('l10n_br_fiscal.cst_cofins_98'),
            },
            'icmsfcp': {
                'tax': False,
                'cst': False,
            },
        }

        self.FISCAL_DEFS = {
            CFOP_DESTINATION_INTERNAL: {
                self.fsc_op_line_purchase.name: {
                    'cfop': self.env.ref('l10n_br_fiscal.cfop_1101'),
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
                self.fsc_op_line_purchase_resale.name: {
                    'cfop': self.env.ref('l10n_br_fiscal.cfop_1102'),
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
                self.fsc_op_line_purchase_asset.name: {
                    'cfop': self.env.ref('l10n_br_fiscal.cfop_1551'),
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
                self.fsc_op_line_purchase_use.name: {
                    'cfop': self.env.ref('l10n_br_fiscal.cfop_1556'),
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
            },
            CFOP_DESTINATION_EXTERNAL: {
                self.fsc_op_line_purchase.name: {
                    'cfop': self.env.ref('l10n_br_fiscal.cfop_2101'),
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
                self.fsc_op_line_purchase_resale.name: {
                    'cfop': self.env.ref('l10n_br_fiscal.cfop_2102'),
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
                self.fsc_op_line_purchase_asset.name: {
                    'cfop': self.env.ref('l10n_br_fiscal.cfop_2551'),
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
                self.fsc_op_line_purchase_use.name: {
                    'cfop': self.env.ref('l10n_br_fiscal.cfop_2556'),
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
            },
            'service': {
                self.fsc_op_line_serv.name: {
                    'cfop': False,
                    TAX_FRAMEWORK_SIMPLES: TAXES_SIMPLES,
                    TAX_FRAMEWORK_NORMAL: TAXES_NORMAL,
                },
            },
        }

    def _change_user_company(self, company):
        self.env.user.company_ids += company
        self.env.user.company_id = company

    def _run_purchase_order_onchanges(self, purchase_order):
        purchase_order.onchange_partner_id()
        purchase_order._onchange_fiscal_operation_id()

    def _run_purchase_line_onchanges(self, purchase_line):
        purchase_line._onchange_product_id_fiscal()
        purchase_line._onchange_fiscal_operation_id()
        purchase_line._onchange_fiscal_operation_line_id()
        purchase_line._onchange_fiscal_taxes()

    def _invoice_purchase_order(self, order):
        order.with_context(tracking_disable=True).button_confirm()

        invoice_values = {
            'partner_id': order.partner_id.id,
            'purchase_id': order.id,
            'account_id': order.partner_id.property_account_payable_id.id,
            'type': 'in_invoice',
        }

        invoice_values.update(order._prepare_br_fiscal_dict())
        document_type_id = order._context.get('document_type_id')

        if document_type_id:
            document_type = self.env['l10n_br_fiscal.document.type'].browse(
                document_type_id)
        else:
            document_type = order.company_id.document_type_id
            document_type_id = order.company_id.document_type_id.id

        document_serie = document_type.get_document_serie(
            order.company_id, order.fiscal_operation_id)

        invoice_values['document_serie_id'] = document_serie.id
        invoice_values['document_type_id'] = document_type_id
        invoice_values['issuer'] = DOCUMENT_ISSUER_PARTNER
        self.invoice = self.env['account.invoice'].with_context(
            tracking_disable=True).create(invoice_values)
        self.invoice.purchase_order_change()

        self.assertEqual(
            order.order_line.mapped('qty_invoiced'), [4.0, 2.0],
            'Purchase: all products should be invoiced"'
        )

        self.assertEqual(
            order.state, 'purchase', 'Error to confirm Purchase Order.'
        )

        for invoice in order.invoice_ids:
            self.assertTrue(
                invoice.fiscal_operation_id,
                "Error to included Operation on invoice "
                "dictionary from Purchase Order.",
            )

            self.assertTrue(
                invoice.fiscal_operation_type,
                "Error to included Operation Type on invoice"
                " dictionary from Purchase Order.",
            )

            for line in invoice.invoice_line_ids:
                self.assertTrue(
                    line.fiscal_operation_line_id,
                    "Error to included Operation "
                    "Line from Purchase Order Line.",
                )

        self.invoice_action = order.action_view_invoice()
        self.assertTrue(
            self.invoice_action,
            "Error opening invoice !",
        )

    def test_l10n_br_purchase_products(self):
        """Test brazilian Purchase Order with only Products."""
        self._change_user_company(self.company)
        self._run_purchase_order_onchanges(self.po_products)
        self.assertTrue(
            self.po_products.fiscal_operation_id,
            'Error to mapping Operation on Purchase Order.',
        )

        self.assertEqual(
            self.po_products.fiscal_operation_id.name,
            self.fsc_op_purchase.name,
            "Error to mapping correct Operation on Purchase Order "
            "after change fiscal category.",
        )

        for line in self.po_products.order_line:
            self._run_purchase_line_onchanges(line)

            self.assertTrue(
                line.fiscal_operation_id,
                "Error to mapping Fiscal Operation on Purchase Order Line.",
            )

            self.assertTrue(
                line.fiscal_operation_line_id,
                "Error to mapping Fiscal Operation"
                " Line on Purchase Order Line.",
            )

            cfop = self.FISCAL_DEFS[line.cfop_id.destination][
                line.fiscal_operation_line_id.name]['cfop']

            taxes = self.FISCAL_DEFS[line.cfop_id.destination][
                line.fiscal_operation_line_id.name][
                    line.company_id.tax_framework]

            self.assertEqual(
                line.cfop_id.code, cfop.code,
                "Error to mapping CFOP {} for {}.".format(cfop.code, cfop.name)
            )

            if line.company_id.tax_framework in TAX_FRAMEWORK_SIMPLES_ALL:
                icms_tax = line.icmssn_tax_id
            else:
                icms_tax = line.icms_tax_id

            # ICMS
            self.assertEqual(
                icms_tax.name, taxes['icms']['tax'].name,
                "Error to mapping Tax {} for {}.".format(
                    taxes['icms']['tax'].name,
                    line.fiscal_operation_line_id.name)
            )

            self.assertEqual(
                line.icms_cst_id.code, taxes['icms']['cst'].code,
                "Error to mapping CST {} from {} for {}.".format(
                    taxes['icms']['cst'].code,
                    taxes['icms']['tax'].name,
                    line.fiscal_operation_line_id.name)
                )

            # ICMS FCP
            self.assertFalse(
                line.icmsfcp_tax_id,
                "Error to mapping ICMS FCP 2%"
                " for Venda de Contribuinte Dentro do Estado.")

            # IPI
            self.assertEqual(
                line.ipi_tax_id.name, taxes['ipi']['tax'].name,
                "Error to mapping Tax {} for {}.".format(
                    taxes['ipi']['tax'].name,
                    line.fiscal_operation_line_id.name)
            )

            self.assertEqual(
                line.ipi_cst_id.code, taxes['ipi']['cst'].code,
                "Error to mapping CST {} from {} for {}.".format(
                    taxes['ipi']['cst'].code,
                    taxes['ipi']['tax'].name,
                    line.fiscal_operation_line_id.name)
            )

            # PIS
            self.assertEqual(
                line.pis_tax_id.name, taxes['pis']['tax'].name,
                "Error to mapping Tax {} for {}.".format(
                    taxes['pis']['tax'].name,
                    line.fiscal_operation_line_id.name)
            )

            self.assertEqual(
                line.pis_cst_id.code, taxes['pis']['cst'].code,
                "Error to mapping CST {} from {} for {}.".format(
                    taxes['pis']['cst'].code,
                    taxes['pis']['tax'].name,
                    line.fiscal_operation_line_id.name)
            )

            # COFINS
            self.assertEqual(
                line.cofins_tax_id.name, taxes['cofins']['tax'].name,
                "Error to mapping Tax {} for {}.".format(
                    taxes['cofins']['tax'].name,
                    line.fiscal_operation_line_id.name)
            )

            self.assertEqual(
                line.cofins_cst_id.code, taxes['cofins']['cst'].code,
                "Error to mapping CST {} from {} for {}.".format(
                    taxes['cofins']['cst'].code,
                    taxes['cofins']['tax'].name,
                    line.fiscal_operation_line_id.name)
            )

        self._invoice_purchase_order(self.po_products)
        self._change_user_company(self.main_company)

    def test_fields_view_get(self):
        """Test Purchase Order fields_view_get."""
        view_arch = etree.fromstring(
            self.po_products.fields_view_get()["arch"])

        self.assertTrue(
            view_arch.findall(".//field[@name='fiscal_operation_id']"),
            "Error to included Operation "
            "Line from Purchase Order Line.",
        )
<|endoftext|>


<commit_before>import collections

from urllib.parse import urljoin, urlparse
from io import BytesIO
import hashlib
import hmac
import json
import logging
from urllib.parse import parse_qs, quote

import requests
import grpc

from .grpcservice import service_pb2
from .grpcservice import service_pb2_grpc
from .grpcservice import generic_client_interceptor
from .utils import str2bool, ensure_binary

logger = logging.getLogger(__name__)

class SpiderTask:
    """
    Spider task description data fetched from server.
    """
    id = None
    spider_id = None
    project_name = None
    spider_name = None
    project_version = None
    spider_parameters = None
    extra_requirements = None
    settings = None
    figure = None
    package = None

class HearbeatResponse:
    def __init__(self, kill_jobs=None, has_new_job=None):
        self.kill_jobs = kill_jobs
        self.has_new_job = has_new_job

    @property
    def newJobAvailable(self):
        return self.has_new_job

    def __str__(self):
        return f'HearbeatResponse {{newJobAvailable: {self.newJobAvailable}}}'

class InsecureHeaderAuth(requests.auth.AuthBase):
    def __init__(self, node_id):
        self._node_id = str(node_id)

    def __call__(self, r):
        r.headers['X-Dd-Nodeid'] = self._node_id
        return r

def generate_digest(secret, method, path, query, body):
    parsed_query = parse_qs(query, keep_blank_values=True)

    canonical_query = []

    for key in sorted(parsed_query.keys()):
        for value in sorted(parsed_query[key]):
            canonical_query.append("=".join((key, quote(value))))
    body = ensure_binary(body)
    return hmac.new(
        secret.encode("utf-8"),
        "\n".join((method, path, "&".join(canonical_query), "")).encode(
            "utf-8") +
        body,
        hashlib.sha256).hexdigest()

class HmacAuth(requests.auth.AuthBase):
    def __init__(self, key, secret_key):
        self._key = key
        self._secret_key = secret_key

    def __call__(self, request):
        parsed_url = urlparse(request.url)
        path = parsed_url.path
        query = parsed_url.query
        method = request.method
        body = request.body or b''
        digest = generate_digest(self._secret_key, method, path, query,
                                 body)
        authorization_header = '%s %s %s' % ('HMAC', self._key, digest)
        request.headers['Authorization'] = authorization_header
        return request

class NodeRestClient:
    def __init__(self, base_url, auth=None):
        self._session = requests.Session()
        self._auth = auth
        self._session.auth = auth
        self._base_url = base_url
        self._node_id = None
        self._session_id = None

    def register_node(self, node_key):
        url = urljoin(self._base_url, '/nodes/register')
        post_data = {'node_key': node_key}
        res = self._session.post(url, data=post_data)
        res.raise_for_status()
        res_data = res.json()
        return res_data['id']

    def login(self):
        url = urljoin(self._base_url, '/api/nodeSessions')
        res = self._session.post(url)
        res.raise_for_status()
        res_data = res.json()
        self._session_id = res_data['id']
        self._node_id = res_data['node']['id']
        if self._auth is None:
            self._auth = InsecureHeaderAuth(self._node_id)
            self._session.auth = self._auth

    def heartbeat(self, running_job_ids=None):
        url = urljoin(self._base_url,
                      '/api/nodeSessions/%s:heartbeat' % self._session_id)
        running_tasks = ''
        if running_job_ids:
            running_tasks = ','.join(running_job_ids)
        response = self._session.post(url=url,
                                      data='',
                                      headers={
                                          'X-DD-RunningJobs': running_tasks})
        response.raise_for_status()
        new_job = str2bool(response.headers.get('X-DD-New-Task', 'false'))
        heartbeat_res = HearbeatResponse(has_new_job=new_job)
        return heartbeat_res

    def get_next_job(self):
        url = urljoin(self._base_url, '/executing/next_task')
        response = self._session.post(url, data={'node_id': self._node_id})
        response.raise_for_status()
        response_data = response.json()
        task = SpiderTask()
        task.id = response_data['data']['task']['task_id']
        task.spider_id = response_data['data']['task']['spider_id']
        task.project_name = response_data['data']['task']['project_name']
        task.project_version = response_data['data']['task']['version']
        task.spider_name = response_data['data']['task']['spider_name']
        if 'extra_requirements' in response_data['data']['task'] and \
                response_data['data']['task']['extra_requirements']:
            task.extra_requirements = response_data['data']['task'][
                'extra_requirements']
        if 'spider_parameters' in response_data['data']['task']:
            task.spider_parameters = response_data['data']['task'][
                'spider_parameters']
        else:
            task.spider_parameters = {}
        task.settings = response_data['data']
        if 'figure' in response_data['data']['task']:
            task.figure = response_data['data']['task']['figure']
        return task

    def get_job_egg(self, job_id):
        egg_request_url = urljoin(self._base_url, '/jobs/%s/egg' % job_id)
        response = self._session.get(egg_request_url)
        response.raise_for_status()
        return BytesIO(response.content)

    def complete_job(self, job_id, status='success', items_file=None,
                     logs_file=None):
        url = urljoin(self._base_url, '/executing/complete')
        post_data = {
            'task_id': job_id,
            'status': status,
        }
        post_files = {}
        f_items = None
        f_logs = None
        if items_file:
            f_items = open(items_file, 'rb')
            post_files['items'] = f_items
        if logs_file:
            f_logs = open(logs_file, 'rb')
            post_files['log'] = f_logs
        try:
            response = self._session.post(url, data=post_data,
                                          files=post_files)
            response.raise_for_status()
        finally:
            if f_items:
                f_items.close()
            if f_logs:
                f_logs.close()

class UsernamePasswordCallCredentials(grpc.AuthMetadataPlugin):
    def __init__(self, username):
        self._username = username

    def __call__(self, context, callback):
        metadata = (('x-custom-auth-ticket', self._username),)
        callback(metadata, None)

class NodeCallCredentials(grpc.AuthMetadataPlugin):
    def __init__(self, node_id):
        self._node_id = node_id

    def __call__(self, context, callback):
        metadata = (('x-node-id', str(self._node_id)),)
        logger.debug(metadata)
        callback(metadata, None)

class NodeExpiredException(Exception):
    pass

class LoginFailedException(Exception):
    pass

class _ClientCallDetails(
    collections.namedtuple(
        '_ClientCallDetails',
        ('method', 'timeout', 'metadata', 'credentials')),
    grpc.ClientCallDetails):
    pass

def header_adder_interceptor(header, value):
    def intercept_call(client_call_details, request_iterator,
                       request_streaming,
                       response_streaming):
        metadata = []
        if client_call_details.metadata is not None:
            metadata = list(client_call_details.metadata)
        metadata.append((
            header,
            value,
        ))
        client_call_details = _ClientCallDetails(
            client_call_details.method, client_call_details.timeout, metadata,
            client_call_details.credentials)
        return client_call_details, request_iterator, None

    return generic_client_interceptor.create(intercept_call)

class NodeGrpcClient:
    def __init__(self, base_url, server_cert=None, ca_cert=None, node_id=None):
        urlparts = urlparse(base_url)
        if urlparts.scheme == 'grpcs':
            use_tls = True
        else:
            use_tls = False
        host = urlparts.hostname
        port = urlparts.port
        logger.debug('node_id: %s', node_id)
        # logger.debug('token: %s', token)
        # with open('server.crt', 'rb') as f:
        #     server_cert = f.read()
        max_message_length = 100 * 1024 * 1024
        channel_opt = [('grpc.max_send_message_length', max_message_length),
                       ('grpc.max_receive_message_length', max_message_length)]

        if use_tls:
            credentials = grpc.ssl_channel_credentials(
                root_certificates=server_cert)
            call_creds = grpc.metadata_call_credentials(
                NodeCallCredentials(node_id))
            channel_creds = grpc.composite_channel_credentials(credentials,
                                                               call_creds)

            channel = grpc.secure_channel('{}:{}'.format(host, port),
                                          channel_creds, options=channel_opt)
        else:
            channel = grpc.insecure_channel('{}:{}'.format(host, port),
                                            options=channel_opt)

        self._channel = channel
        self._node_stub = service_pb2_grpc.NodeServiceStub(channel)

    def connect(self):
        pass

    def login(self):
        request = service_pb2.LoginRequest()
        try:
            response = self._node_stub.Login(request)
        except grpc.RpcError as ex:
            if ex.code() == grpc.StatusCode.UNAUTHENTICATED:
                raise LoginFailedException()
            raise
        if response.nodeId == 0:
            raise LoginFailedException()

        node_id = response.nodeId
        logger.info('node_id %s logged in.', node_id)

        self._node_id = node_id
        interceptor = header_adder_interceptor('x-node-id', str(node_id))
        self._channel = grpc.intercept_channel(self._channel, interceptor)
        self._node_stub = service_pb2_grpc.NodeServiceStub(self._channel)

    def heartbeat(self, running_job_ids: None):
        request = service_pb2.HeartbeatRequest()
        if running_job_ids:
            for running_job_id in running_job_ids:
                request.runningJobs.append(running_job_id)
        response = self._node_stub.Heartbeat(request)
        if response.nodeExpired:
            raise NodeExpiredException()
        logger.debug(f'HeartbeatResponse, nodeExpired:{response.nodeExpired}, '
                     f'killJobs:{response.killJobs}, newJobAvailable:{response.newJobAvailable}')
        return response

    def get_next_job(self):
        request = service_pb2.GetNextJobRequest()
        response = self._node_stub.GetNextJob(request)
        if not response.jobId:
            return None

        task = SpiderTask()
        task.id = response.jobId
        task.figure = json.loads(response.figure)
        task.package = BytesIO(response.package)
        return task

    def complete_job(self, job_id, status='success', items_file=None,
                     logs_file=None):
        request = service_pb2.CompleteJobRequest()
        request.jobId = job_id
        request.status = status
        if items_file:
            with open(items_file, 'rb') as f:
                request.items = f.read()

        if logs_file:
            with open(logs_file, 'rb') as f:
                request.logs = f.read()

        response = self._node_stub.CompleteJob(request)
        return response

    def close(self):
        self._channel = None

    def __del__(self):
        self._channel = None

def get_client(config, app_key=None, app_secret=None):
    server_url = config.get('server', 'http://localhost:6800')
    server_url_parts = urlparse(server_url)
    if not server_url_parts.scheme:
        auth = None
        if app_key and app_secret:
            auth = HmacAuth(app_key, app_secret)
        client = NodeRestClient('http://%s:6800' % server_url,
                                auth=auth)
        return client
    if server_url_parts.scheme in ('http', 'https'):
        auth = None
        if app_key and app_secret:
            auth = HmacAuth(app_key, app_secret)
        client = NodeRestClient(server_url, auth=auth)
        return client

    if server_url_parts.scheme in ('grpc', 'grpcs'):
        client = NodeGrpcClient(server_url)
        return client

    raise Exception('Unsupported scheme: %s' % server_url_parts.scheme)
<commit_msg>RestClient support login with tags<commit_after>import collections

from urllib.parse import urljoin, urlparse
from io import BytesIO
import hashlib
import hmac
import json
import logging
from urllib.parse import parse_qs, quote

import requests
import grpc

from .grpcservice import service_pb2
from .grpcservice import service_pb2_grpc
from .grpcservice import generic_client_interceptor
from .utils import str2bool, ensure_binary

logger = logging.getLogger(__name__)

class SpiderTask:
    """
    Spider task description data fetched from server.
    """
    id = None
    spider_id = None
    project_name = None
    spider_name = None
    project_version = None
    spider_parameters = None
    extra_requirements = None
    settings = None
    figure = None
    package = None

class HearbeatResponse:
    def __init__(self, kill_jobs=None, has_new_job=None):
        self.kill_jobs = kill_jobs
        self.has_new_job = has_new_job

    @property
    def newJobAvailable(self):
        return self.has_new_job

    def __str__(self):
        return f'HearbeatResponse {{newJobAvailable: {self.newJobAvailable}}}'

class InsecureHeaderAuth(requests.auth.AuthBase):
    def __init__(self, node_id):
        self._node_id = str(node_id)

    def __call__(self, r):
        r.headers['X-Dd-Nodeid'] = self._node_id
        return r

def generate_digest(secret, method, path, query, body):
    parsed_query = parse_qs(query, keep_blank_values=True)

    canonical_query = []

    for key in sorted(parsed_query.keys()):
        for value in sorted(parsed_query[key]):
            canonical_query.append("=".join((key, quote(value))))
    body = ensure_binary(body)
    return hmac.new(
        secret.encode("utf-8"),
        "\n".join((method, path, "&".join(canonical_query), "")).encode(
            "utf-8") +
        body,
        hashlib.sha256).hexdigest()

class HmacAuth(requests.auth.AuthBase):
    def __init__(self, key, secret_key):
        self._key = key
        self._secret_key = secret_key

    def __call__(self, request):
        parsed_url = urlparse(request.url)
        path = parsed_url.path
        query = parsed_url.query
        method = request.method
        body = request.body or b''
        digest = generate_digest(self._secret_key, method, path, query,
                                 body)
        authorization_header = '%s %s %s' % ('HMAC', self._key, digest)
        request.headers['Authorization'] = authorization_header
        return request

class NodeRestClient:
    def __init__(self, base_url, auth=None, tags=None):
        self._session = requests.Session()
        self._auth = auth
        self._session.auth = auth
        self._base_url = base_url
        self._node_id = None
        self._session_id = None
        self._tags = tags

    def register_node(self, node_key):
        url = urljoin(self._base_url, '/nodes/register')
        post_data = {'node_key': node_key}

        res = self._session.post(url, data=post_data)
        res.raise_for_status()
        res_data = res.json()
        return res_data['id']

    def login(self):
        url = urljoin(self._base_url, '/api/nodeSessions')
        post_data = {'tags': self._tags or ''}
        res = self._session.post(url, data=post_data)
        res.raise_for_status()
        res_data = res.json()
        self._session_id = res_data['id']
        self._node_id = res_data['node']['id']
        if self._auth is None:
            self._auth = InsecureHeaderAuth(self._node_id)
            self._session.auth = self._auth

    def heartbeat(self, running_job_ids=None):
        url = urljoin(self._base_url,
                      '/api/nodeSessions/%s:heartbeat' % self._session_id)
        running_tasks = ''
        if running_job_ids:
            running_tasks = ','.join(running_job_ids)
        response = self._session.post(url=url,
                                      data='',
                                      headers={
                                          'X-DD-RunningJobs': running_tasks})
        response.raise_for_status()
        new_job = str2bool(response.headers.get('X-DD-New-Task', 'false'))
        heartbeat_res = HearbeatResponse(has_new_job=new_job)
        return heartbeat_res

    def get_next_job(self):
        url = urljoin(self._base_url, '/executing/next_task')
        response = self._session.post(url, data={'node_id': self._node_id})
        response.raise_for_status()
        response_data = response.json()
        task = SpiderTask()
        task.id = response_data['data']['task']['task_id']
        task.spider_id = response_data['data']['task']['spider_id']
        task.project_name = response_data['data']['task']['project_name']
        task.project_version = response_data['data']['task']['version']
        task.spider_name = response_data['data']['task']['spider_name']
        if 'extra_requirements' in response_data['data']['task'] and \
                response_data['data']['task']['extra_requirements']:
            task.extra_requirements = response_data['data']['task'][
                'extra_requirements']
        if 'spider_parameters' in response_data['data']['task']:
            task.spider_parameters = response_data['data']['task'][
                'spider_parameters']
        else:
            task.spider_parameters = {}
        task.settings = response_data['data']
        if 'figure' in response_data['data']['task']:
            task.figure = response_data['data']['task']['figure']
        return task

    def get_job_egg(self, job_id):
        egg_request_url = urljoin(self._base_url, '/jobs/%s/egg' % job_id)
        response = self._session.get(egg_request_url)
        response.raise_for_status()
        return BytesIO(response.content)

    def complete_job(self, job_id, status='success', items_file=None,
                     logs_file=None):
        url = urljoin(self._base_url, '/executing/complete')
        post_data = {
            'task_id': job_id,
            'status': status,
        }
        post_files = {}
        f_items = None
        f_logs = None
        if items_file:
            f_items = open(items_file, 'rb')
            post_files['items'] = f_items
        if logs_file:
            f_logs = open(logs_file, 'rb')
            post_files['log'] = f_logs
        try:
            response = self._session.post(url, data=post_data,
                                          files=post_files)
            response.raise_for_status()
        finally:
            if f_items:
                f_items.close()
            if f_logs:
                f_logs.close()

class UsernamePasswordCallCredentials(grpc.AuthMetadataPlugin):
    def __init__(self, username):
        self._username = username

    def __call__(self, context, callback):
        metadata = (('x-custom-auth-ticket', self._username),)
        callback(metadata, None)

class NodeCallCredentials(grpc.AuthMetadataPlugin):
    def __init__(self, node_id):
        self._node_id = node_id

    def __call__(self, context, callback):
        metadata = (('x-node-id', str(self._node_id)),)
        logger.debug(metadata)
        callback(metadata, None)

class NodeExpiredException(Exception):
    pass

class LoginFailedException(Exception):
    pass

class _ClientCallDetails(
    collections.namedtuple(
        '_ClientCallDetails',
        ('method', 'timeout', 'metadata', 'credentials')),
    grpc.ClientCallDetails):
    pass

def header_adder_interceptor(header, value):
    def intercept_call(client_call_details, request_iterator,
                       request_streaming,
                       response_streaming):
        metadata = []
        if client_call_details.metadata is not None:
            metadata = list(client_call_details.metadata)
        metadata.append((
            header,
            value,
        ))
        client_call_details = _ClientCallDetails(
            client_call_details.method, client_call_details.timeout, metadata,
            client_call_details.credentials)
        return client_call_details, request_iterator, None

    return generic_client_interceptor.create(intercept_call)

class NodeGrpcClient:
    def __init__(self, base_url, server_cert=None, ca_cert=None, node_id=None):
        urlparts = urlparse(base_url)
        if urlparts.scheme == 'grpcs':
            use_tls = True
        else:
            use_tls = False
        host = urlparts.hostname
        port = urlparts.port
        logger.debug('node_id: %s', node_id)
        # logger.debug('token: %s', token)
        # with open('server.crt', 'rb') as f:
        #     server_cert = f.read()
        max_message_length = 100 * 1024 * 1024
        channel_opt = [('grpc.max_send_message_length', max_message_length),
                       ('grpc.max_receive_message_length', max_message_length)]

        if use_tls:
            credentials = grpc.ssl_channel_credentials(
                root_certificates=server_cert)
            call_creds = grpc.metadata_call_credentials(
                NodeCallCredentials(node_id))
            channel_creds = grpc.composite_channel_credentials(credentials,
                                                               call_creds)

            channel = grpc.secure_channel('{}:{}'.format(host, port),
                                          channel_creds, options=channel_opt)
        else:
            channel = grpc.insecure_channel('{}:{}'.format(host, port),
                                            options=channel_opt)

        self._channel = channel
        self._node_stub = service_pb2_grpc.NodeServiceStub(channel)

    def connect(self):
        pass

    def login(self):
        request = service_pb2.LoginRequest()
        try:
            response = self._node_stub.Login(request)
        except grpc.RpcError as ex:
            if ex.code() == grpc.StatusCode.UNAUTHENTICATED:
                raise LoginFailedException()
            raise
        if response.nodeId == 0:
            raise LoginFailedException()

        node_id = response.nodeId
        logger.info('node_id %s logged in.', node_id)

        self._node_id = node_id
        interceptor = header_adder_interceptor('x-node-id', str(node_id))
        self._channel = grpc.intercept_channel(self._channel, interceptor)
        self._node_stub = service_pb2_grpc.NodeServiceStub(self._channel)

    def heartbeat(self, running_job_ids: None):
        request = service_pb2.HeartbeatRequest()
        if running_job_ids:
            for running_job_id in running_job_ids:
                request.runningJobs.append(running_job_id)
        response = self._node_stub.Heartbeat(request)
        if response.nodeExpired:
            raise NodeExpiredException()
        logger.debug(f'HeartbeatResponse, nodeExpired:{response.nodeExpired}, '
                     f'killJobs:{response.killJobs}, newJobAvailable:{response.newJobAvailable}')
        return response

    def get_next_job(self):
        request = service_pb2.GetNextJobRequest()
        response = self._node_stub.GetNextJob(request)
        if not response.jobId:
            return None

        task = SpiderTask()
        task.id = response.jobId
        task.figure = json.loads(response.figure)
        task.package = BytesIO(response.package)
        return task

    def complete_job(self, job_id, status='success', items_file=None,
                     logs_file=None):
        request = service_pb2.CompleteJobRequest()
        request.jobId = job_id
        request.status = status
        if items_file:
            with open(items_file, 'rb') as f:
                request.items = f.read()

        if logs_file:
            with open(logs_file, 'rb') as f:
                request.logs = f.read()

        response = self._node_stub.CompleteJob(request)
        return response

    def close(self):
        self._channel = None

    def __del__(self):
        self._channel = None

def get_client(config, app_key=None, app_secret=None):
    server_url = config.get('server', 'http://localhost:6800')
    server_url_parts = urlparse(server_url)
    tags = config.get('tags', '')
    if not server_url_parts.scheme:
        auth = None
        if app_key and app_secret:
            auth = HmacAuth(app_key, app_secret)
        client = NodeRestClient('http://%s:6800' % server_url,
                                auth=auth, tags=tags)
        return client
    if server_url_parts.scheme in ('http', 'https'):
        auth = None
        if app_key and app_secret:
            auth = HmacAuth(app_key, app_secret)
        client = NodeRestClient(server_url, auth=auth, tags=tags)
        return client

    if server_url_parts.scheme in ('grpc', 'grpcs'):
        client = NodeGrpcClient(server_url)
        return client

    raise Exception('Unsupported scheme: %s' % server_url_parts.scheme)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.9.2 on 2016-03-19 04:54
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('reviews', '0008_auto_20160307_1659'),
    ]

    operations = [
        migrations.AlterField(
            model_name='review',
            name='score',
            field=models.SmallIntegerField(choices=[(2, '+1 (strong accept)'), (1, '+0 (weakly accept)'), (0, '----------'), (-1, '-0 (weakly reject)'), (-2, '-1 (strong reject)')], default=0, help_text='Your score to accept or reject this talk. More information about the scoring and acceptance criteria can be found at the google doc Review Guideline.', verbose_name='score'),
        ),
    ]
<commit_msg>Remove conflict migration<commit_after><|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
import json
import sys
from optparse import make_option

from django.core.management.base import BaseCommand, CommandError

from conference import utils
from conference import settings

class Command(BaseCommand):
    """
    """
    option_list = BaseCommand.option_list + (
        make_option('--count',
            action='store',
            dest='count',
            default=1,
            help='How many tweet retrieve',
        ),
        make_option('--output',
            action='store',
            dest='output',
            default=settings.LATEST_TWEETS_FILE,
            help='Write the output in this. If an error occurs output file is not overwritten. - means stdout',
        ),
    )
    def handle(self, *args, **options):
        try:
            screen_name = args[0]
        except IndexError:
            raise CommandError('screen_name missing')

        data = json.dumps(utils.latest_tweets(screen_name, count=options['count']))
        if data:
            if options['output'] in ('-', None):
                sys.stdout.write(data)
            else:
                file(options['output'], 'w').write(data)
<commit_msg>sigalarm per evitare di rimanere imprigionati dentro twitter<commit_after># -*- coding: UTF-8 -*-
import json
import signal
import sys
from optparse import make_option

from django.core.management.base import BaseCommand, CommandError

from conference import utils
from conference import settings

signal.alarm(5*60)

class Command(BaseCommand):
    """
    """
    option_list = BaseCommand.option_list + (
        make_option('--count',
            action='store',
            dest='count',
            default=1,
            help='How many tweet retrieve',
        ),
        make_option('--output',
            action='store',
            dest='output',
            default=settings.LATEST_TWEETS_FILE,
            help='Write the output in this. If an error occurs output file is not overwritten. - means stdout',
        ),
    )
    def handle(self, *args, **options):
        try:
            screen_name = args[0]
        except IndexError:
            raise CommandError('screen_name missing')

        data = json.dumps(utils.latest_tweets(screen_name, count=options['count']))
        if data:
            if options['output'] in ('-', None):
                sys.stdout.write(data)
            else:
                file(options['output'], 'w').write(data)
<|endoftext|>


<commit_before>__author__ = 'Robert Hayek and Keith Neyman'

from gopigo import *
import time

#Global Variable that says how far butler can get to a wall
STOP_DIST = 50

#print variables
ERROR = "ERROR"
VOLT = "HAZARDOUS VOLTAGE"
STOP = "STOPPING"
MOVE = "MOVING"

class Butler:

    ############
    ######## BASIC STATUS AND METHODS
    ############
    status = {'isMoving' : False, 'servo': 90, 'leftSpeed' : 175, 'rightSpeed' : 175, "distance" : 100}

    def __init__(self):
        print "BUTLERPI IS NOW ON"
        self.status['distance'] = us_dist(15) #update distance  with the current distance through ultrasonic sensor in (mm)
    def stop(self):
        self.status["isMoving"] = False
        print STOP #debugging message (delete later after)
        for x in range(3):
            stop()

    def fwd(self):
        self.isMoving = True
        print MOVE#debugging message (delete later)
        for x in range(3):
            fwd()
    #Check if conditions are safe for ButlerPi to continue
    def keepGoing(self):
        if self.status['distance'] < STOP_DIST:
            return False
        elif volt() > 14 or volt() < 6:
            print VOLT
            return False
        else:
            return True

    def checkDistance(self):
        self.status['distance'] = us_dist(15)
        print "CHECKING DISTANCE" + "SOMETHING IS " + str(self.status['distance']) + "mm away"

    def spin(self):
        right_rot()
        time.sleep(6)
        self.stop()

    ############
    ######## COMPLEX METHODS
    ############
    def dance(self):
        print "STARTING DANCE METHOD" #Dance Method
        self.spin()
        self.shuffle()
        self.shakeServo()
        self.rightTurn()
        self.leftTurn()
        self.led()

############
######## MAIN APP STARTS HERE
############
butler = Butler()
while butler.keepGoing():
    butler.checkDistance()
    butler.fwd()
    time.sleep(2)
    butler.stop()
butler.stop()
butler.checkDistance()
<commit_msg>Changed safe distance from 15mm to 50mm:<commit_after>__author__ = 'Robert Hayek and Keith Neyman'

from gopigo import *
import time

#Global Variable that says how far butler can get to a wall
STOP_DIST = 50

#print variables
ERROR = "ERROR"
VOLT = "HAZARDOUS VOLTAGE"
STOP = "STOPPING"
MOVE = "MOVING"

class Butler:

    ############
    ######## BASIC STATUS AND METHODS
    ############
    status = {'isMoving' : False, 'servo': 90, 'leftSpeed' : 175, 'rightSpeed' : 175, "distance" : 100}

    def __init__(self):
        print "BUTLERPI IS NOW ON"
        self.status['distance'] = us_dist(15) #update distance  with the current distance through ultrasonic sensor in (mm)
    def stop(self):
        self.status["isMoving"] = False
        print STOP #debugging message (delete later after)
        for x in range(3):
            stop()

    def fwd(self):
        self.isMoving = True
        print MOVE#debugging message (delete later)
        for x in range(3):
            fwd()
    #Check if conditions are safe for ButlerPi to continue
    def keepGoing(self):
        if self.status['distance'] < STOP_DIST:
            return False
        elif volt() > 14 or volt() < 6:
            print VOLT
            return False
        else:
            return True

    def checkDistance(self):
        self.status['distance'] = us_dist(50)
        print "CHECKING DISTANCE" + "SOMETHING IS " + str(self.status['distance']) + "mm away"

    def spin(self):
        right_rot()
        time.sleep(6)
        self.stop()

    ############
    ######## COMPLEX METHODS
    ############
    def dance(self):
        print "STARTING DANCE METHOD" #Dance Method
        self.spin()
        self.shuffle()
        self.shakeServo()
        self.rightTurn()
        self.leftTurn()
        self.led()

############
######## MAIN APP STARTS HERE
############
butler = Butler()
while butler.keepGoing():
    butler.checkDistance()
    butler.fwd()
    time.sleep(2)
    butler.stop()
butler.stop()
butler.checkDistance()
<|endoftext|>