<commit_before>from bottle import request, response, HTTPError
from webrecorder.basecontroller import BaseController

# ============================================================================
class InfoController(BaseController):
    def init_routes(self):
        @self.app.get(['/anonymous', '/anonymous/'])
        def anon_coll_info():
            user = self.get_session().anon_user

            return self.get_info(user, 'anonymous')

        @self.app.get(['/anonymous/<rec>', '/anonymous/<rec>/'])
        def anon_rec_info(rec):
            user = self.get_session().anon_user

            return self.get_info(user, 'anonymous', rec)

        @self.app.get(['/<user>/<coll>', '/<user>/<coll>/'])
        def coll_info(user, coll):

            return self.get_info(user, coll)

        @self.app.get(['/<user>/<coll>/<rec>', '/<user>/<coll>/<rec>/'])
        def rec_info(user, coll, rec):

            return self.get_info(user, coll, rec)

    def get_info(self, user, coll, rec=None):
        result = {}
        result['size_remaining'] = self.manager.get_size_remaining(user)
        result['collection'] = self.manager.get_collection(user, coll)
        if rec:
            result['recording'] = self.manager.get_recording(user, coll, rec)
            result['pages'] = self.manager.list_pages(user, coll, rec)

        return result

<commit_msg>infocontroller: disable general info paths for now to avoid edge case conflict (will revisit soon)<commit_after>from bottle import request, response, HTTPError
from webrecorder.basecontroller import BaseController

# ============================================================================
class InfoController(BaseController):
    def init_routes(self):
        @self.app.get(['/anonymous', '/anonymous/'])
        def anon_coll_info():
            user = self.get_session().anon_user

            return self.get_info(user, 'anonymous')

        @self.app.get(['/anonymous/<rec>', '/anonymous/<rec>/'])
        def anon_rec_info(rec):
            user = self.get_session().anon_user

            return self.get_info(user, 'anonymous', rec)

        #@self.app.get(['/<user>/<coll>', '/<user>/<coll>/'])
        def coll_info(user, coll):

            return self.get_info(user, coll)

        #@self.app.get(['/<user>/<coll>/<rec>', '/<user>/<coll>/<rec>/'])
        def rec_info(user, coll, rec):

            return self.get_info(user, coll, rec)

    def get_info(self, user, coll, rec=None):
        result = {}
        result['size_remaining'] = self.manager.get_size_remaining(user)
        result['collection'] = self.manager.get_collection(user, coll)
        if rec:
            result['recording'] = self.manager.get_recording(user, coll, rec)
            result['pages'] = self.manager.list_pages(user, coll, rec)

        return result

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.db.models import F
from django_filters import rest_framework as filters, STRICTNESS

from server.models import Event, Category, Reference, Tour, State

class ActivityFilter(filters.FilterSet):

    ACTIVITY_CHOICES = (
        ('tour', 'tour'),
        ('topic', 'topic'),
        ('collective', 'collective'),
        ('talk', 'talk')
    )

    DIVISION_CHOICES = (
        ('winter', 'winter'),
        ('summer', 'summer'),
        ('indoor', 'indoor'),
    )

    STATE_CHOICES = (
        ('done', 'done'),
        ('moved', 'moved'),
        ('canceled', 'canceled'),
        ('unfeasible', 'unfeasible'),
        ('public', 'public'),
    )

    # CATEGORY_CHOICES = [
    #    (c.lower(), c.lower()) for c in Category.objects
    #        .filter(deprecated=False, seasons__current=True)
    #        .exclude(deadline=True).exclude(preliminary=True)
    #        .order_by('code')
    #        .values_list('code', flat=True)
    # ]

    activity = filters.ChoiceFilter(label='activity', method='activity_filter', choices=ACTIVITY_CHOICES)
    division = filters.ChoiceFilter(label='division', method='division_filter', choices=DIVISION_CHOICES)
    category = filters.CharFilter(label='category', method='category_filter')  # , choices=CATEGORY_CHOICES)
    guide = filters.CharFilter(label='guide', method='guide_filter')
    month = filters.NumberFilter(label='month', method='month_filter', min_value=1, max_value=12)
    ladiesOnly = filters.BooleanFilter(label='ladiesOnly', method='ladies_only_filter')
    publicTransport = filters.BooleanFilter(label='publicTransport', method='public_transport_filter')
    lowEmissionAdventure = filters.BooleanFilter(label='lowEmissionAdventure', method='low_emission_adventure_filter')
    state = filters.ChoiceFilter(label='state', method='state_filter', choices=STATE_CHOICES)
    open =filters.BooleanFilter(label='open', method='open_filter')

    def activity_filter(self, queryset, name, value):
        if value in ("tour", "topic", "collective", "talk"):
            return queryset.filter(**{"reference__category__{}".format(value): True})

    def division_filter(self, queryset, name, value):
        if value in ("winter", "summer", "indoor"):
            return queryset.filter(**{"reference__category__{}".format(value): True})

    def category_filter(self, queryset, name, value):
        return queryset.filter(reference__category__code__iexact=value)

    def guide_filter(self, queryset, name, value):
        return queryset.filter(tour__guide__user__username__iexact=value)

    def month_filter(self, queryset, name, value):
        return queryset.filter(start_date__month=value)

    def ladies_only_filter(self, queryset, name, value):
        return queryset.filter(tour__ladies_only=value)

    def public_transport_filter(self, queryset, name, value):
        return queryset.filter(public_transport=value)

    def low_emission_adventure_filter(self, queryset, name, value):
        return queryset.filter(lea=value)

    def state_filter(self, queryset, name, value):
        return queryset.filter(**{"tour__state__{}".format(value): True})

    def open_filter(self, queryset, name, value):
        if value:
            return queryset.filter(tour__cur_quantity__lt=F('tour__max_quantity'))
        else:
            return queryset.filter(tour__cur_quantity__gte=F('tour__max_quantity'))

    class Meta:
        model = Event
        fields = (
            'activity',
            'division',
            'category',
            'guide',
            'month',
            'ladiesOnly',
            'publicTransport',
            'lowEmissionAdventure',
            'state',
            'open'
        )
        strict = STRICTNESS.RAISE_VALIDATION_ERROR
<commit_msg>Improve category filter for activities<commit_after># -*- coding: utf-8 -*-
from django.db.models import F, Q
from django_filters import rest_framework as filters, STRICTNESS

from server.models import Event, Category, Reference, Tour, State

class ActivityFilter(filters.FilterSet):

    ACTIVITY_CHOICES = (
        ('tour', 'tour'),
        ('topic', 'topic'),
        ('collective', 'collective'),
        ('talk', 'talk')
    )

    DIVISION_CHOICES = (
        ('winter', 'winter'),
        ('summer', 'summer'),
        ('indoor', 'indoor'),
    )

    STATE_CHOICES = (
        ('done', 'done'),
        ('moved', 'moved'),
        ('canceled', 'canceled'),
        ('unfeasible', 'unfeasible'),
        ('public', 'public'),
    )

    # CATEGORY_CHOICES = [
    #    (c.lower(), c.lower()) for c in Category.objects
    #        .filter(deprecated=False, seasons__current=True)
    #        .exclude(deadline=True).exclude(preliminary=True)
    #        .order_by('code')
    #        .values_list('code', flat=True)
    # ]

    activity = filters.ChoiceFilter(label='activity', method='activity_filter', choices=ACTIVITY_CHOICES)
    division = filters.ChoiceFilter(label='division', method='division_filter', choices=DIVISION_CHOICES)
    category = filters.CharFilter(label='category', method='category_filter')  # , choices=CATEGORY_CHOICES)
    guide = filters.CharFilter(label='guide', method='guide_filter')
    month = filters.NumberFilter(label='month', method='month_filter', min_value=1, max_value=12)
    ladiesOnly = filters.BooleanFilter(label='ladiesOnly', method='ladies_only_filter')
    publicTransport = filters.BooleanFilter(label='publicTransport', method='public_transport_filter')
    lowEmissionAdventure = filters.BooleanFilter(label='lowEmissionAdventure', method='low_emission_adventure_filter')
    state = filters.ChoiceFilter(label='state', method='state_filter', choices=STATE_CHOICES)
    open =filters.BooleanFilter(label='open', method='open_filter')

    def activity_filter(self, queryset, name, value):
        if value in ("tour", "topic", "collective", "talk"):
            return queryset.filter(**{"reference__category__{}".format(value): True})

    def division_filter(self, queryset, name, value):
        if value in ("winter", "summer", "indoor"):
            return queryset.filter(**{"reference__category__{}".format(value): True})

    def category_filter(self, queryset, name, value):
        return queryset.filter(Q(reference__category__code__iexact=value) | Q(tour__categories__code__iexact=value))

    def guide_filter(self, queryset, name, value):
        return queryset.filter(tour__guide__user__username__iexact=value)

    def month_filter(self, queryset, name, value):
        return queryset.filter(start_date__month=value)

    def ladies_only_filter(self, queryset, name, value):
        return queryset.filter(tour__ladies_only=value)

    def public_transport_filter(self, queryset, name, value):
        return queryset.filter(public_transport=value)

    def low_emission_adventure_filter(self, queryset, name, value):
        return queryset.filter(lea=value)

    def state_filter(self, queryset, name, value):
        return queryset.filter(**{"tour__state__{}".format(value): True})

    def open_filter(self, queryset, name, value):
        if value:
            return queryset.filter(tour__cur_quantity__lt=F('tour__max_quantity'))
        else:
            return queryset.filter(tour__cur_quantity__gte=F('tour__max_quantity'))

    class Meta:
        model = Event
        fields = (
            'activity',
            'division',
            'category',
            'guide',
            'month',
            'ladiesOnly',
            'publicTransport',
            'lowEmissionAdventure',
            'state',
            'open'
        )
        strict = STRICTNESS.RAISE_VALIDATION_ERROR
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    Donation Recurring module for Odoo
#    Copyright (C) 2014 Artisanat Monastique de Provence www.barroux.org
#    @author: Alexis de Lattre <alexis.delattre@akretion.com>
#    @author: Brother Bernard <informatique@barroux.org>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp import models, fields, api, _
from openerp.exceptions import Warning

class DonationRecurringGenerate(models.TransientModel):
    _name = 'donation.recurring.generate'
    _description = 'Generate Recurring Donations'
    _rec_name = 'date'

    date = fields.Date(required=True, default=fields.Date.context_today)
    payment_ref = fields.Char(string='Payment Reference', size=32)

    @api.model
    def _prepare_donation_default(self, donation):
        default = {
            'donation_date': self.date,
            'source_recurring_id': donation.id,
            'payment_ref': self.payment_ref,
            }
        return default

    @api.multi
    def generate(self):
        self.ensure_one()
        doo = self.env['donation.donation']
        donations = doo.search([('recurring_template', '=', 'active')])
        new_donation_ids = []
        existing_recur_donations = doo.search([
            ('donation_date', '=', self.date),
            ('source_recurring_id', '!=', False)])
        if existing_recur_donations:
            raise Warning(
                _('Recurring donations have already been generated for %s.')
                % self.date)
        for donation in donations:
            default = self._prepare_donation_default(donation)
            new_donation = donation.copy(default=default)
            new_donation_ids.append(new_donation.id)
        action = self.env.ref('donation.donation_action').read()[0]
        action.update({
            'view_mode': 'tree,form,graph',
            'domain': [('id', 'in', new_donation_ids)],
            'target': 'current',
            'limit': 500,
            })
        return action
<commit_msg>Generation recurring donation only for the company of the user<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    Donation Recurring module for Odoo
#    Copyright (C) 2014 Artisanat Monastique de Provence www.barroux.org
#    @author: Alexis de Lattre <alexis.delattre@akretion.com>
#    @author: Brother Bernard <informatique@barroux.org>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp import models, fields, api, _
from openerp.exceptions import Warning

class DonationRecurringGenerate(models.TransientModel):
    _name = 'donation.recurring.generate'
    _description = 'Generate Recurring Donations'
    _rec_name = 'date'

    date = fields.Date(required=True, default=fields.Date.context_today)
    payment_ref = fields.Char(string='Payment Reference', size=32)

    @api.model
    def _prepare_donation_default(self, donation):
        default = {
            'donation_date': self.date,
            'source_recurring_id': donation.id,
            'payment_ref': self.payment_ref,
            }
        return default

    @api.multi
    def generate(self):
        self.ensure_one()
        doo = self.env['donation.donation']
        donations = doo.search([
            ('recurring_template', '=', 'active'),
            ('company_id', '=', self.env.user.company_id.id)])
        new_donation_ids = []
        existing_recur_donations = doo.search([
            ('donation_date', '=', self.date),
            ('source_recurring_id', '!=', False)])
        if existing_recur_donations:
            raise Warning(
                _('Recurring donations have already been generated for %s.')
                % self.date)
        for donation in donations:
            default = self._prepare_donation_default(donation)
            new_donation = donation.copy(default=default)
            new_donation_ids.append(new_donation.id)
        action = self.env.ref('donation.donation_action').read()[0]
        action.update({
            'view_mode': 'tree,form,graph',
            'domain': [('id', 'in', new_donation_ids)],
            'target': 'current',
            'limit': 500,
            })
        return action
<|endoftext|>


<commit_before>"""Adapter layer to distributed job manager server api
"""
import os
import os.path
import logging
import datetime

from fabric.api import *
from fabric.operations import put
import fabric.network

from file_locator import FileLocator
import utils
from dist_job_mgr.client import get_local_connection
from dist_job_mgr.version import VERSION
import dist_job_mgr.common as common

class DjmAdapterError(Exception):
    pass

logger = logging.getLogger(__name__)

def get_djm_connection():
    fl = FileLocator()
    return get_local_connection(fl.get_djm_server_dir())

class DjmJob(object):
    def __init__(self, c, job_id, nodes):
        self.c = c
        self.job_id = job_id
        self.nodes = nodes
        self.nodes_by_name = {}
        for node in nodes:
            if node["private_ip"]!=None:
                ip_address = node["private_ip"]
            else:
                ip_address = node["public_ip"]
            if not ip_address:
                raise Exception("Neither public ip address nor private ip address specified for node %s, need to specify at least one" % node["name"])
            node["datablox_ip_address"] = ip_address
            self.nodes_by_name[node["name"]] = node
        self.nodes_except_master = filter(lambda name: name!="master",
                                          [node["name"] for node in self.nodes])
        # set up the fabric nodes
        env.hosts = [node["name"] for node in self.nodes]
        env.roledefs['workers'] = self.nodes_except_master
        for node in nodes:
            env.hostdefs[node["name"]] = "%s@%s" % (node["os_username"],
                                                    node["contact_address"])
            logger.debug("Node %s defined as %s" % (node["name"],
                                                    env.hostdefs[node["name"]]))
        if "master" in env.hosts:
            env.roledefs['master'] = ['master',]
        else:
            env.roledefs['master'] = []

    def has_node(self, node_name):
        return self.nodes_by_name.has_key(node_name)

    def get_node(self, node_name):
        return self.nodes_by_name[node_name]

    def stop_job(self, successful=True, msg=None):
        if successful:
            self.c.stop_job(self.job_id,
                            common.JobStatus.JOB_SUCCESSFUL,
                            comment=msg)
            logger.debug("Stopped job %s, status=JOB_SUCESSFUL" % self.job_id)
        else:
            self.c.stop_job(self.job_id,
                            common.JobStatus.JOB_FAILED,
                            comment=msg)
            logger.debug("Stopped job %s, status=FAILED" % self.job_id)
        fabric.network.disconnect_all()

@task
@parallel
@roles("workers")
def setup_worker_node(reuse_existing_installs):
    fl = FileLocator()
    dist_path = fl.get_engage_distribution_file()
    # todo: don't copy engage if existing install can be reused
    put(dist_path, "~/" + os.path.basename(dist_path))
    setup_script = os.path.join(fl.get_sw_packages_dir(), "setup_caretaker.sh")
    put(setup_script, "~/setup_caretaker.sh")
    run("chmod 755 ~/setup_caretaker.sh")
    run("~/setup_caretaker.sh")
    if reuse_existing_installs:
        run("~/setup_caretaker.sh --reuse-existing-install")
    else:
        run("~/setup_caretaker.sh")

def start_job_and_get_nodes(node_list, config_file_name, total_nodes=None,
                            reuse_existing_installs=True):
    """Given a node list and optional number of nodes, try to get the
    requested nodes and start a job.
    """
    if not total_nodes:
        total_nodes = len(node_list)
    if total_nodes<1:
        raise DjmAdapterError("Must have at least one node")
    c = get_djm_connection()
    # make sure there aren't any dead jobs laying around
    c.cleanup_dead_coordinators()
    pool = None
    for node_name in node_list:
        n = c.find_node_by_name(node_name)
        if not n:
            raise DjmAdapterError("Node '%s' not defined" % node_name)
        if n["pool"]:
            if pool and pool!=n["pool"]:
                raise DjmAdapterError("Cannot take nodes from both pool %s and pool %s"%
                                      (pool, n["pool"]))
            pool = n["pool"]
    start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
    j = c.start_job(config_file_name, common.JobType.ONE_TIME_JOB,
                    total_nodes, "Datablox job started %s" % start_time,
                    node_pool_name=pool, requested_nodes=node_list)
    logger.info("Started DJM job %s" % j)
    try:
        fl = FileLocator()
        allocated_nodes = c.query_nodes(job_id=j)
        djm_job = DjmJob(c, j, allocated_nodes)
        logger.info("Setting up nodes")
        # for all the non-master nodes, we setup the caretaker
        nodes_except_master = djm_job.nodes_except_master
        if len(nodes_except_master)>0:
            execute(setup_worker_node, reuse_existing_installs)
        # make sure the master node has the caretaker running
        if djm_job.has_node("master"):
            utils.run_svcctl(fl, ["start", "all"])
        return djm_job
    except KeyboardInterrupt:
        logger.exception("Got keyboard interrupt in node initialization")
        c.stop_job(j, common.JobStatus.JOB_FAILED,
                   comment="Got keyboard interrupt in node initialization")
        raise
    except Exception, e:
        logger.exception("DJM problem in node initialization: %s" % e)
        c.stop_job(j, common.JobStatus.JOB_FAILED,
                   comment="DJM problem in node initialization: %s" % e)
        raise

<commit_msg>Fix issue with re-install of datablox even when we tell it not to<commit_after>"""Adapter layer to distributed job manager server api
"""
import os
import os.path
import logging
import datetime

from fabric.api import *
from fabric.operations import put
import fabric.network

from file_locator import FileLocator
import utils
from dist_job_mgr.client import get_local_connection
from dist_job_mgr.version import VERSION
import dist_job_mgr.common as common

class DjmAdapterError(Exception):
    pass

logger = logging.getLogger(__name__)

def get_djm_connection():
    fl = FileLocator()
    return get_local_connection(fl.get_djm_server_dir())

class DjmJob(object):
    def __init__(self, c, job_id, nodes):
        self.c = c
        self.job_id = job_id
        self.nodes = nodes
        self.nodes_by_name = {}
        for node in nodes:
            if node["private_ip"]!=None:
                ip_address = node["private_ip"]
            else:
                ip_address = node["public_ip"]
            if not ip_address:
                raise Exception("Neither public ip address nor private ip address specified for node %s, need to specify at least one" % node["name"])
            node["datablox_ip_address"] = ip_address
            self.nodes_by_name[node["name"]] = node
        self.nodes_except_master = filter(lambda name: name!="master",
                                          [node["name"] for node in self.nodes])
        # set up the fabric nodes
        env.hosts = [node["name"] for node in self.nodes]
        env.roledefs['workers'] = self.nodes_except_master
        for node in nodes:
            env.hostdefs[node["name"]] = "%s@%s" % (node["os_username"],
                                                    node["contact_address"])
            logger.debug("Node %s defined as %s" % (node["name"],
                                                    env.hostdefs[node["name"]]))
        if "master" in env.hosts:
            env.roledefs['master'] = ['master',]
        else:
            env.roledefs['master'] = []

    def has_node(self, node_name):
        return self.nodes_by_name.has_key(node_name)

    def get_node(self, node_name):
        return self.nodes_by_name[node_name]

    def stop_job(self, successful=True, msg=None):
        if successful:
            self.c.stop_job(self.job_id,
                            common.JobStatus.JOB_SUCCESSFUL,
                            comment=msg)
            logger.debug("Stopped job %s, status=JOB_SUCESSFUL" % self.job_id)
        else:
            self.c.stop_job(self.job_id,
                            common.JobStatus.JOB_FAILED,
                            comment=msg)
            logger.debug("Stopped job %s, status=FAILED" % self.job_id)
        fabric.network.disconnect_all()

@task
@parallel
@roles("workers")
def setup_worker_node(reuse_existing_installs):
    fl = FileLocator()
    dist_path = fl.get_engage_distribution_file()
    # todo: don't copy engage if existing install can be reused
    put(dist_path, "~/" + os.path.basename(dist_path))
    setup_script = os.path.join(fl.get_sw_packages_dir(), "setup_caretaker.sh")
    put(setup_script, "~/setup_caretaker.sh")
    run("chmod 755 ~/setup_caretaker.sh")
    if reuse_existing_installs:
        run("~/setup_caretaker.sh --reuse-existing-install")
    else:
        run("~/setup_caretaker.sh")

def start_job_and_get_nodes(node_list, config_file_name, total_nodes=None,
                            reuse_existing_installs=True):
    """Given a node list and optional number of nodes, try to get the
    requested nodes and start a job.
    """
    if not total_nodes:
        total_nodes = len(node_list)
    if total_nodes<1:
        raise DjmAdapterError("Must have at least one node")
    c = get_djm_connection()
    # make sure there aren't any dead jobs laying around
    c.cleanup_dead_coordinators()
    pool = None
    for node_name in node_list:
        n = c.find_node_by_name(node_name)
        if not n:
            raise DjmAdapterError("Node '%s' not defined" % node_name)
        if n["pool"]:
            if pool and pool!=n["pool"]:
                raise DjmAdapterError("Cannot take nodes from both pool %s and pool %s"%
                                      (pool, n["pool"]))
            pool = n["pool"]
    start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
    j = c.start_job(config_file_name, common.JobType.ONE_TIME_JOB,
                    total_nodes, "Datablox job started %s" % start_time,
                    node_pool_name=pool, requested_nodes=node_list)
    logger.info("Started DJM job %s" % j)
    try:
        fl = FileLocator()
        allocated_nodes = c.query_nodes(job_id=j)
        djm_job = DjmJob(c, j, allocated_nodes)
        logger.info("Setting up nodes")
        # for all the non-master nodes, we setup the caretaker
        nodes_except_master = djm_job.nodes_except_master
        if len(nodes_except_master)>0:
            execute(setup_worker_node, reuse_existing_installs)
        # make sure the master node has the caretaker running
        if djm_job.has_node("master"):
            utils.run_svcctl(fl, ["start", "all"])
        return djm_job
    except KeyboardInterrupt:
        logger.exception("Got keyboard interrupt in node initialization")
        c.stop_job(j, common.JobStatus.JOB_FAILED,
                   comment="Got keyboard interrupt in node initialization")
        raise
    except Exception, e:
        logger.exception("DJM problem in node initialization: %s" % e)
        c.stop_job(j, common.JobStatus.JOB_FAILED,
                   comment="DJM problem in node initialization: %s" % e)
        raise

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
from frappe import _
from frappe.model.document import Document
import copy
import json

class DuplicateAttribute(frappe.ValidationError): pass

class ManageVariants(Document):

	def get_item_details(self):
		self.clear_tables()
		if self.item_code:
			self.get_attributes()
			self.get_variants()

	def generate_combinations(self):
		self.validate_attributes()
		self.validate_template_item()
		self.validate_attribute_values()
		self.validate_attributes_are_unique()
		self.get_variant_item_codes()

	def create_variants(self):
		self.sync_variants()

	def clear_tables(self):
		self.set('attributes', [])
		self.set('variants', [])

	def get_attributes(self):
		attributes = {}
		self.set('attributes', [])
		for d in frappe.db.sql("""select attribute, attribute_value from `tabVariant Attribute` as attr,
			`tabItem` as item where attr.parent= item.name and item.variant_of = %s""", self.item_code, as_dict=1):
				attributes.setdefault(d.attribute, []).append(d.attribute_value)
		for d in attributes:
			attribute_values = set(attributes[d])
			for value in attribute_values:
				self.append('attributes',{"attribute": d, "attribute_value": value})

	def get_variants(self):
		variants = [d.name for d in frappe.get_all("Item",
			filters={"variant_of":self.item_code})]
		data = frappe.db.sql("""select parent, attribute, attribute_value from `tabVariant Attribute`""", as_dict=1)
		for d in variants:
			variant_attributes, attributes = "", []
			for attribute in data:
				if attribute.parent == d:
					variant_attributes += attribute.attribute_value + " | "
					attributes.append([attribute.attribute, attribute.attribute_value])
			self.append('variants',{"variant": d, "variant_attributes": variant_attributes[: -3], "attributes": json.dumps(attributes)})

	def validate_attributes(self):
		if not self.attributes:
			frappe.throw(_("Enter atleast one Attribute & its Value in Attribute table."))

	def validate_template_item(self):
		if not frappe.db.get_value("Item", self.item_code, "has_variants"):
			frappe.throw(_("Selected Item cannot have Variants."))

		if frappe.db.get_value("Item", self.item_code, "variant_of"):
			frappe.throw(_("Item cannot be a variant of a variant"))

	def validate_attribute_values(self):
		attributes = {}
		for t in frappe.db.get_all("Item Attribute Value", fields=["parent", "attribute_value"]):
			attributes.setdefault(t.parent, []).append(t.attribute_value)

		for d in self.attributes:
			if d.attribute_value not in attributes.get(d.attribute):
				frappe.throw(_("Attribute value {0} does not exist in Item Attribute Master.").format(d.attribute_value))

	def validate_attributes_are_unique(self):
		attributes = []
		for d in self.attributes:
			key = (d.attribute, d.attribute_value)
			if key in attributes:
				frappe.throw(_("{0} {1} is entered more than once in Attributes table")
					.format(d.attribute, d.attribute_value), DuplicateAttribute)
			attributes.append(key)

	def get_variant_item_codes(self):
		"""Get all possible suffixes for variants"""
		variant_dict = {}
		self.set('variants', [])

		for d in self.attributes:
			variant_dict.setdefault(d.attribute, []).append(d.attribute_value)

		all_attributes = [d.name for d in frappe.get_all("Item Attribute", order_by = "priority asc")]

		# sort attributes by their priority
		attributes = filter(None, map(lambda d: d if d in variant_dict else None, all_attributes))

		def add_attribute_suffixes(item_code, my_attributes, attributes):
			attr = frappe.get_doc("Item Attribute", attributes[0])
			for value in attr.item_attribute_values:
				if value.attribute_value in variant_dict[attr.name]:
					_my_attributes = copy.deepcopy(my_attributes)
					_my_attributes.append([attr.name, value.attribute_value])
					if len(attributes) > 1:
						add_attribute_suffixes(item_code + "-" + value.abbr, _my_attributes, attributes[1:])
					else:
						variant_attributes = ""
						for d in _my_attributes:
							variant_attributes += d[1] + " | "
						self.append('variants', {"variant": item_code + "-" + value.abbr,
							"attributes": json.dumps(_my_attributes), "variant_attributes": variant_attributes[: -3]})
		add_attribute_suffixes(self.item_code, [], attributes)

	def sync_variants(self):
		variant_item_codes = []
		item_variants_attributes = {}
		inserted, updated, old_variant_name, new_variant_name, deleted = [], [], [], [], []

		for v in self.variants:
			variant_item_codes.append(v.variant)

		existing_variants = [d.name for d in frappe.get_all("Item",
			filters={"variant_of":self.item_code})]

		for d in existing_variants:
			attributes = []
			for attribute in frappe.db.sql("""select attribute, attribute_value from `tabVariant Attribute` where parent = %s""", d):
				attributes.append([attribute[0], attribute[1]])
			item_variants_attributes.setdefault(d, []).append(attributes)

		for existing_variant in existing_variants:
			if existing_variant not in variant_item_codes:
				att = item_variants_attributes[existing_variant][0]
				for variant in self.variants:
					if sorted(json.loads(variant.attributes) ,key=lambda x: x[0]) == \
						sorted(att ,key=lambda x: x[0]):
							rename_variant(existing_variant, variant.variant)
							old_variant_name.append(existing_variant)
							new_variant_name.append(variant.variant)

				if existing_variant not in old_variant_name:
					delete_variant(existing_variant)
					deleted.append(existing_variant)

		for item_code in variant_item_codes:
			if item_code not in existing_variants:
				if item_code not in new_variant_name:
					make_variant(self.item_code, item_code, self.variants)
					inserted.append(item_code)
			else:
				update_variant(self.item_code, item_code, self.variants)
				updated.append(item_code)

		if inserted:
			frappe.msgprint(_("Item Variants {0} created").format(", ".join(inserted)))

		if updated:
			frappe.msgprint(_("Item Variants {0} updated").format(", ".join(updated)))

		if old_variant_name:
			frappe.msgprint(_("Item Variants {0} renamed").format(", ".join(old_variant_name)))

		if deleted:
			frappe.msgprint(_("Item Variants {0} deleted").format(", ".join(deleted)))

def make_variant(item, variant_code, variant_attribute):
	variant = frappe.new_doc("Item")
	variant.item_code = variant_code
	copy_attributes_to_variant(item, variant, variant_attribute, insert=True)
	variant.insert()

def update_variant(item, variant_code, variant_attribute=None):
	variant = frappe.get_doc("Item", variant_code)
	copy_attributes_to_variant(item, variant, variant_attribute, insert=True)
	variant.save()

def rename_variant(old_variant_code, new_variant_code):
	frappe.rename_doc("Item", old_variant_code, new_variant_code)

def delete_variant(variant_code):
	frappe.delete_doc("Item", variant_code)

def copy_attributes_to_variant(item, variant, variant_attribute=None, insert=False):
	template = frappe.get_doc("Item", item)
	from frappe.model import no_value_fields
	for field in template.meta.fields:
		if field.fieldtype not in no_value_fields and (insert or not field.no_copy)\
			and field.fieldname not in ("item_code", "item_name"):
			if variant.get(field.fieldname) != template.get(field.fieldname):
				variant.set(field.fieldname, template.get(field.fieldname))
	variant.item_name = template.item_name + variant.item_code[len(template.name):]
	variant.variant_of = template.name
	variant.has_variants = 0
	variant.show_in_website = 0
	if variant_attribute:
		for d in variant_attribute:
			if d.variant == variant.item_code:
				variant.attributes= []
				for a in json.loads(d.attributes):
					variant.append('attributes', {"attribute": a[0], "attribute_value": a[1]})
	if variant.attributes:
		variant.description += "\n"
		for d in variant.attributes:
			variant.description += "
" + d.attribute + ": " + d.attribute_value + "
"<commit_msg>minor fix<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
from frappe import _
from frappe.model.document import Document
import copy
import json

class DuplicateAttribute(frappe.ValidationError): pass

class ManageVariants(Document):

	def get_item_details(self):
		self.clear_tables()
		if self.item_code:
			self.get_attributes()
			self.get_variants()

	def generate_combinations(self):
		self.validate_attributes()
		self.validate_template_item()
		self.validate_attribute_values()
		self.validate_attributes_are_unique()
		self.get_variant_item_codes()

	def create_variants(self):
		self.sync_variants()

	def clear_tables(self):
		self.set('attributes', [])
		self.set('variants', [])

	def get_attributes(self):
		attributes = {}
		self.set('attributes', [])
		for d in frappe.db.sql("""select attr.attribute, attr.attribute_value from `tabVariant Attribute` as attr,
			`tabItem` as item where attr.parent = item.name and item.variant_of = %s""", self.item_code, as_dict=1):
				attributes.setdefault(d.attribute, []).append(d.attribute_value)
		for d in attributes:
			attribute_values = set(attributes[d])
			for value in attribute_values:
				self.append('attributes',{"attribute": d, "attribute_value": value})

	def get_variants(self):
		variants = [d.name for d in frappe.get_all("Item",
			filters={"variant_of":self.item_code})]
		data = frappe.db.sql("""select parent, attribute, attribute_value from `tabVariant Attribute`""", as_dict=1)
		for d in variants:
			variant_attributes, attributes = "", []
			for attribute in data:
				if attribute.parent == d:
					variant_attributes += attribute.attribute_value + " | "
					attributes.append([attribute.attribute, attribute.attribute_value])
			self.append('variants',{"variant": d, "variant_attributes": variant_attributes[: -3], "attributes": json.dumps(attributes)})

	def validate_attributes(self):
		if not self.attributes:
			frappe.throw(_("Enter atleast one Attribute & its Value in Attribute table."))

	def validate_template_item(self):
		if not frappe.db.get_value("Item", self.item_code, "has_variants"):
			frappe.throw(_("Selected Item cannot have Variants."))

		if frappe.db.get_value("Item", self.item_code, "variant_of"):
			frappe.throw(_("Item cannot be a variant of a variant"))

	def validate_attribute_values(self):
		attributes = {}
		for t in frappe.db.get_all("Item Attribute Value", fields=["parent", "attribute_value"]):
			attributes.setdefault(t.parent, []).append(t.attribute_value)

		for d in self.attributes:
			if d.attribute_value not in attributes.get(d.attribute):
				frappe.throw(_("Attribute value {0} does not exist in Item Attribute Master.").format(d.attribute_value))

	def validate_attributes_are_unique(self):
		attributes = []
		for d in self.attributes:
			key = (d.attribute, d.attribute_value)
			if key in attributes:
				frappe.throw(_("{0} {1} is entered more than once in Attributes table")
					.format(d.attribute, d.attribute_value), DuplicateAttribute)
			attributes.append(key)

	def get_variant_item_codes(self):
		"""Get all possible suffixes for variants"""
		variant_dict = {}
		self.set('variants', [])

		for d in self.attributes:
			variant_dict.setdefault(d.attribute, []).append(d.attribute_value)

		all_attributes = [d.name for d in frappe.get_all("Item Attribute", order_by = "priority asc")]

		# sort attributes by their priority
		attributes = filter(None, map(lambda d: d if d in variant_dict else None, all_attributes))

		def add_attribute_suffixes(item_code, my_attributes, attributes):
			attr = frappe.get_doc("Item Attribute", attributes[0])
			for value in attr.item_attribute_values:
				if value.attribute_value in variant_dict[attr.name]:
					_my_attributes = copy.deepcopy(my_attributes)
					_my_attributes.append([attr.name, value.attribute_value])
					if len(attributes) > 1:
						add_attribute_suffixes(item_code + "-" + value.abbr, _my_attributes, attributes[1:])
					else:
						variant_attributes = ""
						for d in _my_attributes:
							variant_attributes += d[1] + " | "
						self.append('variants', {"variant": item_code + "-" + value.abbr,
							"attributes": json.dumps(_my_attributes), "variant_attributes": variant_attributes[: -3]})
		add_attribute_suffixes(self.item_code, [], attributes)

	def sync_variants(self):
		variant_item_codes = []
		item_variants_attributes = {}
		inserted, updated, old_variant_name, new_variant_name, deleted = [], [], [], [], []

		for v in self.variants:
			variant_item_codes.append(v.variant)

		existing_variants = [d.name for d in frappe.get_all("Item",
			filters={"variant_of":self.item_code})]

		for d in existing_variants:
			attributes = []
			for attribute in frappe.db.sql("""select attribute, attribute_value from `tabVariant Attribute` where parent = %s""", d):
				attributes.append([attribute[0], attribute[1]])
			item_variants_attributes.setdefault(d, []).append(attributes)

		for existing_variant in existing_variants:
			if existing_variant not in variant_item_codes:
				att = item_variants_attributes[existing_variant][0]
				for variant in self.variants:
					if sorted(json.loads(variant.attributes) ,key=lambda x: x[0]) == \
						sorted(att ,key=lambda x: x[0]):
							rename_variant(existing_variant, variant.variant)
							old_variant_name.append(existing_variant)
							new_variant_name.append(variant.variant)

				if existing_variant not in old_variant_name:
					delete_variant(existing_variant)
					deleted.append(existing_variant)

		for item_code in variant_item_codes:
			if item_code not in existing_variants:
				if item_code not in new_variant_name:
					make_variant(self.item_code, item_code, self.variants)
					inserted.append(item_code)
			else:
				update_variant(self.item_code, item_code, self.variants)
				updated.append(item_code)

		if inserted:
			frappe.msgprint(_("Item Variants {0} created").format(", ".join(inserted)))

		if updated:
			frappe.msgprint(_("Item Variants {0} updated").format(", ".join(updated)))

		if old_variant_name:
			frappe.msgprint(_("Item Variants {0} renamed").format(", ".join(old_variant_name)))

		if deleted:
			frappe.msgprint(_("Item Variants {0} deleted").format(", ".join(deleted)))

def make_variant(item, variant_code, variant_attribute):
	variant = frappe.new_doc("Item")
	variant.item_code = variant_code
	copy_attributes_to_variant(item, variant, variant_attribute, insert=True)
	variant.insert()

def update_variant(item, variant_code, variant_attribute=None):
	variant = frappe.get_doc("Item", variant_code)
	copy_attributes_to_variant(item, variant, variant_attribute, insert=True)
	variant.save()

def rename_variant(old_variant_code, new_variant_code):
	frappe.rename_doc("Item", old_variant_code, new_variant_code)

def delete_variant(variant_code):
	frappe.delete_doc("Item", variant_code)

def copy_attributes_to_variant(item, variant, variant_attribute=None, insert=False):
	template = frappe.get_doc("Item", item)
	from frappe.model import no_value_fields
	for field in template.meta.fields:
		if field.fieldtype not in no_value_fields and (insert or not field.no_copy)\
			and field.fieldname not in ("item_code", "item_name"):
			if variant.get(field.fieldname) != template.get(field.fieldname):
				variant.set(field.fieldname, template.get(field.fieldname))
	variant.item_name = template.item_name + variant.item_code[len(template.name):]
	variant.variant_of = template.name
	variant.has_variants = 0
	variant.show_in_website = 0
	if variant_attribute:
		for d in variant_attribute:
			if d.variant == variant.item_code:
				variant.attributes= []
				for a in json.loads(d.attributes):
					variant.append('attributes', {"attribute": a[0], "attribute_value": a[1]})
	if variant.attributes:
		variant.description += "\n"
		for d in variant.attributes:
			variant.description += "
" + d.attribute + ": " + d.attribute_value + "
"<|endoftext|>


<commit_before>#
# This file is part of pygimplib.
#
# Copyright (C) 2014, 2015 khalim19 <khalim19@gmail.com>
#
# pygimplib is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# pygimplib is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with pygimplib.  If not, see <http://www.gnu.org/licenses/>.
#

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

str = unicode

import StringIO
import unittest

from ..lib import mock

from . import gimpmocks
from .. import pgsetting
from .. import pgsettingpersistor
from .test_pgsettinggroup import create_test_settings
from .test_pgsettinggroup import create_test_settings_hierarchical

#===============================================================================

LIB_NAME = ".".join(__name__.split(".")[:-2])

#===============================================================================

class MockStringIO(StringIO.StringIO):
  def read(self):
    return self.getvalue()

#===============================================================================

@mock.patch(LIB_NAME + ".pgsettingpersistor.gimpshelf.shelf", new_callable=gimpmocks.MockGimpShelf)
class TestSessionPersistentSettingSource(unittest.TestCase):

  @mock.patch(LIB_NAME + ".pgsettingpersistor.gimpshelf.shelf", new=gimpmocks.MockGimpShelf())
  def setUp(self):
    self.source_name = 'test_settings_'
    self.source = pgsettingpersistor.SessionPersistentSettingSource(self.source_name)
    self.settings = create_test_settings()

  def test_write(self, mock_session_source):
    self.settings['file_extension'].set_value("png")
    self.settings['ignore_invisible'].set_value(True)
    self.source.write(self.settings)

    self.assertEqual(pgsettingpersistor.gimpshelf.shelf[self.source_name + 'file_extension'], "png")
    self.assertEqual(pgsettingpersistor.gimpshelf.shelf[self.source_name + 'ignore_invisible'], True)

  def test_write_multiple_settings_separately(self, mock_session_source):
    self.settings['file_extension'].set_value("jpg")
    self.source.write([self.settings['file_extension']])
    self.settings['ignore_invisible'].set_value(True)
    self.source.write([self.settings['ignore_invisible']])
    self.source.read([self.settings['file_extension']])
    self.source.read([self.settings['ignore_invisible']])

    self.assertEqual(self.settings['file_extension'].value, "jpg")
    self.assertEqual(self.settings['ignore_invisible'].value, True)

  def test_read(self, mock_session_source):
    pgsettingpersistor.gimpshelf.shelf[self.source_name + 'file_extension'] = "png"
    pgsettingpersistor.gimpshelf.shelf[self.source_name + 'ignore_invisible'] = True
    self.source.read([self.settings['file_extension'], self.settings['ignore_invisible']])
    self.assertEqual(self.settings['file_extension'].value, "png")
    self.assertEqual(self.settings['ignore_invisible'].value, True)

  def test_read_settings_not_found(self, mock_session_source):
    with self.assertRaises(pgsettingpersistor.SettingsNotFoundInSourceError):
      self.source.read(self.settings)

  def test_read_invalid_setting_value_set_to_default_value(self, mock_session_source):
    setting_with_invalid_value = pgsetting.IntSetting('int', -1)
    self.source.write([setting_with_invalid_value])

    setting = pgsetting.IntSetting('int', 2, min_value=0)
    self.source.read([setting])

    self.assertEqual(setting.value, setting.default_value)

@mock.patch(LIB_NAME + ".pgsettingpersistor.gimp", new_callable=gimpmocks.MockGimpParasite)
class TestPersistentSettingSource(unittest.TestCase):

  @mock.patch(LIB_NAME + ".pgsettingpersistor.gimp", new=gimpmocks.MockGimpParasite())
  def setUp(self):
    self.source_name = 'test_settings_'
    self.source = pgsettingpersistor.PersistentSettingSource(self.source_name)
    self.settings = create_test_settings()

  def test_write_read(self, mock_persistent_source):
    self.settings['file_extension'].set_value("jpg")
    self.settings['ignore_invisible'].set_value(True)
    self.source.write(self.settings)
    self.source.read(self.settings)
    self.assertEqual(self.settings['file_extension'].value, "jpg")
    self.assertEqual(self.settings['ignore_invisible'].value, True)

  def test_write_multiple_settings_separately(self, mock_persistent_source):
    self.settings['file_extension'].set_value("jpg")
    self.source.write([self.settings['file_extension']])
    self.settings['ignore_invisible'].set_value(True)
    self.source.write([self.settings['ignore_invisible']])
    self.source.read([self.settings['file_extension']])
    self.source.read([self.settings['ignore_invisible']])

    self.assertEqual(self.settings['file_extension'].value, "jpg")
    self.assertEqual(self.settings['ignore_invisible'].value, True)

  def test_read_source_not_found(self, mock_persistent_source):
    with self.assertRaises(pgsettingpersistor.SettingSourceNotFoundError):
      self.source.read(self.settings)

  def test_read_settings_not_found(self, mock_persistent_source):
    self.source.write([self.settings['file_extension']])
    with self.assertRaises(pgsettingpersistor.SettingsNotFoundInSourceError):
      self.source.read(self.settings)

  def test_read_settings_invalid_format(self, mock_persistent_source):
    self.source.write(self.settings)

    # Simulate formatting error
    parasite = pgsettingpersistor.gimp.parasite_find(self.source_name)
    parasite.data = parasite.data[:-1]
    pgsettingpersistor.gimp.parasite_attach(parasite)

    with self.assertRaises(pgsettingpersistor.SettingSourceInvalidFormatError):
      self.source.read(self.settings)

  def test_read_invalid_setting_value_set_to_default_value(self, mock_persistent_source):
    setting_with_invalid_value = pgsetting.IntSetting('int', -1)
    self.source.write([setting_with_invalid_value])

    setting = pgsetting.IntSetting('int', 2, min_value=0)
    self.source.read([setting])

    self.assertEqual(setting.value, setting.default_value)

#===============================================================================

@mock.patch(LIB_NAME + ".pgsettingpersistor.gimpshelf.shelf", new_callable=gimpmocks.MockGimpShelf)
@mock.patch(LIB_NAME + ".pgsettingpersistor.gimp", new_callable=gimpmocks.MockGimpParasite)
class TestSettingPersistor(unittest.TestCase):

  @mock.patch(LIB_NAME + ".pgsettingpersistor.gimpshelf.shelf", new=gimpmocks.MockGimpShelf())
  @mock.patch(LIB_NAME + ".pgsettingpersistor.gimp", new=gimpmocks.MockGimpParasite())
  def setUp(self):
    self.settings = create_test_settings()
    self.session_source = pgsettingpersistor.SessionPersistentSettingSource('')
    self.persistent_source = pgsettingpersistor.PersistentSettingSource('')

  def test_load_save(self, mock_session_source, mock_persistent_source):
    self.settings['file_extension'].set_value("png")
    self.settings['ignore_invisible'].set_value(True)

    status, unused_ = pgsettingpersistor.SettingPersistor.save(
      [self.settings], [self.session_source, self.persistent_source])
    self.assertEqual(status, pgsettingpersistor.SettingPersistor.SUCCESS)

    self.settings['file_extension'].set_value("jpg")
    self.settings['ignore_invisible'].set_value(False)

    status, unused_ = pgsettingpersistor.SettingPersistor.load(
      [self.settings], [self.session_source, self.persistent_source])
    self.assertEqual(status, pgsettingpersistor.SettingPersistor.SUCCESS)
    self.assertEqual(self.settings['file_extension'].value, "png")
    self.assertEqual(self.settings['ignore_invisible'].value, True)

  def test_load_combine_settings_from_multiple_sources(self, mock_session_source, mock_persistent_source):
    self.settings['file_extension'].set_value("png")
    self.settings['ignore_invisible'].set_value(True)
    self.session_source.write([self.settings['file_extension']])
    self.settings['file_extension'].set_value("jpg")
    self.persistent_source.write([self.settings['ignore_invisible'], self.settings['file_extension']])
    self.settings['file_extension'].set_value("gif")
    self.settings['ignore_invisible'].set_value(False)

    pgsettingpersistor.SettingPersistor.load([self.settings], [self.session_source, self.persistent_source])

    self.assertEqual(self.settings['file_extension'].value, "png")
    self.assertEqual(self.settings['ignore_invisible'].value, True)

    for setting in self.settings:
      if setting not in [self.settings['file_extension'], self.settings['ignore_invisible']]:
        self.assertEqual(setting.value, setting.default_value)

  def test_load_setting_groups(self, mock_session_source, mock_persistent_source):
    settings = create_test_settings_hierarchical()

    settings['main']['file_extension'].set_value("png")
    settings['advanced']['ignore_invisible'].set_value(True)
    self.session_source.write(list(settings.iterate_all()))
    settings['main']['file_extension'].set_value("gif")
    settings['advanced']['ignore_invisible'].set_value(False)

    pgsettingpersistor.SettingPersistor.load([settings], [self.session_source])

    self.assertEqual(settings['main']['file_extension'].value, "png")
    self.assertEqual(settings['advanced']['ignore_invisible'].value, True)

  def test_load_settings_source_not_found(self, mock_session_source, mock_persistent_source):
    status, unused_ = pgsettingpersistor.SettingPersistor.load(
      [self.settings], [self.session_source, self.persistent_source])
    self.assertEqual(status, pgsettingpersistor.SettingPersistor.NOT_ALL_SETTINGS_FOUND)

  def test_load_settings_not_found(self, mock_session_source, mock_persistent_source):
    self.session_source.write([self.settings['ignore_invisible']])
    self.persistent_source.write([self.settings['file_extension'], self.settings['ignore_invisible']])

    status, unused_ = pgsettingpersistor.SettingPersistor.load(
      [self.settings['overwrite_mode']], [self.session_source, self.persistent_source])
    self.assertEqual(status, pgsettingpersistor.SettingPersistor.NOT_ALL_SETTINGS_FOUND)

  def test_load_read_fail(self, mock_session_source, mock_persistent_source):
    self.persistent_source.write(self.settings)

    # Simulate formatting error
    parasite = pgsettingpersistor.gimp.parasite_find(self.persistent_source.source_name)
    parasite.data = parasite.data[:-1]
    pgsettingpersistor.gimp.parasite_attach(parasite)

    status, unused_ = pgsettingpersistor.SettingPersistor.load(
      [self.settings], [self.session_source, self.persistent_source])
    self.assertEqual(status, pgsettingpersistor.SettingPersistor.READ_FAIL)
<commit_msg>pgsettingpersistor: Update tests<commit_after>#
# This file is part of pygimplib.
#
# Copyright (C) 2014, 2015 khalim19 <khalim19@gmail.com>
#
# pygimplib is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# pygimplib is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with pygimplib.  If not, see <http://www.gnu.org/licenses/>.
#

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

str = unicode

import StringIO
import unittest

from ..lib import mock

from . import gimpmocks
from .. import pgsetting
from .. import pgsettingpersistor
from .test_pgsettinggroup import create_test_settings
from .test_pgsettinggroup import create_test_settings_hierarchical

#===============================================================================

LIB_NAME = ".".join(__name__.split(".")[:-2])

#===============================================================================

class MockStringIO(StringIO.StringIO):
  def read(self):
    return self.getvalue()

#===============================================================================

@mock.patch(LIB_NAME + ".pgsettingpersistor.gimpshelf.shelf", new_callable=gimpmocks.MockGimpShelf)
class TestSessionPersistentSettingSource(unittest.TestCase):

  @mock.patch(LIB_NAME + ".pgsettingpersistor.gimpshelf.shelf", new=gimpmocks.MockGimpShelf())
  def setUp(self):
    self.source_name = 'test_settings_'
    self.source = pgsettingpersistor.SessionPersistentSettingSource(self.source_name)
    self.settings = create_test_settings()

  def test_write(self, mock_session_source):
    self.settings['file_extension'].set_value("png")
    self.settings['ignore_invisible'].set_value(True)
    self.source.write(self.settings)

    self.assertEqual(pgsettingpersistor.gimpshelf.shelf[self.source_name + 'file_extension'], "png")
    self.assertEqual(pgsettingpersistor.gimpshelf.shelf[self.source_name + 'ignore_invisible'], True)

  def test_write_multiple_settings_separately(self, mock_session_source):
    self.settings['file_extension'].set_value("jpg")
    self.source.write([self.settings['file_extension']])
    self.settings['ignore_invisible'].set_value(True)
    self.source.write([self.settings['ignore_invisible']])
    self.source.read([self.settings['file_extension']])
    self.source.read([self.settings['ignore_invisible']])

    self.assertEqual(self.settings['file_extension'].value, "jpg")
    self.assertEqual(self.settings['ignore_invisible'].value, True)

  def test_read(self, mock_session_source):
    pgsettingpersistor.gimpshelf.shelf[self.source_name + 'file_extension'] = "png"
    pgsettingpersistor.gimpshelf.shelf[self.source_name + 'ignore_invisible'] = True
    self.source.read([self.settings['file_extension'], self.settings['ignore_invisible']])
    self.assertEqual(self.settings['file_extension'].value, "png")
    self.assertEqual(self.settings['ignore_invisible'].value, True)

  def test_read_settings_not_found(self, mock_session_source):
    with self.assertRaises(pgsettingpersistor.SettingsNotFoundInSourceError):
      self.source.read(self.settings)

  def test_read_invalid_setting_value_set_to_default_value(self, mock_session_source):
    setting_with_invalid_value = pgsetting.IntSetting('int', -1)
    self.source.write([setting_with_invalid_value])

    setting = pgsetting.IntSetting('int', 2, min_value=0)
    self.source.read([setting])

    self.assertEqual(setting.value, setting.default_value)

@mock.patch(LIB_NAME + ".pgsettingpersistor.gimp", new_callable=gimpmocks.MockGimpParasite)
class TestPersistentSettingSource(unittest.TestCase):

  @mock.patch(LIB_NAME + ".pgsettingpersistor.gimp", new=gimpmocks.MockGimpParasite())
  def setUp(self):
    self.source_name = 'test_settings_'
    self.source = pgsettingpersistor.PersistentSettingSource(self.source_name)
    self.settings = create_test_settings()

  def test_write_read(self, mock_persistent_source):
    self.settings['file_extension'].set_value("jpg")
    self.settings['ignore_invisible'].set_value(True)
    self.source.write(self.settings)
    self.source.read(self.settings)
    self.assertEqual(self.settings['file_extension'].value, "jpg")
    self.assertEqual(self.settings['ignore_invisible'].value, True)

  def test_write_multiple_settings_separately(self, mock_persistent_source):
    self.settings['file_extension'].set_value("jpg")
    self.source.write([self.settings['file_extension']])
    self.settings['ignore_invisible'].set_value(True)
    self.source.write([self.settings['ignore_invisible']])
    self.source.read([self.settings['file_extension']])
    self.source.read([self.settings['ignore_invisible']])

    self.assertEqual(self.settings['file_extension'].value, "jpg")
    self.assertEqual(self.settings['ignore_invisible'].value, True)

  def test_read_source_not_found(self, mock_persistent_source):
    with self.assertRaises(pgsettingpersistor.SettingSourceNotFoundError):
      self.source.read(self.settings)

  def test_read_settings_not_found(self, mock_persistent_source):
    self.source.write([self.settings['file_extension']])
    with self.assertRaises(pgsettingpersistor.SettingsNotFoundInSourceError):
      self.source.read(self.settings)

  def test_read_settings_invalid_format(self, mock_persistent_source):
    self.source.write(self.settings)

    # Simulate formatting error
    parasite = pgsettingpersistor.gimp.parasite_find(self.source_name)
    parasite.data = parasite.data[:-1]
    pgsettingpersistor.gimp.parasite_attach(parasite)

    with self.assertRaises(pgsettingpersistor.SettingSourceInvalidFormatError):
      self.source.read(self.settings)

  def test_read_invalid_setting_value_set_to_default_value(self, mock_persistent_source):
    setting_with_invalid_value = pgsetting.IntSetting('int', -1)
    self.source.write([setting_with_invalid_value])

    setting = pgsetting.IntSetting('int', 2, min_value=0)
    self.source.read([setting])

    self.assertEqual(setting.value, setting.default_value)

#===============================================================================

@mock.patch(LIB_NAME + ".pgsettingpersistor.gimpshelf.shelf", new_callable=gimpmocks.MockGimpShelf)
@mock.patch(LIB_NAME + ".pgsettingpersistor.gimp", new_callable=gimpmocks.MockGimpParasite)
class TestSettingPersistor(unittest.TestCase):

  @mock.patch(LIB_NAME + ".pgsettingpersistor.gimpshelf.shelf", new=gimpmocks.MockGimpShelf())
  @mock.patch(LIB_NAME + ".pgsettingpersistor.gimp", new=gimpmocks.MockGimpParasite())
  def setUp(self):
    self.settings = create_test_settings()
    self.session_source = pgsettingpersistor.SessionPersistentSettingSource('')
    self.persistent_source = pgsettingpersistor.PersistentSettingSource('')

  def test_load_save(self, mock_session_source, mock_persistent_source):
    self.settings['file_extension'].set_value("png")
    self.settings['ignore_invisible'].set_value(True)

    status, unused_ = pgsettingpersistor.SettingPersistor.save(
      [self.settings], [self.session_source, self.persistent_source])
    self.assertEqual(status, pgsettingpersistor.SettingPersistor.SUCCESS)

    self.settings['file_extension'].set_value("jpg")
    self.settings['ignore_invisible'].set_value(False)

    status, unused_ = pgsettingpersistor.SettingPersistor.load(
      [self.settings], [self.session_source, self.persistent_source])
    self.assertEqual(status, pgsettingpersistor.SettingPersistor.SUCCESS)
    self.assertEqual(self.settings['file_extension'].value, "png")
    self.assertEqual(self.settings['ignore_invisible'].value, True)

  def test_load_combine_settings_from_multiple_sources(self, mock_session_source, mock_persistent_source):
    self.settings['file_extension'].set_value("png")
    self.settings['ignore_invisible'].set_value(True)
    self.session_source.write([self.settings['file_extension']])
    self.settings['file_extension'].set_value("jpg")
    self.persistent_source.write([self.settings['ignore_invisible'], self.settings['file_extension']])
    self.settings['file_extension'].set_value("gif")
    self.settings['ignore_invisible'].set_value(False)

    pgsettingpersistor.SettingPersistor.load([self.settings], [self.session_source, self.persistent_source])

    self.assertEqual(self.settings['file_extension'].value, "png")
    self.assertEqual(self.settings['ignore_invisible'].value, True)

    for setting in self.settings:
      if setting not in [self.settings['file_extension'], self.settings['ignore_invisible']]:
        self.assertEqual(setting.value, setting.default_value)

  def test_load_setting_groups(self, mock_session_source, mock_persistent_source):
    settings = create_test_settings_hierarchical()

    settings['main']['file_extension'].set_value("png")
    settings['advanced']['ignore_invisible'].set_value(True)
    self.session_source.write(list(settings.iterate_all()))
    settings['main']['file_extension'].set_value("gif")
    settings['advanced']['ignore_invisible'].set_value(False)

    pgsettingpersistor.SettingPersistor.load([settings], [self.session_source])

    self.assertEqual(settings['main']['file_extension'].value, "png")
    self.assertEqual(settings['advanced']['ignore_invisible'].value, True)

  def test_load_settings_source_not_found(self, mock_session_source, mock_persistent_source):
    status, unused_ = pgsettingpersistor.SettingPersistor.load(
      [self.settings], [self.session_source, self.persistent_source])
    self.assertEqual(status, pgsettingpersistor.SettingPersistor.NOT_ALL_SETTINGS_FOUND)

  def test_load_settings_not_found(self, mock_session_source, mock_persistent_source):
    self.session_source.write([self.settings['ignore_invisible']])
    self.persistent_source.write([self.settings['file_extension'], self.settings['ignore_invisible']])

    status, unused_ = pgsettingpersistor.SettingPersistor.load(
      [self.settings['overwrite_mode']], [self.session_source, self.persistent_source])
    self.assertEqual(status, pgsettingpersistor.SettingPersistor.NOT_ALL_SETTINGS_FOUND)

  def test_load_read_fail(self, mock_session_source, mock_persistent_source):
    self.persistent_source.write(self.settings)

    # Simulate formatting error
    parasite = pgsettingpersistor.gimp.parasite_find(self.persistent_source.source_name)
    parasite.data = parasite.data[:-1]
    pgsettingpersistor.gimp.parasite_attach(parasite)

    status, unused_ = pgsettingpersistor.SettingPersistor.load(
      [self.settings], [self.session_source, self.persistent_source])
    self.assertEqual(status, pgsettingpersistor.SettingPersistor.READ_FAIL)

  def test_load_write_fail(self, mock_session_source, mock_persistent_source):
    with mock.patch(LIB_NAME + ".pgsettingpersistor.gimp") as temp_mock_persistent_source:
      temp_mock_persistent_source.parasite_find.side_effect = pgsettingpersistor.SettingSourceWriteError
      status, _unused = pgsettingpersistor.SettingPersistor.save(
        [self.settings], [self.session_source, self.persistent_source])

    self.assertEqual(status, pgsettingpersistor.SettingPersistor.WRITE_FAIL)
<|endoftext|>


<commit_before>"""End to end tests for auto_gpa.py """

__authors__ = ["Martha Brennich"]
__license__ = "MIT"
__date__ = "27/12/2020"

import unittest
import pathlib
import logging
from subprocess import run, PIPE, STDOUT
from os import linesep
from os.path import normpath
from platform import system

logger = logging.getLogger(__name__)

class TestFreeSAS(unittest.TestCase):
    cwd = pathlib.Path.cwd()
    test_location = pathlib.Path(__file__)
    test_data_location = pathlib.Path(test_location.parent, "e2etest_data")
    bsa_filename = pathlib.Path(test_data_location, "bsa_005_sub.dat")
    sas_curve2_filename = pathlib.Path(test_data_location, "SASDF52.dat")

    def __init__(self, testName, **extra_kwargs):
        super(TestFreeSAS, self).__init__(testName)
        self.extra_arg = extra_kwargs

    def test_one_bm29_bsa_without_arguments(self):
        """
        Test whether guinier fit apps on BSA data from BM29 returns one line.
        """
        app_name: str = self.extra_arg["app"]
        run_app = run(
            [app_name, normpath(str(self.bsa_filename))],
            stdout=PIPE,
            stderr=STDOUT,
            check=True,
        )
        self.assertEqual(
            run_app.returncode, 0, msg=f"{app_name} completed well"
        )
        if system() == "Windows":
            run_app_output = str(run_app.stdout)[:-1].replace("\\\\", "\\")
        else:
            run_app_output = str(run_app.stdout, "utf-8")[:-1]
        self.assertFalse(
            linesep in run_app_output,
            "One line of output of {app_name}",
        )
        self.assertTrue(
            str(self.bsa_filename) in run_app_output,
            msg="filename of testdata found in output of {app_name}",
        )
        self.assertTrue("I0" in run_app_output, msg=f"I0 in {app_name}")
        self.assertTrue("Rg" in run_app_output, msg=f"Rg in {app_name}")

    def test_two_easy_files_without_arguments(self):
        """
        Test whether auto_gpa.py with two datasets returns two lines.
        """
        app_name: str = self.extra_arg["app"]
        run_app = run(
            [
                app_name,
                str(self.bsa_filename),
                str(self.sas_curve2_filename),
            ],
            stdout=PIPE,
            stderr=STDOUT,
            check=True,
        )
        self.assertEqual(
            run_app.returncode,
            0,
            msg=f"{app_name} for 2 files completed well",
        )
        if system() == "Windows":
            run_app_output_raw = str(run_app.stdout)[:-1].replace("\\\\", "\\")
            run_app_output = run_app_output_raw.split("\\n")[:-1]
        else:
            run_app_output_raw = str(run_app.stdout, "utf-8")[:-1]
            run_app_output = run_app_output_raw.split(linesep)
        self.assertTrue(
            str(self.bsa_filename) in run_app_output_raw
            and str(self.sas_curve2_filename) in run_app_output_raw,
            f"filename of testdata found in output in output of {app_name}",
        )

        self.assertEqual(
            len(run_app_output),
            2,
            msg=f"Number of output lines is 2 in output of {app_name}",
        )
        for result in run_app_output:
            self.assertTrue(
                str(self.bsa_filename) in result
                or str(self.sas_curve2_filename) in result,
                f"filename of testdata not found in result line for {app_name}",
            )
            self.assertTrue("I" in result, f"I0 in {app_name} output")
            self.assertTrue("Rg" in result, f"Rg in {app_name} output")

    def test_csv_output_with_two_files(self):
        """Test whether the output in csv format is consistent"""
        app_name: str = self.extra_arg["app"]
        test_output_name = pathlib.Path(self.cwd, f"{app_name}.csv")
        try:
            test_output_name.unlink()
        except FileNotFoundError:
            pass
        run_app = run(
            [
                app_name,
                str(self.bsa_filename),
                str(self.sas_curve2_filename),
                "-o",
                test_output_name,
                "-f",
                "csv",
            ],
            stdout=PIPE,
            stderr=STDOUT,
            check=True,
        )
        self.assertEqual(
            run_app.returncode,
            0,
            msg=f"{app_name} for 2 files to csv completed well",
        )
        self.assertEqual(
            run_app.stdout,
            b"",
            msg=f"{app_name} for 2 files to csv provided no output to stdout",
        )
        with open(str(test_output_name), "r") as test_output_file:
            test_output_header = test_output_file.readline()[:-1].split(",")
            test_output_result = test_output_file.readlines()
        test_output_result = [
            line[:-1].split(",") for line in test_output_result
        ]
        expected_header = {
            "File",
            "Rg",
            "Rg StDev",
            "I(0)",
            "I(0) StDev",
            "First point",
            "Last point",
            "Quality",
            "Aggregated",
        }
        self.assertEqual(
            expected_header ^ set(test_output_header),
            set(),
            msg=f"header created by {app_name} contains the expected element",
        )
        self.assertEqual(
            len(test_output_header),
            len(set(test_output_header)),
            msg=f"header created by {app_name} contains no duplicates",
        )
        index = {}
        for header_item in expected_header:
            index[header_item] = test_output_header.index(header_item)

        self.assertEqual(
            len(test_output_result),
            2,
            msg=f"result file created by {app_name} contains 2 results",
        )
        self.assertEqual(
            {str(self.bsa_filename), str(self.sas_curve2_filename)},
            {
                test_output_result[0][index["File"]],
                test_output_result[1][index["File"]],
            },
            msg=f"result file created by {app_name} contains results for the right files",
        )
        for result in test_output_result:
            if result[index["File"]] == str(self.bsa_filename):
                self.assertTrue(
                    abs(float(result[index["Rg"]]) - 2.95) <= 0.1,
                    msg=f"Rg for BSA by {app_name} between 2.85 and 3.05",
                )
                self.assertTrue(
                    float(result[index["Rg StDev"]]) < 0.3,
                    msg=f"Rg std. for BSA by {app_name} below 0.3",
                )
                self.assertTrue(
                    abs(float(result[index["I(0)"]]) - 62.5) <= 2.5,
                    msg=f"I0 for BSA by {app_name} between 60 and 65",
                )
                self.assertTrue(
                    float(result[index["I(0) StDev"]]) < 5,
                    msg=f"I0 std. for BSA by {app_name} below 5",
                )
                self.assertTrue(
                    float(result[index["First point"]]) < 70,
                    msg=f"Guinier region start for BSA by {app_name} below 70",
                )
                self.assertTrue(
                    float(result[index["Last point"]]) > 85,
                    msg=f"Guinier region end for BSA by {app_name} above 85",
                )
                self.assertTrue(
                    float(result[index["Quality"]]) > 0.8,
                    msg=f"Fit quality BSA by {app_name} above 0.8",
                )
                self.assertTrue(
                    float(result[index["Aggregated"]]) < 0.05,
                    msg=f"Aggregation number for BSA by {app_name} below 0.05",
                )
            elif result[index["File"]] == str(self.sas_curve2_filename):
                self.assertTrue(
                    abs(float(result[index["Rg"]]) - 3.05) <= 0.2,
                    msg=f"Rg for BSA by {app_name} between 2.85 and 3.25",
                )
                self.assertTrue(
                    float(result[index["Rg StDev"]]) < 0.15,
                    msg=f"Rg std. for SASDF52 by {app_name} below 0.5",
                )
                self.assertTrue(
                    abs(float(result[index["I(0)"]]) - 64.5) <= 2.5,
                    msg=f"I0 for SASDF52 by {app_name} between 62 and 67",
                )
                self.assertTrue(
                    float(result[index["I(0) StDev"]]) < 5,
                    msg=f"I0 std. for SASDF52 by {app_name} below 5",
                )
                self.assertTrue(
                    float(result[index["First point"]]) < 35,
                    msg=f"Guinier region start for SASDF52 by {app_name} below 35",
                )
                self.assertTrue(
                    float(result[index["Last point"]]) > 80,
                    msg=f"Guinier region end for SASDF52 by {app_name} above 80",
                )
                self.assertTrue(
                    float(result[index["Quality"]]) > 0.8,
                    msg=f"Fit quality SASDF52 by {app_name} above 0.8",
                )
                self.assertTrue(
                    float(result[index["Aggregated"]]) < 0.1,
                    msg=f"Aggregation number for SASDF52 by {app_name} below 0.1",
                )

    def test_ssv_output_with_two_files(self):
        """Test wether the output in the ssv table has the correct dimensions"""
        app_name: str = self.extra_arg["app"]
        test_output_name = pathlib.Path(self.cwd, f"{app_name}.ssv")
        try:
            test_output_name.unlink()
        except FileNotFoundError:
            pass
        run_app = run(
            [
                app_name,
                str(self.bsa_filename),
                str(self.sas_curve2_filename),
                "-o",
                test_output_name,
                "-f",
                "ssv",
            ],
            stdout=PIPE,
            stderr=STDOUT,
            check=True,
        )
        self.assertEqual(
            run_app.returncode,
            0,
            msg=f"{app_name} for 2 files to csv completed well",
        )
        self.assertEqual(
            run_app.stdout,
            b"",
            msg=f"{app_name} for 2 files to ssv provided no output to stdout",
        )
        with open(str(test_output_name), "r") as test_output_file:
            test_output_result = test_output_file.readlines()
        test_output_result = [line[:-1].split() for line in test_output_result]

        self.assertEqual(
            len(test_output_result),
            2,
            msg=f"results created by {app_name} in native have the expected number of rows",
        )
        self.assertTrue(
            len(test_output_result[0]) == 9
            and len(test_output_result[1]) == 9,
            msg=f"results created by {app_name} in ssv have the expected number of elements",
        )
        self.assertEqual(
            {str(self.bsa_filename), str(self.sas_curve2_filename)},
            {
                test_output_result[0][-1],
                test_output_result[1][-1],
            },
            msg=f"ssv file created by {app_name} contains the filenames at the end of each row",
        )

    def test_native_output_with_two_files(self):
        """Test wether the output in the antive format has the correct dimensions"""
        app_name: str = self.extra_arg["app"]
        test_output_name = pathlib.Path(self.cwd, f"{app_name}.txt")
        try:
            test_output_name.unlink()
        except FileNotFoundError:
            pass
        run_app = run(
            [
                app_name,
                str(self.bsa_filename),
                str(self.sas_curve2_filename),
                "-o",
                test_output_name,
            ],
            stdout=PIPE,
            stderr=STDOUT,
            check=True,
        )
        self.assertEqual(
            run_app.returncode,
            0,
            msg=f"{app_name} for 2 files to native completed well",
        )
        self.assertEqual(
            run_app.stdout,
            b"",
            msg=f"{app_name} for 2 files to native provided no output to stdout",
        )
        with open(str(test_output_name), "r") as test_output_file:
            test_output_result = test_output_file.readlines()
        test_output_result = [line[:-1].split() for line in test_output_result]

        self.assertEqual(
            len(test_output_result),
            2,
            msg=f"results created by {app_name} in native have the expected number of rows",
        )
        self.assertTrue(
            len(test_output_result[0]) == 5
            and len(test_output_result[1]) == 5,
            msg=f"results created by {app_name} in native have the expected number of elements",
        )
        self.assertEqual(
            {str(self.bsa_filename), str(self.sas_curve2_filename)},
            {
                test_output_result[0][0],
                test_output_result[1][0],
            },
            msg=f"native file created by {app_name} contains "
            "the filenames at the start of each row",
        )

def suite():
    test_suite = unittest.TestSuite()
    for app in ["free_gpa", "free_guinier", "free_rg"]:
        if system() == "Windows":
            app += ".exe"
        test_suite.addTest(
            TestFreeSAS("test_one_bm29_bsa_without_arguments", app=app)
        )
        test_suite.addTest(
            TestFreeSAS("test_two_easy_files_without_arguments", app=app)
        )
        test_suite.addTest(
            TestFreeSAS("test_csv_output_with_two_files", app=app)
        )
        test_suite.addTest(
            TestFreeSAS("test_ssv_output_with_two_files", app=app)
        )
        test_suite.addTest(
            TestFreeSAS("test_native_output_with_two_files", app=app)
        )
    return test_suite

if __name__ == "__main__":
    runner = unittest.TextTestRunner()
    runner.run(suite())
<commit_msg>more test fixing<commit_after>"""End to end tests for auto_gpa.py """

__authors__ = ["Martha Brennich"]
__license__ = "MIT"
__date__ = "27/12/2020"

import unittest
import pathlib
import logging
from subprocess import run, PIPE, STDOUT
from os import linesep
from os.path import normpath
from platform import system

logger = logging.getLogger(__name__)

class TestFreeSAS(unittest.TestCase):
    cwd = pathlib.Path.cwd()
    test_location = pathlib.Path(__file__)
    test_data_location = pathlib.Path(test_location.parent, "e2etest_data")
    bsa_filename = pathlib.Path(test_data_location, "bsa_005_sub.dat")
    sas_curve2_filename = pathlib.Path(test_data_location, "SASDF52.dat")

    def __init__(self, testName, **extra_kwargs):
        super(TestFreeSAS, self).__init__(testName)
        self.extra_arg = extra_kwargs

    def test_one_bm29_bsa_without_arguments(self):
        """
        Test whether guinier fit apps on BSA data from BM29 returns one line.
        """
        app_name: str = self.extra_arg["app"]
        run_app = run(
            [app_name, normpath(str(self.bsa_filename))],
            stdout=PIPE,
            stderr=STDOUT,
            check=True,
        )
        self.assertEqual(
            run_app.returncode, 0, msg=f"{app_name} completed well"
        )
        if system() == "Windows":
            run_app_output = str(run_app.stdout)[:-1].replace("\\\\", "\\")
        else:
            run_app_output = str(run_app.stdout, "utf-8")[:-1]
        self.assertFalse(
            linesep in run_app_output,
            "One line of output of {app_name}",
        )
        self.assertTrue(
            str(self.bsa_filename) in run_app_output,
            msg="filename of testdata found in output of {app_name}",
        )
        self.assertTrue("I0" in run_app_output, msg=f"I0 in {app_name}")
        self.assertTrue("Rg" in run_app_output, msg=f"Rg in {app_name}")

    def test_two_easy_files_without_arguments(self):
        """
        Test whether auto_gpa.py with two datasets returns two lines.
        """
        app_name: str = self.extra_arg["app"]
        run_app = run(
            [
                app_name,
                str(self.bsa_filename),
                str(self.sas_curve2_filename),
            ],
            stdout=PIPE,
            stderr=STDOUT,
            check=True,
        )
        self.assertEqual(
            run_app.returncode,
            0,
            msg=f"{app_name} for 2 files completed well",
        )
        if system() == "Windows":
            run_app_output_raw = str(run_app.stdout)[:-1].replace("\\\\", "\\")
            run_app_output = run_app_output_raw.split("\\n")[:-1]
        else:
            run_app_output_raw = str(run_app.stdout, "utf-8")[:-1]
            run_app_output = run_app_output_raw.split(linesep)
        self.assertTrue(
            str(self.bsa_filename) in run_app_output_raw
            and str(self.sas_curve2_filename) in run_app_output_raw,
            f"filename of testdata found in output in output of {app_name}",
        )

        self.assertEqual(
            len(run_app_output),
            2,
            msg=f"Number of output lines is 2 in output of {app_name}",
        )
        for result in run_app_output:
            self.assertTrue(
                str(self.bsa_filename) in result
                or str(self.sas_curve2_filename) in result,
                f"filename of testdata not found in result line for {app_name}",
            )
            self.assertTrue("I" in result, f"I0 in {app_name} output")
            self.assertTrue("Rg" in result, f"Rg in {app_name} output")

    def test_csv_output_with_two_files(self):
        """Test whether the output in csv format is consistent"""
        app_name: str = self.extra_arg["app"]
        test_output_name = pathlib.Path(self.cwd, f"{app_name}.csv")
        try:
            test_output_name.unlink()
        except FileNotFoundError:
            pass
        run_app = run(
            [
                app_name,
                str(self.bsa_filename),
                str(self.sas_curve2_filename),
                "-o",
                str(test_output_name),
                "-f",
                "csv",
            ],
            stdout=PIPE,
            stderr=STDOUT,
            check=True,
        )
        self.assertEqual(
            run_app.returncode,
            0,
            msg=f"{app_name} for 2 files to csv completed well",
        )
        self.assertEqual(
            run_app.stdout,
            b"",
            msg=f"{app_name} for 2 files to csv provided no output to stdout",
        )
        with open(str(test_output_name), "r") as test_output_file:
            test_output_header = test_output_file.readline()[:-1].split(",")
            test_output_result = test_output_file.readlines()
        test_output_result = [
            line[:-1].split(",") for line in test_output_result
        ]
        expected_header = {
            "File",
            "Rg",
            "Rg StDev",
            "I(0)",
            "I(0) StDev",
            "First point",
            "Last point",
            "Quality",
            "Aggregated",
        }
        self.assertEqual(
            expected_header ^ set(test_output_header),
            set(),
            msg=f"header created by {app_name} contains the expected element",
        )
        self.assertEqual(
            len(test_output_header),
            len(set(test_output_header)),
            msg=f"header created by {app_name} contains no duplicates",
        )
        index = {}
        for header_item in expected_header:
            index[header_item] = test_output_header.index(header_item)

        self.assertEqual(
            len(test_output_result),
            2,
            msg=f"result file created by {app_name} contains 2 results",
        )
        self.assertEqual(
            {str(self.bsa_filename), str(self.sas_curve2_filename)},
            {
                test_output_result[0][index["File"]],
                test_output_result[1][index["File"]],
            },
            msg=f"result file created by {app_name} contains results for the right files",
        )
        for result in test_output_result:
            if result[index["File"]] == str(self.bsa_filename):
                self.assertTrue(
                    abs(float(result[index["Rg"]]) - 2.95) <= 0.1,
                    msg=f"Rg for BSA by {app_name} between 2.85 and 3.05",
                )
                self.assertTrue(
                    float(result[index["Rg StDev"]]) < 0.3,
                    msg=f"Rg std. for BSA by {app_name} below 0.3",
                )
                self.assertTrue(
                    abs(float(result[index["I(0)"]]) - 62.5) <= 2.5,
                    msg=f"I0 for BSA by {app_name} between 60 and 65",
                )
                self.assertTrue(
                    float(result[index["I(0) StDev"]]) < 5,
                    msg=f"I0 std. for BSA by {app_name} below 5",
                )
                self.assertTrue(
                    float(result[index["First point"]]) < 70,
                    msg=f"Guinier region start for BSA by {app_name} below 70",
                )
                self.assertTrue(
                    float(result[index["Last point"]]) > 85,
                    msg=f"Guinier region end for BSA by {app_name} above 85",
                )
                self.assertTrue(
                    float(result[index["Quality"]]) > 0.8,
                    msg=f"Fit quality BSA by {app_name} above 0.8",
                )
                self.assertTrue(
                    float(result[index["Aggregated"]]) < 0.05,
                    msg=f"Aggregation number for BSA by {app_name} below 0.05",
                )
            elif result[index["File"]] == str(self.sas_curve2_filename):
                self.assertTrue(
                    abs(float(result[index["Rg"]]) - 3.05) <= 0.2,
                    msg=f"Rg for BSA by {app_name} between 2.85 and 3.25",
                )
                self.assertTrue(
                    float(result[index["Rg StDev"]]) < 0.15,
                    msg=f"Rg std. for SASDF52 by {app_name} below 0.5",
                )
                self.assertTrue(
                    abs(float(result[index["I(0)"]]) - 64.5) <= 2.5,
                    msg=f"I0 for SASDF52 by {app_name} between 62 and 67",
                )
                self.assertTrue(
                    float(result[index["I(0) StDev"]]) < 5,
                    msg=f"I0 std. for SASDF52 by {app_name} below 5",
                )
                self.assertTrue(
                    float(result[index["First point"]]) < 35,
                    msg=f"Guinier region start for SASDF52 by {app_name} below 35",
                )
                self.assertTrue(
                    float(result[index["Last point"]]) > 80,
                    msg=f"Guinier region end for SASDF52 by {app_name} above 80",
                )
                self.assertTrue(
                    float(result[index["Quality"]]) > 0.8,
                    msg=f"Fit quality SASDF52 by {app_name} above 0.8",
                )
                self.assertTrue(
                    float(result[index["Aggregated"]]) < 0.1,
                    msg=f"Aggregation number for SASDF52 by {app_name} below 0.1",
                )

    def test_ssv_output_with_two_files(self):
        """Test wether the output in the ssv table has the correct dimensions"""
        app_name: str = self.extra_arg["app"]
        test_output_name = pathlib.Path(self.cwd, f"{app_name}.ssv")
        try:
            test_output_name.unlink()
        except FileNotFoundError:
            pass
        run_app = run(
            [
                app_name,
                str(self.bsa_filename),
                str(self.sas_curve2_filename),
                "-o",
                str(test_output_name),
                "-f",
                "ssv",
            ],
            stdout=PIPE,
            stderr=STDOUT,
            check=True,
        )
        self.assertEqual(
            run_app.returncode,
            0,
            msg=f"{app_name} for 2 files to csv completed well",
        )
        self.assertEqual(
            run_app.stdout,
            b"",
            msg=f"{app_name} for 2 files to ssv provided no output to stdout",
        )
        with open(str(test_output_name), "r") as test_output_file:
            test_output_result = test_output_file.readlines()
        test_output_result = [line[:-1].split() for line in test_output_result]

        self.assertEqual(
            len(test_output_result),
            2,
            msg=f"results created by {app_name} in native have the expected number of rows",
        )
        self.assertTrue(
            len(test_output_result[0]) == 9
            and len(test_output_result[1]) == 9,
            msg=f"results created by {app_name} in ssv have the expected number of elements",
        )
        self.assertEqual(
            {str(self.bsa_filename), str(self.sas_curve2_filename)},
            {
                test_output_result[0][-1],
                test_output_result[1][-1],
            },
            msg=f"ssv file created by {app_name} contains the filenames at the end of each row",
        )

    def test_native_output_with_two_files(self):
        """Test wether the output in the antive format has the correct dimensions"""
        app_name: str = self.extra_arg["app"]
        test_output_name = pathlib.Path(self.cwd, f"{app_name}.txt")
        try:
            test_output_name.unlink()
        except FileNotFoundError:
            pass
        run_app = run(
            [
                app_name,
                str(self.bsa_filename),
                str(self.sas_curve2_filename),
                "-o",
                str(test_output_name),
            ],
            stdout=PIPE,
            stderr=STDOUT,
            check=True,
        )
        self.assertEqual(
            run_app.returncode,
            0,
            msg=f"{app_name} for 2 files to native completed well",
        )
        self.assertEqual(
            run_app.stdout,
            b"",
            msg=f"{app_name} for 2 files to native provided no output to stdout",
        )
        with open(str(test_output_name), "r") as test_output_file:
            test_output_result = test_output_file.readlines()
        test_output_result = [line[:-1].split() for line in test_output_result]

        self.assertEqual(
            len(test_output_result),
            2,
            msg=f"results created by {app_name} in native have the expected number of rows",
        )
        self.assertTrue(
            len(test_output_result[0]) == 5
            and len(test_output_result[1]) == 5,
            msg=f"results created by {app_name} in native have the expected number of elements",
        )
        self.assertEqual(
            {str(self.bsa_filename), str(self.sas_curve2_filename)},
            {
                test_output_result[0][0],
                test_output_result[1][0],
            },
            msg=f"native file created by {app_name} contains "
            "the filenames at the start of each row",
        )

def suite():
    test_suite = unittest.TestSuite()
    for app in ["free_gpa", "free_guinier", "free_rg"]:
        if system() == "Windows":
            app += ".exe"
        test_suite.addTest(
            TestFreeSAS("test_one_bm29_bsa_without_arguments", app=app)
        )
        test_suite.addTest(
            TestFreeSAS("test_two_easy_files_without_arguments", app=app)
        )
        test_suite.addTest(
            TestFreeSAS("test_csv_output_with_two_files", app=app)
        )
        test_suite.addTest(
            TestFreeSAS("test_ssv_output_with_two_files", app=app)
        )
        test_suite.addTest(
            TestFreeSAS("test_native_output_with_two_files", app=app)
        )
    return test_suite

if __name__ == "__main__":
    runner = unittest.TextTestRunner()
    runner.run(suite())
<|endoftext|>