<commit_before>from crowdsourcing.serializers.task import *
from rest_framework import status, viewsets
from rest_framework.response import Response
from rest_framework.decorators import detail_route, list_route
from django.shortcuts import get_object_or_404
from crowdsourcing.permissions.project import IsProjectOwnerOrCollaborator
from crowdsourcing.models import Task, TaskWorker, TaskWorkerResult
from django.utils import timezone

class TaskViewSet(viewsets.ModelViewSet):
    queryset = Task.objects.all()
    serializer_class = TaskSerializer

    @detail_route(methods=['post'],permission_classes=[IsProjectOwnerOrCollaborator])
    def update_task(self, request, id=None):
        task_serializer = TaskSerializer(data=request.data)
        task = self.get_object()
        if task_serializer.is_valid():
            task_serializer.update(task,task_serializer.validated_data)
            return Response({'status': 'updated task'})
        else:
            return Response(task_serializer.errors,status=status.HTTP_400_BAD_REQUEST)

    def list(self, request, *args, **kwargs):
        try:
            module = request.query_params.get('module')
            task = Task.objects.filter(module=module)
            task_serialized = TaskSerializer(task, many=True)
            return Response(task_serialized.data)
        except:
            return Response([])

    def destroy(self, request, *args, **kwargs):
        task_serializer = TaskSerializer()
        task = self.get_object()
        task_serializer.delete(task)
        return Response({'status': 'deleted task'})

    @detail_route(methods=['get'])
    def retrieve_with_data(self, request, *args, **kwargs):
        task = self.get_object()
        serializer = TaskSerializer(instance=task, fields=('id', 'task_template', 'status'))
        return Response(serializer.data, status.HTTP_200_OK)

    @list_route(methods=['get'])
    def list_by_module(self, request, **kwargs):
        tasks = Task.objects.filter(module=request.query_params.get('module_id'))
        task_serializer = TaskSerializer(instance=tasks, many=True, fields=('id', 'status', \
                                        'template_items_monitoring', 'task_workers_monitoring'))
        response_data = {
            'project_name': tasks[0].module.project.name,
            'project_id': tasks[0].module.project.id,
            'module_name': tasks[0].module.name,
            'module_id': tasks[0].module.id,
            'tasks': task_serializer.data
        }
        return Response(response_data, status.HTTP_200_OK)

class TaskWorkerViewSet(viewsets.ModelViewSet):
    queryset = TaskWorker.objects.all()
    serializer_class = TaskWorkerSerializer
    #permission_classes = [IsAuthenticated]
    lookup_field = 'task__id'

    def create(self, request, *args, **kwargs):
        serializer = TaskWorkerSerializer(data=request.data)
        if serializer.is_valid():
            instance = serializer.create(worker=request.user.userprofile.worker, module=request.data.get('module', None))
            serialized_data = TaskWorkerSerializer(instance=instance)
            return Response(serialized_data.data, 200)
        else:
            return Response(serializer.errors,
                            status=status.HTTP_400_BAD_REQUEST)

    def destroy(self, request, *args, **kwargs):
        serializer = TaskWorkerSerializer()
        obj = self.queryset.get(task=kwargs['task__id'], worker=request.user.userprofile.worker.id)
        obj.task_status = 6
        obj.save()
        instance = serializer.create(worker=request.user.userprofile.worker, module=obj.task.module_id)
        serialized_data = TaskWorkerSerializer(instance=instance)
        return Response(serialized_data.data, status.HTTP_200_OK)

    @list_route(methods=['post'])
    def bulk_update_status(self, request, *args, **kwargs):
        task_status = request.data.get('task_status', -1)
        task_workers = TaskWorker.objects.filter(id__in=tuple(request.data.get('task_workers', [])))
        task_workers.update(task_status=task_status, last_updated=timezone.now())
        return Response(TaskWorkerSerializer(instance=task_workers, many=True,
                        fields=('id', 'task', 'task_status', 'task_worker_results_monitoring',
                                'worker_alias', 'updated_delta')).data, status.HTTP_200_OK)

    @list_route(methods=['get'])
    def list_by_status(self, request, *args, **kwargs):
        task_status = request.query_params.get('task_status')
        task_workers = TaskWorker.objects.filter(worker=request.user.userprofile.worker, task_status=task_status)
        serializer = TaskWorkerSerializer(instance=task_workers, many=True,
                    fields=('id', 'task_status', 'task', 'requester_alias', 'module', 'project_name'))
        return Response(serializer.data, status.HTTP_200_OK)

    @detail_route(methods=['get'])
    def retrieve_with_data_and_results(self, request, *args, **kwargs):
        task_worker = TaskWorker.objects.get(id=request.query_params['id'])
        serializer = TaskWorkerSerializer(instance=task_worker,
                        fields=('id', 'task_status', 'task_with_data_and_results'))
        return Response(serializer.data, status.HTTP_200_OK)

class TaskWorkerResultViewSet(viewsets.ModelViewSet):
    queryset = TaskWorkerResult.objects.all()
    serializer_class = TaskWorkerResultSerializer
    #permission_classes = [IsOwnerOrReadOnly]

    def update(self, request, *args, **kwargs):
        task_worker_result_serializer = TaskWorkerResultSerializer(data=request.data)
        task_worker_result = self.queryset.filter(id=kwargs['pk'])[0]
        status = 1
        if 'status' in request.data:
            status = request.data['status']
        task_worker_result.status = status
        task_worker_result.save()
        return Response("Success")

    def retrieve(self, request, *args, **kwargs):
        worker = get_object_or_404(self.queryset, worker=request.worker)
        serializer = TaskWorkerResultSerializer(instance=worker)
        return Response(serializer.data)

    @list_route(methods=['post'], url_path="submit-results")
    def submit_results(self, request, *args, **kwargs):
        task = request.data.get('task', None)
        template_items = request.data.get('template_items', [])
        task_status = request.data.get('task_status', None)
        task_worker = TaskWorker.objects.get(worker=request.user.userprofile.worker, task=task)
        task_worker.task_status = task_status
        task_worker.save()
        serializer = TaskWorkerResultSerializer(data=template_items, many=True)
        if serializer.is_valid():
            serializer.create(task_worker=task_worker)
            if task_status == 1:
                return Response('Success', status.HTTP_200_OK)
            elif task_status == 2:
                task_worker_serializer = TaskWorkerSerializer()
                instance = task_worker_serializer.create(worker=request.user.userprofile.worker, module=task_worker.task.module_id)
                serialized_data = TaskWorkerSerializer(instance=instance)
                return Response(serialized_data.data, status.HTTP_200_OK)
        else:
            return Response(serializer.errors, status.HTTP_400_BAD_REQUEST)

class CurrencyViewSet(viewsets.ModelViewSet):
  from crowdsourcing.models import Currency
  queryset = Currency.objects.all()
  serializer_class = CurrencySerializer<commit_msg>task list_comments and post_comment endpoints added<commit_after>from crowdsourcing.serializers.task import *
from rest_framework import status, viewsets
from rest_framework.response import Response
from rest_framework.decorators import detail_route, list_route
from django.shortcuts import get_object_or_404
from crowdsourcing.permissions.project import IsProjectOwnerOrCollaborator
from crowdsourcing.models import Task, TaskWorker, TaskWorkerResult
from django.utils import timezone

class TaskViewSet(viewsets.ModelViewSet):
    queryset = Task.objects.all()
    serializer_class = TaskSerializer

    @detail_route(methods=['post'],permission_classes=[IsProjectOwnerOrCollaborator])
    def update_task(self, request, id=None):
        task_serializer = TaskSerializer(data=request.data)
        task = self.get_object()
        if task_serializer.is_valid():
            task_serializer.update(task,task_serializer.validated_data)
            return Response({'status': 'updated task'})
        else:
            return Response(task_serializer.errors,status=status.HTTP_400_BAD_REQUEST)

    def list(self, request, *args, **kwargs):
        try:
            module = request.query_params.get('module')
            task = Task.objects.filter(module=module)
            task_serialized = TaskSerializer(task, many=True)
            return Response(task_serialized.data)
        except:
            return Response([])

    def destroy(self, request, *args, **kwargs):
        task_serializer = TaskSerializer()
        task = self.get_object()
        task_serializer.delete(task)
        return Response({'status': 'deleted task'})

    @detail_route(methods=['get'])
    def retrieve_with_data(self, request, *args, **kwargs):
        task = self.get_object()
        serializer = TaskSerializer(instance=task, fields=('id', 'task_template', 'status'))
        return Response(serializer.data, status.HTTP_200_OK)

    @list_route(methods=['get'])
    def list_by_module(self, request, **kwargs):
        tasks = Task.objects.filter(module=request.query_params.get('module_id'))
        task_serializer = TaskSerializer(instance=tasks, many=True, fields=('id', 'status', \
                                        'template_items_monitoring', 'task_workers_monitoring'))
        response_data = {
            'project_name': tasks[0].module.project.name,
            'project_id': tasks[0].module.project.id,
            'module_name': tasks[0].module.name,
            'module_id': tasks[0].module.id,
            'tasks': task_serializer.data
        }
        return Response(response_data, status.HTTP_200_OK)

    @detail_route(methods=['get'])
    def list_comments(self, request, **kwargs):
        comments = models.ModuleComment.objects.filter(module=kwargs['pk'])
        serializer = TaskCommentSerializer(instance=comments, many=True, fields=('comment', 'id',))
        response_data = {
            'task': kwargs['pk'],
            'comments': serializer.data
        }
        return Response(response_data, status.HTTP_200_OK)

    @detail_route(methods=['post'])
    def post_comment(self, request, **kwargs):
        serializer = TaskCommentSerializer(data=request.data)
        module_comment_data = {}
        if serializer.is_valid():
            comment = serializer.create(module=kwargs['pk'], sender=request.user.userprofile)
            module_comment_data = TaskCommentSerializer(comment, fields=('id', 'comment',)).data

        return Response(module_comment_data, status.HTTP_200_OK)

class TaskWorkerViewSet(viewsets.ModelViewSet):
    queryset = TaskWorker.objects.all()
    serializer_class = TaskWorkerSerializer
    #permission_classes = [IsAuthenticated]
    lookup_field = 'task__id'

    def create(self, request, *args, **kwargs):
        serializer = TaskWorkerSerializer(data=request.data)
        if serializer.is_valid():
            instance = serializer.create(worker=request.user.userprofile.worker, module=request.data.get('module', None))
            serialized_data = TaskWorkerSerializer(instance=instance)
            return Response(serialized_data.data, 200)
        else:
            return Response(serializer.errors,
                            status=status.HTTP_400_BAD_REQUEST)

    def destroy(self, request, *args, **kwargs):
        serializer = TaskWorkerSerializer()
        obj = self.queryset.get(task=kwargs['task__id'], worker=request.user.userprofile.worker.id)
        obj.task_status = 6
        obj.save()
        instance = serializer.create(worker=request.user.userprofile.worker, module=obj.task.module_id)
        serialized_data = TaskWorkerSerializer(instance=instance)
        return Response(serialized_data.data, status.HTTP_200_OK)

    @list_route(methods=['post'])
    def bulk_update_status(self, request, *args, **kwargs):
        task_status = request.data.get('task_status', -1)
        task_workers = TaskWorker.objects.filter(id__in=tuple(request.data.get('task_workers', [])))
        task_workers.update(task_status=task_status, last_updated=timezone.now())
        return Response(TaskWorkerSerializer(instance=task_workers, many=True,
                        fields=('id', 'task', 'task_status', 'task_worker_results_monitoring',
                                'worker_alias', 'updated_delta')).data, status.HTTP_200_OK)

    @list_route(methods=['get'])
    def list_by_status(self, request, *args, **kwargs):
        task_status = request.query_params.get('task_status')
        task_workers = TaskWorker.objects.filter(worker=request.user.userprofile.worker, task_status=task_status)
        serializer = TaskWorkerSerializer(instance=task_workers, many=True,
                    fields=('id', 'task_status', 'task', 'requester_alias', 'module', 'project_name'))
        return Response(serializer.data, status.HTTP_200_OK)

    @detail_route(methods=['get'])
    def retrieve_with_data_and_results(self, request, *args, **kwargs):
        task_worker = TaskWorker.objects.get(id=request.query_params['id'])
        serializer = TaskWorkerSerializer(instance=task_worker,
                        fields=('id', 'task_status', 'task_with_data_and_results'))
        return Response(serializer.data, status.HTTP_200_OK)

class TaskWorkerResultViewSet(viewsets.ModelViewSet):
    queryset = TaskWorkerResult.objects.all()
    serializer_class = TaskWorkerResultSerializer
    #permission_classes = [IsOwnerOrReadOnly]

    def update(self, request, *args, **kwargs):
        task_worker_result_serializer = TaskWorkerResultSerializer(data=request.data)
        task_worker_result = self.queryset.filter(id=kwargs['pk'])[0]
        status = 1
        if 'status' in request.data:
            status = request.data['status']
        task_worker_result.status = status
        task_worker_result.save()
        return Response("Success")

    def retrieve(self, request, *args, **kwargs):
        worker = get_object_or_404(self.queryset, worker=request.worker)
        serializer = TaskWorkerResultSerializer(instance=worker)
        return Response(serializer.data)

    @list_route(methods=['post'], url_path="submit-results")
    def submit_results(self, request, *args, **kwargs):
        task = request.data.get('task', None)
        template_items = request.data.get('template_items', [])
        task_status = request.data.get('task_status', None)
        task_worker = TaskWorker.objects.get(worker=request.user.userprofile.worker, task=task)
        task_worker.task_status = task_status
        task_worker.save()
        serializer = TaskWorkerResultSerializer(data=template_items, many=True)
        if serializer.is_valid():
            serializer.create(task_worker=task_worker)
            if task_status == 1:
                return Response('Success', status.HTTP_200_OK)
            elif task_status == 2:
                task_worker_serializer = TaskWorkerSerializer()
                instance = task_worker_serializer.create(worker=request.user.userprofile.worker, module=task_worker.task.module_id)
                serialized_data = TaskWorkerSerializer(instance=instance)
                return Response(serialized_data.data, status.HTTP_200_OK)
        else:
            return Response(serializer.errors, status.HTTP_400_BAD_REQUEST)

class CurrencyViewSet(viewsets.ModelViewSet):
  from crowdsourcing.models import Currency
  queryset = Currency.objects.all()
  serializer_class = CurrencySerializer<|endoftext|>


<commit_before><commit_msg>Renamed latest_path to prior_path (cosmetic change).<commit_after><|endoftext|>


<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals
import frappe, erpnext
from frappe import _
from frappe.utils import flt
from frappe.model.meta import get_field_precision
from frappe.utils.xlsxutils import handle_html
from erpnext.accounts.report.sales_register.sales_register import get_mode_of_payments

def execute(filters=None):
	return _execute(filters)

def _execute(filters=None, additional_table_columns=None, additional_query_columns=None):
	if not filters: filters = {}
	columns = get_columns(additional_table_columns)

	company_currency = erpnext.get_company_currency(filters.company)

	item_list = get_items(filters, additional_query_columns)
	if item_list:
		itemised_tax, tax_columns = get_tax_accounts(item_list, columns, company_currency)
	columns.append({
		"fieldname": "currency",
		"label": _("Currency"),
		"fieldtype": "Data",
		"width": 80
	})
	mode_of_payments = get_mode_of_payments(set([d.parent for d in item_list]))
	so_dn_map = get_delivery_notes_against_sales_order(item_list)

	data = []
	for d in item_list:
		delivery_note = None
		if d.delivery_note:
			delivery_note = d.delivery_note
		elif d.so_detail:
			delivery_note = ", ".join(so_dn_map.get(d.so_detail, []))

		if not delivery_note and d.update_stock:
			delivery_note = d.parent

		row = [d.item_code, d.item_name, d.item_group, d.parent, d.posting_date, d.customer, d.customer_name]

		if additional_query_columns:
			for col in additional_query_columns:
				row.append(d.get(col))

		row += [
			d.customer_group, d.debit_to, ", ".join(mode_of_payments.get(d.parent, [])),
			d.territory, d.project, d.company, d.sales_order,
			delivery_note, d.income_account, d.cost_center, d.stock_qty, d.stock_uom
		]

		row += [(d.base_net_rate * d.qty)/d.stock_qty, d.base_net_amount] \
			if d.stock_uom != d.uom else [d.base_net_rate, d.base_net_amount]

		total_tax = 0
		for tax in tax_columns:
			item_tax = itemised_tax.get(d.name, {}).get(tax, {})
			row += [item_tax.get("tax_rate", 0), item_tax.get("tax_amount", 0)]
			total_tax += flt(item_tax.get("tax_amount"))

		row += [total_tax, d.base_net_amount + total_tax, company_currency]

		data.append(row)

	return columns, data

def get_columns(additional_table_columns):
	columns = [
		_("Item Code") + ":Link/Item:120", _("Item Name") + "::120",
		_("Item Group") + ":Link/Item Group:100", _("Invoice") + ":Link/Sales Invoice:120",
		_("Posting Date") + ":Date:80", _("Customer") + ":Link/Customer:120",
		_("Customer Name") + "::120"]

	if additional_table_columns:
		columns += additional_table_columns

	columns += [
		_("Customer Group") + ":Link/Customer Group:120",
		_("Receivable Account") + ":Link/Account:120",
		_("Mode of Payment") + "::120", _("Territory") + ":Link/Territory:80",
		_("Project") + ":Link/Project:80", _("Company") + ":Link/Company:100",
		_("Sales Order") + ":Link/Sales Order:100", _("Delivery Note") + ":Link/Delivery Note:100",
		_("Income Account") + ":Link/Account:140", _("Cost Center") + ":Link/Cost Center:140",
		_("Stock Qty") + ":Float:120", _("Stock UOM") + "::100",
		_("Rate") + ":Currency/currency:120",
		_("Amount") + ":Currency/currency:120"
	]

	return columns

def get_conditions(filters):
	conditions = ""

	for opts in (("company", " and company=%(company)s"),
		("customer", " and `tabSales Invoice`.customer = %(customer)s"),
		("item_code", " and `tabSales Invoice Item`.item_code = %(item_code)s"),
		("from_date", " and `tabSales Invoice`.posting_date>=%(from_date)s"),
		("to_date", " and `tabSales Invoice`.posting_date<=%(to_date)s")):
			if filters.get(opts[0]):
				conditions += opts[1]

	if filters.get("mode_of_payment"):
		conditions += """ and exists(select name from `tabSales Invoice Payment`
			where parent=`tabSales Invoice`.name
				and ifnull(`tabSales Invoice Payment`.mode_of_payment, '') = %(mode_of_payment)s)"""

	return conditions

def get_items(filters, additional_query_columns):
	conditions = get_conditions(filters)
	match_conditions = frappe.build_match_conditions("Sales Invoice")

	if match_conditions:
		match_conditions = " and {0} ".format(match_conditions)

	if additional_query_columns:
		additional_query_columns = ', ' + ', '.join(additional_query_columns)

	return frappe.db.sql("""
		select
			`tabSales Invoice Item`.name, `tabSales Invoice Item`.parent,
			`tabSales Invoice`.posting_date, `tabSales Invoice`.debit_to,
			`tabSales Invoice`.project, `tabSales Invoice`.customer, `tabSales Invoice`.remarks,
			`tabSales Invoice`.territory, `tabSales Invoice`.company, `tabSales Invoice`.base_net_total,
			`tabSales Invoice Item`.item_code, `tabSales Invoice Item`.item_name,
			`tabSales Invoice Item`.item_group, `tabSales Invoice Item`.sales_order,
			`tabSales Invoice Item`.delivery_note, `tabSales Invoice Item`.income_account,
			`tabSales Invoice Item`.cost_center, `tabSales Invoice Item`.stock_qty,
			`tabSales Invoice Item`.stock_uom, `tabSales Invoice Item`.base_net_rate,
			`tabSales Invoice Item`.base_net_amount, `tabSales Invoice`.customer_name,
			`tabSales Invoice`.customer_group, `tabSales Invoice Item`.so_detail,
			`tabSales Invoice`.update_stock, `tabSales Invoice Item`.uom, `tabSales Invoice Item`.qty {0}
		from `tabSales Invoice`, `tabSales Invoice Item`
		where `tabSales Invoice`.name = `tabSales Invoice Item`.parent
			and `tabSales Invoice`.docstatus = 1 %s %s
		order by `tabSales Invoice`.posting_date desc, `tabSales Invoice Item`.item_code desc
		""".format(additional_query_columns or '') % (conditions, match_conditions), filters, as_dict=1)

def get_delivery_notes_against_sales_order(item_list):
	so_dn_map = frappe._dict()
	so_item_rows = list(set([d.so_detail for d in item_list]))

	if so_item_rows:
		delivery_notes = frappe.db.sql("""
			select parent, so_detail
			from `tabDelivery Note Item`
			where docstatus=1 and so_detail in (%s)
			group by so_detail, parent
		""" % (', '.join(['%s']*len(so_item_rows))), tuple(so_item_rows), as_dict=1)

		for dn in delivery_notes:
			so_dn_map.setdefault(dn.so_detail, []).append(dn.parent)

	return so_dn_map

def get_deducted_taxes():
	return frappe.db.sql_list("select name from `tabPurchase Taxes and Charges` where add_deduct_tax = 'Deduct'")

def get_tax_accounts(item_list, columns, company_currency,
		doctype="Sales Invoice", tax_doctype="Sales Taxes and Charges"):
	import json
	item_row_map = {}
	tax_columns = []
	invoice_item_row = {}
	itemised_tax = {}

	tax_amount_precision = get_field_precision(frappe.get_meta(tax_doctype).get_field("tax_amount"),
		currency=company_currency) or 2

	for d in item_list:
		invoice_item_row.setdefault(d.parent, []).append(d)
		item_row_map.setdefault(d.parent, {}).setdefault(d.item_code or d.item_name, []).append(d)

	conditions = ""
	if doctype == "Purchase Invoice":
		conditions = " and category in ('Total', 'Valuation and Total') and base_tax_amount_after_discount_amount != 0"

	deducted_tax = get_deducted_taxes()
	tax_details = frappe.db.sql("""
		select
			name, parent, description, item_wise_tax_detail,
			charge_type, base_tax_amount_after_discount_amount
		from `tab%s`
		where
			parenttype = %s and docstatus = 1
			and (description is not null and description != '')
			and parent in (%s)
			%s
		order by description
	""" % (tax_doctype, '%s', ', '.join(['%s']*len(invoice_item_row)), conditions),
		tuple([doctype] + invoice_item_row.keys()))

	for name, parent, description, item_wise_tax_detail, charge_type, tax_amount in tax_details:
		description = handle_html(description)
		if description not in tax_columns and tax_amount:
			# as description is text editor earlier and markup can break the column convention in reports
			tax_columns.append(description)

		if item_wise_tax_detail:
			try:
				item_wise_tax_detail = json.loads(item_wise_tax_detail)

				for item_code, tax_data in item_wise_tax_detail.items():
					itemised_tax.setdefault(item_code, frappe._dict())

					if isinstance(tax_data, list):
						tax_rate, tax_amount = tax_data
					else:
						tax_rate = tax_data
						tax_amount = 0

					if charge_type == "Actual" and not tax_rate:
						tax_rate = "NA"

					item_net_amount = sum([flt(d.base_net_amount)
						for d in item_row_map.get(parent, {}).get(item_code, [])])

					for d in item_row_map.get(parent, {}).get(item_code, []):
						item_tax_amount = flt((tax_amount * d.base_net_amount) / item_net_amount) \
							if item_net_amount else 0
						if item_tax_amount:
							tax_value = flt(item_tax_amount, tax_amount_precision)
							tax_value = (tax_value * -1
								if (doctype == 'Purchase Invoice' and name in deducted_tax) else tax_value)

							itemised_tax.setdefault(d.name, {})[description] = frappe._dict({
								"tax_rate": tax_rate,
								"tax_amount": tax_value
							})

			except ValueError:
				continue
		elif charge_type == "Actual" and tax_amount:
			for d in invoice_item_row.get(parent, []):
				itemised_tax.setdefault(d.name, {})[description] = frappe._dict({
					"tax_rate": "NA",
					"tax_amount": flt((tax_amount * d.base_net_amount) / d.base_net_total,
						tax_amount_precision)
				})

	tax_columns.sort()
	for desc in tax_columns:
		columns.append(desc + " Rate:Data:80")
		columns.append(desc + " Amount:Currency/currency:100")

	columns += ["Total Tax:Currency/currency:80", "Total:Currency/currency:100"]

	return itemised_tax, tax_columns
<commit_msg>fixed  divided by zero error (#15885)<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals
import frappe, erpnext
from frappe import _
from frappe.utils import flt
from frappe.model.meta import get_field_precision
from frappe.utils.xlsxutils import handle_html
from erpnext.accounts.report.sales_register.sales_register import get_mode_of_payments

def execute(filters=None):
	return _execute(filters)

def _execute(filters=None, additional_table_columns=None, additional_query_columns=None):
	if not filters: filters = {}
	columns = get_columns(additional_table_columns)

	company_currency = erpnext.get_company_currency(filters.company)

	item_list = get_items(filters, additional_query_columns)
	if item_list:
		itemised_tax, tax_columns = get_tax_accounts(item_list, columns, company_currency)
	columns.append({
		"fieldname": "currency",
		"label": _("Currency"),
		"fieldtype": "Data",
		"width": 80
	})
	mode_of_payments = get_mode_of_payments(set([d.parent for d in item_list]))
	so_dn_map = get_delivery_notes_against_sales_order(item_list)

	data = []
	for d in item_list:
		delivery_note = None
		if d.delivery_note:
			delivery_note = d.delivery_note
		elif d.so_detail:
			delivery_note = ", ".join(so_dn_map.get(d.so_detail, []))

		if not delivery_note and d.update_stock:
			delivery_note = d.parent

		row = [d.item_code, d.item_name, d.item_group, d.parent, d.posting_date, d.customer, d.customer_name]

		if additional_query_columns:
			for col in additional_query_columns:
				row.append(d.get(col))

		row += [
			d.customer_group, d.debit_to, ", ".join(mode_of_payments.get(d.parent, [])),
			d.territory, d.project, d.company, d.sales_order,
			delivery_note, d.income_account, d.cost_center, d.stock_qty, d.stock_uom
		]

		row += [(d.base_net_rate * d.qty)/d.stock_qty, d.base_net_amount] \
			if d.stock_uom != d.uom and d.stock_qty != 0 else [d.base_net_rate, d.base_net_amount]

		total_tax = 0
		for tax in tax_columns:
			item_tax = itemised_tax.get(d.name, {}).get(tax, {})
			row += [item_tax.get("tax_rate", 0), item_tax.get("tax_amount", 0)]
			total_tax += flt(item_tax.get("tax_amount"))

		row += [total_tax, d.base_net_amount + total_tax, company_currency]

		data.append(row)

	return columns, data

def get_columns(additional_table_columns):
	columns = [
		_("Item Code") + ":Link/Item:120", _("Item Name") + "::120",
		_("Item Group") + ":Link/Item Group:100", _("Invoice") + ":Link/Sales Invoice:120",
		_("Posting Date") + ":Date:80", _("Customer") + ":Link/Customer:120",
		_("Customer Name") + "::120"]

	if additional_table_columns:
		columns += additional_table_columns

	columns += [
		_("Customer Group") + ":Link/Customer Group:120",
		_("Receivable Account") + ":Link/Account:120",
		_("Mode of Payment") + "::120", _("Territory") + ":Link/Territory:80",
		_("Project") + ":Link/Project:80", _("Company") + ":Link/Company:100",
		_("Sales Order") + ":Link/Sales Order:100", _("Delivery Note") + ":Link/Delivery Note:100",
		_("Income Account") + ":Link/Account:140", _("Cost Center") + ":Link/Cost Center:140",
		_("Stock Qty") + ":Float:120", _("Stock UOM") + "::100",
		_("Rate") + ":Currency/currency:120",
		_("Amount") + ":Currency/currency:120"
	]

	return columns

def get_conditions(filters):
	conditions = ""

	for opts in (("company", " and company=%(company)s"),
		("customer", " and `tabSales Invoice`.customer = %(customer)s"),
		("item_code", " and `tabSales Invoice Item`.item_code = %(item_code)s"),
		("from_date", " and `tabSales Invoice`.posting_date>=%(from_date)s"),
		("to_date", " and `tabSales Invoice`.posting_date<=%(to_date)s")):
			if filters.get(opts[0]):
				conditions += opts[1]

	if filters.get("mode_of_payment"):
		conditions += """ and exists(select name from `tabSales Invoice Payment`
			where parent=`tabSales Invoice`.name
				and ifnull(`tabSales Invoice Payment`.mode_of_payment, '') = %(mode_of_payment)s)"""

	return conditions

def get_items(filters, additional_query_columns):
	conditions = get_conditions(filters)
	match_conditions = frappe.build_match_conditions("Sales Invoice")

	if match_conditions:
		match_conditions = " and {0} ".format(match_conditions)

	if additional_query_columns:
		additional_query_columns = ', ' + ', '.join(additional_query_columns)

	return frappe.db.sql("""
		select
			`tabSales Invoice Item`.name, `tabSales Invoice Item`.parent,
			`tabSales Invoice`.posting_date, `tabSales Invoice`.debit_to,
			`tabSales Invoice`.project, `tabSales Invoice`.customer, `tabSales Invoice`.remarks,
			`tabSales Invoice`.territory, `tabSales Invoice`.company, `tabSales Invoice`.base_net_total,
			`tabSales Invoice Item`.item_code, `tabSales Invoice Item`.item_name,
			`tabSales Invoice Item`.item_group, `tabSales Invoice Item`.sales_order,
			`tabSales Invoice Item`.delivery_note, `tabSales Invoice Item`.income_account,
			`tabSales Invoice Item`.cost_center, `tabSales Invoice Item`.stock_qty,
			`tabSales Invoice Item`.stock_uom, `tabSales Invoice Item`.base_net_rate,
			`tabSales Invoice Item`.base_net_amount, `tabSales Invoice`.customer_name,
			`tabSales Invoice`.customer_group, `tabSales Invoice Item`.so_detail,
			`tabSales Invoice`.update_stock, `tabSales Invoice Item`.uom, `tabSales Invoice Item`.qty {0}
		from `tabSales Invoice`, `tabSales Invoice Item`
		where `tabSales Invoice`.name = `tabSales Invoice Item`.parent
			and `tabSales Invoice`.docstatus = 1 %s %s
		order by `tabSales Invoice`.posting_date desc, `tabSales Invoice Item`.item_code desc
		""".format(additional_query_columns or '') % (conditions, match_conditions), filters, as_dict=1)

def get_delivery_notes_against_sales_order(item_list):
	so_dn_map = frappe._dict()
	so_item_rows = list(set([d.so_detail for d in item_list]))

	if so_item_rows:
		delivery_notes = frappe.db.sql("""
			select parent, so_detail
			from `tabDelivery Note Item`
			where docstatus=1 and so_detail in (%s)
			group by so_detail, parent
		""" % (', '.join(['%s']*len(so_item_rows))), tuple(so_item_rows), as_dict=1)

		for dn in delivery_notes:
			so_dn_map.setdefault(dn.so_detail, []).append(dn.parent)

	return so_dn_map

def get_deducted_taxes():
	return frappe.db.sql_list("select name from `tabPurchase Taxes and Charges` where add_deduct_tax = 'Deduct'")

def get_tax_accounts(item_list, columns, company_currency,
		doctype="Sales Invoice", tax_doctype="Sales Taxes and Charges"):
	import json
	item_row_map = {}
	tax_columns = []
	invoice_item_row = {}
	itemised_tax = {}

	tax_amount_precision = get_field_precision(frappe.get_meta(tax_doctype).get_field("tax_amount"),
		currency=company_currency) or 2

	for d in item_list:
		invoice_item_row.setdefault(d.parent, []).append(d)
		item_row_map.setdefault(d.parent, {}).setdefault(d.item_code or d.item_name, []).append(d)

	conditions = ""
	if doctype == "Purchase Invoice":
		conditions = " and category in ('Total', 'Valuation and Total') and base_tax_amount_after_discount_amount != 0"

	deducted_tax = get_deducted_taxes()
	tax_details = frappe.db.sql("""
		select
			name, parent, description, item_wise_tax_detail,
			charge_type, base_tax_amount_after_discount_amount
		from `tab%s`
		where
			parenttype = %s and docstatus = 1
			and (description is not null and description != '')
			and parent in (%s)
			%s
		order by description
	""" % (tax_doctype, '%s', ', '.join(['%s']*len(invoice_item_row)), conditions),
		tuple([doctype] + invoice_item_row.keys()))

	for name, parent, description, item_wise_tax_detail, charge_type, tax_amount in tax_details:
		description = handle_html(description)
		if description not in tax_columns and tax_amount:
			# as description is text editor earlier and markup can break the column convention in reports
			tax_columns.append(description)

		if item_wise_tax_detail:
			try:
				item_wise_tax_detail = json.loads(item_wise_tax_detail)

				for item_code, tax_data in item_wise_tax_detail.items():
					itemised_tax.setdefault(item_code, frappe._dict())

					if isinstance(tax_data, list):
						tax_rate, tax_amount = tax_data
					else:
						tax_rate = tax_data
						tax_amount = 0

					if charge_type == "Actual" and not tax_rate:
						tax_rate = "NA"

					item_net_amount = sum([flt(d.base_net_amount)
						for d in item_row_map.get(parent, {}).get(item_code, [])])

					for d in item_row_map.get(parent, {}).get(item_code, []):
						item_tax_amount = flt((tax_amount * d.base_net_amount) / item_net_amount) \
							if item_net_amount else 0
						if item_tax_amount:
							tax_value = flt(item_tax_amount, tax_amount_precision)
							tax_value = (tax_value * -1
								if (doctype == 'Purchase Invoice' and name in deducted_tax) else tax_value)

							itemised_tax.setdefault(d.name, {})[description] = frappe._dict({
								"tax_rate": tax_rate,
								"tax_amount": tax_value
							})

			except ValueError:
				continue
		elif charge_type == "Actual" and tax_amount:
			for d in invoice_item_row.get(parent, []):
				itemised_tax.setdefault(d.name, {})[description] = frappe._dict({
					"tax_rate": "NA",
					"tax_amount": flt((tax_amount * d.base_net_amount) / d.base_net_total,
						tax_amount_precision)
				})

	tax_columns.sort()
	for desc in tax_columns:
		columns.append(desc + " Rate:Data:80")
		columns.append(desc + " Amount:Currency/currency:100")

	columns += ["Total Tax:Currency/currency:80", "Total:Currency/currency:100"]

	return itemised_tax, tax_columns
<|endoftext|>


<commit_before>from oldowan.mtconvert.sites2seq import sites2seq
from oldowan.mtdna import rCRS
from oldowan.polymorphism import Polymorphism

def test_single_site_as_string():
    sites = '16129-'
    seq   = sites2seq(sites)
    assert '-' in seq

def test_single_site_as_string_in_list():
    sites = ['16129-']
    seq   = sites2seq(sites)
    assert '-' in seq

def test_single_site_as_Polymorphism_in_list():
    sites = [Polymorphism(16129,0,'-')]
    seq   = sites2seq(sites)
    assert '-' in seq

def test_two_sites_in_string():
    sites = '316C 316.1A'
    seq = sites2seq(sites, region=range(314,319))
    assert seq == 'CCCACT'

def test_two_sites_as_string_in_list():
    sites = ['316C', '316.1A']
    seq = sites2seq(sites, region=range(314,319))
    assert seq == 'CCCACT'

def test_two_sites_as_Polymorphisms_in_list():
    sites = [Polymorphism(316,0,'C'), Polymorphism(316,1,'A')]
    seq = sites2seq(sites, region=range(314,319))
    assert seq == 'CCCACT'

def test_mix_of_string_and_Polymorphisms_in_list():
    sites = [Polymorphism(316,0,'C'), '316.1A']
    seq = sites2seq(sites, region=range(314,319))
    assert seq == 'CCCACT'
    # reverse the order
    sites = ['316.1A', Polymorphism(316,0,'C')]
    seq = sites2seq(sites, region=range(314,319))
    assert seq == 'CCCACT'

def test_default_sequence_range_is_HVR1():
    assert sites2seq('') == rCRS[16023:16365]

def test_rCRS_argument_returns_rCRS():
    assert sites2seq('rCRS') == rCRS[16023:16365]

def test_HVR2_range_argument():
    assert sites2seq('', region='HVR2') == rCRS[72:340]

def test_HVR1andHVR2_range_argument():
    assert sites2seq('', region='HVR1and2') == rCRS[16023:16365]+rCRS[72:340]

def test_HVR1toHVR2_range_argument():
    assert sites2seq('', region='HVR1to2') == rCRS[16023:]+rCRS[:340]

def test_coding_range_argument():
    assert sites2seq('', region='coding') == rCRS[576:15992]

def test_all_range_argument():
    assert sites2seq('', region='all') == rCRS

def test_single_substition_in_correct_place():
    # rCRS string is 0-based count
    # rCRS as DNA sequence is 1-based
    # the the numbering discrepancy
    assert 'A' == sites2seq('16129A', region='all')[16128]

def test_custom_range():
    assert 'GAG' == sites2seq('', region=[1,5,9])

def test_insertion_in_custom_range():
    assert 'GCAT' == sites2seq('1.1C', region=[1,2,3])

def test_two_insertions_in_custom_range():
    assert 'GCAGT' == sites2seq('1.1C 2.1G', region=[1,2,3])

def test_deletion_at_start_of_custom_range():
    assert '-AT' == sites2seq('1-', region=[1,2,3])

def test_deletion_in_middle_of_custom_range():
    assert 'G-T' == sites2seq('2-', region=[1,2,3])

<commit_msg>converted some tests to use new region=(start,stop) of sites2seq<commit_after>from oldowan.mtconvert.sites2seq import sites2seq
from oldowan.mtdna import rCRS
from oldowan.polymorphism import Polymorphism

def test_single_site_as_string():
    sites = '16129-'
    seq   = sites2seq(sites)
    assert '-' in seq

def test_single_site_as_string_in_list():
    sites = ['16129-']
    seq   = sites2seq(sites)
    assert '-' in seq

def test_single_site_as_Polymorphism_in_list():
    sites = [Polymorphism(16129,0,'-')]
    seq   = sites2seq(sites)
    assert '-' in seq

def test_two_sites_in_string():
    sites = '316C 316.1A'
    seq = sites2seq(sites, region=range(314,319))
    assert seq == 'CCCACT'

def test_two_sites_as_string_in_list():
    sites = ['316C', '316.1A']
    seq = sites2seq(sites, region=(314,318))
    assert seq == 'CCCACT'

def test_two_sites_as_Polymorphisms_in_list():
    sites = [Polymorphism(316,0,'C'), Polymorphism(316,1,'A')]
    seq = sites2seq(sites, region=(314,318))
    assert seq == 'CCCACT'

def test_mix_of_string_and_Polymorphisms_in_list():
    sites = [Polymorphism(316,0,'C'), '316.1A']
    seq = sites2seq(sites, region=(314,318))
    assert seq == 'CCCACT'
    # reverse the order
    sites = ['316.1A', Polymorphism(316,0,'C')]
    seq = sites2seq(sites, region=(314,318))
    assert seq == 'CCCACT'

def test_default_sequence_range_is_HVR1():
    assert sites2seq('') == rCRS[16023:16365]
    assert sites2seq('', region=(16024,16365)) == rCRS[16023:16365]

def test_rCRS_argument_returns_rCRS():
    assert sites2seq('rCRS') == rCRS[16023:16365]

def test_HVR2_range_argument():
    assert sites2seq('', region='HVR2') == rCRS[72:340]

def test_HVR1andHVR2_range_argument():
    assert sites2seq('', region='HVR1and2') == rCRS[16023:16365]+rCRS[72:340]

def test_HVR1toHVR2_range_argument():
    assert sites2seq('', region='HVR1to2') == rCRS[16023:]+rCRS[:340]

def test_coding_range_argument():
    assert sites2seq('', region='coding') == rCRS[576:15992]

def test_all_range_argument():
    assert sites2seq('', region='all') == rCRS

def test_single_substition_in_correct_place():
    # rCRS string is 0-based count
    # rCRS as DNA sequence is 1-based
    # the the numbering discrepancy
    assert 'A' == sites2seq('16129A', region='all')[16128]

def test_custom_range():
    assert 'GAG' == sites2seq('', region=[1,5,9])

def test_insertion_in_custom_range():
    assert 'GCAT' == sites2seq('1.1C', region=[1,2,3])

def test_two_insertions_in_custom_range():
    assert 'GCAGT' == sites2seq('1.1C 2.1G', region=[1,2,3])

def test_deletion_at_start_of_custom_range():
    assert '-AT' == sites2seq('1-', region=[1,2,3])

def test_deletion_in_middle_of_custom_range():
    assert 'G-T' == sites2seq('2-', region=[1,2,3])

<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst

"""
Optimization algorithms used in `~astropy.modeling.fitting`.
"""

from __future__ import (absolute_import, unicode_literals, division,
                        print_function)
import warnings
import abc
import numpy as np
from ..extern import six
from ..utils.exceptions import AstropyUserWarning

__all__ = ["Optimization", "SLSQP", "Simplex"]

# Maximum number of iterations
DEFAULT_MAXITER = 100

# Step for the forward difference approximation of the Jacobian
DEFAULT_EPS = np.sqrt(np.finfo(float).eps)

#Default requested accuracy
DEFAULT_ACC = 1e-07

DEFAULT_BOUNDS = (-10 ** 12, 10 ** 12)

@six.add_metaclass(abc.ABCMeta)
class Optimization(object):
    """
    Base class for optimizers.

    Parameters
    ----------
    opt_method : callable
        Implements optimization method

    Notes
    -----
    The base Optimizer does not support any constraints by default; individual
    optimizers should explicitly set this list to the specific constraints
    it supports.

    """

    supported_constraints = []

    def __init__(self, opt_method):
        self._opt_method = opt_method
        self._maxiter = DEFAULT_MAXITER
        self._eps = DEFAULT_EPS
        self._acc = DEFAULT_ACC

    @property
    def maxiter(self):
        """Maximum number of iterations"""
        return self._maxiter

    @maxiter.setter
    def maxiter(self, val):
        """Set maxiter"""
        self._maxiter = val

    @property
    def eps(self):
        """Step for the forward difference approximation of the Jacobian"""
        return self._eps

    @eps.setter
    def eps(self, val):
        """Set eps value"""
        self._eps = val

    @property
    def acc(self):
        """Requested accuracy"""
        return self._acc

    @acc.setter
    def acc(self, val):
        """Set accuracy"""
        self._acc = val

    def __repr__(self):
        fmt = "{0}()".format(self.__class__.__name__)
        return fmt

    @property
    def opt_method(self):
        return self._opt_method

    @abc.abstractmethod
    def __call__(self):
        raise NotImplementedError("Subclasses should implement this method")

class SLSQP(Optimization):
    """
    Sequential Least Squares Programming optimization algorithm.

    The algorithm is described in [1]_. It supports tied and fixed
    parameters, as well as bounded constraints. Uses
    `scipy.optimize.fmin_slsqp`.

    References
    ----------
    .. [1] http://www.netlib.org/toms/733
    """
    supported_constraints = ['bounds', 'eqcons', 'ineqcons', 'fixed', 'tied']

    def __init__(self):
        from scipy.optimize import fmin_slsqp
        super(SLSQP, self).__init__(fmin_slsqp)
        self.fit_info = {
            'final_func_val': None,
            'numiter': None,
            'exit_mode': None,
            'message': None
        }

    def __call__(self, objfunc, initval, fargs, **kwargs):
        """
        Run the solver.

        Parameters
        ----------
        objfunc : callable
            objection function
        initval : iterable
            initial guess for the parameter values
        fargs : tuple
            other arguments to be passed to the statistic function
        kwargs : dict
            other keyword arguments to be passed to the solver

        """
        kwargs['iter'] = kwargs.pop('maxiter', self._maxiter)

        if 'epsilon' not in kwargs:
            kwargs['epsilon'] = self._eps
        if 'acc' not in kwargs:
            kwargs['acc'] = self._acc

        # set the values of constraints to match the requirements of fmin_slsqp
        model = fargs[0]
        pars = [getattr(model, name) for name in model.param_names]
        bounds = [par.bounds for par in pars if par.fixed != True and
                  par.tied == False]
        bounds = np.asarray(bounds)
        for i in bounds:
            if i[0] is None:
                i[0] =  DEFAULT_BOUNDS[0]
            if i[1] is None:
                i[1] = DEFAULT_BOUNDS[1]
        # older versions of scipy require this array to be float
        bounds = np.asarray(bounds, dtype=np.float)
        eqcons = np.array(model.eqcons)
        ineqcons = np.array(model.ineqcons)
        fitparams, final_func_val, numiter, exit_mode, mess = self.opt_method(
            objfunc, initval, args=fargs, full_output=True,
            bounds=bounds, eqcons=eqcons, ieqcons=ineqcons,
            **kwargs)

        self.fit_info['final_func_val'] = final_func_val
        self.fit_info['numiter'] = numiter
        self.fit_info['exit_mode'] = exit_mode
        self.fit_info['message'] = mess

        if exit_mode != 0:
            warnings.warn("The fit may be unsuccessful; check "
                          "fit_info['message'] for more information.",
                          AstropyUserWarning)

        return fitparams, self.fit_info

class Simplex(Optimization):
    """
    Neald-Mead (downhill simplex) algorithm [1].

    This algorithm only uses function values, not derivatives.
    Uses `scipy.optimize.fmin`.

    .. [1] Nelder, J.A. and Mead, R. (1965), "A simplex method for function
           minimization", The Computer Journal, 7, pp. 308-313
    """

    supported_constraints = ['bounds', 'fixed', 'tied']

    def __init__(self):
        from scipy.optimize import fmin as simplex
        super(Simplex, self).__init__(simplex)
        self.fit_info = {
            'final_func_val': None,
            'numiter': None,
            'exit_mode': None,
            'num_function_calls': None
        }

    def __call__(self, objfunc, initval, fargs, **kwargs):
        """
        Run the solver.

        Parameters
        ----------
        objfunc : callable
            objection function
        initval : iterable
            initial guess for the parameter values
        fargs : tuple
            other arguments to be passed to the statistic function
        kwargs : dict
            other keyword arguments to be passed to the solver

        """
        if 'maxiter' not in kwargs:
            kwargs['maxiter'] = self._maxiter

        fitparams, final_func_val, numiter, funcalls, exit_mode = self.opt_method(
            objfunc, initval, args=fargs, xtol=self._acc,
            full_output=True, **kwargs)
        self.fit_info['final_func_val'] = final_func_val
        self.fit_info['numiter'] = numiter
        self.fit_info['exit_mode'] = exit_mode
        self.fit_info['num_function_calls'] = funcalls
        if self.fit_info['exit_mode'] == 1:
            warnings.warn("The fit may be unsuccessful; "
                          "Maximum number of function evaluations reached.",
                          AstropyUserWarning)
        if self.fit_info['exit_mode'] == 2:
            warnings.warn("The fit may be unsuccessful; "
                          "Maximum number of iterations reached.",
                          AstropyUserWarning)
        return fitparams, self.fit_info
<commit_msg>Update optimizers.py<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst

"""
Optimization algorithms used in `~astropy.modeling.fitting`.
"""

from __future__ import (absolute_import, unicode_literals, division,
                        print_function)
import warnings
import abc
import numpy as np
from ..extern import six
from ..utils.exceptions import AstropyUserWarning

__all__ = ["Optimization", "SLSQP", "Simplex"]

# Maximum number of iterations
DEFAULT_MAXITER = 100

# Step for the forward difference approximation of the Jacobian
DEFAULT_EPS = np.sqrt(np.finfo(float).eps)

#Default requested accuracy
DEFAULT_ACC = 1e-07

DEFAULT_BOUNDS = (-10 ** 12, 10 ** 12)

@six.add_metaclass(abc.ABCMeta)
class Optimization(object):
    """
    Base class for optimizers.

    Parameters
    ----------
    opt_method : callable
        Implements optimization method

    Notes
    -----
    The base Optimizer does not support any constraints by default; individual
    optimizers should explicitly set this list to the specific constraints
    it supports.

    """

    supported_constraints = []

    def __init__(self, opt_method):
        self._opt_method = opt_method
        self._maxiter = DEFAULT_MAXITER
        self._eps = DEFAULT_EPS
        self._acc = DEFAULT_ACC

    @property
    def maxiter(self):
        """Maximum number of iterations"""
        return self._maxiter

    @maxiter.setter
    def maxiter(self, val):
        """Set maxiter"""
        self._maxiter = val

    @property
    def eps(self):
        """Step for the forward difference approximation of the Jacobian"""
        return self._eps

    @eps.setter
    def eps(self, val):
        """Set eps value"""
        self._eps = val

    @property
    def acc(self):
        """Requested accuracy"""
        return self._acc

    @acc.setter
    def acc(self, val):
        """Set accuracy"""
        self._acc = val

    def __repr__(self):
        fmt = "{0}()".format(self.__class__.__name__)
        return fmt

    @property
    def opt_method(self):
        return self._opt_method

    @abc.abstractmethod
    def __call__(self):
        raise NotImplementedError("Subclasses should implement this method")

class SLSQP(Optimization):
    """
    Sequential Least Squares Programming optimization algorithm.

    The algorithm is described in [1]_. It supports tied and fixed
    parameters, as well as bounded constraints. Uses
    `scipy.optimize.fmin_slsqp`.

    References
    ----------
    .. [1] http://www.netlib.org/toms/733
    """
    supported_constraints = ['bounds', 'eqcons', 'ineqcons', 'fixed', 'tied']

    def __init__(self):
        from scipy.optimize import fmin_slsqp
        super(SLSQP, self).__init__(fmin_slsqp)
        self.fit_info = {
            'final_func_val': None,
            'numiter': None,
            'exit_mode': None,
            'message': None
        }

    def __call__(self, objfunc, initval, fargs, **kwargs):
        """
        Run the solver.

        Parameters
        ----------
        objfunc : callable
            objection function
        initval : iterable
            initial guess for the parameter values
        fargs : tuple
            other arguments to be passed to the statistic function
        kwargs : dict
            other keyword arguments to be passed to the solver

        """
        kwargs['iter'] = kwargs.pop('maxiter', self._maxiter)

        if 'epsilon' not in kwargs:
            kwargs['epsilon'] = self._eps
        if 'acc' not in kwargs:
            kwargs['acc'] = self._acc

        # set the values of constraints to match the requirements of fmin_slsqp
        model = fargs[0]
        pars = [getattr(model, name) for name in model.param_names]
        bounds = [par.bounds for par in pars if par.fixed != True and
                  par.tied == False]
        bounds = np.asarray(bounds)
        for i in bounds:
            if i[0] is None:
                i[0] =  DEFAULT_BOUNDS[0]
            if i[1] is None:
                i[1] = DEFAULT_BOUNDS[1]
        # older versions of scipy require this array to be float
        bounds = np.asarray(bounds, dtype=np.float)
        eqcons = np.array(model.eqcons)
        ineqcons = np.array(model.ineqcons)
        fitparams, final_func_val, numiter, exit_mode, mess = self.opt_method(
            objfunc, initval, args=fargs, full_output=True,
            bounds=bounds, eqcons=eqcons, ieqcons=ineqcons,
            **kwargs)

        self.fit_info['final_func_val'] = final_func_val
        self.fit_info['numiter'] = numiter
        self.fit_info['exit_mode'] = exit_mode
        self.fit_info['message'] = mess

        if exit_mode != 0:
            warnings.warn("The fit may be unsuccessful; check "
                          "fit_info['message'] for more information.",
                          AstropyUserWarning)

        return fitparams, self.fit_info

class Simplex(Optimization):
    """
    Neald-Mead (downhill simplex) algorithm [1].

    This algorithm only uses function values, not derivatives.
    Uses `scipy.optimize.fmin`.

    .. [1] Nelder, J.A. and Mead, R. (1965), "A simplex method for function
           minimization", The Computer Journal, 7, pp. 308-313
    """

    supported_constraints = ['bounds', 'fixed', 'tied']

    def __init__(self):
        from scipy.optimize import fmin as simplex
        super(Simplex, self).__init__(simplex)
        self.fit_info = {
            'final_func_val': None,
            'numiter': None,
            'exit_mode': None,
            'num_function_calls': None
        }

    def __call__(self, objfunc, initval, fargs, **kwargs):
        """
        Run the solver.

        Parameters
        ----------
        objfunc : callable
            objection function
        initval : iterable
            initial guess for the parameter values
        fargs : tuple
            other arguments to be passed to the statistic function
        kwargs : dict
            other keyword arguments to be passed to the solver

        """
        if 'maxiter' not in kwargs:
            kwargs['maxiter'] = self._maxiter
        if 'acc' in kwargs:
            self._acc = kwargs['acc']
            kwargs.pop['acc']
        if 'xtol' in kwargs:
            self._acc = kwargs['xtol']
            kwargs.pop['xtol']

        fitparams, final_func_val, numiter, funcalls, exit_mode = self.opt_method(
            objfunc, initval, args=fargs, xtol=self._acc,
            full_output=True, **kwargs)
        self.fit_info['final_func_val'] = final_func_val
        self.fit_info['numiter'] = numiter
        self.fit_info['exit_mode'] = exit_mode
        self.fit_info['num_function_calls'] = funcalls
        if self.fit_info['exit_mode'] == 1:
            warnings.warn("The fit may be unsuccessful; "
                          "Maximum number of function evaluations reached.",
                          AstropyUserWarning)
        if self.fit_info['exit_mode'] == 2:
            warnings.warn("The fit may be unsuccessful; "
                          "Maximum number of iterations reached.",
                          AstropyUserWarning)
        return fitparams, self.fit_info
<|endoftext|>


<commit_before>#!/usr/bin/python

# Remap each CraftBukkit commit

import subprocess
import os
import shutil
import xml.dom.minidom
import re
import sys
import optparse

global repoURL, outDirGitRepo, inOriginalDir, inRemappedDir, defaultStartCommit, defaultStartMessage

def runRemap():
    print "Starting remap script..."
    run(sys.executable + " remap-craftbukkit.py --cb-dir "+inOriginalDir+" --fml-dir fml --skip-finish-cleanup --skip-compile")
    print "Remap script finished"

def run(cmd):
    print ">",cmd
    #raw_input()
    status = os.system(cmd)
    assert status == 0, "Failed to run '%s' status=%s" % (cmd, status)

def runOutput(cmd):
    print ">",cmd
    return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.read()

DIR_STACK = []
def pushd(newDir):
    print "cd",newDir
    DIR_STACK.append(os.getcwd())
    os.chdir(newDir)

def popd():
    print "cd",DIR_STACK[-1]
    os.chdir(DIR_STACK.pop())

"""Clean out even non-repository or moved files."""
def clean():
    run("rm -rf src")
    run("git reset --hard HEAD")

"""Get commit IDs and short messages after the starting commit, in reverse chronological order."""
def readCommitLog(startCommit):
    commits = []
    for line in runOutput(("git", "log", "--format=oneline")).split("\n"):
        assert len(line) != 0, "Reached end of commit log without finding starting commit "+startCommit
        commit, message = line.split(" ", 1)
        if commit == startCommit: break
        print commit, message
        commits.append((commit, message))
    commits.reverse()
    return commits

"""Get detailed information on a commit."""
def getCommitInfo(commit):
    out = runOutput(("git", "show", commit, "--format=format:%an <%ae>%n%aD%n%s%n%n%b%n---END---"))
    lines = out.split("\n")
    author = lines[0]
    date = lines[1]
    messageLines = []
    for i in range(2,len(lines)):
        if lines[i] == "---END---": break
        messageLines.append(lines[i])
    message = "\n".join(messageLines)

    if commit == defaultStartCommit:
        # First commit!
        return None, None, defaultStartMessage

    return author, date, message

def getCreditMessagePrefix():
    return "\n\nRemapped by Srg2Source from "+repoURL.replace("http://github.com/","")+"@"  # github markup, Bukkit/CraftBukkit@hash

"""Get the last remapped commit, if any"""
def getStartCommit():
    if not os.path.exists(outDirGitRepo):
        print "Creating",outDirGitRepo
        os.mkdir(outDirGitRepo)

    pushd(outDirGitRepo)

    if not os.path.exists(".git"):
        print "No git repository found in "+outDirGitRepo
        if defaultStartCommit is None or defaultStartMessage is None:
            print "--startCommit and --startMessage required for initializing new repository"
            sys.exit(1)

        print "Initializing new repository"
        run("git init")
        popd()
        print "Starting at default "+defaultStartCommit
        return defaultStartCommit

    message = runOutput(("git", "show", "--format=%s%n%n%b"))

    r = re.compile(getCreditMessagePrefix().strip() + "([0-9a-f]+)")
    match = r.search(message)
    if not match:
        print "Unrecognized commit message: >>>\n"+message+"\n<<< - no match for '"+getCreditMessagePrefix()+"'"
        if defaultStartCommit is None or defaultStartMessage is None:
            print "No remapped commits found; --startCommit and --startMessage required!"
            sys.exit(1)
        print "Starting at default "+defaultStartCommit
        popd()
        return defaultStartCommit

    commit = match.group(1)
    popd()

    print "Resuming from upstream commit",repoURL+"/commit/"+commit
    return commit

def main():
    global outDirGitRepo, repoURL, inOriginalDir, inRemappedDir, defaultStartCommit, defaultStartMessage

    parser = optparse.OptionParser()
    parser.add_option('-o', '--outDirGitRepo',  action='store', dest='outDirGitRepo', help='Output directory git repository')
    parser.add_option('-p', '--pushGit',  action='store_true', dest='shouldPushGit', help='Run git push after each commit remap', default=False)
    parser.add_option('-n', '--no-cloneRepo', action='store_false', dest='shouldCloneRepo', help='Disable cloning upstream repository', default=True)
    parser.add_option('-u', '--no-fetchLatestChanges', action='store_false', dest='shouldFetchLatestChanges', help='Disable fetching latest upstream changes', default=True)
    parser.add_option('-c', '--no-checkoutMaster', action='store_false', dest='shouldCheckoutMaster', help='Disable checking out master branch', default=True)
    parser.add_option('-s', '--remoteSource', action='store', dest='remoteSource', help='git remote source name', default='origin')
    parser.add_option('-b', '--masterBranch', action='store', dest='masterBranch', help='git branch name', default='master')
    parser.add_option('-r', '--repoURL', action='store', dest='repoURL', help='git repository URL', default='http://github.com/Bukkit/CraftBukkit')
    parser.add_option('-i', '--inOriginalDir', action='store', dest='inOriginalDir', help='Original source directory', default='CraftBukkit')
    parser.add_option('-I', '--inRemappedDir', action='store', dest='inRemappedDir', help='Remapper output directory', default='../output')
    parser.add_option('-C', '--startCommit', action='store', dest='defaultStartCommit', help='Starting commit if no existing remapped commits found', default=None)
    parser.add_option('-M', '--startMessage', action='store', dest='defaultStartMessage', help='Initial commit message for --startCommit', default=None)

    options, args = parser.parse_args()

    if options.outDirGitRepo is None:
        print "--outDirGitRepo required"
        parser.print_help()
        sys.exit(1)

    outDirGitRepo = os.path.abspath(options.outDirGitRepo)
    repoURL = options.repoURL
    inOriginalDir = os.path.abspath(options.inOriginalDir)
    inRemappedDir = os.path.abspath(options.inRemappedDir)
    defaultStartCommit = options.defaultStartCommit
    defaultStartMessage = options.defaultStartMessage

    if options.shouldCloneRepo:
        if os.path.exists(inOriginalDir):
            shutil.rmtree(inOriginalDir)

        run("git clone "+repoURL+" "+inOriginalDir)

    pushd(inOriginalDir)
    if options.shouldFetchLatestChanges:
        # Get all the latest changes, overwriting anything locally
        run("git fetch --all")
        run("git reset --hard "+options.remoteSource+"/"+options.masterBranch)

    if options.shouldCheckoutMaster:
        clean()
        run("git checkout "+options.masterBranch)

    # Get commits beyond our last remapped commit, the new commits to be remapped
    startCommit = getStartCommit()
    commits = readCommitLog(startCommit)
    popd()

    for commitInfo in commits:
        # Remap this commit
        commit, shortMessage = commitInfo
        print "\n\n*** %s %s" % (commit, shortMessage)
        pushd(inOriginalDir)
        clean()
        run("git checkout "+commit)
        author, date, message = getCommitInfo(commit)
        popd()
        try:
            runRemap()
        except Exception as e:
            print "WARNING!!! Remapping failed with exception:",e
            print "Continuing anyways"

        # Append message to commit
        # TODO: former-commit in 'commit notes' like bfg? http://rtyley.github.com/bfg-repo-cleaner/
        message += getCreditMessagePrefix()+commit

        # Copy remapper output to git repository
        inRemappedDirNames = ("patches", "src/org", "src/jline") # subdirectories in inRemappedDir to add
        for part in inRemappedDirNames:
            a = os.path.join(inRemappedDir, part)
            b = os.path.join(outDirGitRepo, part)
            print "Copying %s -> %s" % (a, b)
            if os.path.exists(b): shutil.rmtree(b)
            shutil.copytree(a, b)

        # For copying full NMS source (vs patches) with --skip-output-archive; might want this in a separate repo later
        ## Copy to target
        #a = os.path.join(inOriginalDir, "src")
        #b = os.path.join(outDirGitRepo, "src")
        #print "Copying %s -> %s" % (a, b)
        #if os.path.exists(b): shutil.rmtree(b)
        #shutil.copytree(a, b)

        # Generate the new remapped commit
        pushd(outDirGitRepo)
        for part in inRemappedDirNames:
            run("git add "+part)
        stagedDiff = runOutput(("git", "diff", "--staged"))
        if len(stagedDiff.strip()) == 0:
            print "Nothing changed in this commit; skipping"  # docs, 'add for diff visibility', etc.
            popd()
            continue
        commitFile = os.path.join(os.getcwd(), "commit.msg")
        file(commitFile,"w").write(message)
        cmd = "git commit --file='%s' --all" % (commitFile,)
        if author is not None:
            cmd += " --author='%s'" % (author,)
        if date is not None:
            cmd += " --date='%s'" % (date,)
        run(cmd)
        os.unlink(commitFile)

        # Push
        if options.shouldPushGit:
            run("git push")

        popd()

if __name__ == "__main__":
    main()

<commit_msg>Change how subprocesses are run/read, and tell git to not use less, should help some commands stalling.<commit_after>#!/usr/bin/python

# Remap each CraftBukkit commit

import subprocess
import os
import shutil
import xml.dom.minidom
import re
import sys
import optparse

global repoURL, outDirGitRepo, inOriginalDir, inRemappedDir, defaultStartCommit, defaultStartMessage

def runRemap():
    print "Starting remap script..."
    run(sys.executable + " remap-craftbukkit.py --cb-dir "+inOriginalDir+" --fml-dir fml --skip-finish-cleanup --skip-compile")
    print "Remap script finished"

def run(cmd):
    print ">",cmd
    #raw_input()
    status = os.system(cmd)
    assert status == 0, "Failed to run '%s' status=%s" % (cmd, status)

def runOutput(cmd):
    print ">",cmd
    return subprocess.check_output(cmd, stderr=subprocess.STDOUT)

DIR_STACK = []
def pushd(newDir):
    print "cd",newDir
    DIR_STACK.append(os.getcwd())
    os.chdir(newDir)

def popd():
    print "cd",DIR_STACK[-1]
    os.chdir(DIR_STACK.pop())

"""Clean out even non-repository or moved files."""
def clean():
    run("rm -rf src")
    run("git reset --hard HEAD")

"""Get commit IDs and short messages after the starting commit, in reverse chronological order."""
def readCommitLog(startCommit):
    commits = []
    for line in runOutput(['git', '--no-pager', 'log', '--format=oneline']).split("\n"):
        assert len(line) != 0, "Reached end of commit log without finding starting commit "+startCommit
        commit, message = line.split(" ", 1)
        if commit == startCommit: break
        print commit, message
        commits.append((commit, message))
    commits.reverse()
    return commits

"""Get detailed information on a commit."""
def getCommitInfo(commit):
    lines = runOutput(['git', '--no-pager', 'show', commit, '--format=format:%an <%ae>%n%aD%n%s%n%n%b%n---END---']).split("\n")
    author = lines[0]
    date = lines[1]
    messageLines = []
    for i in range(2,len(lines)):
        if lines[i] == "---END---": break
        messageLines.append(lines[i])
    message = "\n".join(messageLines)

    if commit == defaultStartCommit:
        # First commit!
        return None, None, defaultStartMessage

    return author, date, message

def getCreditMessagePrefix():
    return "\n\nRemapped by Srg2Source from "+repoURL.replace("http://github.com/","")+"@"  # github markup, Bukkit/CraftBukkit@hash

"""Get the last remapped commit, if any"""
def getStartCommit():
    if not os.path.exists(outDirGitRepo):
        print "Creating",outDirGitRepo
        os.mkdir(outDirGitRepo)

    pushd(outDirGitRepo)

    if not os.path.exists(".git"):
        print "No git repository found in "+outDirGitRepo
        if defaultStartCommit is None or defaultStartMessage is None:
            print "--startCommit and --startMessage required for initializing new repository"
            sys.exit(1)

        print "Initializing new repository"
        run('git init')
        popd()
        print "Starting at default "+defaultStartCommit
        return defaultStartCommit

    message = runOutput(['git', '--no-pager', 'show', '--format=%s%n%n%b'])

    r = re.compile(getCreditMessagePrefix().strip() + "([0-9a-f]+)")
    match = r.search(message)
    if not match:
        print "Unrecognized commit message: >>>\n"+message+"\n<<< - no match for '"+getCreditMessagePrefix()+"'"
        if defaultStartCommit is None or defaultStartMessage is None:
            print "No remapped commits found; --startCommit and --startMessage required!"
            sys.exit(1)
        print "Starting at default "+defaultStartCommit
        popd()
        return defaultStartCommit

    commit = match.group(1)
    popd()

    print "Resuming from upstream commit",repoURL+"/commit/"+commit
    return commit

def main():
    global outDirGitRepo, repoURL, inOriginalDir, inRemappedDir, defaultStartCommit, defaultStartMessage

    parser = optparse.OptionParser()
    parser.add_option('-o', '--outDirGitRepo',  action='store', dest='outDirGitRepo', help='Output directory git repository')
    parser.add_option('-p', '--pushGit',  action='store_true', dest='shouldPushGit', help='Run git push after each commit remap', default=False)
    parser.add_option('-n', '--no-cloneRepo', action='store_false', dest='shouldCloneRepo', help='Disable cloning upstream repository', default=True)
    parser.add_option('-u', '--no-fetchLatestChanges', action='store_false', dest='shouldFetchLatestChanges', help='Disable fetching latest upstream changes', default=True)
    parser.add_option('-c', '--no-checkoutMaster', action='store_false', dest='shouldCheckoutMaster', help='Disable checking out master branch', default=True)
    parser.add_option('-s', '--remoteSource', action='store', dest='remoteSource', help='git remote source name', default='origin')
    parser.add_option('-b', '--masterBranch', action='store', dest='masterBranch', help='git branch name', default='master')
    parser.add_option('-r', '--repoURL', action='store', dest='repoURL', help='git repository URL', default='http://github.com/Bukkit/CraftBukkit')
    parser.add_option('-i', '--inOriginalDir', action='store', dest='inOriginalDir', help='Original source directory', default='CraftBukkit')
    parser.add_option('-I', '--inRemappedDir', action='store', dest='inRemappedDir', help='Remapper output directory', default='../output')
    parser.add_option('-C', '--startCommit', action='store', dest='defaultStartCommit', help='Starting commit if no existing remapped commits found', default=None)
    parser.add_option('-M', '--startMessage', action='store', dest='defaultStartMessage', help='Initial commit message for --startCommit', default=None)

    options, args = parser.parse_args()

    if options.outDirGitRepo is None:
        print "--outDirGitRepo required"
        parser.print_help()
        sys.exit(1)

    outDirGitRepo = os.path.abspath(options.outDirGitRepo)
    repoURL = options.repoURL
    inOriginalDir = os.path.abspath(options.inOriginalDir)
    inRemappedDir = os.path.abspath(options.inRemappedDir)
    defaultStartCommit = options.defaultStartCommit
    defaultStartMessage = options.defaultStartMessage

    if options.shouldCloneRepo:
        if os.path.exists(inOriginalDir):
            shutil.rmtree(inOriginalDir)

        run("git clone "+repoURL+" "+inOriginalDir)

    pushd(inOriginalDir)
    if options.shouldFetchLatestChanges:
        # Get all the latest changes, overwriting anything locally
        run("git fetch --all")
        run("git reset --hard "+options.remoteSource+"/"+options.masterBranch)

    if options.shouldCheckoutMaster:
        clean()
        run("git checkout "+options.masterBranch)

    # Get commits beyond our last remapped commit, the new commits to be remapped
    startCommit = getStartCommit()
    commits = readCommitLog(startCommit)
    popd()

    for commitInfo in commits:
        # Remap this commit
        commit, shortMessage = commitInfo
        print "\n\n*** %s %s" % (commit, shortMessage)
        pushd(inOriginalDir)
        clean()
        run("git checkout "+commit)
        author, date, message = getCommitInfo(commit)
        popd()
        try:
            runRemap()
        except Exception as e:
            print "WARNING!!! Remapping failed with exception:",e
            print "Continuing anyways"
            sys.exit(1)

        # Append message to commit
        # TODO: former-commit in 'commit notes' like bfg? http://rtyley.github.com/bfg-repo-cleaner/
        message += getCreditMessagePrefix()+commit

        # Copy remapper output to git repository
        inRemappedDirNames = ("patches", "src") # subdirectories in inRemappedDir to add
        for part in inRemappedDirNames:
            a = os.path.join(inRemappedDir, part)
            b = os.path.join(outDirGitRepo, part)
            print "Copying %s -> %s" % (a, b)
            if os.path.exists(b): shutil.rmtree(b)
            shutil.copytree(a, b)

        # For copying full NMS source (vs patches) with --skip-output-archive; might want this in a separate repo later
        ## Copy to target
        #a = os.path.join(inOriginalDir, "src")
        #b = os.path.join(outDirGitRepo, "src")
        #print "Copying %s -> %s" % (a, b)
        #if os.path.exists(b): shutil.rmtree(b)
        #shutil.copytree(a, b)

        # Generate the new remapped commit
        pushd(outDirGitRepo)
        for part in inRemappedDirNames:
            run("git add "+part)
        stagedDiff = runOutput(['git', '--no-pager', 'diff', '--staged'])
        if len(stagedDiff.strip()) == 0:
            print "Nothing changed in this commit; skipping"  # docs, 'add for diff visibility', etc.
            popd()
            continue
        commitFile = os.path.join(os.getcwd(), "commit.msg")
        file(commitFile,"w").write(message)
        cmd = 'git commit --file "%s" --all' % commitFile
        if author is not None:
            cmd += ' --author "%s"' % author
        if date is not None:
            cmd += ' --date "%s"' % date
        run(cmd)
        os.unlink(commitFile)

        # Push
        if options.shouldPushGit:
            run("git push")

        popd()

if __name__ == "__main__":
    main()

<|endoftext|>