<commit_before># Copyright (c) 2013, Web Notes Technologies Pvt. Ltd.
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals
import webnotes

from webnotes.utils import cstr, flt, nowdate
from webnotes.model.code import get_obj
from webnotes import msgprint, _

sql = webnotes.conn.sql

class OverProductionError(webnotes.ValidationError): pass

class DocType:
	def __init__(self, doc, doclist=[]):
		self.doc = doc
		self.doclist = doclist

	def validate(self):
		import utilities
		utilities.validate_status(self.doc.status, ["Draft", "Submitted", "Stopped",
			"In Process", "Completed", "Cancelled"])

		if self.doc.production_item :
			item_detail = sql("select name from `tabItem` where name = '%s' and docstatus != 2"
			 	% self.doc.production_item, as_dict = 1)
			if not item_detail:
				msgprint("Item '%s' does not exist or cancelled in the system."
					% cstr(self.doc.production_item), raise_exception=1)

		if self.doc.bom_no:
			bom = sql("""select name from `tabBOM` where name=%s and docstatus=1
				and is_active=1 and item=%s"""
				, (self.doc.bom_no, self.doc.production_item), as_dict =1)
			if not bom:
				msgprint("""Incorrect BOM: %s entered.
					May be BOM not exists or inactive or not submitted
					or for some other item.""" % cstr(self.doc.bom_no), raise_exception=1)

		if self.doc.sales_order:
			if not webnotes.conn.sql("""select name from `tabSales Order`
					where name=%s and docstatus = 1""", self.doc.sales_order):
				msgprint("Sales Order: %s is not valid" % self.doc.sales_order, raise_exception=1)

			self.validate_production_order_against_so()

		from utilities.transaction_base import validate_uom_is_integer
		validate_uom_is_integer(self.doclist, "stock_uom", ["qty", "produced_qty"])

	def validate_production_order_against_so(self):
		# already ordered qty
		ordered_qty_against_so = webnotes.conn.sql("""select sum(qty) from `tabProduction Order`
			where production_item = %s and sales_order = %s and docstatus < 2 and name != %s""",
			(self.doc.production_item, self.doc.sales_order, self.doc.name))[0][0]

		total_qty = flt(ordered_qty_against_so) + flt(self.doc.qty)

		# get qty from Sales Order Item table
		so_item_qty = webnotes.conn.sql("""select sum(qty) from `tabSales Order Item`
			where parent = %s and item_code = %s""",
			(self.doc.sales_order, self.doc.production_item))[0][0]
		# get qty from Packing Item table
		dnpi_qty = webnotes.conn.sql("""select sum(qty) from `tabDelivery Note Packing Item`
			where parent = %s and parenttype = 'Sales Order' and item_code = %s""",
			(self.doc.sales_order, self.doc.production_item))[0][0]
		# total qty in SO
		so_qty = flt(so_item_qty) + flt(dnpi_qty)

		if total_qty > so_qty:
			webnotes.msgprint(_("Total production order qty for item") + ": " +
				cstr(self.doc.production_item) + _(" against sales order") + ": " +
				cstr(self.doc.sales_order) + _(" will be ") + cstr(total_qty) + ", " +
				_("which is greater than sales order qty ") + "(" + cstr(so_qty) + ")" +
				_("Please reduce qty."), raise_exception=OverProductionError)

	def stop_unstop(self, status):
		""" Called from client side on Stop/Unstop event"""
		self.update_status(status)
		qty = (flt(self.doc.qty)-flt(self.doc.produced_qty)) * ((status == 'Stopped') and -1 or 1)
		self.update_planned_qty(qty)
		msgprint("Production Order has been %s" % status)

	def update_status(self, status):
		if status == 'Stopped':
			webnotes.conn.set(self.doc, 'status', cstr(status))
		else:
			if flt(self.doc.qty) == flt(self.doc.produced_qty):
				webnotes.conn.set(self.doc, 'status', 'Completed')
			if flt(self.doc.qty) > flt(self.doc.produced_qty):
				webnotes.conn.set(self.doc, 'status', 'In Process')
			if flt(self.doc.produced_qty) == 0:
				webnotes.conn.set(self.doc, 'status', 'Submitted')

	def on_submit(self):
		if not self.doc.wip_warehouse:
			webnotes.throw(_("WIP Warehouse required before Submit"))
		webnotes.conn.set(self.doc,'status', 'Submitted')
		self.update_planned_qty(self.doc.qty)

	def on_cancel(self):
		# Check whether any stock entry exists against this Production Order
		stock_entry = sql("""select name from `tabStock Entry`
			where production_order = %s and docstatus = 1""", self.doc.name)
		if stock_entry:
			msgprint("""Submitted Stock Entry %s exists against this production order.
				Hence can not be cancelled.""" % stock_entry[0][0], raise_exception=1)

		webnotes.conn.set(self.doc,'status', 'Cancelled')
		self.update_planned_qty(-self.doc.qty)

	def update_planned_qty(self, qty):
		"""update planned qty in bin"""
		args = {
			"item_code": self.doc.production_item,
			"posting_date": nowdate(),
			"planned_qty": flt(qty)
		}
		get_obj('Warehouse', self.doc.fg_warehouse).update_bin(args)

@webnotes.whitelist()
def get_item_details(item):
	res = webnotes.conn.sql("""select stock_uom
		from `tabItem` where (ifnull(end_of_life, "")="" or end_of_life > now())
		and name=%s""", (item,), as_dict=1)

	if not res:
		return {}

	res = res[0]
	bom = webnotes.conn.sql("""select name from `tabBOM` where item=%s
		and ifnull(is_default, 0)=1""", (item,))
	if bom:
		res.bom_no = bom[0][0]

	return res

@webnotes.whitelist()
def make_stock_entry(production_order_id, purpose):
	production_order = webnotes.bean("Production Order", production_order_id)

	stock_entry = webnotes.new_bean("Stock Entry")
	stock_entry.doc.purpose = purpose
	stock_entry.doc.production_order = production_order_id
	stock_entry.doc.company = production_order.doc.company
	stock_entry.doc.bom_no = production_order.doc.bom_no
	stock_entry.doc.fg_completed_qty = flt(production_order.doc.qty) - flt(production_order.doc.produced_qty)

	if purpose=="Material Transfer":
		stock_entry.doc.to_warehouse = production_order.doc.wip_warehouse
	else:
		stock_entry.doc.from_warehouse = production_order.doc.wip_warehouse
		stock_entry.doc.to_warehouse = production_order.doc.fg_warehouse

	return [d.fields for d in stock_entry.doclist]
<commit_msg>[fix] [minor] map use_multi_level_bom from production order to stoc entry<commit_after># Copyright (c) 2013, Web Notes Technologies Pvt. Ltd.
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals
import webnotes

from webnotes.utils import cstr, flt, nowdate
from webnotes.model.code import get_obj
from webnotes import msgprint, _

sql = webnotes.conn.sql

class OverProductionError(webnotes.ValidationError): pass

class DocType:
	def __init__(self, doc, doclist=[]):
		self.doc = doc
		self.doclist = doclist

	def validate(self):
		import utilities
		utilities.validate_status(self.doc.status, ["Draft", "Submitted", "Stopped",
			"In Process", "Completed", "Cancelled"])

		if self.doc.production_item :
			item_detail = sql("select name from `tabItem` where name = '%s' and docstatus != 2"
			 	% self.doc.production_item, as_dict = 1)
			if not item_detail:
				msgprint("Item '%s' does not exist or cancelled in the system."
					% cstr(self.doc.production_item), raise_exception=1)

		if self.doc.bom_no:
			bom = sql("""select name from `tabBOM` where name=%s and docstatus=1
				and is_active=1 and item=%s"""
				, (self.doc.bom_no, self.doc.production_item), as_dict =1)
			if not bom:
				msgprint("""Incorrect BOM: %s entered.
					May be BOM not exists or inactive or not submitted
					or for some other item.""" % cstr(self.doc.bom_no), raise_exception=1)

		if self.doc.sales_order:
			if not webnotes.conn.sql("""select name from `tabSales Order`
					where name=%s and docstatus = 1""", self.doc.sales_order):
				msgprint("Sales Order: %s is not valid" % self.doc.sales_order, raise_exception=1)

			self.validate_production_order_against_so()

		from utilities.transaction_base import validate_uom_is_integer
		validate_uom_is_integer(self.doclist, "stock_uom", ["qty", "produced_qty"])

	def validate_production_order_against_so(self):
		# already ordered qty
		ordered_qty_against_so = webnotes.conn.sql("""select sum(qty) from `tabProduction Order`
			where production_item = %s and sales_order = %s and docstatus < 2 and name != %s""",
			(self.doc.production_item, self.doc.sales_order, self.doc.name))[0][0]

		total_qty = flt(ordered_qty_against_so) + flt(self.doc.qty)

		# get qty from Sales Order Item table
		so_item_qty = webnotes.conn.sql("""select sum(qty) from `tabSales Order Item`
			where parent = %s and item_code = %s""",
			(self.doc.sales_order, self.doc.production_item))[0][0]
		# get qty from Packing Item table
		dnpi_qty = webnotes.conn.sql("""select sum(qty) from `tabDelivery Note Packing Item`
			where parent = %s and parenttype = 'Sales Order' and item_code = %s""",
			(self.doc.sales_order, self.doc.production_item))[0][0]
		# total qty in SO
		so_qty = flt(so_item_qty) + flt(dnpi_qty)

		if total_qty > so_qty:
			webnotes.msgprint(_("Total production order qty for item") + ": " +
				cstr(self.doc.production_item) + _(" against sales order") + ": " +
				cstr(self.doc.sales_order) + _(" will be ") + cstr(total_qty) + ", " +
				_("which is greater than sales order qty ") + "(" + cstr(so_qty) + ")" +
				_("Please reduce qty."), raise_exception=OverProductionError)

	def stop_unstop(self, status):
		""" Called from client side on Stop/Unstop event"""
		self.update_status(status)
		qty = (flt(self.doc.qty)-flt(self.doc.produced_qty)) * ((status == 'Stopped') and -1 or 1)
		self.update_planned_qty(qty)
		msgprint("Production Order has been %s" % status)

	def update_status(self, status):
		if status == 'Stopped':
			webnotes.conn.set(self.doc, 'status', cstr(status))
		else:
			if flt(self.doc.qty) == flt(self.doc.produced_qty):
				webnotes.conn.set(self.doc, 'status', 'Completed')
			if flt(self.doc.qty) > flt(self.doc.produced_qty):
				webnotes.conn.set(self.doc, 'status', 'In Process')
			if flt(self.doc.produced_qty) == 0:
				webnotes.conn.set(self.doc, 'status', 'Submitted')

	def on_submit(self):
		if not self.doc.wip_warehouse:
			webnotes.throw(_("WIP Warehouse required before Submit"))
		webnotes.conn.set(self.doc,'status', 'Submitted')
		self.update_planned_qty(self.doc.qty)

	def on_cancel(self):
		# Check whether any stock entry exists against this Production Order
		stock_entry = sql("""select name from `tabStock Entry`
			where production_order = %s and docstatus = 1""", self.doc.name)
		if stock_entry:
			msgprint("""Submitted Stock Entry %s exists against this production order.
				Hence can not be cancelled.""" % stock_entry[0][0], raise_exception=1)

		webnotes.conn.set(self.doc,'status', 'Cancelled')
		self.update_planned_qty(-self.doc.qty)

	def update_planned_qty(self, qty):
		"""update planned qty in bin"""
		args = {
			"item_code": self.doc.production_item,
			"posting_date": nowdate(),
			"planned_qty": flt(qty)
		}
		get_obj('Warehouse', self.doc.fg_warehouse).update_bin(args)

@webnotes.whitelist()
def get_item_details(item):
	res = webnotes.conn.sql("""select stock_uom
		from `tabItem` where (ifnull(end_of_life, "")="" or end_of_life > now())
		and name=%s""", (item,), as_dict=1)

	if not res:
		return {}

	res = res[0]
	bom = webnotes.conn.sql("""select name from `tabBOM` where item=%s
		and ifnull(is_default, 0)=1""", (item,))
	if bom:
		res.bom_no = bom[0][0]

	return res

@webnotes.whitelist()
def make_stock_entry(production_order_id, purpose):
	production_order = webnotes.bean("Production Order", production_order_id)

	stock_entry = webnotes.new_bean("Stock Entry")
	stock_entry.doc.purpose = purpose
	stock_entry.doc.production_order = production_order_id
	stock_entry.doc.company = production_order.doc.company
	stock_entry.doc.bom_no = production_order.doc.bom_no
	stock_entry.doc.use_multi_level_bom = production_order.doc.use_multi_level_bom
	stock_entry.doc.fg_completed_qty = flt(production_order.doc.qty) - flt(production_order.doc.produced_qty)

	if purpose=="Material Transfer":
		stock_entry.doc.to_warehouse = production_order.doc.wip_warehouse
	else:
		stock_entry.doc.from_warehouse = production_order.doc.wip_warehouse
		stock_entry.doc.to_warehouse = production_order.doc.fg_warehouse

	return [d.fields for d in stock_entry.doclist]
<|endoftext|>


<commit_before># fmt: off
__title__ = "spacy-nightly"
__version__ = "3.0.0a24"
__release__ = True
__download_url__ = "https://github.com/explosion/spacy-models/releases/download"
__compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"
__projects__ = "https://github.com/explosion/projects"
__projects_branch__ = "v3"
<commit_msg>Increment version [ci skip]<commit_after># fmt: off
__title__ = "spacy-nightly"
__version__ = "3.0.0a25"
__release__ = True
__download_url__ = "https://github.com/explosion/spacy-models/releases/download"
__compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"
__projects__ = "https://github.com/explosion/projects"
__projects_branch__ = "v3"
<|endoftext|>


<commit_before>import numpy

def confusion_matrix(rater_a, rater_b,
		 min_rating=None, max_rating=None):
    assert(len(rater_a)==len(rater_b))
    if min_rating is None:
        min_rating = min(reduce(min, rater_a), reduce(min, rater_b))
    if max_rating is None:
        max_rating = max(reduce(max, rater_a), reduce(max, rater_b))
    num_ratings = max_rating - min_rating + 1
    conf_mat = [[0 for i in range(num_ratings)]
                for j in range(num_ratings)]
    for a,b in zip(rater_a,rater_b):
        conf_mat[a-min_rating][b-min_rating] += 1
    return conf_mat

def histogram(ratings, min_rating=None, max_rating=None):
    if min_rating is None: min_rating = reduce(min, ratings)
    if max_rating is None: max_rating = reduce(max, ratings)
    num_ratings = max_rating - min_rating + 1
    hist_ratings = [0 for x in range(num_ratings)]
    for r in ratings:
	    hist_ratings[r-min_rating] += 1
    return hist_ratings

def quadratic_weighted_kappa(rater_a, rater_b,
                             min_rating = None, max_rating = None):
    assert(len(rater_a) == len(rater_b))
    if min_rating is None:
	min_rating = min(reduce(min, rater_a), reduce(min, rater_b))
    if max_rating is None:
	max_rating = max(reduce(max, rater_a), reduce(max, rater_b))
    conf_mat = confusion_matrix(rater_a, rater_b,
				     min_rating, max_rating)
    num_ratings = len(conf_mat)
    num_scored_items = float(len(rater_a))

    hist_rater_a = histogram(rater_a, min_rating, max_rating)
    hist_rater_b = histogram(rater_b, min_rating, max_rating)

    numerator = 0.0
    denominator = 0.0

    for i in range(num_ratings):
	for j in range(num_ratings):
	    expected_count = (hist_rater_a[i]*hist_rater_b[j]
			      / num_scored_items)
	    d = pow(i-j,2.0) / pow(num_ratings-1, 2.0)
	    numerator += d*conf_mat[i][j] / num_scored_items
	    denominator += d*expected_count / num_scored_items

    return 1.0 - numerator / denominator

def mean_quadratic_weighted_kappa(kappas, weights=None):
    """
    Calculates the mean kappa in the z-space
    """
    kappas = numpy.array(kappas, dtype=float)
    if weights is None:
        weights = numpy.ones(numpy.shape(kappas))
    else:
        weights = weights / numpy.mean(weights)

    # ensure that kappas are in the range [-.999, .999]
    kappas = numpy.array([min(x, .999) for x in kappas])
    kappas = numpy.array([max(x, -.999) for x in kappas])

    z = 0.5 * numpy.log( (1+kappas)/(1-kappas) ) * weights
    z = numpy.mean(z)
    kappa = (numpy.exp(2*z)-1) / (numpy.exp(2*z)+1)
    return kappa
<commit_msg>Added some comments<commit_after>import numpy

def confusion_matrix(rater_a, rater_b,
		 min_rating=None, max_rating=None):
    """
    Returns the confusion matrix between rater's ratings
    """
    assert(len(rater_a)==len(rater_b))
    if min_rating is None:
        min_rating = min(reduce(min, rater_a), reduce(min, rater_b))
    if max_rating is None:
        max_rating = max(reduce(max, rater_a), reduce(max, rater_b))
    num_ratings = max_rating - min_rating + 1
    conf_mat = [[0 for i in range(num_ratings)]
                for j in range(num_ratings)]
    for a,b in zip(rater_a,rater_b):
        conf_mat[a-min_rating][b-min_rating] += 1
    return conf_mat

def histogram(ratings, min_rating=None, max_rating=None):
    """
    Returns the counts of each type of rating that a rater made
    """
    if min_rating is None: min_rating = reduce(min, ratings)
    if max_rating is None: max_rating = reduce(max, ratings)
    num_ratings = max_rating - min_rating + 1
    hist_ratings = [0 for x in range(num_ratings)]
    for r in ratings:
	    hist_ratings[r-min_rating] += 1
    return hist_ratings

def quadratic_weighted_kappa(rater_a, rater_b,
                             min_rating = None, max_rating = None):
    """
    Calculates the quadratic weighted kappa, a measure of
    agreement between two raters
    """
    assert(len(rater_a) == len(rater_b))
    if min_rating is None:
	min_rating = min(reduce(min, rater_a), reduce(min, rater_b))
    if max_rating is None:
	max_rating = max(reduce(max, rater_a), reduce(max, rater_b))
    conf_mat = confusion_matrix(rater_a, rater_b,
				     min_rating, max_rating)
    num_ratings = len(conf_mat)
    num_scored_items = float(len(rater_a))

    hist_rater_a = histogram(rater_a, min_rating, max_rating)
    hist_rater_b = histogram(rater_b, min_rating, max_rating)

    numerator = 0.0
    denominator = 0.0

    for i in range(num_ratings):
	for j in range(num_ratings):
	    expected_count = (hist_rater_a[i]*hist_rater_b[j]
			      / num_scored_items)
	    d = pow(i-j,2.0) / pow(num_ratings-1, 2.0)
	    numerator += d*conf_mat[i][j] / num_scored_items
	    denominator += d*expected_count / num_scored_items

    return 1.0 - numerator / denominator

def mean_quadratic_weighted_kappa(kappas, weights=None):
    """
    Calculates the mean kappa in the z-space
    """
    kappas = numpy.array(kappas, dtype=float)
    if weights is None:
        weights = numpy.ones(numpy.shape(kappas))
    else:
        weights = weights / numpy.mean(weights)

    # ensure that kappas are in the range [-.999, .999]
    kappas = numpy.array([min(x, .999) for x in kappas])
    kappas = numpy.array([max(x, -.999) for x in kappas])

    z = 0.5 * numpy.log( (1+kappas)/(1-kappas) ) * weights
    z = numpy.mean(z)
    kappa = (numpy.exp(2*z)-1) / (numpy.exp(2*z)+1)
    return kappa
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals

import random
from .gold import GoldParse
from .scorer import Scorer

class Trainer(object):
    def __init__(self, nlp, gold_tuples):
        self.nlp = nlp
        self.gold_tuples = gold_tuples

    def epochs(self, nr_epoch, augment_data=None):
        def _epoch():
            for raw_text, paragraph_tuples in self.gold_tuples:
                if augment_data is not None:
                    raw_text, paragraph_tuples = augment_data(raw_text, paragraph_tuples)
                docs = self.make_docs(raw_text, paragraph_tuples)
                golds = self.make_golds(docs, paragraph_tuples)
                for doc, gold in zip(docs, golds):
                    yield doc, gold

        for itn in range(nr_epoch):
            random.shuffle(self.gold_tuples)
            yield _epoch()

    def update(self, doc, gold):
        for process in self.nlp.pipeline[1:]:
            if hasattr(process, 'update'):
                process.update(doc, gold)
            process(doc)
        return doc

    def evaluate(self, dev_sents):
        scorer = Scorer()
        for raw_text, paragraph_tuples in dev_sents:
            docs = self.make_docs(raw_text, paragraph_tuples)
            golds = self.make_golds(docs, paragraph_tuples)
            for doc, gold in zip(docs, golds):
                for process in self.nlp.pipeline[1:]:
                    process(doc)
                scorer.score(doc, gold)
        return scorer

    def make_docs(self, raw_text, paragraph_tuples):
        if raw_text is not None:
            return [self.nlp.tokenizer(raw_text)]
        else:
            return [self.nlp.tokenizer.tokens_from_list(sent_tuples[0][1])
                    for sent_tuples in paragraph_tuples]

    def make_golds(self, docs, paragraph_tuples):
        if len(docs) == 1:
            return [GoldParse(docs[0], sent_tuples[0])
                    for sent_tuples in paragraph_tuples]
        else:
            return [GoldParse(doc, sent_tuples[0])
                    for doc, sent_tuples in zip(docs, paragraph_tuples)]
<commit_msg>Add docstring for Trainer class.<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals

import random
from .gold import GoldParse
from .scorer import Scorer

class Trainer(object):
    '''Manage training of an NLP pipeline.'''
    def __init__(self, nlp, gold_tuples):
        self.nlp = nlp
        self.gold_tuples = gold_tuples

    def epochs(self, nr_epoch, augment_data=None):
        def _epoch():
            for raw_text, paragraph_tuples in self.gold_tuples:
                if augment_data is not None:
                    raw_text, paragraph_tuples = augment_data(raw_text, paragraph_tuples)
                docs = self.make_docs(raw_text, paragraph_tuples)
                golds = self.make_golds(docs, paragraph_tuples)
                for doc, gold in zip(docs, golds):
                    yield doc, gold

        for itn in range(nr_epoch):
            random.shuffle(self.gold_tuples)
            yield _epoch()

    def update(self, doc, gold):
        for process in self.nlp.pipeline[1:]:
            if hasattr(process, 'update'):
                process.update(doc, gold)
            process(doc)
        return doc

    def evaluate(self, dev_sents):
        scorer = Scorer()
        for raw_text, paragraph_tuples in dev_sents:
            docs = self.make_docs(raw_text, paragraph_tuples)
            golds = self.make_golds(docs, paragraph_tuples)
            for doc, gold in zip(docs, golds):
                for process in self.nlp.pipeline[1:]:
                    process(doc)
                scorer.score(doc, gold)
        return scorer

    def make_docs(self, raw_text, paragraph_tuples):
        if raw_text is not None:
            return [self.nlp.tokenizer(raw_text)]
        else:
            return [self.nlp.tokenizer.tokens_from_list(sent_tuples[0][1])
                    for sent_tuples in paragraph_tuples]

    def make_golds(self, docs, paragraph_tuples):
        if len(docs) == 1:
            return [GoldParse(docs[0], sent_tuples[0])
                    for sent_tuples in paragraph_tuples]
        else:
            return [GoldParse(doc, sent_tuples[0])
                    for doc, sent_tuples in zip(docs, paragraph_tuples)]
<|endoftext|>


<commit_before>import random

from Orange.data import Table
from django.db.models import Prefetch

from workflows.engine import ValueNotSet
from workflows.engine.widget_runner import WidgetRunner
from workflows.models import *
from collections import defaultdict
import sklearn.model_selection as skl

class WorkflowRunner():
    def __init__(self, workflow, final_widget=None, parent_workflow_runner=None,representing_widget=None): # clean=True,
        self.workflow = workflow
        self.connections = workflow.connections.all()
        self.widgets = workflow.get_runnable_widgets(last_runnable_widget_id=final_widget and final_widget.id)

        #used for storing results
        self.input_id_to_input= {}
        self.output_id_to_output= {}

        #used for discovery of connections
        self.inputs_per_widget_id = defaultdict(list)
        self.outputs_per_widget_id = defaultdict(list)

        for w in self.workflow.widgets.prefetch_related(
                Prefetch('outputs', queryset=Output.objects.all()),
                Prefetch('inputs', queryset=Input.objects.filter(parameter=True),to_attr="parameter_inputs"),
                Prefetch('inputs', queryset=Input.objects.filter(parameter=False).defer("value"),to_attr="connection_inputs")):
            for i in w.parameter_inputs+w.connection_inputs:
                self.input_id_to_input[i.id] = i
                if not i.parameter:
                    i.value=ValueNotSet
                self.inputs_per_widget_id[w.id].append(i)
            for o in w.outputs.all():
                if not (w.save_results and w.finished):
                    o.value=ValueNotSet
                self.output_id_to_output[o.id] = o
                self.outputs_per_widget_id[w.id].append(o)

        #preassing input values
        relevant_input_ids=set()
        for w in self.widgets:
            relevant_input_ids.update([i.id for i in self.inputs_per_widget_id[w.id]])
        for c in self.connections:
            if c.input_id in relevant_input_ids:
                self.input_id_to_input[c.input_id].value=self.output_id_to_output[c.output_id].value

        #self.clean = clean
        self.parent_workflow_runner = parent_workflow_runner
        self.representing_widget = representing_widget

    def is_for_loop(self):
        for w in self.widgets:
            if w.type=='for_input':
                return True
        return False

    def is_cross_validation(self):
        for w in self.widgets:
            if w.type=='cv_input':
                return True
        return False

    def cleanup(self):
        for w in self.widgets:
            #if self.clean:
            if self.representing_widget:
                w.finished = False
            w.error = False

    def get_connections_for_output(self, output):
        return [c for c in self.connections if c.output_id==output.id]

    def get_connection_for_input(self,input):
        for c in self.connections:
            if c.input_id==input.id:
                return c
        return None
    #
    # @property
    # def finished_widgets(self):
    #     finished_widgets = []
    #     for w in self.widgets:
    #         if w.finished:
    #             finished_widgets.append(w)
    #     return finished_widgets

    @property
    def unfinished_widgets(self):
        unfinished_widgets = []
        for w in self.widgets:
            if not w.finished and not w.running and not w.error:
                unfinished_widgets.append(w)
        return unfinished_widgets

    @property
    def runnable_widgets(self):
        """ a widget is runnable if all widgets connected before
            it are finished (i.e. widgets that have outputs that
            are connected to this widget's input) """

        # finished_widget_ids = [w.id for w in self.finished_widgets]
        runnable = []
        # ufw=self.unfinished_widgets
        for w in self.unfinished_widgets:
            incoming_widget_connections= [c for c in self.connections if self.input_id_to_input[c.input_id].widget_id == w.id]
            # incoming_widget_connection_inputs=[self.input_id_to_input[c.input_id].value for c in incoming_widget_connections]
            # widget_connections= [c for c in self.connections if c.input_id in self.inputs_per_widget_id]

            #check if all outputs are connection outputs are calculated
            # c.output_id in self.outputs and
            # if all([self.output_id_to_output[c.output_id].value!=ValueNotSet for c in widget_connections]):
            #     runnable.append(w)
            if not any([self.input_id_to_input[c.input_id].value is ValueNotSet for c in incoming_widget_connections]):
                runnable.append(w)
        return runnable

    def run_all_unfinished_widgets(self):
        runnable_widgets = self.runnable_widgets
        while len(runnable_widgets)>0:
            print(runnable_widgets)

            for w in runnable_widgets:
                try:
                    if w.type == 'subprocess':
                        WorkflowRunner(w.workflow_link, parent_workflow_runner=self,representing_widget=w).run()
                    else:
                        WidgetRunner(w,self.inputs_per_widget_id[w.id],self.outputs_per_widget_id[w.id],self).run()
                except NotImplementedError as c:
                    w.set_as_faulty()
                    self.save()
                    raise c
            runnable_widgets = self.runnable_widgets

    def run(self):
        self.cleanup()
        if self.is_for_loop():
            for_input_widget = None
            for_output_widget = None
            for w in self.widgets:
                if w.type=='for_input':
                    for_input_widget = w
                if w.type=='for_output':
                    for_output_widget = w
            outer_output = self.parent_workflow_runner.outputs[for_output_widget.inputs.all()[0].outer_output_id]
            outer_output.value = []
            input_list = self.parent_workflow_runner.inputs[for_input_widget.outputs.all()[0].outer_input_id].value
            for i in input_list:
                self.cleanup()
                proper_output = for_input_widget.outputs.all()[0]
                proper_output.value = i
                for_input_widget.finished = True
                self.run_all_unfinished_widgets()
        elif self.is_cross_validation():
            cv_input_widget = None
            cv_output_widget = None
            for w in self.widgets:
                if w.type=='cv_input':
                    cv_input_widget = w
                if w.type=='cv_output':
                    cv_output_widget = w

            outer_output = self.parent_workflow_runner.output_id_to_output[self.inputs_per_widget_id[cv_output_widget.id][0].outer_output_id]
            outer_output.value = []

            list_output,fold_output,seed_output=self.outputs_per_widget_id[cv_input_widget.id]

            #input_list = self.parent_workflow_runner.inputs[cv_input_widget.outputs.all()[0].outer_input_id].value
            #input_fold = self.parent_workflow_runner.inputs[cv_input_widget.outputs.all()[1].outer_input_id].value
            #input_seed = self.parent_workflow_runner.inputs[cv_input_widget.outputs.all()[2].outer_input_id].value

            #GET INNER INPUTS
            input_list = self.parent_workflow_runner.input_id_to_input[list_output.outer_input_id].value
            input_fold = self.parent_workflow_runner.input_id_to_input[fold_output.outer_input_id].value
            input_seed = self.parent_workflow_runner.input_id_to_input[seed_output.outer_input_id].value
            if input_fold is not ValueNotSet:
                input_fold = int(input_fold)
            else:
                input_fold = 10

            if input_seed is not ValueNotSet:
                input_seed = int(input_seed)
            else:
                input_seed = random.randint(0,10**9)
            proper_output = cv_input_widget.outputs.all()[2]
            proper_output.value = input_seed

            input_type = input_list.__class__.__name__
            context = None
            if input_type == 'DBContext':
                context = input_list
                input_list = context.orng_tables.get(context.target_table,None)
            elif input_type == 'DocumentCorpus':
                document_corpus = input_list
                input_list = document_corpus.documents

            if not input_list:
                raise Exception('CrossValidation: Empty input list!')

            #SPLIT INPUT DATA INTO FOLDS
            folds = []

            if input_type == 'Table': #input_list is orange table
                indices = None
                if input_list.domain.has_discrete_class:
                    try:
                        splitter = skl.StratifiedKFold(
                            input_fold, shuffle=True, random_state=input_seed
                        )
                        splitter.get_n_splits(input_list.X, input_list.Y)
                        self.indices = list(splitter.split(input_list.X, input_list.Y))
                    except ValueError:
                        self.warnings.append("Using non-stratified sampling.")
                        indices = None
                    if indices is None:
                        splitter = skl.KFold(
                            input_fold, shuffle=True, random_state=input_seed
                        )
                        splitter.get_n_splits(input_list)
                        indices = list(splitter.split(input_list))

                for i in range(input_fold):
                    output_train = Table.from_table_rows(input_list,indices[i][0])
                    output_test = Table.from_table_rows(input_list,indices[i][1])
                    output_train.name = input_list.name
                    output_test.name = input_list.name
                    folds.append((output_train, output_test))

            elif input_type == 'DocumentCorpus':
                from sklearn.model_selection import StratifiedKFold, KFold

                if 'Labels' in document_corpus.features:
                    labels = document_corpus.get_document_labels()
                    # print "Seed:"+str(input_seed)
                    stf = StratifiedKFold(labels, n_folds=input_fold, random_state=input_seed)
                else:
                    stf = KFold(len(document_corpus.documents), n_folds=input_fold, random_state=input_seed)

                folds = [(list(train_index), list(test_index)) for train_index, test_index in stf]
            else:
                random.seed(input_seed)
                random.shuffle(input_list)
                folds = [input_list[i::input_fold] for i in range(input_fold)]

            for i in range(len(folds)):
                if input_type == 'Table':
                    output_test = folds[i][1]
                    output_train = folds[i][0]
                elif input_type == 'DocumentCorpus':
                    train_indices, test_indices= folds[i]
                    print("engine")
                    print("TRAIN:", train_indices, "TEST:", test_indices)

                    output_train, output_test = document_corpus.split(train_indices,test_indices)
                else:
                    output_train = folds[:i] + folds[i+1:]
                    output_test = folds[i]
                if input_type == 'DBContext':
                    output_train_obj = context.copy()
                    output_train_obj.orng_tables[context.target_table] = output_train
                    output_test_obj = context.copy()
                    output_test_obj.orng_tables[context.target_table] = output_test
                    output_train = output_train_obj
                    output_test = output_test_obj

                self.cleanup()

                #proper_output = cv_input_widget.outputs.all()[0] # inner output
                #proper_output.value = output_train
                #proper_output = cv_input_widget.outputs.all()[1] # inner output
                #proper_output.value = output_test
                cv_input_widget.finished=True # set the input widget as finished
                self.run_all_unfinished_widgets()
        else:
            self.run_all_unfinished_widgets()
        self.save()

        if self.representing_widget:
            self.representing_widget.set_as_finished()

    def save(self):
        for w in self.widgets:
            w.save_with_inputs_and_outputs(force_update=True,
                inputs=w.save_results and [self.input_id_to_input[i.id] for i in self.inputs_per_widget_id[w.id]],
                outputs=w.save_results and [self.output_id_to_output[i.id] for i in self.outputs_per_widget_id[w.id]])<commit_msg>Cross-validation ported<commit_after>import random

from Orange.data import Table
from django.db.models import Prefetch

from workflows.engine import ValueNotSet
from workflows.engine.widget_runner import WidgetRunner
from workflows.models import *
from collections import defaultdict
import sklearn.model_selection as skl

class WorkflowRunner():
    def __init__(self, workflow, final_widget=None, parent_workflow_runner=None,representing_widget=None): # clean=True,
        self.workflow = workflow
        self.connections = workflow.connections.all()
        self.widgets = workflow.get_runnable_widgets(last_runnable_widget_id=final_widget and final_widget.id)

        #used for storing results
        self.input_id_to_input= {}
        self.output_id_to_output= {}

        #used for discovery of connections
        self.inputs_per_widget_id = defaultdict(list)
        self.outputs_per_widget_id = defaultdict(list)

        for w in self.workflow.widgets.prefetch_related(
                Prefetch('outputs', queryset=Output.objects.all()),
                Prefetch('inputs', queryset=Input.objects.filter(parameter=True),to_attr="parameter_inputs"),
                Prefetch('inputs', queryset=Input.objects.filter(parameter=False).defer("value"),to_attr="connection_inputs")):
            for i in w.parameter_inputs+w.connection_inputs:
                self.input_id_to_input[i.id] = i
                if not i.parameter:
                    i.value=ValueNotSet
                self.inputs_per_widget_id[w.id].append(i)
            for o in w.outputs.all():
                if not (w.save_results and w.finished):
                    o.value=ValueNotSet
                self.output_id_to_output[o.id] = o
                self.outputs_per_widget_id[w.id].append(o)

        #preassing input values
        relevant_input_ids=set()
        for w in self.widgets:
            relevant_input_ids.update([i.id for i in self.inputs_per_widget_id[w.id]])
        for c in self.connections:
            if c.input_id in relevant_input_ids:
                self.input_id_to_input[c.input_id].value=self.output_id_to_output[c.output_id].value

        #self.clean = clean
        self.parent_workflow_runner = parent_workflow_runner
        self.representing_widget = representing_widget

    def is_for_loop(self):
        for w in self.widgets:
            if w.type=='for_input':
                return True
        return False

    def is_cross_validation(self):
        for w in self.widgets:
            if w.type=='cv_input':
                return True
        return False

    def cleanup(self):
        for w in self.widgets:
            #if self.clean:
            if self.representing_widget or not w.save_results:
                w.finished = False
                for i in self.inputs_per_widget_id[w.id]:
                    if not i.parameter:
                        i.value = ValueNotSet
            w.error = False

    def get_connections_for_output(self, output):
        return [c for c in self.connections if c.output_id==output.id]

    def get_connection_for_input(self,input):
        for c in self.connections:
            if c.input_id==input.id:
                return c
        return None

    def unfinished_widgets(self):
        unfinished_widgets = []
        for w in self.widgets:
            if not w.finished and not w.running and not w.error:
                unfinished_widgets.append(w)
        return unfinished_widgets

    def runnable_widgets(self):
        """ a widget is runnable if all widgets connected before
            it are finished (i.e. widgets that have outputs that
            are connected to this widget's input) """

        # finished_widget_ids = [w.id for w in self.finished_widgets]
        runnable = []
        # ufw=self.unfinished_widgets
        for w in self.unfinished_widgets():
            incoming_widget_connections= [c for c in self.connections if self.input_id_to_input[c.input_id].widget_id == w.id]
            # incoming_widget_connection_inputs=[self.input_id_to_input[c.input_id].value for c in incoming_widget_connections]
            # widget_connections= [c for c in self.connections if c.input_id in self.inputs_per_widget_id]

            #check if all outputs are connection outputs are calculated
            # c.output_id in self.outputs and
            # if all([self.output_id_to_output[c.output_id].value!=ValueNotSet for c in widget_connections]):
            #     runnable.append(w)
            if not any([self.input_id_to_input[c.input_id].value is ValueNotSet for c in incoming_widget_connections]):
                runnable.append(w)
        return runnable

    def run_all_unfinished_widgets(self):
        runnable_widgets = self.runnable_widgets()
        while len(runnable_widgets)>0:
            print(runnable_widgets)

            for w in runnable_widgets:
                try:
                    if w.type == 'subprocess':
                        WorkflowRunner(w.workflow_link, parent_workflow_runner=self,representing_widget=w).run()
                    else:
                        WidgetRunner(w,self.inputs_per_widget_id[w.id],self.outputs_per_widget_id[w.id],self).run()
                except NotImplementedError as c:
                    w.set_as_faulty()
                    self.save()
                    raise c
            runnable_widgets = self.runnable_widgets()

    def run(self):
        self.cleanup()
        if self.is_for_loop():
            for_input_widget = None
            for_output_widget = None
            for w in self.widgets:
                if w.type=='for_input':
                    for_input_widget = w
                if w.type=='for_output':
                    for_output_widget = w
            outer_output = self.parent_workflow_runner.outputs[for_output_widget.inputs.all()[0].outer_output_id]
            outer_output.value = []
            outer_input_data = self.parent_workflow_runner.inputs[for_input_widget.outputs.all()[0].outer_input_id].value
            for i in outer_input_data:
                self.cleanup()
                proper_output = for_input_widget.outputs.all()[0]
                proper_output.value = i
                for_input_widget.finished = True
                self.run_all_unfinished_widgets()
        elif self.is_cross_validation():
            cv_input_widget = None
            cv_output_widget = None
            for w in self.widgets:
                if w.type=='cv_input':
                    cv_input_widget = w
                if w.type=='cv_output':
                    cv_output_widget = w

            outer_output = self.parent_workflow_runner.output_id_to_output[self.inputs_per_widget_id[cv_output_widget.id][0].outer_output_id]
            outer_output.value = []

            train_output, test_output, seed_output=self.outputs_per_widget_id[cv_input_widget.id]

            #GET INNER INPUTS
            #
            # !!!!
            # E.g., outer input for fold corresponds to inner output for the test fold
            # !!!!
            outer_input_data = self.parent_workflow_runner.input_id_to_input[train_output.outer_input_id].value
            outer_input_fold = self.parent_workflow_runner.input_id_to_input[test_output.outer_input_id].value
            outer_input_seed = self.parent_workflow_runner.input_id_to_input[seed_output.outer_input_id].value

            outer_input_fold = int(outer_input_fold) if outer_input_fold is not ValueNotSet else 10
            outer_input_seed = int(outer_input_seed) if outer_input_seed is not ValueNotSet else random.randint(0, 10**9)

            #proper_output = cv_input_widget.outputs.all()[2]
            seed_output.value = outer_input_seed

            input_type = outer_input_data.__class__.__name__
            if input_type == 'DBContext':
                context = outer_input_data
                outer_input_data = context.orng_tables.get(context.target_table, None)
            elif input_type == 'DocumentCorpus':
                document_corpus = outer_input_data
                outer_input_data = document_corpus.documents

            if not outer_input_data:
                raise Exception('CrossValidation: Empty input data!')

            #SPLIT INPUT DATA INTO FOLDS
            folds = []

            if input_type in {'Table', 'DBContext'}: #input_list is orange table
                indices = None
                if outer_input_data.domain.has_discrete_class:
                    try:
                        splitter = skl.StratifiedKFold(
                            outer_input_fold, shuffle=True, random_state=outer_input_seed
                        )
                        splitter.get_n_splits(outer_input_data.X, outer_input_data.Y)
                        self.indices = list(splitter.split(outer_input_data.X, outer_input_data.Y))
                    except ValueError:
                        self.warnings.append("Using non-stratified sampling.")
                        indices = None
                if indices is None:
                    splitter = skl.KFold(
                        outer_input_fold, shuffle=True, random_state=outer_input_seed
                    )
                    splitter.get_n_splits(outer_input_data)
                    indices = list(splitter.split(outer_input_data))

                for i in range(outer_input_fold):
                    output_train = Table.from_table_rows(outer_input_data,indices[i][0])
                    output_test = Table.from_table_rows(outer_input_data,indices[i][1])
                    output_train.name = outer_input_data.name
                    output_test.name = outer_input_data.name
                    folds.append((output_train, output_test))

            elif input_type == 'DocumentCorpus':
                from sklearn.model_selection import StratifiedKFold, KFold

                if 'Labels' in document_corpus.features:
                    labels = document_corpus.get_document_labels()
                    # print "Seed:"+str(input_seed)
                    stf = StratifiedKFold(labels, n_folds=outer_input_fold, random_state=outer_input_seed)
                else:
                    stf = KFold(len(document_corpus.documents), n_folds=outer_input_fold, random_state=outer_input_seed)

                folds = [(list(train_index), list(test_index)) for train_index, test_index in stf]
            else:
                random.seed(outer_input_seed)
                random.shuffle(outer_input_data)
                folds = [outer_input_data[i::outer_input_fold] for i in range(outer_input_fold)]

            for i in range(len(folds)):
                if input_type in {'Table', 'DBContext'}:
                    output_test = folds[i][1]
                    output_train = folds[i][0]
                elif input_type == 'DocumentCorpus':
                    train_indices, test_indices= folds[i]
                    print("engine")
                    print("TRAIN:", train_indices, "TEST:", test_indices)

                    output_train, output_test = document_corpus.split(train_indices,test_indices)
                else:
                    output_train = folds[:i] + folds[i+1:]
                    output_test = folds[i]
                if input_type == 'DBContext':
                    output_train_obj = context.copy()
                    output_train_obj.orng_tables[context.target_table] = output_train
                    output_test_obj = context.copy()
                    output_test_obj.orng_tables[context.target_table] = output_test
                    output_train = output_train_obj
                    output_test = output_test_obj

                self.cleanup()

                train_output.value = output_train
                test_output.value = output_test

                for o in [train_output, test_output]:
                    for con in self.get_connections_for_output(o):
                        self.input_id_to_input[con.input_id].value = o.value

                cv_input_widget.finished=True # set the input widget as finished
                self.run_all_unfinished_widgets()
        else:
            self.run_all_unfinished_widgets()
        self.save()

        if self.representing_widget:
            self.representing_widget.set_as_finished()

    def save(self):
        for w in self.widgets:
            w.save_with_inputs_and_outputs(force_update=True,
                inputs=w.save_results and [self.input_id_to_input[i.id] for i in self.inputs_per_widget_id[w.id]],
                outputs=w.save_results and [self.output_id_to_output[i.id] for i in self.outputs_per_widget_id[w.id]])<|endoftext|>


<commit_before>#!/usr/bin/env python
import os, unittest, subprocess, re, signal, time
import socket
import threading
import queue
import sys
import os

from socketserver import ThreadingMixIn, UnixStreamServer

from HyREPL.bencode import encode, decode, decode_multiple
from HyREPL.server import ReplRequestHandler

sock = "/tmp/HyREPL-test"
class TestServer(ThreadingMixIn, UnixStreamServer): pass
def start_server():
    os.unlink(sock)
    s = TestServer(sock, ReplRequestHandler)
    t = threading.Thread(target=s.serve_forever)
    t.daemon = True
    t.start()
    def stop():
        s.shutdown()
        t.join()
    return stop

def soc_send(message, return_reply=True):
    s = socket.socket(family=socket.AF_UNIX)
    s.connect(sock)
    s.sendall(encode(message))
    if not return_reply:
        s.close()
        return []
    s.setblocking(False)

    ret = []
    buf = b""
    while True:
        try:
            resp = s.recv(1)
            buf += resp
        except BlockingIOError:
            pass
        try:
            response, rest = decode(buf)
        except:
            continue
        buf = rest

        ret.append(response)
        if "done" in response.get("status", []):
            break
    s.close()
    return ret

def test_bencode():
    d = { "foo": 42, "spam": [1, 2, 'a'] }
    assert d == decode(encode(d))[0]

    d = decode_multiple(b"d5:value1:47:session36:31594b80-7f2e-4915-9969-f1127d562cc42:ns2:Hye" +\
            b"d6:statusl4:donee7:session36:31594b80-7f2e-4915-9969-f1127d562cc4e")
    assert len(d) == 2
    assert isinstance(d[0], dict)
    assert isinstance(d[1], dict)
    assert d[0]["value"] == "4"
    assert d[0]["ns"] == "Hy"
    assert isinstance(d[1]["status"], list)
    assert len(d[1]["status"]) == 1
    assert d[1]["status"][0] == "done"

def test_code_eval():
    """
    simple eval
    Example output from the server:
        [{'session': '0361c419-ef89-4a86-ae1a-48388be56041', 'ns': 'Hy', 'value': '4'},
         {'status': ['done'], 'session': '0361c419-ef89-4a86-ae1a-48388be56041'}]
    """
    stop = start_server()
    code = {"op": "eval", "code": "(+ 2 2)"}
    ret = soc_send(code)
    value, status = ret
    stop()

    assert len(ret) == 2
    assert value["value"] == '4'
    assert "done" in status["status"]
    assert value["session"] == status["session"]

def test_stdout_eval():
    """
    stdout eval
    Example output from the server:
        [{'session': '2d6b48d8-4a3e-49a6-9131-3321a11f70d4', 'ns': 'Hy', 'value': 'None'},
         {'session': '2d6b48d8-4a3e-49a6-9131-3321a11f70d4', 'out': 'Hello World\n'},
         {'status': ['done'], 'session': '2d6b48d8-4a3e-49a6-9131-3321a11f70d4'}]
    """

    stop = start_server()
    code = {"op": "eval", "code": '(print "Hello World")'}
    ret = soc_send(code)
    value, out, status = ret
    stop()

    assert len(ret) == 3
    assert value["value"] == 'None'
    assert out["out"] == "Hello World\n"
    assert "done" in status["status"]
    assert value["session"] == out["session"] == status["session"]

def stdin_send(code, my_queue):
    ret = soc_send(code)
    my_queue.put(ret)

def test_stdin_eval():
    """
    stdin eval
    The current implementation will send all the responses back
    into the first thread which dispatched the (def...), so we throw
    it into a thread and add a Queue to get it.
    Bad hack. But it works.

    Example output from the server:
        [{'status': ['need-input'], 'session': 'ec100813-8e76-4d69-9116-6460c1db4428'},
         {'session': 'ec100813-8e76-4d69-9116-6460c1db4428', 'ns': 'Hy', 'value': 'test'},
         {'status': ['done'], 'session': 'ec100813-8e76-4d69-9116-6460c1db4428'}]
    """

    stop = start_server()

    my_queue = queue.Queue()

    code = {"op": "eval", "code": '(def a (input))'}
    t = threading.Thread(target=stdin_send, args=(code, my_queue))
    t.start()

    # Might encounter a race condition where
    # we send stdin before we eval (input)
    time.sleep(0.5)

    code = {"op": "stdin", "value": "test"}
    soc_send(code, return_reply=False)

    t.join()
    ret = my_queue.get()

    input_request, value, status = ret
    stop()

    assert len(ret) == 3
    assert value["value"] == 'test'
    assert input_request["status"] == ["need-input"]
    assert "done" in status["status"]
    assert value["session"] == input_request["session"] == status["session"]
<commit_msg>Ignore ENOENT when unlinking test socket<commit_after>#!/usr/bin/env python
import os, unittest, subprocess, re, signal, time
import socket
import threading
import queue
import sys
import os

from socketserver import ThreadingMixIn, UnixStreamServer

from HyREPL.bencode import encode, decode, decode_multiple
from HyREPL.server import ReplRequestHandler

sock = "/tmp/HyREPL-test"
class TestServer(ThreadingMixIn, UnixStreamServer): pass
def start_server():
    try:
        os.unlink(sock)
    except FileNotFoundError:
        pass
    s = TestServer(sock, ReplRequestHandler)
    t = threading.Thread(target=s.serve_forever)
    t.daemon = True
    t.start()
    def stop():
        s.shutdown()
        t.join()
    return stop

def soc_send(message, return_reply=True):
    s = socket.socket(family=socket.AF_UNIX)
    s.connect(sock)
    s.sendall(encode(message))
    if not return_reply:
        s.close()
        return []
    s.setblocking(False)

    ret = []
    buf = b""
    while True:
        try:
            resp = s.recv(1)
            buf += resp
        except BlockingIOError:
            pass
        try:
            response, rest = decode(buf)
        except:
            continue
        buf = rest

        ret.append(response)
        if "done" in response.get("status", []):
            break
    s.close()
    return ret

def test_bencode():
    d = { "foo": 42, "spam": [1, 2, 'a'] }
    assert d == decode(encode(d))[0]

    d = decode_multiple(b"d5:value1:47:session36:31594b80-7f2e-4915-9969-f1127d562cc42:ns2:Hye" +\
            b"d6:statusl4:donee7:session36:31594b80-7f2e-4915-9969-f1127d562cc4e")
    assert len(d) == 2
    assert isinstance(d[0], dict)
    assert isinstance(d[1], dict)
    assert d[0]["value"] == "4"
    assert d[0]["ns"] == "Hy"
    assert isinstance(d[1]["status"], list)
    assert len(d[1]["status"]) == 1
    assert d[1]["status"][0] == "done"

def test_code_eval():
    """
    simple eval
    Example output from the server:
        [{'session': '0361c419-ef89-4a86-ae1a-48388be56041', 'ns': 'Hy', 'value': '4'},
         {'status': ['done'], 'session': '0361c419-ef89-4a86-ae1a-48388be56041'}]
    """
    stop = start_server()
    code = {"op": "eval", "code": "(+ 2 2)"}
    ret = soc_send(code)
    value, status = ret
    stop()

    assert len(ret) == 2
    assert value["value"] == '4'
    assert "done" in status["status"]
    assert value["session"] == status["session"]

def test_stdout_eval():
    """
    stdout eval
    Example output from the server:
        [{'session': '2d6b48d8-4a3e-49a6-9131-3321a11f70d4', 'ns': 'Hy', 'value': 'None'},
         {'session': '2d6b48d8-4a3e-49a6-9131-3321a11f70d4', 'out': 'Hello World\n'},
         {'status': ['done'], 'session': '2d6b48d8-4a3e-49a6-9131-3321a11f70d4'}]
    """

    stop = start_server()
    code = {"op": "eval", "code": '(print "Hello World")'}
    ret = soc_send(code)
    value, out, status = ret
    stop()

    assert len(ret) == 3
    assert value["value"] == 'None'
    assert out["out"] == "Hello World\n"
    assert "done" in status["status"]
    assert value["session"] == out["session"] == status["session"]

def stdin_send(code, my_queue):
    ret = soc_send(code)
    my_queue.put(ret)

def test_stdin_eval():
    """
    stdin eval
    The current implementation will send all the responses back
    into the first thread which dispatched the (def...), so we throw
    it into a thread and add a Queue to get it.
    Bad hack. But it works.

    Example output from the server:
        [{'status': ['need-input'], 'session': 'ec100813-8e76-4d69-9116-6460c1db4428'},
         {'session': 'ec100813-8e76-4d69-9116-6460c1db4428', 'ns': 'Hy', 'value': 'test'},
         {'status': ['done'], 'session': 'ec100813-8e76-4d69-9116-6460c1db4428'}]
    """

    stop = start_server()

    my_queue = queue.Queue()

    code = {"op": "eval", "code": '(def a (input))'}
    t = threading.Thread(target=stdin_send, args=(code, my_queue))
    t.start()

    # Might encounter a race condition where
    # we send stdin before we eval (input)
    time.sleep(0.5)

    code = {"op": "stdin", "value": "test"}
    soc_send(code, return_reply=False)

    t.join()
    ret = my_queue.get()

    input_request, value, status = ret
    stop()

    assert len(ret) == 3
    assert value["value"] == 'test'
    assert input_request["status"] == ["need-input"]
    assert "done" in status["status"]
    assert value["session"] == input_request["session"] == status["session"]
<|endoftext|>


<commit_before>from PIL import Image, ImageDraw
import numpy as np
from analyzer import get_mode_colors

height = 40
im = Image.new('RGB', (400, height))

color_list = [20, 0, 0, 240, 230, 200, 250, 240, 210, 50, 40, 30, 50, 30, 30, 40, 30, 20, 220, 230, 240, 260, 260, 240, 20, 10, 0, 80, 80, 50]
# from barefoot blue jean night
colors = []
for i in range(0,len(color_list),3):
    colors.append((color_list[i],color_list[i+1],color_list[i+2]))

draw = ImageDraw.Draw(im)
i = 10
new_coord = [0,0,40,40]
prev_coord = []
# for color in colors:
width = 10 + i * 2
# draw.rectangle(new_coord,colors[1])
prev_coord = new_coord
new_coord = [0,20,0,height]
print new_coord
draw.rectangle(new_coord,colors[2])
i -= 1
im.show()
<commit_msg>finished image generation function (it saves a jpeg output).<commit_after>from PIL import Image, ImageDraw
import numpy as np
from analyzer import get_mode_colors

def generate_image(vid_path,im_path):
    height = 40
    width = 420
    im = Image.new('RGB', (width, height))

    color_list = get_mode_colors(vid_path)

    colors = []
    for i in range(0,len(color_list),3):
        colors.append((color_list[i],color_list[i+1],color_list[i+2]))

    draw = ImageDraw.Draw(im)
    i = 10

    width = 10 + i * 5
    new_coord = [0,0,width,height]
    for color in colors:
        print new_coord
        print width
        width = 10 + i * 5
        draw.rectangle(new_coord,color)
        prev_coord = new_coord
        new_coord = [prev_coord[2],0,prev_coord[2]+width,height]
        i -= 1
    # im.show()
    im.save(im_path)

<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4
import json
import random
import string
import unittest2

from roush import webapp
from roush.db.database import init_db

class RoushTestCase(unittest2.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.app = webapp.Thing('roush', configfile='test.conf', debug=True)
        init_db(cls.app.config['database_uri'])
        cls.client = cls.app.test_client()

    @classmethod
    def tearDownClass(cls):
        pass

    def _clean_all(self):
        for what in ['task', 'node', 'fact']:
            all_results = self._model_get_all(what)
            for what_id in [x['id'] for x in all_results]:
                self._model_delete(what, what_id)

    def _valid_rand(self, var_type):
        if var_type == 'INTEGER':
            return random.randrange(1,100)
        elif var_type == 'TEXT':
            return self._random_str(250)
        elif var_type == 'JSON_ENTRY':
            choices = [lambda: self._random_str(10),
                       lambda: self._valid_rand('INTEGER'),
                       lambda: [self._random_str(10) for x in range(10)]]
            return choices()
        elif var_type == 'JSON':
            choices = [lambda: [self._random_str(10) for x in range(10)],
                       lambda: dict([(self._random_str(5),
                                      self._valid_rand('INTEGER'))
                                     for x in range(10)])]
            return choices()
        elif var_type.startswith('VARCHAR'):
            str_len = int(var_type.split('(')[1][:-1])
            return self._random_str(str_len)

        raise ValueError('bad rand type: %s' % var_type)

    def _random_str(self, size=20):
        return "".join(random.choice(string.ascii_lowercase)
                       for x in range(size))

    def _pluralize(self, what):
        return what + 's'

    def _singularize(self, what):
        return what[:-1]

    def _client_request(self, method, uri, **kwargs):
        fn = getattr(self.client, method)
        self.assertIsNotNone(fn)
        return fn(uri, content_type='application/json', **kwargs)

    def _model_create(self, model, **kwargs):
        resp = self.client.post('/%s/' % self._pluralize(model),
                                content_type='application/json',
                                data=json.dumps(kwargs))

        self.assertEquals(resp.status_code, 201)
        return json.loads(resp.data)[model]

    def _model_update(self, model, id, **kwargs):
        resp = self.client.put('/%s/%s' % (self._pluralize(model), id),
                               content_type='application/json',
                               data=json.dumps(kwargs))
        self.assertEquals(resp.status_code, 200)
        return json.loads(resp.data)[model]

    def _model_delete(self, model, id):
        resp = self.client.delete('/%s/%s' % (self._pluralize(model), id))
        self.assertEquals(resp.status_code, 200)
        out = json.loads(resp.data)
        self.app.logger.debug('model_delete response: %s' % out)
        self.assertEquals(out['status'], 200)
        self.assertEquals(out['message'], '%s deleted' %
                          model.capitalize())

    def _model_get_by_id(self, model, id):
        resp = self.client.get('/%s/%s' % (self._pluralize(model), id))
        self.assertEquals(resp.status_code, 200)
        out = json.loads(resp.data)
        self.app.logger.debug('model_get_by_id response: %s' % out)
        return out[model]

    def _model_filter(self, model, filter_str):
        resp = self.client.post('/%s/filter' % self._pluralize(model),
                                content_type='application/json',
                                data=json.dumps({'filter': filter_str}))
        self.app.logger.debug('response: %s' % resp)
        self.assertEquals(resp.status_code, 200)
        out = json.loads(resp.data)
        self.app.logger.debug('model_get_by_filter response: %s' % out)
        return out[self._pluralize(model)]

    def _model_get_all(self, model):
        resp = self.client.get('/%s/' % self._pluralize(model))
        self.assertEquals(resp.status_code, 200)
        out = json.loads(resp.data)
        self.app.logger.debug('get_all response: %s' % out)
        return out[self._pluralize(model)]

    def _model_get_schema(self, model):
        resp = self.client.get('/%s/schema' % self._pluralize(model))
        self.app.logger.debug('get_schema response: %s' % resp.data)
        self.assertEquals(resp.status_code, 200)
        out = json.loads(resp.data)
        return out['schema']

    def test_get_schema(self):
        if not hasattr(self, 'base_object'):
            self.skipTest('no base_object')

        self._model_get_schema(self.base_object)

    def test_create(self):
        if not hasattr(self, 'base_object'):
            self.skipTest('no base_object')

        bo = self.base_object
        bop = self._pluralize(bo)

        schema = self._model_get_schema(self.base_object)

        req_fields = [x for x in schema if (schema[x]['required'] == True) and
                      (schema[x]['primary_key'] == False)]
        ro_fields = [x for x in schema if schema[x]['updatable'] == False]

        ids = []

        # make sure we can't create without a non-updatable field
        for field in req_fields:
            full_field_list = req_fields
            full_field_list.remove(field)

            data = dict(zip(full_field_list,
                            [self._valid_rand(schema[x]['type'])
                             for x in full_field_list]))

            self.app.logger.debug('Creating new %s with %s' % (bo, data))

            res = self._client_request('post', '/%s/' % bop, data=data)
            self.app.logger.debug('result: %s: %s' %
                                  (res.status_code, res.data))
            self.assertEquals(res.status_code, 400)
<commit_msg><3 pep8<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4
import json
import random
import string
import unittest2

from roush import webapp
from roush.db.database import init_db

class RoushTestCase(unittest2.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.app = webapp.Thing('roush', configfile='test.conf', debug=True)
        init_db(cls.app.config['database_uri'])
        cls.client = cls.app.test_client()

    @classmethod
    def tearDownClass(cls):
        pass

    def _clean_all(self):
        for what in ['task', 'node', 'fact']:
            all_results = self._model_get_all(what)
            for what_id in [x['id'] for x in all_results]:
                self._model_delete(what, what_id)

    def _valid_rand(self, var_type):
        if var_type == 'INTEGER':
            return random.randrange(1, 100)
        elif var_type == 'TEXT':
            return self._random_str(250)
        elif var_type == 'JSON_ENTRY':
            choices = [lambda: self._random_str(10),
                       lambda: self._valid_rand('INTEGER'),
                       lambda: [self._random_str(10) for x in range(10)]]
            return choices()
        elif var_type == 'JSON':
            choices = [lambda: [self._random_str(10) for x in range(10)],
                       lambda: dict([(self._random_str(5),
                                      self._valid_rand('INTEGER'))
                                     for x in range(10)])]
            return choices()
        elif var_type.startswith('VARCHAR'):
            str_len = int(var_type.split('(')[1][:-1])
            return self._random_str(str_len)

        raise ValueError('bad rand type: %s' % var_type)

    def _random_str(self, size=20):
        return "".join(random.choice(string.ascii_lowercase)
                       for x in range(size))

    def _pluralize(self, what):
        return what + 's'

    def _singularize(self, what):
        return what[:-1]

    def _client_request(self, method, uri, **kwargs):
        fn = getattr(self.client, method)
        self.assertIsNotNone(fn)
        return fn(uri, content_type='application/json', **kwargs)

    def _model_create(self, model, **kwargs):
        resp = self.client.post('/%s/' % self._pluralize(model),
                                content_type='application/json',
                                data=json.dumps(kwargs))

        self.assertEquals(resp.status_code, 201)
        return json.loads(resp.data)[model]

    def _model_update(self, model, id, **kwargs):
        resp = self.client.put('/%s/%s' % (self._pluralize(model), id),
                               content_type='application/json',
                               data=json.dumps(kwargs))
        self.assertEquals(resp.status_code, 200)
        return json.loads(resp.data)[model]

    def _model_delete(self, model, id):
        resp = self.client.delete('/%s/%s' % (self._pluralize(model), id))
        self.assertEquals(resp.status_code, 200)
        out = json.loads(resp.data)
        self.app.logger.debug('model_delete response: %s' % out)
        self.assertEquals(out['status'], 200)
        self.assertEquals(out['message'], '%s deleted' %
                          model.capitalize())

    def _model_get_by_id(self, model, id):
        resp = self.client.get('/%s/%s' % (self._pluralize(model), id))
        self.assertEquals(resp.status_code, 200)
        out = json.loads(resp.data)
        self.app.logger.debug('model_get_by_id response: %s' % out)
        return out[model]

    def _model_filter(self, model, filter_str):
        resp = self.client.post('/%s/filter' % self._pluralize(model),
                                content_type='application/json',
                                data=json.dumps({'filter': filter_str}))
        self.app.logger.debug('response: %s' % resp)
        self.assertEquals(resp.status_code, 200)
        out = json.loads(resp.data)
        self.app.logger.debug('model_get_by_filter response: %s' % out)
        return out[self._pluralize(model)]

    def _model_get_all(self, model):
        resp = self.client.get('/%s/' % self._pluralize(model))
        self.assertEquals(resp.status_code, 200)
        out = json.loads(resp.data)
        self.app.logger.debug('get_all response: %s' % out)
        return out[self._pluralize(model)]

    def _model_get_schema(self, model):
        resp = self.client.get('/%s/schema' % self._pluralize(model))
        self.app.logger.debug('get_schema response: %s' % resp.data)
        self.assertEquals(resp.status_code, 200)
        out = json.loads(resp.data)
        return out['schema']

    def test_get_schema(self):
        if not hasattr(self, 'base_object'):
            self.skipTest('no base_object')

        self._model_get_schema(self.base_object)

    def test_create(self):
        if not hasattr(self, 'base_object'):
            self.skipTest('no base_object')

        bo = self.base_object
        bop = self._pluralize(bo)

        schema = self._model_get_schema(self.base_object)

        req_fields = [x for x in schema if (schema[x]['required'] is True) and
                      (schema[x]['primary_key'] is False)]
        ro_fields = [x for x in schema if schema[x]['updatable'] is False]

        ids = []

        # make sure we can't create without a non-updatable field
        for field in req_fields:
            full_field_list = req_fields
            full_field_list.remove(field)

            data = dict(zip(full_field_list,
                            [self._valid_rand(schema[x]['type'])
                             for x in full_field_list]))

            self.app.logger.debug('Creating new %s with %s' % (bo, data))

            res = self._client_request('post', '/%s/' % bop, data=data)
            self.app.logger.debug('result: %s: %s' %
                                  (res.status_code, res.data))
            self.assertEquals(res.status_code, 400)
<|endoftext|>


<commit_before># -*- coding: utf8 -*-
__author__ = 'Clemens Prescher'

from PyQt4 import QtCore, QtGui

from ScatteringFactors import scattering_factor_param

class ControlWidget(QtGui.QWidget):
    def __init__(self, *args, **kwargs):
        super(ControlWidget, self).__init__(*args, **kwargs)
        self.vertical_layout = QtGui.QVBoxLayout()
        self.vertical_layout.setSpacing(0)
        self.vertical_layout.setContentsMargins(5, 5, 5, 5)

        self.file_widget = FileWidget()
        self.background_options_gb = BackgroundOptionsGroupBox()
        self.smooth_gb = SmoothGroupBox()
        self.composition_gb = CompositionGroupBox()
        self.calculation_gb = CalculationGroupBox()

        self.vertical_layout.addWidget(self.file_widget)
        self.vertical_layout.addWidget(self.background_options_gb)
        self.vertical_layout.addWidget(self.smooth_gb)
        self.vertical_layout.addSpacerItem(QtGui.QSpacerItem(10, 10, QtGui.QSizePolicy.Fixed,
                                                             QtGui.QSizePolicy.Fixed))
        self.vertical_layout.addWidget(self.composition_gb)
        self.vertical_layout.addWidget(self.calculation_gb)

        self.vertical_layout.addSpacerItem(QtGui.QSpacerItem(20, 20, QtGui.QSizePolicy.Fixed,
                                                             QtGui.QSizePolicy.Expanding))
        self.setLayout(self.vertical_layout)

class FileWidget(QtGui.QWidget):
    def __init__(self, *args, **kwargs):
        super(FileWidget, self).__init__(*args, **kwargs)

        self.vertical_layout = QtGui.QVBoxLayout()
        self.vertical_layout.setContentsMargins(0, 0, 0, 0)

        self.load_data_btn = QtGui.QPushButton("Load Data")
        self.data_filename_lbl = QtGui.QLabel("None")
        self.data_filename_lbl.setAlignment(QtCore.Qt.AlignRight)
        self.load_background_btn = QtGui.QPushButton("Load Bkg")
        self.background_filename_lbl = QtGui.QLabel("None")
        self.background_filename_lbl.setAlignment(QtCore.Qt.AlignRight)

        self.vertical_layout.addWidget(self.load_data_btn)
        self.vertical_layout.addWidget(self.data_filename_lbl)
        self.vertical_layout.addWidget(self.load_background_btn)
        self.vertical_layout.addWidget(self.background_filename_lbl)

        self.setLayout(self.vertical_layout)

class BackgroundOptionsGroupBox(QtGui.QGroupBox):
    def __init__(self, *args):
        super(BackgroundOptionsGroupBox, self).__init__(*args)

        self.grid_layout = QtGui.QGridLayout()
        self.grid_layout.setContentsMargins(3, 5, 5, 3)
        self.grid_layout.setSpacing(5)

        self.scale_lbl = QtGui.QLabel("Scale:")
        self.offset_lbl = QtGui.QLabel("Offset:")
        self.scale_lbl.setAlignment(QtCore.Qt.AlignVCenter | QtCore.Qt.AlignRight)
        self.offset_lbl.setAlignment(QtCore.Qt.AlignVCenter | QtCore.Qt.AlignRight)

        self.scale_sb = QtGui.QDoubleSpinBox()
        self.scale_sb.setValue(1.0)
        self.offset_sb = QtGui.QDoubleSpinBox()

        self.scale_sb.setAlignment(QtCore.Qt.AlignRight)
        self.offset_sb.setAlignment(QtCore.Qt.AlignRight)

        self.scale_sb.setSingleStep(0.01)
        self.offset_sb.setSingleStep(10)

        self.scale_step_txt = QtGui.QLineEdit("0.01")
        self.scale_step_txt.setMaximumWidth(60)
        self.offset_step_txt = QtGui.QLineEdit("10")
        self.offset_step_txt.setMaximumWidth(60)

        self.scale_step_txt.setAlignment(QtCore.Qt.AlignRight)
        self.offset_step_txt.setAlignment(QtCore.Qt.AlignRight)

        self.scale_step_txt.setValidator(QtGui.QDoubleValidator())
        self.offset_step_txt.setValidator(QtGui.QDoubleValidator())

        self.grid_layout.addWidget(self.scale_lbl, 0, 0)
        self.grid_layout.addWidget(self.scale_sb, 0, 1)
        self.grid_layout.addWidget(self.scale_step_txt, 0, 2)

        self.grid_layout.addWidget(self.offset_lbl, 1, 0)
        self.grid_layout.addWidget(self.offset_sb, 1, 1)
        self.grid_layout.addWidget(self.offset_step_txt, 1, 2)

        self.setLayout(self.grid_layout)

class SmoothGroupBox(QtGui.QGroupBox):
    def __init__(self, *args):
        super(SmoothGroupBox, self).__init__(*args)
        self.grid_layout = QtGui.QGridLayout()
        self.grid_layout.setContentsMargins(5, 5, 5, 5)
        self.grid_layout.setSpacing(5)

        self.smooth_lbl = QtGui.QLabel("Smooth:")
        self.smooth_lbl.setAlignment(QtCore.Qt.AlignVCenter | QtCore.Qt.AlignRight)

        self.smooth_sb = QtGui.QDoubleSpinBox()
        self.smooth_sb.setAlignment(QtCore.Qt.AlignRight)
        self.smooth_sb.setSingleStep(0.01)

        self.smooth_step_txt = QtGui.QLineEdit("0.01")
        self.smooth_step_txt.setAlignment(QtCore.Qt.AlignRight)
        self.smooth_step_txt.setValidator(QtGui.QDoubleValidator())
        self.smooth_step_txt.setMaximumWidth(60)

        self.grid_layout.addWidget(self.smooth_lbl, 0, 0)
        self.grid_layout.addWidget(self.smooth_sb, 0, 1)
        self.grid_layout.addWidget(self.smooth_step_txt, 0, 2)

        self.setLayout(self.grid_layout)

class CompositionGroupBox(QtGui.QGroupBox):
    composition_changed = QtCore.pyqtSignal(dict, float)

    def __init__(self, *args):
        super(CompositionGroupBox, self).__init__("Composition", *args)
        self.create_widgets()

    def create_widgets(self):
        self.main_layout = QtGui.QVBoxLayout()
        self.main_layout.setContentsMargins(5, 5, 5, 5)
        self.main_layout.setSpacing(5)

        self.button_layout = QtGui.QHBoxLayout()
        self.button_layout.setSpacing(15)
        self.add_element_btn = QtGui.QPushButton("Add")
        self.delete_element_btn = QtGui.QPushButton("Delete")
        self.button_layout.addWidget(self.add_element_btn)
        self.button_layout.addWidget(self.delete_element_btn)

        self.density_layout = QtGui.QHBoxLayout()
        self.density_lbl = QtGui.QLabel("Density:")
        self.density_lbl.setAlignment(QtCore.Qt.AlignVCenter | QtCore.Qt.AlignRight)
        self.density_txt = QtGui.QLineEdit("2.2")
        self.density_txt.setAlignment(QtCore.Qt.AlignRight)
        self.density_txt.setValidator(QtGui.QDoubleValidator())
        self.density_txt.editingFinished.connect(self.emit_composition_changed_signal)
        self.density_txt.setMaximumWidth(100)
        self.density_layout.addWidget(self.density_lbl)
        self.density_layout.addWidget(self.density_txt)

        self.composition_tw = QtGui.QTableWidget()
        self.composition_tw.setFixedHeight(100)
        self.composition_tw.setColumnCount(2)
        self.composition_tw.horizontalHeader().setVisible(False)
        self.composition_tw.verticalHeader().setVisible(False)
        self.composition_tw.setColumnWidth(0, 80)
        self.composition_tw.setColumnWidth(1, 80)
        self.composition_tw.setItemDelegate(TextDoubleDelegate(self))
        self.composition_tw.cellChanged.connect(self.emit_composition_changed_signal)

        self.main_layout.addLayout(self.button_layout)
        self.main_layout.addWidget(self.composition_tw)
        self.main_layout.addLayout(self.density_layout)

        self.setLayout(self.main_layout)

    def add_element(self, element=None, value=None):
        current_rows = self.composition_tw.rowCount()
        self.composition_tw.setRowCount(current_rows + 1)
        self.composition_tw.blockSignals(True)
        element_cb = QtGui.QComboBox(self)
        element_cb.setStyle(QtGui.QStyleFactory.create('cleanlooks'))

        for ind, ele in enumerate(scattering_factor_param.index):
            element_cb.insertItem(ind, ele)

        if element is not None:
            element_cb.setCurrentIndex(element_cb.findText(element))
        self.composition_tw.setCellWidget(current_rows, 0, element_cb)

        element_cb.currentIndexChanged.connect(self.emit_composition_changed_signal)
        if value is not None:
            value_item = QtGui.QTableWidgetItem(str(value))
        else:
            value_item = QtGui.QTableWidgetItem(str(1))
        value_item.setTextAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)
        self.composition_tw.setItem(current_rows, 1, value_item)
        self.composition_tw.blockSignals(False)
        self.emit_composition_changed_signal()

    def delete_element(self, ind):
        self.composition_tw.blockSignals(True)
        self.composition_tw.removeRow(ind)
        self.composition_tw.blockSignals(False)
        self.emit_composition_changed_signal()

    def get_composition(self):
        composition = {}
        for row_ind in range(self.composition_tw.rowCount()):
            cb_item = self.composition_tw.cellWidget(row_ind, 0)
            value_item = self.composition_tw.item(row_ind, 1)
            composition[str(cb_item.currentText())] = float(str(value_item.text()))
        return composition

    def get_density(self):
        return float(str(self.density_txt.text()))

    def emit_composition_changed_signal(self):
        print self.get_composition()
        self.composition_changed.emit(self.get_composition(), self.get_density())

class TextDoubleDelegate(QtGui.QStyledItemDelegate):
    def __init__(self, parent):
        super(TextDoubleDelegate, self).__init__(parent)

    def createEditor(self, parent, _, model):
        self.editor = QtGui.QLineEdit(parent)
        self.editor.setFrame(False)
        self.editor.setValidator(QtGui.QDoubleValidator())
        self.editor.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)
        return self.editor

    def setEditorData(self, parent, index):
        value = index.model().data(index, QtCore.Qt.EditRole)
        if value.toString() != '':
            self.editor.setText("{:g}".format(float(str(value.toString()))))

    def setModelData(self, parent, model, index):
        value = self.editor.text()
        model.setData(index, value, QtCore.Qt.EditRole)

    def updateEditorGeometry(self, editor, option, _):
        editor.setGeometry(option.rect)

class CalculationGroupBox(QtGui.QGroupBox):
    calculation_parameters_changed = QtCore.pyqtSignal(float, float, float)

    def __init__(self, *args):
        super(CalculationGroupBox, self).__init__(*args)

        self.create_widgets()
        self.style_widgets()
        self.create_layout()
        self.create_signals()

    def create_widgets(self):
        self.q_min_lbl = QtGui.QLabel('Q min:')
        self.q_max_lbl = QtGui.QLabel('Q max:')
        self.r_cutoff_lbl = QtGui.QLabel('r cutoff:')

        self.q_min_txt = QtGui.QLineEdit('0')
        self.q_max_txt = QtGui.QLineEdit('10')
        self.r_cutoff_txt = QtGui.QLineEdit('1')

        self.optimize_btn = QtGui.QPushButton("Optimize")
        self.optimize_r_cutoff_btn = QtGui.QPushButton("Optimize r cutoff")

    def style_widgets(self):
        self.q_min_lbl.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)
        self.q_max_lbl.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)
        self.r_cutoff_lbl.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)

        self.q_min_txt.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)
        self.q_max_txt.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)
        self.r_cutoff_txt.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)

        self.q_min_txt.setMaximumWidth(80)
        self.q_max_txt.setMaximumWidth(80)
        self.r_cutoff_txt.setMaximumWidth(80)

        self.q_min_txt.setValidator(QtGui.QDoubleValidator())
        self.q_max_txt.setValidator(QtGui.QDoubleValidator())
        self.r_cutoff_txt.setValidator(QtGui.QDoubleValidator())

        self.optimize_btn.setFlat(True)
        self.optimize_r_cutoff_btn.setFlat(True)

    def create_layout(self):
        self.grid_layout = QtGui.QGridLayout()
        self.grid_layout.setSpacing(5)
        self.grid_layout.addWidget(self.q_min_lbl, 0, 0)
        self.grid_layout.addWidget(self.q_min_txt, 0, 1)
        self.grid_layout.addWidget(self.q_max_lbl, 1, 0)
        self.grid_layout.addWidget(self.q_max_txt, 1, 1)
        self.grid_layout.addWidget(self.r_cutoff_lbl, 2, 0)
        self.grid_layout.addWidget(self.r_cutoff_txt, 2, 1)

        self.grid_layout.addWidget(self.optimize_btn, 3, 0, 1, 2)
        self.grid_layout.addWidget(self.optimize_r_cutoff_btn, 4, 0, 1, 2)

        self.setLayout(self.grid_layout)

    def create_signals(self):
        self.q_max_txt.editingFinished.connect(self.emit_calculation_changed_signal)
        self.q_min_txt.editingFinished.connect(self.emit_calculation_changed_signal)
        self.r_cutoff_txt.editingFinished.connect(self.emit_calculation_changed_signal)

    def emit_calculation_changed_signal(self):
        q_min = float(str(self.q_min_txt.text()))
        q_max = float(str(self.q_max_txt.text()))
        r_cutoff = float(str(self.r_cutoff_txt.text()))
        self.calculation_parameters_changed.emit(q_min, q_max, r_cutoff)

<commit_msg>added plot subtracted checkbox<commit_after># -*- coding: utf8 -*-
__author__ = 'Clemens Prescher'

from PyQt4 import QtCore, QtGui

from ScatteringFactors import scattering_factor_param

class ControlWidget(QtGui.QWidget):
    def __init__(self, *args, **kwargs):
        super(ControlWidget, self).__init__(*args, **kwargs)
        self.vertical_layout = QtGui.QVBoxLayout()
        self.vertical_layout.setSpacing(0)
        self.vertical_layout.setContentsMargins(5, 5, 5, 5)

        self.file_widget = FileWidget()
        self.background_options_gb = BackgroundOptionsGroupBox()
        self.smooth_gb = SmoothGroupBox()
        self.composition_gb = CompositionGroupBox()
        self.calculation_gb = CalculationGroupBox()

        self.vertical_layout.addWidget(self.file_widget)
        self.vertical_layout.addWidget(self.background_options_gb)
        self.vertical_layout.addWidget(self.smooth_gb)
        self.vertical_layout.addSpacerItem(QtGui.QSpacerItem(10, 10, QtGui.QSizePolicy.Fixed,
                                                             QtGui.QSizePolicy.Fixed))
        self.vertical_layout.addWidget(self.composition_gb)
        self.vertical_layout.addWidget(self.calculation_gb)

        self.vertical_layout.addSpacerItem(QtGui.QSpacerItem(20, 20, QtGui.QSizePolicy.Fixed,
                                                             QtGui.QSizePolicy.Expanding))
        self.setLayout(self.vertical_layout)

class FileWidget(QtGui.QWidget):
    def __init__(self, *args, **kwargs):
        super(FileWidget, self).__init__(*args, **kwargs)

        self.vertical_layout = QtGui.QVBoxLayout()
        self.vertical_layout.setContentsMargins(0, 0, 0, 0)

        self.load_data_btn = QtGui.QPushButton("Load Data")
        self.data_filename_lbl = QtGui.QLabel("None")
        self.data_filename_lbl.setAlignment(QtCore.Qt.AlignRight)
        self.load_background_btn = QtGui.QPushButton("Load Bkg")
        self.background_filename_lbl = QtGui.QLabel("None")
        self.background_filename_lbl.setAlignment(QtCore.Qt.AlignRight)
        self.plot_subtracted = QtGui.QCheckBox("Plot subtracted")
        self.plot_subtracted.setChecked(True)

        self.vertical_layout.addWidget(self.load_data_btn)
        self.vertical_layout.addWidget(self.data_filename_lbl)
        self.vertical_layout.addWidget(self.load_background_btn)
        self.vertical_layout.addWidget(self.background_filename_lbl)
        self.vertical_layout.addWidget(self.plot_subtracted)

        self.setLayout(self.vertical_layout)

class BackgroundOptionsGroupBox(QtGui.QGroupBox):
    def __init__(self, *args):
        super(BackgroundOptionsGroupBox, self).__init__(*args)

        self.grid_layout = QtGui.QGridLayout()
        self.grid_layout.setContentsMargins(3, 5, 5, 3)
        self.grid_layout.setSpacing(5)

        self.scale_lbl = QtGui.QLabel("Scale:")
        self.offset_lbl = QtGui.QLabel("Offset:")
        self.scale_lbl.setAlignment(QtCore.Qt.AlignVCenter | QtCore.Qt.AlignRight)
        self.offset_lbl.setAlignment(QtCore.Qt.AlignVCenter | QtCore.Qt.AlignRight)

        self.scale_sb = QtGui.QDoubleSpinBox()
        self.scale_sb.setValue(1.0)
        self.offset_sb = QtGui.QDoubleSpinBox()

        self.scale_sb.setAlignment(QtCore.Qt.AlignRight)
        self.offset_sb.setAlignment(QtCore.Qt.AlignRight)

        self.scale_sb.setSingleStep(0.01)
        self.offset_sb.setSingleStep(10)

        self.scale_step_txt = QtGui.QLineEdit("0.01")
        self.scale_step_txt.setMaximumWidth(60)
        self.offset_step_txt = QtGui.QLineEdit("10")
        self.offset_step_txt.setMaximumWidth(60)

        self.scale_step_txt.setAlignment(QtCore.Qt.AlignRight)
        self.offset_step_txt.setAlignment(QtCore.Qt.AlignRight)

        self.scale_step_txt.setValidator(QtGui.QDoubleValidator())
        self.offset_step_txt.setValidator(QtGui.QDoubleValidator())

        self.grid_layout.addWidget(self.scale_lbl, 0, 0)
        self.grid_layout.addWidget(self.scale_sb, 0, 1)
        self.grid_layout.addWidget(self.scale_step_txt, 0, 2)

        self.grid_layout.addWidget(self.offset_lbl, 1, 0)
        self.grid_layout.addWidget(self.offset_sb, 1, 1)
        self.grid_layout.addWidget(self.offset_step_txt, 1, 2)

        self.setLayout(self.grid_layout)

class SmoothGroupBox(QtGui.QGroupBox):
    def __init__(self, *args):
        super(SmoothGroupBox, self).__init__(*args)
        self.grid_layout = QtGui.QGridLayout()
        self.grid_layout.setContentsMargins(5, 5, 5, 5)
        self.grid_layout.setSpacing(5)

        self.smooth_lbl = QtGui.QLabel("Smooth:")
        self.smooth_lbl.setAlignment(QtCore.Qt.AlignVCenter | QtCore.Qt.AlignRight)

        self.smooth_sb = QtGui.QDoubleSpinBox()
        self.smooth_sb.setAlignment(QtCore.Qt.AlignRight)
        self.smooth_sb.setSingleStep(0.01)

        self.smooth_step_txt = QtGui.QLineEdit("0.01")
        self.smooth_step_txt.setAlignment(QtCore.Qt.AlignRight)
        self.smooth_step_txt.setValidator(QtGui.QDoubleValidator())
        self.smooth_step_txt.setMaximumWidth(60)

        self.grid_layout.addWidget(self.smooth_lbl, 0, 0)
        self.grid_layout.addWidget(self.smooth_sb, 0, 1)
        self.grid_layout.addWidget(self.smooth_step_txt, 0, 2)

        self.setLayout(self.grid_layout)

class CompositionGroupBox(QtGui.QGroupBox):
    composition_changed = QtCore.pyqtSignal(dict, float)

    def __init__(self, *args):
        super(CompositionGroupBox, self).__init__("Composition", *args)
        self.create_widgets()

    def create_widgets(self):
        self.main_layout = QtGui.QVBoxLayout()
        self.main_layout.setContentsMargins(5, 5, 5, 5)
        self.main_layout.setSpacing(5)

        self.button_layout = QtGui.QHBoxLayout()
        self.button_layout.setSpacing(15)
        self.add_element_btn = QtGui.QPushButton("Add")
        self.delete_element_btn = QtGui.QPushButton("Delete")
        self.button_layout.addWidget(self.add_element_btn)
        self.button_layout.addWidget(self.delete_element_btn)

        self.density_layout = QtGui.QHBoxLayout()
        self.density_lbl = QtGui.QLabel("Density:")
        self.density_lbl.setAlignment(QtCore.Qt.AlignVCenter | QtCore.Qt.AlignRight)
        self.density_txt = QtGui.QLineEdit("2.2")
        self.density_txt.setAlignment(QtCore.Qt.AlignRight)
        self.density_txt.setValidator(QtGui.QDoubleValidator())
        self.density_txt.editingFinished.connect(self.emit_composition_changed_signal)
        self.density_txt.setMaximumWidth(100)
        self.density_layout.addWidget(self.density_lbl)
        self.density_layout.addWidget(self.density_txt)

        self.composition_tw = QtGui.QTableWidget()
        self.composition_tw.setFixedHeight(100)
        self.composition_tw.setColumnCount(2)
        self.composition_tw.horizontalHeader().setVisible(False)
        self.composition_tw.verticalHeader().setVisible(False)
        self.composition_tw.setColumnWidth(0, 80)
        self.composition_tw.setColumnWidth(1, 80)
        self.composition_tw.setItemDelegate(TextDoubleDelegate(self))
        self.composition_tw.cellChanged.connect(self.emit_composition_changed_signal)

        self.main_layout.addLayout(self.button_layout)
        self.main_layout.addWidget(self.composition_tw)
        self.main_layout.addLayout(self.density_layout)

        self.setLayout(self.main_layout)

    def add_element(self, element=None, value=None):
        current_rows = self.composition_tw.rowCount()
        self.composition_tw.setRowCount(current_rows + 1)
        self.composition_tw.blockSignals(True)
        element_cb = QtGui.QComboBox(self)
        element_cb.setStyle(QtGui.QStyleFactory.create('cleanlooks'))

        for ind, ele in enumerate(scattering_factor_param.index):
            element_cb.insertItem(ind, ele)

        if element is not None:
            element_cb.setCurrentIndex(element_cb.findText(element))
        self.composition_tw.setCellWidget(current_rows, 0, element_cb)

        element_cb.currentIndexChanged.connect(self.emit_composition_changed_signal)
        if value is not None:
            value_item = QtGui.QTableWidgetItem(str(value))
        else:
            value_item = QtGui.QTableWidgetItem(str(1))
        value_item.setTextAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)
        self.composition_tw.setItem(current_rows, 1, value_item)
        self.composition_tw.blockSignals(False)
        self.emit_composition_changed_signal()

    def delete_element(self, ind):
        self.composition_tw.blockSignals(True)
        self.composition_tw.removeRow(ind)
        self.composition_tw.blockSignals(False)
        self.emit_composition_changed_signal()

    def get_composition(self):
        composition = {}
        for row_ind in range(self.composition_tw.rowCount()):
            cb_item = self.composition_tw.cellWidget(row_ind, 0)
            value_item = self.composition_tw.item(row_ind, 1)
            composition[str(cb_item.currentText())] = float(str(value_item.text()))
        return composition

    def get_density(self):
        return float(str(self.density_txt.text()))

    def emit_composition_changed_signal(self):
        print self.get_composition()
        self.composition_changed.emit(self.get_composition(), self.get_density())

class TextDoubleDelegate(QtGui.QStyledItemDelegate):
    def __init__(self, parent):
        super(TextDoubleDelegate, self).__init__(parent)

    def createEditor(self, parent, _, model):
        self.editor = QtGui.QLineEdit(parent)
        self.editor.setFrame(False)
        self.editor.setValidator(QtGui.QDoubleValidator())
        self.editor.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)
        return self.editor

    def setEditorData(self, parent, index):
        value = index.model().data(index, QtCore.Qt.EditRole)
        if value.toString() != '':
            self.editor.setText("{:g}".format(float(str(value.toString()))))

    def setModelData(self, parent, model, index):
        value = self.editor.text()
        model.setData(index, value, QtCore.Qt.EditRole)

    def updateEditorGeometry(self, editor, option, _):
        editor.setGeometry(option.rect)

class CalculationGroupBox(QtGui.QGroupBox):
    calculation_parameters_changed = QtCore.pyqtSignal(float, float, float)

    def __init__(self, *args):
        super(CalculationGroupBox, self).__init__(*args)

        self.create_widgets()
        self.style_widgets()
        self.create_layout()
        self.create_signals()

    def create_widgets(self):
        self.q_min_lbl = QtGui.QLabel('Q min:')
        self.q_max_lbl = QtGui.QLabel('Q max:')
        self.r_cutoff_lbl = QtGui.QLabel('r cutoff:')

        self.q_min_txt = QtGui.QLineEdit('0')
        self.q_max_txt = QtGui.QLineEdit('10')
        self.r_cutoff_txt = QtGui.QLineEdit('1')

        self.optimize_btn = QtGui.QPushButton("Optimize")
        self.optimize_r_cutoff_btn = QtGui.QPushButton("Optimize r cutoff")

    def style_widgets(self):
        self.q_min_lbl.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)
        self.q_max_lbl.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)
        self.r_cutoff_lbl.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)

        self.q_min_txt.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)
        self.q_max_txt.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)
        self.r_cutoff_txt.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)

        self.q_min_txt.setMaximumWidth(80)
        self.q_max_txt.setMaximumWidth(80)
        self.r_cutoff_txt.setMaximumWidth(80)

        self.q_min_txt.setValidator(QtGui.QDoubleValidator())
        self.q_max_txt.setValidator(QtGui.QDoubleValidator())
        self.r_cutoff_txt.setValidator(QtGui.QDoubleValidator())

        self.optimize_btn.setFlat(True)
        self.optimize_r_cutoff_btn.setFlat(True)

    def create_layout(self):
        self.grid_layout = QtGui.QGridLayout()
        self.grid_layout.setSpacing(5)
        self.grid_layout.addWidget(self.q_min_lbl, 0, 0)
        self.grid_layout.addWidget(self.q_min_txt, 0, 1)
        self.grid_layout.addWidget(self.q_max_lbl, 1, 0)
        self.grid_layout.addWidget(self.q_max_txt, 1, 1)
        self.grid_layout.addWidget(self.r_cutoff_lbl, 2, 0)
        self.grid_layout.addWidget(self.r_cutoff_txt, 2, 1)

        self.grid_layout.addWidget(self.optimize_btn, 3, 0, 1, 2)
        self.grid_layout.addWidget(self.optimize_r_cutoff_btn, 4, 0, 1, 2)

        self.setLayout(self.grid_layout)

    def create_signals(self):
        self.q_max_txt.editingFinished.connect(self.emit_calculation_changed_signal)
        self.q_min_txt.editingFinished.connect(self.emit_calculation_changed_signal)
        self.r_cutoff_txt.editingFinished.connect(self.emit_calculation_changed_signal)

    def emit_calculation_changed_signal(self):
        q_min = float(str(self.q_min_txt.text()))
        q_max = float(str(self.q_max_txt.text()))
        r_cutoff = float(str(self.r_cutoff_txt.text()))
        self.calculation_parameters_changed.emit(q_min, q_max, r_cutoff)

<|endoftext|>