<commit_before>import random

### DATA ###

data = {
	"APPLE" : {
			"sentences" :

				[

				{"string": "It is MOD DESC", "type": 0}, {"string": "This light is MOD DESC", "type": 0}, {"string": "Oh, it's MOD DESC", "type": 0}, {"string": "Oh, that's MOD DESC", "type": 0}, {"string": "MOD DESC now", "type": 0}, {"string": "Well, that's MOD DESC", "type": 0}, {"string": "I am MOD DESC", "type": 1}, {"string": "Am I DESC?", "type": 2}, {"string": "Is it DESC?", "type": 3}, {"string": "Well now I'm MOD DESC", "type": 4}, {"string": "My skin is MOD DESC", "type": 4}, {"string": "I feel DESC me", "type": 5}

				]

			,

			"descriptors" :

				[

				["light", 0, lambda m: m > 0.5 and m < 1.0], ["bright", 0, lambda m: m > 0.6 and m < 1.0], ["beaming", 0, lambda m: m > 0.8 and m < 1.0], ["dark", 0, lambda m: m > 0.0 and m < 0.5], ["murky", 0, lambda m: m > 0.0 and m < 0.5], ["shadowy", 0, lambda m: m > 0.0 and m < 0.5], ["basking", 1, lambda m: m > 0.5 and m < 1.0], ["shaded", 1, lambda m: m > 0.0 and m < 0.5], ["ripening", 1, lambda m: m > 0.5 and m < 1.0], ["ripe", 2, lambda m: m > 0.0 and m < 1.0], ["night", 3, lambda m: m > 0.0 and m < 0.5], ["daytime", 3, lambda m: m > 0.5 and m < 1.0], ["bathed in light", 4, lambda m: m > 0.5 and m < 1.0], ["illuminated", 4, lambda m: m > 0.6 and m < 1.0], ["red", 4, lambda m: m > 0.0 and m < 1.0] , ["light shining on", 5, lambda m: m > 0.5 and m < 1.0], ["shadow falling on", 5, lambda m: m > 0.5 and m < 1.0]

				]
			,
			"dreams" :

				[
				"I used to be quite small, but I am larger now", "When I was green, there was much more firmness inside", "The time I fell to the ground, there had been less light that day", "I haven't noticed any bees in a while", "There were big green leaves atop my crown"
				]

	},
	"MUG" : {
			"sentences" :

				[

				{"string": "It is MOD DESC", "type": 0}, {"string": "It is MOD DESC now, would you like some DRINK?", "type": 0}, {"string": "Oh, it's getting MOD DESC. Might be time for DRINK.", "type": 0}, {"string": "Aren't you MOD DESC now? Some DRINK might be a good idea.", "type": 0}, {"string": "Sure is MOD DESC. Why don't you pour youself a cup of DRINK?", "type": 0}

				]

			,

			"descriptors" :

				[

				["warm", 0, lambda m: m > 0.5 and m < 1.0], ["temperate", 0, lambda m: m > 0.4 and m < 0.6], ["cozy", 0, lambda m: m > 0.5 and m < 0.7], ["hot", 0, lambda m: m > 0.7 and m < 1.0], ["sweltering", 0, lambda m: m > 0.8 and m < 1.0], ["cool", 0, lambda m: m > 0.2 and m < 0.5], ["cold", 0, lambda m: m > 0.0 and m < 0.3], ["chilly", 0, lambda m: m > 0.0 and m < 0.3], ["freezing", 0, lambda m: m > 0.0 and m < 0.3], ["brisk", 0, lambda m: m > 0.2 and m < 0.5], ["lukewarm", 0, lambda m: m > 0.4 and m < 0.6]

				]
			,
			"drinks" :

				[

				["tea", lambda m: m > 0.0 and m < 0.5], ["coffee", lambda m: m > 0.0 and m < 0.5], ["cocoa", lambda m: m > 0.0 and m < 0.5], ["hot toddy", lambda m: m > 0.0 and m < 0.5], ["warm milk", lambda m: m > 0.0 and m < 0.5], ["hot cider", lambda m: m > 0.0 and m < 0.5], ["lemonade", lambda m: m > 0.5 and m < 1.0], ["water", lambda m: m > 0.5 and m < 1.0], ["iced tea", lambda m: m > 0.5 and m < 1.0], ["iced coffee", lambda m: m > 0.5 and m < 1.0], ["soda", lambda m: m > 0.5 and m < 1.0], ["seltzer", lambda m: m > 0.5 and m < 1.0], ["cold drink", lambda m: m > 0.5 and m < 1.0], ["hot or cold beverage", lambda m: m > 0.4 and m < 0.6]

				]
			,
			"dreams" :

				[
				"Sometimes I spin around and around and get very hot", "I once sat unused in darkness for a very long time", "There was a day when I was used 15 times. There were many cups of tea.", "When the spoon stirs, it clinks and clanks against my sides", "Ice dropped in makes liquid slosh around me"
				]

	},
	"BLOCKS" : {
			"sentences" :

				[

				{"string": "It is DESC", "type": 0}, {"string": "DESC here", "type": 1}, {"string": "DESC with us now", "type": 1}, {"string": "DESC present", "type": 1}, {"string": "DESC near", "type": 1}, {"string": "We see DESC", "type": 2}, {"string": "We see DESC has arrived", "type": 3}, {"string": "We see DESC is present", "type": 3}, {"string": "We see DESC has come", "type": 3}, {"string": "Oh, DESC now", "type": 4}, {"string": "Hello", "type": 5}, {"string": "Have you been here long?", "type": 5}, {"string": "Are you here to play with us?", "type": 5}, {"string": "Will we be one on top of the other again?", "type": 5}, {"string": "Will we be separated again?", "type": 5}

				]

			,

			"descriptors" :

				[

				["busy", 0, lambda m: m > 0.5 and m < 1.0], ["crowded", 0, lambda m: m > 0.8 and m < 1.0], ["packed", 0, lambda m: m > 0.8 and m < 1.0], ["teeming", 0, lambda m: m > 0.8 and m < 1.0], ["populated", 0, lambda m: m > 0.5 and m < 1.0], ["not empty", 0, lambda m: m > 0.2 and m < 1.0], ["empty", 0, lambda m: m > 0.0 and m < 0.3], ["idle", 0, lambda m: m > 0.0 and m < 0.2], ["clear of people", 0, lambda m: m > 0.0 and m < 0.2], ["devoid of people", 0, lambda m: m > 0.0 and m < 0.2], ["someone is", 1, lambda m: m > 0.3 and m < 1.0], ["people are", 1, lambda m: m > 0.3 and m < 1.0], ["a person is", 1, lambda m: m > 0.3 and m < 1.0], ["nobody is", 1, lambda m: m > 0.0 and m < 0.3], ["some people are", 0, lambda m: m > 0.3 and m < 1.0], ["few people are", 1, lambda m: m > 0.0 and m < 0.3], ["no one", 2, lambda m: m > 0.0 and m < 0.2], ["someone", 2, lambda m: m > 0.2 and m < 1.0], ["people", 2, lambda m: m > 0.2 and m < 1.0], ["you", 2, lambda m: m > 0.2 and m < 1.0], ["a group of people", 2, lambda m: m > 0.4 and m < 1.0], ["an assembly of people", 2, lambda m: m > 0.4 and m < 1.0], ["someone", 3, lambda m: m > 0.3 and m < 1.0], ["no one", 3, lambda m: m > 0.0 and m < 0.3], ["you are here", 4, lambda m: m > 0.3 and m < 1.0], ["we have company", 4, lambda m: m > 0.3 and m < 1.0], ["you have gone", 4, lambda m: m > 0.0 and m < 0.3], ["nobody is here", 4, lambda m: m > 0.0 and m < 0.3], ["", 5, lambda m: m > 0.0 and m < 1.0]

				]
			,
			"dreams" :

				[
				"We have seen many more people today than usual", "A long time ago, we sat next to one another in a container", "62 days ago, we were stacked unusually high", "Once, we moved around for a long time before we rested", "There used to be 26 of us", "Sometimes, nobody comes for a long time"
				]

	},
	"modifiers" :

				[

					["a tiny bit", lambda m: m > 0.0 and m < 0.2], ["a bit", lambda m: m > 0.0 and m < 0.5], ["a little", lambda m: m > 0.0 and m < 0.5], ["quite", lambda m: m > 0.5 and m < 1.0], ["very", lambda m: m > 0.5 and m < 1.0], ["extremely", lambda m: m > 0.8 and m < 1.0], ["fiercely", lambda m: m > 0.8 and m < 1.0], ["somewhat", lambda m: m > 0.3 and m < 0.6], ["fairly", lambda m: m > 0.3 and m < 0.6], ["rather", lambda m: m > 0.5 and m < 0.7]

				]
}

###  FUNCTION ###
def generate(obj, reading, diff, dream=False):
	print 'received %s %s %s %s' % (obj, reading, diff, dream)

	if dream==True:
		output = random.choice(data[obj]["dreams"])
		return output

	else:
		sentence = random.choice(data[obj]["sentences"])
		print sentence
		type = sentence["type"]
		print type
		descriptor = None
		modifier = ""

		while descriptor is None:
			d = random.choice(data[obj]["descriptors"])
			if d[1] == type:
				if d[2](reading):
					descriptor = d[0]

		if obj == "APPLE" or obj == "MUG":
			if random.random() > 0.5:
				while modifier is "":
					m = random.choice(data["modifiers"])
					if m[1](reading):
						modifier = m[0]

		if obj == "MUG":
			drink = None
			while drink is None:
				d = random.choice(data[obj]["drinks"])
				if d[1](reading):
					drink = d[0]
			output = sentence["string"].replace("DESC", descriptor).replace("MOD", modifier).replace("DRINK", drink).lstrip()

		elif obj == "APPLE":
			output = sentence["string"].replace("DESC", descriptor).replace("MOD", modifier).lstrip()

		elif obj == "BLOCKS":
			if "DESC" in sentence["string"]:
				output = sentence["string"].replace("DESC", descriptor).lstrip()
			else:
				output = sentence["string"].lstrip()

		print 'returning %s'%(output[0].upper() + output[1:])
		return output[0].upper() + output[1:]

### EXAMPLE ###
# sentence = generate("BLOCKS", random.random(), random.random())
# sentence = generate("APPLE", random.random(), random.random(), True)
# print sentence

<commit_msg>changed all lambdas to be <= not < and >= not ><commit_after>import random

### DATA ###

data = {
	"APPLE" : {
			"sentences" :

				[

				{"string": "It is MOD DESC", "type": 0}, {"string": "This light is MOD DESC", "type": 0}, {"string": "Oh, it's MOD DESC", "type": 0}, {"string": "Oh, that's MOD DESC", "type": 0}, {"string": "MOD DESC now", "type": 0}, {"string": "Well, that's MOD DESC", "type": 0}, {"string": "I am MOD DESC", "type": 1}, {"string": "Am I DESC?", "type": 2}, {"string": "Is it DESC?", "type": 3}, {"string": "Well now I'm MOD DESC", "type": 4}, {"string": "My skin is MOD DESC", "type": 4}, {"string": "I feel DESC me", "type": 5}

				]

			,

			"descriptors" :

				[

				["light", 0, lambda m: m >= 0.5 and m <= 1.0], ["bright", 0, lambda m: m >= 0.6 and m <= 1.0], ["beaming", 0, lambda m: m >= 0.8 and m <= 1.0], ["dark", 0, lambda m: m >= 0.0 and m <= 0.5], ["murky", 0, lambda m: m >= 0.0 and m <= 0.5], ["shadowy", 0, lambda m: m >= 0.0 and m <= 0.5], ["basking", 1, lambda m: m >= 0.5 and m <= 1.0], ["shaded", 1, lambda m: m >= 0.0 and m <= 0.5], ["ripening", 1, lambda m: m >= 0.5 and m <= 1.0], ["ripe", 2, lambda m: m >= 0.0 and m <= 1.0], ["night", 3, lambda m: m >= 0.0 and m <= 0.5], ["daytime", 3, lambda m: m >= 0.5 and m <= 1.0], ["bathed in light", 4, lambda m: m >= 0.5 and m <= 1.0], ["illuminated", 4, lambda m: m >= 0.6 and m <= 1.0], ["red", 4, lambda m: m >= 0.0 and m <= 1.0] , ["light shining on", 5, lambda m: m >= 0.5 and m <= 1.0], ["shadow falling on", 5, lambda m: m >= 0.5 and m <= 1.0]

				]
			,
			"dreams" :

				[
				"I used to be quite small, but I am larger now", "When I was green, there was much more firmness inside", "The time I fell to the ground, there had been less light that day", "I haven't noticed any bees in a while", "There were big green leaves atop my crown"
				]

	},
	"MUG" : {
			"sentences" :

				[

				{"string": "It is MOD DESC", "type": 0}, {"string": "It is MOD DESC now, would you like some DRINK?", "type": 0}, {"string": "Oh, it's getting MOD DESC. Might be time for DRINK.", "type": 0}, {"string": "Aren't you MOD DESC now? Some DRINK might be a good idea.", "type": 0}, {"string": "Sure is MOD DESC. Why don't you pour youself a cup of DRINK?", "type": 0}

				]

			,

			"descriptors" :

				[

				["warm", 0, lambda m: m >= 0.5 and m <= 1.0], ["temperate", 0, lambda m: m >= 0.4 and m <= 0.6], ["cozy", 0, lambda m: m >= 0.5 and m <= 0.7], ["hot", 0, lambda m: m >= 0.7 and m <= 1.0], ["sweltering", 0, lambda m: m >= 0.8 and m <= 1.0], ["cool", 0, lambda m: m >= 0.2 and m <= 0.5], ["cold", 0, lambda m: m >= 0.0 and m <= 0.3], ["chilly", 0, lambda m: m >= 0.0 and m <= 0.3], ["freezing", 0, lambda m: m >= 0.0 and m <= 0.3], ["brisk", 0, lambda m: m >= 0.2 and m <= 0.5], ["lukewarm", 0, lambda m: m >= 0.4 and m <= 0.6]

				]
			,
			"drinks" :

				[

				["tea", lambda m: m >= 0.0 and m <= 0.5], ["coffee", lambda m: m >= 0.0 and m <= 0.5], ["cocoa", lambda m: m >= 0.0 and m <= 0.5], ["hot toddy", lambda m: m >= 0.0 and m <= 0.5], ["warm milk", lambda m: m >= 0.0 and m <= 0.5], ["hot cider", lambda m: m >= 0.0 and m <= 0.5], ["lemonade", lambda m: m >= 0.5 and m <= 1.0], ["water", lambda m: m >= 0.5 and m <= 1.0], ["iced tea", lambda m: m >= 0.5 and m <= 1.0], ["iced coffee", lambda m: m >= 0.5 and m <= 1.0], ["soda", lambda m: m >= 0.5 and m <= 1.0], ["seltzer", lambda m: m >= 0.5 and m <= 1.0], ["cold drink", lambda m: m >= 0.5 and m <= 1.0], ["hot or cold beverage", lambda m: m >= 0.4 and m <= 0.6]

				]
			,
			"dreams" :

				[
				"Sometimes I spin around and around and get very hot", "I once sat unused in darkness for a very long time", "There was a day when I was used 15 times. There were many cups of tea.", "When the spoon stirs, it clinks and clanks against my sides", "Ice dropped in makes liquid slosh around me"
				]

	},
	"BLOCKS" : {
			"sentences" :

				[

				{"string": "It is DESC", "type": 0}, {"string": "DESC here", "type": 1}, {"string": "DESC with us now", "type": 1}, {"string": "DESC present", "type": 1}, {"string": "DESC near", "type": 1}, {"string": "We see DESC", "type": 2}, {"string": "We see DESC has arrived", "type": 3}, {"string": "We see DESC is present", "type": 3}, {"string": "We see DESC has come", "type": 3}, {"string": "Oh, DESC now", "type": 4}, {"string": "Hello", "type": 5}, {"string": "Have you been here long?", "type": 5}, {"string": "Are you here to play with us?", "type": 5}, {"string": "Will we be one on top of the other again?", "type": 5}, {"string": "Will we be separated again?", "type": 5}

				]

			,

			"descriptors" :

				[

				["busy", 0, lambda m: m >= 0.5 and m <= 1.0], ["crowded", 0, lambda m: m >= 0.8 and m <= 1.0], ["packed", 0, lambda m: m >= 0.8 and m <= 1.0], ["teeming", 0, lambda m: m >= 0.8 and m <= 1.0], ["populated", 0, lambda m: m >= 0.5 and m <= 1.0], ["not empty", 0, lambda m: m >= 0.2 and m <= 1.0], ["empty", 0, lambda m: m >= 0.0 and m <= 0.3], ["idle", 0, lambda m: m >= 0.0 and m <= 0.2], ["clear of people", 0, lambda m: m >= 0.0 and m <= 0.2], ["devoid of people", 0, lambda m: m >= 0.0 and m <= 0.2], ["someone is", 1, lambda m: m >= 0.3 and m <= 1.0], ["people are", 1, lambda m: m >= 0.3 and m <= 1.0], ["a person is", 1, lambda m: m >= 0.3 and m <= 1.0], ["nobody is", 1, lambda m: m >= 0.0 and m <= 0.3], ["some people are", 0, lambda m: m >= 0.3 and m <= 1.0], ["few people are", 1, lambda m: m >= 0.0 and m <= 0.3], ["no one", 2, lambda m: m >= 0.0 and m <= 0.2], ["someone", 2, lambda m: m >= 0.2 and m <= 1.0], ["people", 2, lambda m: m >= 0.2 and m <= 1.0], ["you", 2, lambda m: m >= 0.2 and m <= 1.0], ["a group of people", 2, lambda m: m >= 0.4 and m <= 1.0], ["an assembly of people", 2, lambda m: m >= 0.4 and m <= 1.0], ["someone", 3, lambda m: m >= 0.3 and m <= 1.0], ["no one", 3, lambda m: m >= 0.0 and m <= 0.3], ["you are here", 4, lambda m: m >= 0.3 and m <= 1.0], ["we have company", 4, lambda m: m >= 0.3 and m <= 1.0], ["you have gone", 4, lambda m: m >= 0.0 and m <= 0.3], ["nobody is here", 4, lambda m: m >= 0.0 and m <= 0.3], ["", 5, lambda m: m >= 0.0 and m <= 1.0]

				]
			,
			"dreams" :

				[
				"We have seen many more people today than usual", "A long time ago, we sat next to one another in a container", "62 days ago, we were stacked unusually high", "Once, we moved around for a long time before we rested", "There used to be 26 of us", "Sometimes, nobody comes for a long time"
				]

	},
	"modifiers" :

				[

					["a tiny bit", lambda m: m >= 0.0 and m <= 0.2], ["a bit", lambda m: m >= 0.0 and m <= 0.5], ["a little", lambda m: m >= 0.0 and m <= 0.5], ["quite", lambda m: m >= 0.5 and m <= 1.0], ["very", lambda m: m >= 0.5 and m <= 1.0], ["extremely", lambda m: m >= 0.8 and m <= 1.0], ["fiercely", lambda m: m >= 0.8 and m <= 1.0], ["somewhat", lambda m: m >= 0.3 and m <= 0.6], ["fairly", lambda m: m >= 0.3 and m <= 0.6], ["rather", lambda m: m >= 0.5 and m <= 0.7]

				]
}

###  FUNCTION ###
def generate(obj, reading, diff, dream=False):
	print 'received %s %s %s %s' % (obj, reading, diff, dream)

	if dream==True:
		output = random.choice(data[obj]["dreams"])
		return output

	else:
		sentence = random.choice(data[obj]["sentences"])
		type = sentence["type"]
		descriptor = None
		modifier = ""

		while descriptor is None:
			d = random.choice(data[obj]["descriptors"])
			if d[1] == type: # this is never True; why? Answer, lambdas all had <, needed <= etc
				if d[2](reading):
					descriptor = d[0]

		if obj == "APPLE" or obj == "MUG":
			if random.random() >= 0.5:
				while modifier is "":
					m = random.choice(data["modifiers"])
					if m[1](reading):
						modifier = m[0]

		if obj == "MUG":
			drink = None
			while drink is None:
				d = random.choice(data[obj]["drinks"])
				if d[1](reading):
					drink = d[0]
			output = sentence["string"].replace("DESC", descriptor).replace("MOD", modifier).replace("DRINK", drink).lstrip()

		elif obj == "APPLE":
			output = sentence["string"].replace("DESC", descriptor).replace("MOD", modifier).lstrip()

		elif obj == "BLOCKS":
			if "DESC" in sentence["string"]:
				output = sentence["string"].replace("DESC", descriptor).lstrip()
			else:
				output = sentence["string"].lstrip()

		print 'returning %s'%(output[0].upper() + output[1:])
		return output[0].upper() + output[1:]

### EXAMPLE ###
# sentence = generate("BLOCKS", random.random(), random.random())
# sentence = generate("APPLE", random.random(), random.random(), True)
# print sentence

<|endoftext|>


<commit_before>import operator
import re

import numpy as np
from numpy.random import randn
import pytest

from pandas.core.api import DataFrame
from pandas.core.computation import expressions as expr
import pandas.util.testing as tm

_frame = DataFrame(randn(10000, 4), columns=list("ABCD"), dtype="float64")
_frame2 = DataFrame(randn(100, 4), columns=list("ABCD"), dtype="float64")
_mixed = DataFrame(
    {
        "A": _frame["A"].copy(),
        "B": _frame["B"].astype("float32"),
        "C": _frame["C"].astype("int64"),
        "D": _frame["D"].astype("int32"),
    }
)
_mixed2 = DataFrame(
    {
        "A": _frame2["A"].copy(),
        "B": _frame2["B"].astype("float32"),
        "C": _frame2["C"].astype("int64"),
        "D": _frame2["D"].astype("int32"),
    }
)
_integer = DataFrame(
    np.random.randint(1, 100, size=(10001, 4)), columns=list("ABCD"), dtype="int64"
)
_integer2 = DataFrame(
    np.random.randint(1, 100, size=(101, 4)), columns=list("ABCD"), dtype="int64"
)

@pytest.mark.skipif(not expr._USE_NUMEXPR, reason="not using numexpr")
class TestExpressions:
    def setup_method(self, method):

        self.frame = _frame.copy()
        self.frame2 = _frame2.copy()
        self.mixed = _mixed.copy()
        self.mixed2 = _mixed2.copy()
        self._MIN_ELEMENTS = expr._MIN_ELEMENTS

    def teardown_method(self, method):
        expr._MIN_ELEMENTS = self._MIN_ELEMENTS

    def run_arithmetic(self, df, other):
        expr._MIN_ELEMENTS = 0
        operations = ["add", "sub", "mul", "mod", "truediv", "floordiv"]
        for test_flex in [True, False]:
            for arith in operations:
                # TODO: share with run_binary
                if test_flex:
                    op = lambda x, y: getattr(x, arith)(y)
                    op.__name__ = arith
                else:
                    op = getattr(operator, arith)
                expr.set_use_numexpr(False)
                expected = op(df, other)
                expr.set_use_numexpr(True)

                result = op(df, other)
                if arith == "truediv":
                    if expected.ndim == 1:
                        assert expected.dtype.kind == "f"
                    else:
                        assert all(x.kind == "f" for x in expected.dtypes.values)
                tm.assert_equal(expected, result)

    def run_binary(self, df, other):
        """
        tests solely that the result is the same whether or not numexpr is
        enabled.  Need to test whether the function does the correct thing
        elsewhere.
        """
        expr._MIN_ELEMENTS = 0
        expr.set_test_mode(True)
        operations = ["gt", "lt", "ge", "le", "eq", "ne"]

        for test_flex in [True, False]:
            for arith in operations:
                if test_flex:
                    op = lambda x, y: getattr(x, arith)(y)
                    op.__name__ = arith
                else:
                    op = getattr(operator, arith)
                expr.set_use_numexpr(False)
                expected = op(df, other)
                expr.set_use_numexpr(True)

                expr.get_test_result()
                result = op(df, other)
                used_numexpr = expr.get_test_result()
                assert used_numexpr, "Did not use numexpr as expected."
                tm.assert_equal(expected, result)

    def run_frame(self, df, other, run_binary=True):
        self.run_arithmetic(df, other)
        if run_binary:
            expr.set_use_numexpr(False)
            binary_comp = other + 1
            expr.set_use_numexpr(True)
            self.run_binary(df, binary_comp)

        for i in range(len(df.columns)):
            self.run_arithmetic(df.iloc[:, i], other.iloc[:, i])
            # FIXME: dont leave commented-out
            # series doesn't uses vec_compare instead of numexpr...
            # binary_comp = other.iloc[:, i] + 1
            # self.run_binary(df.iloc[:, i], binary_comp)

    @pytest.mark.parametrize(
        "df",
        [
            _integer,
            _integer2,
            # randint to get a case with zeros
            _integer * np.random.randint(0, 2, size=np.shape(_integer)),
            _frame,
            _frame2,
            _mixed,
            _mixed2,
        ],
    )
    def test_arithmetic(self, df):
        # TODO: FIGURE OUT HOW TO GET RUN_BINARY TO WORK WITH MIXED=...
        # can't do arithmetic because comparison methods try to do *entire*
        # frame instead of by-column
        kinds = {x.kind for x in df.dtypes.values}
        should = len(kinds) == 1

        self.run_frame(df, df, run_binary=should)

    def test_invalid(self):

        # no op
        result = expr._can_use_numexpr(
            operator.add, None, self.frame, self.frame, "evaluate"
        )
        assert not result

        # mixed
        result = expr._can_use_numexpr(
            operator.add, "+", self.mixed, self.frame, "evaluate"
        )
        assert not result

        # min elements
        result = expr._can_use_numexpr(
            operator.add, "+", self.frame2, self.frame2, "evaluate"
        )
        assert not result

        # ok, we only check on first part of expression
        result = expr._can_use_numexpr(
            operator.add, "+", self.frame, self.frame2, "evaluate"
        )
        assert result

    @pytest.mark.parametrize(
        "opname,op_str",
        [("add", "+"), ("sub", "-"), ("mul", "*"), ("truediv", "/"), ("pow", "**")],
    )
    @pytest.mark.parametrize("left,right", [(_frame, _frame2), (_mixed, _mixed2)])
    def test_binary_ops(self, opname, op_str, left, right):
        def testit():

            if opname == "pow":
                # TODO: get this working
                return

            op = getattr(operator, opname)

            result = expr._can_use_numexpr(op, op_str, left, left, "evaluate")
            assert result != left._is_mixed_type

            result = expr.evaluate(op, op_str, left, left, use_numexpr=True)
            expected = expr.evaluate(op, op_str, left, left, use_numexpr=False)

            if isinstance(result, DataFrame):
                tm.assert_frame_equal(result, expected)
            else:
                tm.assert_numpy_array_equal(result, expected.values)

            result = expr._can_use_numexpr(op, op_str, right, right, "evaluate")
            assert not result

        expr.set_use_numexpr(False)
        testit()
        expr.set_use_numexpr(True)
        expr.set_numexpr_threads(1)
        testit()
        expr.set_numexpr_threads()
        testit()

    @pytest.mark.parametrize(
        "opname,op_str",
        [
            ("gt", ">"),
            ("lt", "<"),
            ("ge", ">="),
            ("le", "<="),
            ("eq", "=="),
            ("ne", "!="),
        ],
    )
    @pytest.mark.parametrize("left,right", [(_frame, _frame2), (_mixed, _mixed2)])
    def test_comparison_ops(self, opname, op_str, left, right):
        def testit():
            f12 = left + 1
            f22 = right + 1

            op = getattr(operator, opname)

            result = expr._can_use_numexpr(op, op_str, left, f12, "evaluate")
            assert result != left._is_mixed_type

            result = expr.evaluate(op, op_str, left, f12, use_numexpr=True)
            expected = expr.evaluate(op, op_str, left, f12, use_numexpr=False)
            if isinstance(result, DataFrame):
                tm.assert_frame_equal(result, expected)
            else:
                tm.assert_numpy_array_equal(result, expected.values)

            result = expr._can_use_numexpr(op, op_str, right, f22, "evaluate")
            assert not result

        expr.set_use_numexpr(False)
        testit()
        expr.set_use_numexpr(True)
        expr.set_numexpr_threads(1)
        testit()
        expr.set_numexpr_threads()
        testit()

    @pytest.mark.parametrize("cond", [True, False])
    @pytest.mark.parametrize("df", [_frame, _frame2, _mixed, _mixed2])
    def test_where(self, cond, df):
        def testit():
            c = np.empty(df.shape, dtype=np.bool_)
            c.fill(cond)
            result = expr.where(c, df.values, df.values + 1)
            expected = np.where(c, df.values, df.values + 1)
            tm.assert_numpy_array_equal(result, expected)

        expr.set_use_numexpr(False)
        testit()
        expr.set_use_numexpr(True)
        expr.set_numexpr_threads(1)
        testit()
        expr.set_numexpr_threads()
        testit()

    @pytest.mark.parametrize(
        "op_str,opname", [("/", "truediv"), ("//", "floordiv"), ("**", "pow")]
    )
    def test_bool_ops_raise_on_arithmetic(self, op_str, opname):
        df = DataFrame({"a": np.random.rand(10) > 0.5, "b": np.random.rand(10) > 0.5})

        msg = "operator %r not implemented for bool dtypes"
        f = getattr(operator, opname)
        err_msg = re.escape(msg % op_str)

        with pytest.raises(NotImplementedError, match=err_msg):
            f(df, df)

        with pytest.raises(NotImplementedError, match=err_msg):
            f(df.a, df.b)

        with pytest.raises(NotImplementedError, match=err_msg):
            f(df.a, True)

        with pytest.raises(NotImplementedError, match=err_msg):
            f(False, df.a)

        with pytest.raises(NotImplementedError, match=err_msg):
            f(False, df)

        with pytest.raises(NotImplementedError, match=err_msg):
            f(df, True)

    @pytest.mark.parametrize(
        "op_str,opname", [("+", "add"), ("*", "mul"), ("-", "sub")]
    )
    def test_bool_ops_warn_on_arithmetic(self, op_str, opname):
        n = 10
        df = DataFrame({"a": np.random.rand(n) > 0.5, "b": np.random.rand(n) > 0.5})

        subs = {"+": "|", "*": "&", "-": "^"}
        sub_funcs = {"|": "or_", "&": "and_", "^": "xor"}

        f = getattr(operator, opname)
        fe = getattr(operator, sub_funcs[subs[op_str]])

        if op_str == "-":
            # raises TypeError
            return

        with tm.use_numexpr(True, min_elements=5):
            with tm.assert_produces_warning(check_stacklevel=False):
                r = f(df, df)
                e = fe(df, df)
                tm.assert_frame_equal(r, e)

            with tm.assert_produces_warning(check_stacklevel=False):
                r = f(df.a, df.b)
                e = fe(df.a, df.b)
                tm.assert_series_equal(r, e)

            with tm.assert_produces_warning(check_stacklevel=False):
                r = f(df.a, True)
                e = fe(df.a, True)
                tm.assert_series_equal(r, e)

            with tm.assert_produces_warning(check_stacklevel=False):
                r = f(False, df.a)
                e = fe(False, df.a)
                tm.assert_series_equal(r, e)

            with tm.assert_produces_warning(check_stacklevel=False):
                r = f(False, df)
                e = fe(False, df)
                tm.assert_frame_equal(r, e)

            with tm.assert_produces_warning(check_stacklevel=False):
                r = f(df, True)
                e = fe(df, True)
                tm.assert_frame_equal(r, e)

    @pytest.mark.parametrize(
        "test_input,expected",
        [
            (
                DataFrame(
                    [[0, 1, 2, "aa"], [0, 1, 2, "aa"]], columns=["a", "b", "c", "dtype"]
                ),
                DataFrame([[False, False], [False, False]], columns=["a", "dtype"]),
            ),
            (
                DataFrame(
                    [[0, 3, 2, "aa"], [0, 4, 2, "aa"], [0, 1, 1, "bb"]],
                    columns=["a", "b", "c", "dtype"],
                ),
                DataFrame(
                    [[False, False], [False, False], [False, False]],
                    columns=["a", "dtype"],
                ),
            ),
        ],
    )
    def test_bool_ops_column_name_dtype(self, test_input, expected):
        # GH 22383 - .ne fails if columns containing column name 'dtype'
        result = test_input.loc[:, ["a", "dtype"]].ne(test_input.loc[:, ["a", "dtype"]])
        tm.assert_frame_equal(result, expected)

    @pytest.mark.parametrize(
        "arith", ("add", "sub", "mul", "mod", "truediv", "floordiv")
    )
    @pytest.mark.parametrize("axis", (0, 1))
    def test_frame_series_axis(self, axis, arith):
        # GH#26736 Dataframe.floordiv(Series, axis=1) fails
        if axis == 1 and arith == "floordiv":
            pytest.xfail("'floordiv' does not succeed with axis=1 #27636")

        df = self.frame
        if axis == 1:
            other = self.frame.iloc[0, :]
        else:
            other = self.frame.iloc[:, 0]

        expr._MIN_ELEMENTS = 0

        op_func = getattr(df, arith)

        expr.set_use_numexpr(False)
        expected = op_func(other, axis=axis)
        expr.set_use_numexpr(True)

        result = op_func(other, axis=axis)
        tm.assert_frame_equal(expected, result)
<commit_msg>STYLE: string formatting in test-expressions.py for issue 29886 (#30321)<commit_after>import operator
import re

import numpy as np
from numpy.random import randn
import pytest

from pandas.core.api import DataFrame
from pandas.core.computation import expressions as expr
import pandas.util.testing as tm

_frame = DataFrame(randn(10000, 4), columns=list("ABCD"), dtype="float64")
_frame2 = DataFrame(randn(100, 4), columns=list("ABCD"), dtype="float64")
_mixed = DataFrame(
    {
        "A": _frame["A"].copy(),
        "B": _frame["B"].astype("float32"),
        "C": _frame["C"].astype("int64"),
        "D": _frame["D"].astype("int32"),
    }
)
_mixed2 = DataFrame(
    {
        "A": _frame2["A"].copy(),
        "B": _frame2["B"].astype("float32"),
        "C": _frame2["C"].astype("int64"),
        "D": _frame2["D"].astype("int32"),
    }
)
_integer = DataFrame(
    np.random.randint(1, 100, size=(10001, 4)), columns=list("ABCD"), dtype="int64"
)
_integer2 = DataFrame(
    np.random.randint(1, 100, size=(101, 4)), columns=list("ABCD"), dtype="int64"
)

@pytest.mark.skipif(not expr._USE_NUMEXPR, reason="not using numexpr")
class TestExpressions:
    def setup_method(self, method):

        self.frame = _frame.copy()
        self.frame2 = _frame2.copy()
        self.mixed = _mixed.copy()
        self.mixed2 = _mixed2.copy()
        self._MIN_ELEMENTS = expr._MIN_ELEMENTS

    def teardown_method(self, method):
        expr._MIN_ELEMENTS = self._MIN_ELEMENTS

    def run_arithmetic(self, df, other):
        expr._MIN_ELEMENTS = 0
        operations = ["add", "sub", "mul", "mod", "truediv", "floordiv"]
        for test_flex in [True, False]:
            for arith in operations:
                # TODO: share with run_binary
                if test_flex:
                    op = lambda x, y: getattr(x, arith)(y)
                    op.__name__ = arith
                else:
                    op = getattr(operator, arith)
                expr.set_use_numexpr(False)
                expected = op(df, other)
                expr.set_use_numexpr(True)

                result = op(df, other)
                if arith == "truediv":
                    if expected.ndim == 1:
                        assert expected.dtype.kind == "f"
                    else:
                        assert all(x.kind == "f" for x in expected.dtypes.values)
                tm.assert_equal(expected, result)

    def run_binary(self, df, other):
        """
        tests solely that the result is the same whether or not numexpr is
        enabled.  Need to test whether the function does the correct thing
        elsewhere.
        """
        expr._MIN_ELEMENTS = 0
        expr.set_test_mode(True)
        operations = ["gt", "lt", "ge", "le", "eq", "ne"]

        for test_flex in [True, False]:
            for arith in operations:
                if test_flex:
                    op = lambda x, y: getattr(x, arith)(y)
                    op.__name__ = arith
                else:
                    op = getattr(operator, arith)
                expr.set_use_numexpr(False)
                expected = op(df, other)
                expr.set_use_numexpr(True)

                expr.get_test_result()
                result = op(df, other)
                used_numexpr = expr.get_test_result()
                assert used_numexpr, "Did not use numexpr as expected."
                tm.assert_equal(expected, result)

    def run_frame(self, df, other, run_binary=True):
        self.run_arithmetic(df, other)
        if run_binary:
            expr.set_use_numexpr(False)
            binary_comp = other + 1
            expr.set_use_numexpr(True)
            self.run_binary(df, binary_comp)

        for i in range(len(df.columns)):
            self.run_arithmetic(df.iloc[:, i], other.iloc[:, i])
            # FIXME: dont leave commented-out
            # series doesn't uses vec_compare instead of numexpr...
            # binary_comp = other.iloc[:, i] + 1
            # self.run_binary(df.iloc[:, i], binary_comp)

    @pytest.mark.parametrize(
        "df",
        [
            _integer,
            _integer2,
            # randint to get a case with zeros
            _integer * np.random.randint(0, 2, size=np.shape(_integer)),
            _frame,
            _frame2,
            _mixed,
            _mixed2,
        ],
    )
    def test_arithmetic(self, df):
        # TODO: FIGURE OUT HOW TO GET RUN_BINARY TO WORK WITH MIXED=...
        # can't do arithmetic because comparison methods try to do *entire*
        # frame instead of by-column
        kinds = {x.kind for x in df.dtypes.values}
        should = len(kinds) == 1

        self.run_frame(df, df, run_binary=should)

    def test_invalid(self):

        # no op
        result = expr._can_use_numexpr(
            operator.add, None, self.frame, self.frame, "evaluate"
        )
        assert not result

        # mixed
        result = expr._can_use_numexpr(
            operator.add, "+", self.mixed, self.frame, "evaluate"
        )
        assert not result

        # min elements
        result = expr._can_use_numexpr(
            operator.add, "+", self.frame2, self.frame2, "evaluate"
        )
        assert not result

        # ok, we only check on first part of expression
        result = expr._can_use_numexpr(
            operator.add, "+", self.frame, self.frame2, "evaluate"
        )
        assert result

    @pytest.mark.parametrize(
        "opname,op_str",
        [("add", "+"), ("sub", "-"), ("mul", "*"), ("truediv", "/"), ("pow", "**")],
    )
    @pytest.mark.parametrize("left,right", [(_frame, _frame2), (_mixed, _mixed2)])
    def test_binary_ops(self, opname, op_str, left, right):
        def testit():

            if opname == "pow":
                # TODO: get this working
                return

            op = getattr(operator, opname)

            result = expr._can_use_numexpr(op, op_str, left, left, "evaluate")
            assert result != left._is_mixed_type

            result = expr.evaluate(op, op_str, left, left, use_numexpr=True)
            expected = expr.evaluate(op, op_str, left, left, use_numexpr=False)

            if isinstance(result, DataFrame):
                tm.assert_frame_equal(result, expected)
            else:
                tm.assert_numpy_array_equal(result, expected.values)

            result = expr._can_use_numexpr(op, op_str, right, right, "evaluate")
            assert not result

        expr.set_use_numexpr(False)
        testit()
        expr.set_use_numexpr(True)
        expr.set_numexpr_threads(1)
        testit()
        expr.set_numexpr_threads()
        testit()

    @pytest.mark.parametrize(
        "opname,op_str",
        [
            ("gt", ">"),
            ("lt", "<"),
            ("ge", ">="),
            ("le", "<="),
            ("eq", "=="),
            ("ne", "!="),
        ],
    )
    @pytest.mark.parametrize("left,right", [(_frame, _frame2), (_mixed, _mixed2)])
    def test_comparison_ops(self, opname, op_str, left, right):
        def testit():
            f12 = left + 1
            f22 = right + 1

            op = getattr(operator, opname)

            result = expr._can_use_numexpr(op, op_str, left, f12, "evaluate")
            assert result != left._is_mixed_type

            result = expr.evaluate(op, op_str, left, f12, use_numexpr=True)
            expected = expr.evaluate(op, op_str, left, f12, use_numexpr=False)
            if isinstance(result, DataFrame):
                tm.assert_frame_equal(result, expected)
            else:
                tm.assert_numpy_array_equal(result, expected.values)

            result = expr._can_use_numexpr(op, op_str, right, f22, "evaluate")
            assert not result

        expr.set_use_numexpr(False)
        testit()
        expr.set_use_numexpr(True)
        expr.set_numexpr_threads(1)
        testit()
        expr.set_numexpr_threads()
        testit()

    @pytest.mark.parametrize("cond", [True, False])
    @pytest.mark.parametrize("df", [_frame, _frame2, _mixed, _mixed2])
    def test_where(self, cond, df):
        def testit():
            c = np.empty(df.shape, dtype=np.bool_)
            c.fill(cond)
            result = expr.where(c, df.values, df.values + 1)
            expected = np.where(c, df.values, df.values + 1)
            tm.assert_numpy_array_equal(result, expected)

        expr.set_use_numexpr(False)
        testit()
        expr.set_use_numexpr(True)
        expr.set_numexpr_threads(1)
        testit()
        expr.set_numexpr_threads()
        testit()

    @pytest.mark.parametrize(
        "op_str,opname", [("/", "truediv"), ("//", "floordiv"), ("**", "pow")]
    )
    def test_bool_ops_raise_on_arithmetic(self, op_str, opname):
        df = DataFrame({"a": np.random.rand(10) > 0.5, "b": np.random.rand(10) > 0.5})

        msg = f"operator {repr(op_str)} not implemented for bool dtypes"
        f = getattr(operator, opname)
        err_msg = re.escape(msg)

        with pytest.raises(NotImplementedError, match=err_msg):
            f(df, df)

        with pytest.raises(NotImplementedError, match=err_msg):
            f(df.a, df.b)

        with pytest.raises(NotImplementedError, match=err_msg):
            f(df.a, True)

        with pytest.raises(NotImplementedError, match=err_msg):
            f(False, df.a)

        with pytest.raises(NotImplementedError, match=err_msg):
            f(False, df)

        with pytest.raises(NotImplementedError, match=err_msg):
            f(df, True)

    @pytest.mark.parametrize(
        "op_str,opname", [("+", "add"), ("*", "mul"), ("-", "sub")]
    )
    def test_bool_ops_warn_on_arithmetic(self, op_str, opname):
        n = 10
        df = DataFrame({"a": np.random.rand(n) > 0.5, "b": np.random.rand(n) > 0.5})

        subs = {"+": "|", "*": "&", "-": "^"}
        sub_funcs = {"|": "or_", "&": "and_", "^": "xor"}

        f = getattr(operator, opname)
        fe = getattr(operator, sub_funcs[subs[op_str]])

        if op_str == "-":
            # raises TypeError
            return

        with tm.use_numexpr(True, min_elements=5):
            with tm.assert_produces_warning(check_stacklevel=False):
                r = f(df, df)
                e = fe(df, df)
                tm.assert_frame_equal(r, e)

            with tm.assert_produces_warning(check_stacklevel=False):
                r = f(df.a, df.b)
                e = fe(df.a, df.b)
                tm.assert_series_equal(r, e)

            with tm.assert_produces_warning(check_stacklevel=False):
                r = f(df.a, True)
                e = fe(df.a, True)
                tm.assert_series_equal(r, e)

            with tm.assert_produces_warning(check_stacklevel=False):
                r = f(False, df.a)
                e = fe(False, df.a)
                tm.assert_series_equal(r, e)

            with tm.assert_produces_warning(check_stacklevel=False):
                r = f(False, df)
                e = fe(False, df)
                tm.assert_frame_equal(r, e)

            with tm.assert_produces_warning(check_stacklevel=False):
                r = f(df, True)
                e = fe(df, True)
                tm.assert_frame_equal(r, e)

    @pytest.mark.parametrize(
        "test_input,expected",
        [
            (
                DataFrame(
                    [[0, 1, 2, "aa"], [0, 1, 2, "aa"]], columns=["a", "b", "c", "dtype"]
                ),
                DataFrame([[False, False], [False, False]], columns=["a", "dtype"]),
            ),
            (
                DataFrame(
                    [[0, 3, 2, "aa"], [0, 4, 2, "aa"], [0, 1, 1, "bb"]],
                    columns=["a", "b", "c", "dtype"],
                ),
                DataFrame(
                    [[False, False], [False, False], [False, False]],
                    columns=["a", "dtype"],
                ),
            ),
        ],
    )
    def test_bool_ops_column_name_dtype(self, test_input, expected):
        # GH 22383 - .ne fails if columns containing column name 'dtype'
        result = test_input.loc[:, ["a", "dtype"]].ne(test_input.loc[:, ["a", "dtype"]])
        tm.assert_frame_equal(result, expected)

    @pytest.mark.parametrize(
        "arith", ("add", "sub", "mul", "mod", "truediv", "floordiv")
    )
    @pytest.mark.parametrize("axis", (0, 1))
    def test_frame_series_axis(self, axis, arith):
        # GH#26736 Dataframe.floordiv(Series, axis=1) fails
        if axis == 1 and arith == "floordiv":
            pytest.xfail("'floordiv' does not succeed with axis=1 #27636")

        df = self.frame
        if axis == 1:
            other = self.frame.iloc[0, :]
        else:
            other = self.frame.iloc[:, 0]

        expr._MIN_ELEMENTS = 0

        op_func = getattr(df, arith)

        expr.set_use_numexpr(False)
        expected = op_func(other, axis=axis)
        expr.set_use_numexpr(True)

        result = op_func(other, axis=axis)
        tm.assert_frame_equal(expected, result)
<|endoftext|>


<commit_before>from parcels import FieldSet, ParticleSet, ScipyParticle, JITParticle, AdvectionRK4
from datetime import timedelta as delta
import pytest
from os import path

ptype = {'scipy': ScipyParticle, 'jit': JITParticle}

def set_ofam_fieldset():
    filenames = {'U': path.join(path.dirname(__file__), 'OFAM_example_data', 'OFAM_simple_U.nc'),
                 'V': path.join(path.dirname(__file__), 'OFAM_example_data', 'OFAM_simple_V.nc')}
    variables = {'U': 'u', 'V': 'v'}
    dimensions = {'lat': 'yu_ocean', 'lon': 'xu_ocean', 'depth': 'st_ocean',
                  'time': 'Time'}
    return FieldSet.from_netcdf(filenames, variables, dimensions, allow_time_extrapolation=True)

def test_ofam_fieldset():
    fieldset = set_ofam_fieldset()
    assert(fieldset.U.lon.size == 2001)
    assert(fieldset.U.lat.size == 601)
    assert(fieldset.U.data.shape == (4, 601, 2001))
    assert(fieldset.V.lon.size == 2001)
    assert(fieldset.V.lat.size == 601)
    assert(fieldset.V.data.shape == (4, 601, 2001))

def test_ofam_fieldset_fillvalues():
    fieldset = set_ofam_fieldset()
    # V.data[0, 0, 150] is a landpoint, that makes NetCDF4 generate a masked array, instead of an ndarray
    assert(fieldset.V.data[0, 0, 150] == 0)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_ofam_particles(mode):
    fieldset = set_ofam_fieldset()

    lonstart = [180]
    latstart = [10]
    depstart = [2.5]  # the depth of the first layer in OFAM

    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=lonstart, lat=latstart, depth=depstart)

    pset.execute(AdvectionRK4, runtime=delta(days=10), dt=delta(minutes=5))

    assert(abs(pset[0].lon - 173) < 1)
    assert(abs(pset[0].lat - 11) < 1)
<commit_msg>remove unnecessary py test<commit_after>from parcels import FieldSet, ParticleSet, ScipyParticle, JITParticle, AdvectionRK4
from datetime import timedelta as delta
import pytest
from os import path

ptype = {'scipy': ScipyParticle, 'jit': JITParticle}

def set_ofam_fieldset():
    filenames = {'U': path.join(path.dirname(__file__), 'OFAM_example_data', 'OFAM_simple_U.nc'),
                 'V': path.join(path.dirname(__file__), 'OFAM_example_data', 'OFAM_simple_V.nc')}
    variables = {'U': 'u', 'V': 'v'}
    dimensions = {'lat': 'yu_ocean', 'lon': 'xu_ocean', 'depth': 'st_ocean',
                  'time': 'Time'}
    return FieldSet.from_netcdf(filenames, variables, dimensions, allow_time_extrapolation=True)

def test_ofam_fieldset_fillvalues():
    fieldset = set_ofam_fieldset()
    # V.data[0, 0, 150] is a landpoint, that makes NetCDF4 generate a masked array, instead of an ndarray
    assert(fieldset.V.data[0, 0, 150] == 0)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_ofam_particles(mode):
    fieldset = set_ofam_fieldset()

    lonstart = [180]
    latstart = [10]
    depstart = [2.5]  # the depth of the first layer in OFAM

    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=lonstart, lat=latstart, depth=depstart)

    pset.execute(AdvectionRK4, runtime=delta(days=10), dt=delta(minutes=5))

    assert(abs(pset[0].lon - 173) < 1)
    assert(abs(pset[0].lat - 11) < 1)
<|endoftext|>


<commit_before><commit_msg>More efficient way of dealing with partial/compelete transcripts<commit_after><|endoftext|>


<commit_before><commit_msg>Change invoice events messages<commit_after><|endoftext|>


<commit_before>from datetime import datetime
import pytest
from pytest import approx

import msise00

def test_past():
    t = datetime(2013, 3, 31, 12)
    altkm = 150.0
    glat = 65.0
    glon = -148.0

    try:
        atmos = msise00.run(t, altkm, glat, glon)
    except ConnectionError:
        pytest.skip("unable to download RecentIndices.txt")

    assert atmos["He"].ndim == 4
    assert atmos["He"].size == 1
    dims = list(atmos.dims)
    assert ["alt_km", "lat", "lon", "time"] == dims

    try:  # daily resolution
        assert atmos["He"].item() == approx(4365665440000.0)
        assert atmos["O"].item() == approx(9612878760000000.0)
        assert atmos["N2"].item() == approx(3.15052301e16)
        assert atmos["O2"].item() == approx(2312921490000000.0)
        assert atmos["Ar"].item() == approx(71974164400000.0)
        assert atmos["Total"].item() == approx(1.84757176e-09)
        assert atmos["N"].item() == approx(9424203680000.0)
        assert atmos["AnomalousO"].item() == approx(1.17317104e-14)

        assert atmos["Tn"].item() == approx(682.538, abs=0.01)
        assert atmos["Texo"].item() == approx(948.350, abs=0.01)
    except AssertionError:  # monthly resolution
        assert atmos["He"].item() == approx(4200180480000.0)
        assert atmos["O"].item() == approx(9338048100000000.0)
        assert atmos["N2"].item() == approx(3.23984781e16)
        assert atmos["O2"].item() == approx(2413811350000000.0)
        assert atmos["Ar"].item() == approx(81071685200000.0)
        assert atmos["Total"].item() == approx(1.88774951e-09)
        assert atmos["N"].item() == approx(9310465690000.0)
        assert atmos["AnomalousO"].item() == approx(5.3806201e-15)

        assert atmos["Tn"].item() == approx(699.021, abs=0.01)
        assert atmos["Texo"].item() == approx(1000.513, abs=0.01)

    assert atmos.species == ["He", "O", "N2", "O2", "Ar", "Total", "H", "N", "AnomalousO"]

def test_forecast():
    t = datetime(2023, 3, 31, 12)
    altkm = 150.0
    glat = 65.0
    glon = -148.0

    atmos = msise00.run(t, altkm, glat, glon)

    assert atmos["He"].ndim == 4
    assert atmos["He"].size == 1
    dims = list(atmos.dims)
    assert ["alt_km", "lat", "lon", "time"] == dims

    assert atmos["He"].item() == approx(1240641440000.0)
    assert atmos["O"].item() == approx(3047968100000000.0)
    assert atmos["N2"].item() == approx(3.597249e16)
    assert atmos["O2"].item() == approx(7343665540000000.0)
    assert atmos["Ar"].item() == approx(334703983000000.0)
    assert atmos["Total"].item() == approx(2.16534546e-09)
    assert atmos["N"].item() == approx(2576280450000.0)
    assert atmos["AnomalousO"].item() == approx(8.75042368e-16)

    assert atmos.species == ["He", "O", "N2", "O2", "Ar", "Total", "H", "N", "AnomalousO"]

    assert atmos["Tn"].item() == approx(671.513, abs=0.01)
    assert atmos["Texo"].item() == approx(883.342, abs=0.01)

if __name__ == "__main__":
    pytest.main([__file__])
<commit_msg>remove unused code<commit_after>from datetime import datetime
import pytest
from pytest import approx

import msise00

def test_past():
    t = datetime(2013, 3, 31, 12)
    altkm = 150.0
    glat = 65.0
    glon = -148.0

    try:
        atmos = msise00.run(t, altkm, glat, glon)
    except ConnectionError:
        pytest.skip("unable to download RecentIndices.txt")

    assert atmos["He"].ndim == 4
    assert atmos["He"].size == 1
    dims = list(atmos.dims)
    assert ["alt_km", "lat", "lon", "time"] == dims

    try:  # daily resolution
        assert atmos["He"].item() == approx(4365665440000.0)
        assert atmos["O"].item() == approx(9612878760000000.0)
        assert atmos["N2"].item() == approx(3.15052301e16)
        assert atmos["O2"].item() == approx(2312921490000000.0)
        assert atmos["Ar"].item() == approx(71974164400000.0)
        assert atmos["Total"].item() == approx(1.84757176e-09)
        assert atmos["N"].item() == approx(9424203680000.0)
        assert atmos["AnomalousO"].item() == approx(1.17317104e-14)

        assert atmos["Tn"].item() == approx(682.538, abs=0.01)
        assert atmos["Texo"].item() == approx(948.350, abs=0.01)
    except AssertionError:  # monthly resolution
        assert atmos["He"].item() == approx(4200180480000.0)
        assert atmos["O"].item() == approx(9338048100000000.0)
        assert atmos["N2"].item() == approx(3.23984781e16)
        assert atmos["O2"].item() == approx(2413811350000000.0)
        assert atmos["Ar"].item() == approx(81071685200000.0)
        assert atmos["Total"].item() == approx(1.88774951e-09)
        assert atmos["N"].item() == approx(9310465690000.0)
        assert atmos["AnomalousO"].item() == approx(5.3806201e-15)

        assert atmos["Tn"].item() == approx(699.021, abs=0.01)
        assert atmos["Texo"].item() == approx(1000.513, abs=0.01)

    assert atmos.species == ["He", "O", "N2", "O2", "Ar", "Total", "H", "N", "AnomalousO"]

def test_forecast():
    t = datetime(2023, 3, 31, 12)
    altkm = 150.0
    glat = 65.0
    glon = -148.0

    atmos = msise00.run(t, altkm, glat, glon)

    assert atmos["He"].ndim == 4
    assert atmos["He"].size == 1
    dims = list(atmos.dims)
    assert ["alt_km", "lat", "lon", "time"] == dims

    assert atmos["He"].item() == approx(1240641440000.0)
    assert atmos["O"].item() == approx(3047968100000000.0)
    assert atmos["N2"].item() == approx(3.597249e16)
    assert atmos["O2"].item() == approx(7343665540000000.0)
    assert atmos["Ar"].item() == approx(334703983000000.0)
    assert atmos["Total"].item() == approx(2.16534546e-09)
    assert atmos["N"].item() == approx(2576280450000.0)
    assert atmos["AnomalousO"].item() == approx(8.75042368e-16)

    assert atmos.species == ["He", "O", "N2", "O2", "Ar", "Total", "H", "N", "AnomalousO"]

    assert atmos["Tn"].item() == approx(671.513, abs=0.01)
    assert atmos["Texo"].item() == approx(883.342, abs=0.01)
<|endoftext|>


<commit_before>
from twisted.application import app
from twisted.python import usage
from twisted.python.log import ILogObserver, startLoggingWithObserver
from twisted.python.util import uidFromString, gidFromString
#from twisted.python.runtime import platformType

#from twisted.internet import reactor

import sys, os

from twisted.scripts._twistd_unix import \
            UnixApplicationRunner as _SomeApplicationRunner, _umask, UnixAppLogger

from nodeset.core.config import Configurator
from nodeset.common import log
from nodeset.core import copyright

import logging

def _logLevel(val):
    if val.upper() not in ['INFO', 'DEBUG', 'WARN', 'CRIT', 'NOTICE']:
        raise ValueError("invalid logLevel %s" % val.upper())

    return val.upper()

class NodeSetAppOptions(usage.Options, app.ReactorSelectionMixin):
    """
    Copy of U{ServerOptions<http://twistedmatrix.com/documents/9.0.0/api/twisted.application.app.ServerOptions.html>},
    but without all twistd options.
    """

    optFlags = [['nodaemon','n',  "don't daemonize, don't use default umask of 0077"],
                ['savestats', None,
                 "save the Stats object rather than the text output of "
                 "the profiler."],
                ['quiet', 'q', "No-op for backwards compatibility."],
                ['originalname', None, "Don't try to change the process name"],
                ['syslog', None,   "Log to syslog, not to file"],
                ['euid', '',
                 "Set only effective user-id rather than real user-id. "
                 "(This option has no effect unless the server is running as "
                 "root, in which case it means not to shed all privileges "
                 "after binding ports, retaining the option to regain "
                 "privileges in cases such as spawning processes. "
                 "Use with caution.)"],
               ]

    optParameters = [
                     ['prefix', None,'twisted',
                      "use the given prefix when syslogging"],
                     ['pidfile','','twistd.pid',
                      "Name of the pidfile"],
                     ['chroot', None, None,
                      'Chroot to a supplied directory before running'],
                     ['uid', 'u', None, "The uid to run as.", uidFromString],
                     ['gid', 'g', None, "The gid to run as.", gidFromString],
                     ['umask', None, None,
                      "The (octal) file creation mask to apply.", _umask],
                     ['logfile','l', None,
                      "log to a specified file, - for stdout"],
                     ['loglevel', None, 'info', 'logLevel, one of (info,warn,debug,crit etc.)', _logLevel],
                     ['rundir','d','.',
                      'Change to a supplied directory before running'],
                       ['profile', 'p', None,
                      "Run in profile mode, dumping results to specified file"],
                     ['profiler', None, "hotshot",
                      "Name of the profiler to use (%s)." %
                      ", ".join(app.AppProfiler.profilers)],
                     ['dispatcher-url', None, 'pbu://localhost:5333/dispatcher', "Dispatcher's URL"],
                     ['listen', None, 'localhost:5444',  "Node's listen address (i.e. host:port)"]
                     ]

    def __init__(self, *a, **kw):
        self['debug'] = False
        usage.Options.__init__(self, *a, **kw)

        # set some twisted option to default values
        self['no_save'] = True

    def opt_version(self):
        print "nodeset.core version: %s" % copyright.version
        super(NodeSetAppOptions, self).opt_version()

class NodeSetAppLogger(UnixAppLogger):

    def _getLogObserver(self):
        if self._nodaemon:
            stream = sys.stdout
        else:
            if not self._logfilename:
                self._logfilename = 'nodeset-core.log'

            stream = log.NodeSetLog(os.path.basename(self._logfilename), os.path.dirname(self._logfilename) or '.')

        lvl = getattr(logging, Configurator['loglevel'])

        return log.NodeSetLogObserver(stream, lvl).emit

class NodeSetApplicationRunner(_SomeApplicationRunner):
    """
    Adoption of U{UnixApplicationRunner<http://twistedmatrix.com/documents/9.0.0/api/twisted.scripts._twistd_unix.UnixApplicationRunner.html>}
    for NodeSet, createOrGetApplication redefined only
    """

    loggerFactory = NodeSetAppLogger

    def createOrGetApplication(self):
        """
        Modified to return application instance, early defined
        """
        return self.application

def runApp(config, application):
    """
    twistd runApp modified, added second argument application
    @param config: run option (--option i.e.)
    @type config: NodeSetAppOptions
    @param applicatin: application instance
    @type application: Application
    """
    runner = NodeSetApplicationRunner(config)
    runner.application = application

    #level = getattr(logging, config['loglevel'])

    #if config['logfile']:
    #    import os
    #    logfile = log.NodeSetLog(os.path.basename(config['logfile']), os.path.dirname(config['logfile']) or '.')
    #    application.setComponent(ILogObserver, log.NodeSetLogObserver(logfile, level).emit)

    Configurator._config = config

    runner.run()

def _run(runApp, application, ServerOptions=NodeSetAppOptions):
    """
    instead of run, to handle additional option
    @param runApp: runApp func
    @param ServerOptions: class for options parsing
    @type ServerOptions: NodeSetAppOptions
    """
    config = ServerOptions()
    #config['no_save'] = False
    try:
        config.parseOptions()
    except usage.error, ue:
        print config
        print "%s: %s" % (sys.argv[0], ue)
    else:
        application.config = config
        runApp(config, application)

def run(application, options=NodeSetAppOptions):
    """
    wrapper for _run, application instance added
    @param application: Application instance
    @type application: Application
    """
    _run(runApp, application, options)

__all__ = ['run', 'runApp']

<commit_msg>fixed default logLevel setting<commit_after>
from twisted.application import app
from twisted.python import usage
from twisted.python.log import ILogObserver, startLoggingWithObserver
from twisted.python.util import uidFromString, gidFromString
#from twisted.python.runtime import platformType

#from twisted.internet import reactor

import sys, os

from twisted.scripts._twistd_unix import \
            UnixApplicationRunner as _SomeApplicationRunner, _umask, UnixAppLogger

from nodeset.core.config import Configurator
from nodeset.common import log
from nodeset.core import copyright

import logging

def _logLevel(val):
    if val.upper() not in ['INFO', 'DEBUG', 'WARN', 'CRIT', 'NOTICE']:
        raise ValueError("invalid logLevel %s" % val.upper())

    return val.upper()

class NodeSetAppOptions(usage.Options, app.ReactorSelectionMixin):
    """
    Copy of U{ServerOptions<http://twistedmatrix.com/documents/9.0.0/api/twisted.application.app.ServerOptions.html>},
    but without all twistd options.
    """

    optFlags = [['nodaemon','n',  "don't daemonize, don't use default umask of 0077"],
                ['savestats', None,
                 "save the Stats object rather than the text output of "
                 "the profiler."],
                ['quiet', 'q', "No-op for backwards compatibility."],
                ['originalname', None, "Don't try to change the process name"],
                ['syslog', None,   "Log to syslog, not to file"],
                ['euid', '',
                 "Set only effective user-id rather than real user-id. "
                 "(This option has no effect unless the server is running as "
                 "root, in which case it means not to shed all privileges "
                 "after binding ports, retaining the option to regain "
                 "privileges in cases such as spawning processes. "
                 "Use with caution.)"],
               ]

    optParameters = [
                     ['prefix', None,'twisted',
                      "use the given prefix when syslogging"],
                     ['pidfile','','twistd.pid',
                      "Name of the pidfile"],
                     ['chroot', None, None,
                      'Chroot to a supplied directory before running'],
                     ['uid', 'u', None, "The uid to run as.", uidFromString],
                     ['gid', 'g', None, "The gid to run as.", gidFromString],
                     ['umask', None, None,
                      "The (octal) file creation mask to apply.", _umask],
                     ['logfile','l', None,
                      "log to a specified file, - for stdout"],
                     ['loglevel', None, 'info', 'logLevel, one of (info,warn,debug,crit etc.)', _logLevel],
                     ['rundir','d','.',
                      'Change to a supplied directory before running'],
                       ['profile', 'p', None,
                      "Run in profile mode, dumping results to specified file"],
                     ['profiler', None, "hotshot",
                      "Name of the profiler to use (%s)." %
                      ", ".join(app.AppProfiler.profilers)],
                     ['dispatcher-url', None, 'pbu://localhost:5333/dispatcher', "Dispatcher's URL"],
                     ['listen', None, 'localhost:5444',  "Node's listen address (i.e. host:port)"]
                     ]

    def __init__(self, *a, **kw):
        self['debug'] = False
        usage.Options.__init__(self, *a, **kw)

        # set some twisted option to default values
        self['no_save'] = True

    def opt_version(self):
        print "nodeset.core version: %s" % copyright.version
        super(NodeSetAppOptions, self).opt_version()

class NodeSetAppLogger(UnixAppLogger):

    def _getLogObserver(self):
        if self._nodaemon:
            stream = sys.stdout
        else:
            if not self._logfilename:
                self._logfilename = 'nodeset-core.log'

            stream = log.NodeSetLog(os.path.basename(self._logfilename), os.path.dirname(self._logfilename) or '.')

        lvl = getattr(logging, Configurator['loglevel'].upper())

        return log.NodeSetLogObserver(stream, lvl).emit

class NodeSetApplicationRunner(_SomeApplicationRunner):
    """
    Adoption of U{UnixApplicationRunner<http://twistedmatrix.com/documents/9.0.0/api/twisted.scripts._twistd_unix.UnixApplicationRunner.html>}
    for NodeSet, createOrGetApplication redefined only
    """

    loggerFactory = NodeSetAppLogger

    def createOrGetApplication(self):
        """
        Modified to return application instance, early defined
        """
        return self.application

def runApp(config, application):
    """
    twistd runApp modified, added second argument application
    @param config: run option (--option i.e.)
    @type config: NodeSetAppOptions
    @param applicatin: application instance
    @type application: Application
    """
    runner = NodeSetApplicationRunner(config)
    runner.application = application

    #level = getattr(logging, config['loglevel'])

    #if config['logfile']:
    #    import os
    #    logfile = log.NodeSetLog(os.path.basename(config['logfile']), os.path.dirname(config['logfile']) or '.')
    #    application.setComponent(ILogObserver, log.NodeSetLogObserver(logfile, level).emit)

    Configurator._config = config

    runner.run()

def _run(runApp, application, ServerOptions=NodeSetAppOptions):
    """
    instead of run, to handle additional option
    @param runApp: runApp func
    @param ServerOptions: class for options parsing
    @type ServerOptions: NodeSetAppOptions
    """
    config = ServerOptions()
    #config['no_save'] = False
    try:
        config.parseOptions()
    except usage.error, ue:
        print config
        print "%s: %s" % (sys.argv[0], ue)
    else:
        application.config = config
        runApp(config, application)

def run(application, options=NodeSetAppOptions):
    """
    wrapper for _run, application instance added
    @param application: Application instance
    @type application: Application
    """
    _run(runApp, application, options)

__all__ = ['run', 'runApp']

<|endoftext|>


<commit_before>"""Visualize individual feature channels and their combinations to explore the space of patterns learned by the neural network
Based on http://nbviewer.jupyter.org/github/tensorflow/tensorflow/blob/master/tensorflow/examples/tutorials/deepdream/deepdream.ipynb
"""
# MIT License
#
# Copyright (c) 2016 David Sandberg
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import numpy as np
import sys
import argparse
import tensorflow as tf
import matplotlib.pyplot as plt
import importlib
from scipy import misc

def main(args):

    network = importlib.import_module(args.model_def, 'inference')

    # Start with a gray image with a little noise
    np.random.seed(seed=args.seed)
    img_noise = np.random.uniform(size=(args.image_size,args.image_size,3)) + 100.0

    sess = tf.Session()

    t_input = tf.placeholder(np.float32, shape=(args.image_size,args.image_size,3), name='input') # define the input tensor
    image_mean = 117.0
    t_preprocessed = tf.expand_dims(t_input-image_mean, 0)

    # Build the inference graph
    network.inference(t_preprocessed, 1.0,
            phase_train=True, weight_decay=0.0)

    # Create a saver for restoring variables
    saver = tf.train.Saver(tf.global_variables())

    # Restore the parameters
    saver.restore(sess, args.model_file)

    layers = [op.name for op in tf.get_default_graph().get_operations() if op.type=='Conv2D']
    feature_nums = {layer: int(T(layer).get_shape()[-1]) for layer in layers}

    print('Number of layers: %d' % len(layers))

    for layer in sorted(feature_nums.keys()):
        print('%s%d' % ((layer+': ').ljust(40), feature_nums[layer]))

    # Picking some internal layer. Note that we use outputs before applying the ReLU nonlinearity
    # to have non-zero gradients for features with negative initial activations.
    layer = 'InceptionResnetV1/Repeat_2/block8_3/Conv2d_1x1/Conv2D'
    #layer = 'incept4b/in4_conv1x1_31/Conv2D'
    result_dir = '../data/'
    print('Number of features in layer "%s": %d' % (layer, feature_nums[layer]))
    channels = range(feature_nums[layer])
    np.random.shuffle(channels)
    for i in range(32):
        print('Rendering feature %d' % channels[i])
        channel = channels[i]
        img = render_naive(sess, t_input, T(layer)[:,:,:,channel], img_noise)
        filename = '%s_%03d.png' % (layer.replace('/', '_'), channel)
        misc.imsave(os.path.join(result_dir, filename), img)

def T(layer):
    '''Helper for getting layer output tensor'''
    return tf.get_default_graph().get_tensor_by_name('%s:0' % layer)

def showarray(a):
    a = np.uint8(np.clip(a, 0, 1)*255)
    plt.imshow(a)
    plt.show()

def visstd(a, s=0.1):
    '''Normalize the image range for visualization'''
    return (a-a.mean())/max(a.std(), 1e-4)*s + 0.5

def render_naive(sess, t_input, t_obj, img0, iter_n=20, step=1.0):
    t_score = tf.reduce_mean(t_obj) # defining the optimization objective
    t_grad = tf.gradients(t_score, t_input)[0] # behold the power of automatic differentiation!

    img = img0.copy()
    for _ in range(iter_n):
        g, _ = sess.run([t_grad, t_score], {t_input:img})
        # normalizing the gradient, so the same step size should work
        g /= g.std()+1e-8         # for different layers and networks
        img += g*step
    return visstd(img)

def parse_arguments(argv):
    parser = argparse.ArgumentParser()

    parser.add_argument('model_file', type=str,
        help='Directory containing the graph definition and checkpoint files.')
    parser.add_argument('--model_def', type=str,
        help='Model definition. Points to a module containing the definition of the inference graph.',
        default='models.nn4')
    parser.add_argument('--image_size', type=int,
        help='Image size (height, width) in pixels.', default=96)
    parser.add_argument('--seed', type=int,
        help='Random seed.', default=666)
    return parser.parse_args(argv)

if __name__ == '__main__':
    main(parse_arguments(sys.argv[1:]))
<commit_msg>Remove matplotlib dependency<commit_after>"""Visualize individual feature channels and their combinations to explore the space of patterns learned by the neural network
Based on http://nbviewer.jupyter.org/github/tensorflow/tensorflow/blob/master/tensorflow/examples/tutorials/deepdream/deepdream.ipynb
"""
# MIT License
#
# Copyright (c) 2016 David Sandberg
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import numpy as np
import sys
import argparse
import tensorflow as tf
import importlib
from scipy import misc

def main(args):

    network = importlib.import_module(args.model_def, 'inference')

    # Start with a gray image with a little noise
    np.random.seed(seed=args.seed)
    img_noise = np.random.uniform(size=(args.image_size,args.image_size,3)) + 100.0

    sess = tf.Session()

    t_input = tf.placeholder(np.float32, shape=(args.image_size,args.image_size,3), name='input') # define the input tensor
    image_mean = 117.0
    t_preprocessed = tf.expand_dims(t_input-image_mean, 0)

    # Build the inference graph
    network.inference(t_preprocessed, 1.0,
            phase_train=True, weight_decay=0.0)

    # Create a saver for restoring variables
    saver = tf.train.Saver(tf.global_variables())

    # Restore the parameters
    saver.restore(sess, args.model_file)

    layers = [op.name for op in tf.get_default_graph().get_operations() if op.type=='Conv2D']
    feature_nums = {layer: int(T(layer).get_shape()[-1]) for layer in layers}

    print('Number of layers: %d' % len(layers))

    for layer in sorted(feature_nums.keys()):
        print('%s%d' % ((layer+': ').ljust(40), feature_nums[layer]))

    # Picking some internal layer. Note that we use outputs before applying the ReLU nonlinearity
    # to have non-zero gradients for features with negative initial activations.
    layer = 'InceptionResnetV1/Repeat_2/block8_3/Conv2d_1x1/Conv2D'
    #layer = 'incept4b/in4_conv1x1_31/Conv2D'
    result_dir = '../data/'
    print('Number of features in layer "%s": %d' % (layer, feature_nums[layer]))
    channels = range(feature_nums[layer])
    np.random.shuffle(channels)
    for i in range(32):
        print('Rendering feature %d' % channels[i])
        channel = channels[i]
        img = render_naive(sess, t_input, T(layer)[:,:,:,channel], img_noise)
        filename = '%s_%03d.png' % (layer.replace('/', '_'), channel)
        misc.imsave(os.path.join(result_dir, filename), img)

def T(layer):
    '''Helper for getting layer output tensor'''
    return tf.get_default_graph().get_tensor_by_name('%s:0' % layer)

def visstd(a, s=0.1):
    '''Normalize the image range for visualization'''
    return (a-a.mean())/max(a.std(), 1e-4)*s + 0.5

def render_naive(sess, t_input, t_obj, img0, iter_n=20, step=1.0):
    t_score = tf.reduce_mean(t_obj) # defining the optimization objective
    t_grad = tf.gradients(t_score, t_input)[0] # behold the power of automatic differentiation!

    img = img0.copy()
    for _ in range(iter_n):
        g, _ = sess.run([t_grad, t_score], {t_input:img})
        # normalizing the gradient, so the same step size should work
        g /= g.std()+1e-8         # for different layers and networks
        img += g*step
    return visstd(img)

def parse_arguments(argv):
    parser = argparse.ArgumentParser()

    parser.add_argument('model_file', type=str,
        help='Directory containing the graph definition and checkpoint files.')
    parser.add_argument('--model_def', type=str,
        help='Model definition. Points to a module containing the definition of the inference graph.',
        default='models.nn4')
    parser.add_argument('--image_size', type=int,
        help='Image size (height, width) in pixels.', default=96)
    parser.add_argument('--seed', type=int,
        help='Random seed.', default=666)
    return parser.parse_args(argv)

if __name__ == '__main__':
    main(parse_arguments(sys.argv[1:]))
<|endoftext|>


<commit_before># Orca
#
# Copyright 2013 Igalia, S.L.
#
# Author: Joanmarie Diggs <jdiggs@igalia.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., Franklin Street, Fifth Floor,
# Boston MA  02110-1301 USA.

"""Displays a GUI to present Orca commands."""

__id__        = "$Id$"
__version__   = "$Revision$"
__date__      = "$Date$"
__copyright__ = "Copyright (c) 2013 Igalia, S.L."
__license__   = "LGPL"

from gi.repository import GObject, Gdk, Gtk
from . import orca_state
from .orca_i18n import _

class OrcaCommandListGUI:

    def __init__(self, title, columnHeaders, rows, canPerformCommands):
        self._focusedObject = orca_state.locusOfFocus
        self._activeScript = orca_state.activeScript
        self._tree = None
        self._okButton = None
        self._gui = self._createCommandListDialog(columnHeaders, rows)
        self._gui.set_title(title)
        self._gui.set_modal(True)
        if not canPerformCommands:
            self._okButton.destroy()
        self.showGUI()

    def _createCommandListDialog(self, columnHeaders, rows):
        dialog = Gtk.Dialog()
        dialog.set_default_size(600, 400)

        grid = Gtk.Grid()
        contentArea = dialog.get_content_area()
        contentArea.add(grid)

        scrolledWindow = Gtk.ScrolledWindow()
        grid.add(scrolledWindow)

        self._tree = Gtk.TreeView()
        self._tree.set_hexpand(True)
        self._tree.set_vexpand(True)
        scrolledWindow.add(self._tree)

        cols = len(columnHeaders) * [GObject.TYPE_STRING]
        for i, header in enumerate(columnHeaders):
            cell = Gtk.CellRendererText()
            column = Gtk.TreeViewColumn(header, cell, text=i)
            self._tree.append_column(column)
            if header:
                column.set_sort_column_id(i)

        model = Gtk.ListStore(*cols)
        for row in rows:
            rowIter = model.append(None)
            for i, cell in enumerate(row):
                model.set_value(rowIter, i, str(cell))

        column = self._tree.get_column(0)
        column.set_visible(False)

        btn = dialog.add_button(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL)
        btn.connect('clicked', self._onCancelClicked)

        self._okButton = dialog.add_button(Gtk.STOCK_OK, Gtk.ResponseType.OK)
        self._okButton.grab_default()
        self._okButton.connect('clicked', self._onOKClicked)

        self._tree.set_model(model)
        self._tree.connect('key-release-event', self._onKeyRelease)
        return dialog

    def showGUI(self):
        self._gui.show_all()

        ts = orca_state.lastInputEventTimestamp
        if ts == 0:
            ts = Gtk.get_current_event_time()
        self._gui.present_with_time(ts)

    def _onKeyRelease(self, widget, event):
        keycode = event.hardware_keycode
        keymap = Gdk.Keymap.get_default()
        entries_for_keycode = keymap.get_entries_for_keycode(keycode)
        entries = entries_for_keycode[-1]
        eventString = Gdk.keyval_name(entries[0])
        if eventString == 'Return':
            self._gui.activate_default()

    def _onCancelClicked(self, widget):
        self._gui.destroy()

    def _onOKClicked(self, widget):
        handler = self._getSelectedHandler()
        self._gui.destroy()

    def _getSelectedHandler(self):
        if not self._tree:
            return None

        selection = self._tree.get_selection()
        if not selection:
            return None

        model, paths = selection.get_selected_rows()
        if not paths:
            return None

        return model.get_value(model.get_iter(paths[0]), 0)

def showUI(title='', columnHeaders=[], rows=[()], canPerformCommands=True):
    gui = OrcaCommandListGUI(title, columnHeaders, rows, canPerformCommands)
    gui.showGUI()
<commit_msg>Remove missed, unused import<commit_after># Orca
#
# Copyright 2013 Igalia, S.L.
#
# Author: Joanmarie Diggs <jdiggs@igalia.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., Franklin Street, Fifth Floor,
# Boston MA  02110-1301 USA.

"""Displays a GUI to present Orca commands."""

__id__        = "$Id$"
__version__   = "$Revision$"
__date__      = "$Date$"
__copyright__ = "Copyright (c) 2013 Igalia, S.L."
__license__   = "LGPL"

from gi.repository import GObject, Gdk, Gtk
from . import orca_state

class OrcaCommandListGUI:

    def __init__(self, title, columnHeaders, rows, canPerformCommands):
        self._focusedObject = orca_state.locusOfFocus
        self._activeScript = orca_state.activeScript
        self._tree = None
        self._okButton = None
        self._gui = self._createCommandListDialog(columnHeaders, rows)
        self._gui.set_title(title)
        self._gui.set_modal(True)
        if not canPerformCommands:
            self._okButton.destroy()
        self.showGUI()

    def _createCommandListDialog(self, columnHeaders, rows):
        dialog = Gtk.Dialog()
        dialog.set_default_size(600, 400)

        grid = Gtk.Grid()
        contentArea = dialog.get_content_area()
        contentArea.add(grid)

        scrolledWindow = Gtk.ScrolledWindow()
        grid.add(scrolledWindow)

        self._tree = Gtk.TreeView()
        self._tree.set_hexpand(True)
        self._tree.set_vexpand(True)
        scrolledWindow.add(self._tree)

        cols = len(columnHeaders) * [GObject.TYPE_STRING]
        for i, header in enumerate(columnHeaders):
            cell = Gtk.CellRendererText()
            column = Gtk.TreeViewColumn(header, cell, text=i)
            self._tree.append_column(column)
            if header:
                column.set_sort_column_id(i)

        model = Gtk.ListStore(*cols)
        for row in rows:
            rowIter = model.append(None)
            for i, cell in enumerate(row):
                model.set_value(rowIter, i, str(cell))

        column = self._tree.get_column(0)
        column.set_visible(False)

        btn = dialog.add_button(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL)
        btn.connect('clicked', self._onCancelClicked)

        self._okButton = dialog.add_button(Gtk.STOCK_OK, Gtk.ResponseType.OK)
        self._okButton.grab_default()
        self._okButton.connect('clicked', self._onOKClicked)

        self._tree.set_model(model)
        self._tree.connect('key-release-event', self._onKeyRelease)
        return dialog

    def showGUI(self):
        self._gui.show_all()

        ts = orca_state.lastInputEventTimestamp
        if ts == 0:
            ts = Gtk.get_current_event_time()
        self._gui.present_with_time(ts)

    def _onKeyRelease(self, widget, event):
        keycode = event.hardware_keycode
        keymap = Gdk.Keymap.get_default()
        entries_for_keycode = keymap.get_entries_for_keycode(keycode)
        entries = entries_for_keycode[-1]
        eventString = Gdk.keyval_name(entries[0])
        if eventString == 'Return':
            self._gui.activate_default()

    def _onCancelClicked(self, widget):
        self._gui.destroy()

    def _onOKClicked(self, widget):
        handler = self._getSelectedHandler()
        self._gui.destroy()

    def _getSelectedHandler(self):
        if not self._tree:
            return None

        selection = self._tree.get_selection()
        if not selection:
            return None

        model, paths = selection.get_selected_rows()
        if not paths:
            return None

        return model.get_value(model.get_iter(paths[0]), 0)

def showUI(title='', columnHeaders=[], rows=[()], canPerformCommands=True):
    gui = OrcaCommandListGUI(title, columnHeaders, rows, canPerformCommands)
    gui.showGUI()
<|endoftext|>


<commit_before>"""
Defines view functions and forms for the Sumatra web interface.
"""

from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render_to_response
from django.views.generic import list_detail
from django import forms
from sumatra.recordstore.django_store import models
from sumatra.projects import load_simulation_project
from sumatra.datastore import get_data_store
import mimetypes
import csv

_project = load_simulation_project()
project_name = _project.name
del _project

class TagSearch(forms.Form):
    search = forms.CharField()

class SimulationUpdateForm(forms.ModelForm):
    wide_textarea = forms.Textarea(attrs={'rows': 2, 'cols':80})
    reason = forms.CharField(required=False, widget=wide_textarea)
    outcome = forms.CharField(required=False, widget=wide_textarea)

    class Meta:
        model = models.SimulationRecord
        fields=('reason', 'outcome', 'tags')

def show_project(request):
    project = load_simulation_project()
    return render_to_response('project_detail.html', {'project': project})

def list_simulations(request):
    search_form = TagSearch()
    return list_detail.object_list(request,
                                   queryset=models.SimulationRecord.objects.all(),
                                   template_name="simulation_list.html",
                                   extra_context={
                                    'project_name': project_name,
                                    'search_form': search_form})

def simulation_detail(request, id):
    record = models.SimulationRecord.objects.get(id=id)

    if request.method == 'POST':
        if request.POST.has_key('delete'):
            record.delete()
            return HttpResponseRedirect('/')
        else:
            form = SimulationUpdateForm(request.POST, instance=record)
            if form.is_valid():
                form.save()
    else:
        form = SimulationUpdateForm(instance=record)
    data_store = get_data_store(record.datastore.type, eval(record.datastore.parameters))
    datafiles = data_store.list_files(eval(record.data_key))
    assert isinstance(datafiles, list), type(datafiles)
    parameter_set = record.parameters.to_sumatra()
    return render_to_response('simulation_detail.html', {'record': record,
                                                         'project_name': project_name,
                                                         'parameters': parameter_set.as_dict(),
                                                         'datafiles': datafiles,
                                                         'form': form
                                                         })

def delete_records(request):
    records_to_delete = request.POST.getlist('delete')
    records = models.SimulationRecord.objects.filter(id__in=records_to_delete)
    for record in records:
        record.delete()
    return HttpResponseRedirect('/')

DEFAULT_MAX_DISPLAY_LENGTH = 10*1024

def show_file(request, id):
    path = request.GET['path']
    if 'truncate' in request.GET:
        if request.GET['truncate'].lower() == 'false':
            max_display_length = None
        else:
            max_display_length = int(request.GET['truncate'])*1024
    else:
        max_display_length = DEFAULT_MAX_DISPLAY_LENGTH

    record = models.SimulationRecord.objects.get(id=id)
    data_store = get_data_store(record.datastore.type, eval(record.datastore.parameters))
    # check the file is in the store for a given simulation

    try:
        mimetype, encoding = mimetypes.guess_type(path)
        if mimetype  == "text/csv":
            content = data_store.get_content(record.data_key, path, max_length=max_display_length)
            truncated = False
            if max_display_length is not None and len(content) >= max_display_length:
                truncated = True

                # dump the last truncated line (if any)
                content = content.rpartition('\n')[0]

            lines = content.splitlines()
            reader = csv.reader(lines)

            return render_to_response("show_csv.html",
                                      {'path': path, 'id': id,
                                       'project_name': project_name,
                                       'reader': reader,
                                       'truncated':truncated
                                       })

            print content

        elif mimetype == None or mimetype.split("/")[0] == "text":
            content = data_store.get_content(record.data_key, path, max_length=max_display_length)
            truncated = False
            if max_display_length is not None and len(content) >= max_display_length:
                truncated = True
            return render_to_response("show_file.html",
                                      {'path': path, 'id': id,
                                       'project_name': project_name,
                                       'content': content,
                                       'truncated': truncated,
                                       })
        elif mimetype in ("image/png", "image/jpeg", "image/gif"):
            return render_to_response("show_image.html",
                                      {'path': path, 'id': id,
                                       'project_name': project_name,})
        #elif mimetype == 'application/zip':
        #    import zipfile
        #    if zipfile.is_zipfile(path):
        #        zf = zipfile.ZipFile(path, 'r')
        #        contents = zf.namelist()
        #        zf.close()
        #        return render_to_response("show_file.html",
        #                              {'path': path, 'id': id,
        #                               'content': "\n".join(contents)
        #                               })
        #    else:
        #        raise IOError("Not a valid zip file")
        else:
            return render_to_response("show_file.html", {'path': path, 'id': id,
                                                         'project_name': project_name,
                                                         'content': "Can't display this file (mimetype assumed to be %s)" % mimetype})
    except IOError, e:
        return render_to_response("show_file.html", {'path': path, 'id': id,
                                                     'project_name': project_name,
                                                     'content': "File not found.",
                                                     'errmsg': e})

def show_image(request, id):
    path = request.GET['path']
    mimetype, encoding = mimetypes.guess_type(path)
    if mimetype in ("image/png", "image/jpeg", "image/gif"):
        record = models.SimulationRecord.objects.get(id=id)
        data_store = get_data_store(record.datastore.type, eval(record.datastore.parameters))
        content = data_store.get_content(record.data_key, path)
        response = HttpResponse(mimetype=mimetype)
        response.write(content)
        return response
    else:
        return HttpResponse(mimetype="image/png") # should return a placeholder image?

def show_diff(request, id, package):
    record = models.SimulationRecord.objects.get(id=id)
    if package:
        dependency = record.dependencies.get(name=package)
    else:
        package = "Main script"
        dependency = record
    return render_to_response("show_diff.html", {'id': id,
                                                 'project_name': project_name,
                                                 'package': package,
                                                 'parent_version': dependency.version,
                                                 'diff': dependency.diff})

<commit_msg>In `web.views` we now filter records by project name.<commit_after>"""
Defines view functions and forms for the Sumatra web interface.
"""

from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render_to_response
from django.views.generic import list_detail
from django import forms
from sumatra.recordstore.django_store import models
from sumatra.projects import load_simulation_project
from sumatra.datastore import get_data_store
import mimetypes
import csv

_project = load_simulation_project()
project_name = _project.name
del _project

# Idea: add project as arg to the view functions.
#       When using with smtweb, we use a decorator something like the
#       following to provide the project. When using sumatra_piston, the project
#       is provided from the URL
#       The conditional could be something like whether we successfully load
#       a local project.
#def call_with_project(func):
#    def wrapped(request, *args, **kwargs):
#        return func(request, project, *args, **kwargs)
#    return wrapped

# A much simpler idea: we put the project in the URL for smtweb as well.

class TagSearch(forms.Form):
    search = forms.CharField()

class SimulationUpdateForm(forms.ModelForm):
    wide_textarea = forms.Textarea(attrs={'rows': 2, 'cols':80})
    reason = forms.CharField(required=False, widget=wide_textarea)
    outcome = forms.CharField(required=False, widget=wide_textarea)

    class Meta:
        model = models.SimulationRecord
        fields=('reason', 'outcome', 'tags')

def show_project(request):
    project = load_simulation_project()
    return render_to_response('project_detail.html', {'project': project})

def list_simulations(request):
    search_form = TagSearch()
    return list_detail.object_list(request,
                                   queryset=models.SimulationRecord.objects.filter(project__id=project_name),
                                   template_name="simulation_list.html",
                                   extra_context={
                                    'project_name': project_name,
                                    'search_form': search_form})

def simulation_detail(request, id):
    record = models.SimulationRecord.objects.get(id=id, project__id=project_name)

    if request.method == 'POST':
        if request.POST.has_key('delete'):
            record.delete()
            return HttpResponseRedirect('/')
        else:
            form = SimulationUpdateForm(request.POST, instance=record)
            if form.is_valid():
                form.save()
    else:
        form = SimulationUpdateForm(instance=record)
    data_store = get_data_store(record.datastore.type, eval(record.datastore.parameters))
    datafiles = data_store.list_files(eval(record.data_key))
    assert isinstance(datafiles, list), type(datafiles)
    parameter_set = record.parameters.to_sumatra()
    return render_to_response('simulation_detail.html', {'record': record,
                                                         'project_name': project_name,
                                                         'parameters': parameter_set.as_dict(),
                                                         'datafiles': datafiles,
                                                         'form': form
                                                         })

def delete_records(request):
    records_to_delete = request.POST.getlist('delete')
    records = models.SimulationRecord.objects.filter(id__in=records_to_delete, project__id=project_name)
    for record in records:
        record.delete()
    return HttpResponseRedirect('/')

DEFAULT_MAX_DISPLAY_LENGTH = 10*1024

def show_file(request, id):
    path = request.GET['path']
    if 'truncate' in request.GET:
        if request.GET['truncate'].lower() == 'false':
            max_display_length = None
        else:
            max_display_length = int(request.GET['truncate'])*1024
    else:
        max_display_length = DEFAULT_MAX_DISPLAY_LENGTH

    record = models.SimulationRecord.objects.get(id=id, project__id=project_name)
    data_store = get_data_store(record.datastore.type, eval(record.datastore.parameters))
    # check the file is in the store for a given simulation

    try:
        mimetype, encoding = mimetypes.guess_type(path)
        if mimetype  == "text/csv":
            content = data_store.get_content(record.data_key, path, max_length=max_display_length)
            truncated = False
            if max_display_length is not None and len(content) >= max_display_length:
                truncated = True

                # dump the last truncated line (if any)
                content = content.rpartition('\n')[0]

            lines = content.splitlines()
            reader = csv.reader(lines)

            return render_to_response("show_csv.html",
                                      {'path': path, 'id': id,
                                       'project_name': project_name,
                                       'reader': reader,
                                       'truncated':truncated
                                       })

            print content

        elif mimetype == None or mimetype.split("/")[0] == "text":
            content = data_store.get_content(record.data_key, path, max_length=max_display_length)
            truncated = False
            if max_display_length is not None and len(content) >= max_display_length:
                truncated = True
            return render_to_response("show_file.html",
                                      {'path': path, 'id': id,
                                       'project_name': project_name,
                                       'content': content,
                                       'truncated': truncated,
                                       })
        elif mimetype in ("image/png", "image/jpeg", "image/gif"):
            return render_to_response("show_image.html",
                                      {'path': path, 'id': id,
                                       'project_name': project_name,})
        #elif mimetype == 'application/zip':
        #    import zipfile
        #    if zipfile.is_zipfile(path):
        #        zf = zipfile.ZipFile(path, 'r')
        #        contents = zf.namelist()
        #        zf.close()
        #        return render_to_response("show_file.html",
        #                              {'path': path, 'id': id,
        #                               'content': "\n".join(contents)
        #                               })
        #    else:
        #        raise IOError("Not a valid zip file")
        else:
            return render_to_response("show_file.html", {'path': path, 'id': id,
                                                         'project_name': project_name,
                                                         'content': "Can't display this file (mimetype assumed to be %s)" % mimetype})
    except IOError, e:
        return render_to_response("show_file.html", {'path': path, 'id': id,
                                                     'project_name': project_name,
                                                     'content': "File not found.",
                                                     'errmsg': e})

def show_image(request, id):
    path = request.GET['path']
    mimetype, encoding = mimetypes.guess_type(path)
    if mimetype in ("image/png", "image/jpeg", "image/gif"):
        record = models.SimulationRecord.objects.get(id=id, project__id=project_name)
        data_store = get_data_store(record.datastore.type, eval(record.datastore.parameters))
        content = data_store.get_content(record.data_key, path)
        response = HttpResponse(mimetype=mimetype)
        response.write(content)
        return response
    else:
        return HttpResponse(mimetype="image/png") # should return a placeholder image?

def show_diff(request, id, package):
    record = models.SimulationRecord.objects.get(id=id, project__id=project_name)
    if package:
        dependency = record.dependencies.get(name=package)
    else:
        package = "Main script"
        dependency = record
    return render_to_response("show_diff.html", {'id': id,
                                                 'project_name': project_name,
                                                 'package': package,
                                                 'parent_version': dependency.version,
                                                 'diff': dependency.diff})

<|endoftext|>