<commit_before>"""
Copyright (c) 2018, Arm Limited
SPDX-License-Identifier: Apache-2.0

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

UNIT TEST BUILD & RUN
"""

import os
import logging
import sys
from shutil import copy

from .utils import execute_program
from .get_tools import get_make_tool, \
                       get_cmake_tool, \
                       get_cxx_tool, \
                       get_c_tool, \
                       get_gcov_program, \
                       get_gcovr_program
from .settings import DEFAULT_CMAKE_GENERATORS

class UnitTestTool(object):
    """
    Unit test tool to:
        - prepare build directory
        - create makefiles
        - build unit tests
        - run unit tests
        - generate code coverage reports
    """

    def __init__(self,
                 make_program=None):
        """
        Constructor

        Keyword arguments:
        make_program - Make tool to use
        """

        self.make_program = make_program

        if self.make_program is None:
            self.make_program = get_make_tool()

    def create_makefiles(self,
                         path_to_src=None,
                         generator=None,
                         coverage_output_type=None,
                         debug=False):
        """
        Create Makefiles and prepare targets with CMake.

        Keyword arguments:
        path_to_src - Path to source directory
        generator - Type of Makefiles to generate
        coverage_output_type - Generate HTML, XML or both reports
        debug - Target debug or release build
        """

        if generator is None:
            generator = DEFAULT_CMAKE_GENERATORS.get(self.make_program,
                                                     "Unix Makefiles")

        cmake = get_cmake_tool()

        if cmake is None:
            logging.error(
                "No CMake found in Path. Install all the required tools.")
            sys.exit(1)

        args = [cmake,
                "-G",
                generator,
                "-DCMAKE_MAKE_PROGRAM=%s" % self.make_program,
                "-DCMAKE_CXX_COMPILER=%s" % get_cxx_tool(),
                "-DCMAKE_C_COMPILER=%s" % get_c_tool()]

        if debug:
            args.append("-DCMAKE_BUILD_TYPE=Debug")

        if coverage_output_type:
            args.append("-DCOVERAGE:STRING=%s" % coverage_output_type)

        if path_to_src is not None:
            args.append(path_to_src)

        execute_program(args,
                        "CMake failed to run successfully. See error message.")

    def build_tests(self):
        """
        Build unit tests and libraries to be tested.
        """

        args = [self.make_program]

        if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
            args.append("VERBOSE=1")

        execute_program(args,
                        "Building unit tests failed.",
                        "Unit tests built successfully.")

    def _get_coverage_script(self, coverage_type, excludes):
        args = [get_gcovr_program(),
                "--gcov-executable",
                get_gcov_program(),
                "-r",
                "../..",
                "."]

        if coverage_type == "html":
            args.extend(["--html",
                         "--html-detail",
                         "-o",
                         "./coverage/index.html"])
        elif coverage_type == "xml":
            args.extend(["-x",
                         "-o",
                         "./coverage.xml"])

        for path in excludes:
            args.extend(["-e", path])

        if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
            args.append("-v")

        return args

    def generate_coverage_report(self,
                                 coverage_output_type=None,
                                 excludes=None,
                                 build_path=None):
        """
        Run tests to generate coverage data, and generate coverage reports.
        """

        self.run_tests()

        if get_gcovr_program() is None:
            logging.error("No gcovr tool found in path. \
            Cannot generate coverage report.")
            return

        if build_path is None:
            build_path = os.getcwd()

        if coverage_output_type is None:
            logging.warning("No coverage output type give. \
                            Cannot generate coverage reports.")
            return

        if excludes is None:
            excludes = []

        if coverage_output_type == "html" or coverage_output_type == "both":
            # Create build directory if not exist.
            coverage_path = os.path.join(build_path, "coverage")
            if not os.path.exists(coverage_path):
                os.mkdir(coverage_path)

            args = self._get_coverage_script("html", excludes)

            execute_program(args,
                            "HTML code coverage report generation failed.",
                            "HTML code coverage report created.")

        if coverage_output_type == "xml" or coverage_output_type == "both":
            args = self._get_coverage_script("xml", excludes)

            execute_program(args,
                            "XML code coverage report generation failed.",
                            "XML code coverage report created.")

    def run_tests(self, filter_regex=None):
        """
        Run unit tests.

        Keyword arguments:
        filter_regex - Regular expression to select which tests to run
        """

        args = [self.make_program, "test"]

        if filter_regex:
            args.append("ARGS=-R %s -V -D ExperimentalTest" % filter_regex)
        else:
            args.append("ARGS=-V -D ExperimentalTest")

        if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
            args.append("VERBOSE=1")

        execute_program(args, "Unit test run failed.")

    def prepare_build_directory(self,
                                path_to_src=None,
                                build_path=None,
                                clean=False):
        """
        Create build directory if not exist and
        change current working directory to it

        Keyword arguments:
        path_to_src - Path to source directory
        build_path - Path to build directory
        clean - Clean build directory
        """

        if build_path is None:
            build_path = os.getcwd()

        # Clean CMake data if exists.
        if clean:
            self._clean_build(build_path)

        # Create build directory if not exist.
        if not os.path.exists(build_path):
            os.makedirs(build_path)
            filename = ".mbedignore"
            inputfile = os.path.join(path_to_src, filename)
            outputfile = os.path.join(build_path, filename)
            copy(inputfile, outputfile)

        # Change current working directory to build directory.
        os.chdir(build_path)

    def _clean_build(self, build_path=None):
        """
        Try clean build directory

        Keyword arguments:
        build_path - Path to build directory
        """

        logging.info("Cleaning build directory...")

        if os.path.exists(os.path.join(build_path, "CMakeCache.txt")):
            args = [self.make_program,
                    "-C",
                    build_path,
                    "--no-print-directory",
                    "clean"]

            # Remove coverage files
            for root, _, files in os.walk(build_path):
                for current_file in files:
                    if current_file.endswith((".gcno", ".gcda")):
                        os.remove(os.path.join(root, current_file))

            execute_program(args,
                            "Clean step failed.",
                            "Clean done.")

        else:
            logging.warning("%s does not exist or \
            does not contain previous build data.", build_path)
<commit_msg>fix coverage exclude paths on Windows<commit_after>"""
Copyright (c) 2018, Arm Limited
SPDX-License-Identifier: Apache-2.0

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

UNIT TEST BUILD & RUN
"""

import os
import logging
import sys
from shutil import copy

from .utils import execute_program
from .get_tools import get_make_tool, \
                       get_cmake_tool, \
                       get_cxx_tool, \
                       get_c_tool, \
                       get_gcov_program, \
                       get_gcovr_program
from .settings import DEFAULT_CMAKE_GENERATORS

class UnitTestTool(object):
    """
    Unit test tool to:
        - prepare build directory
        - create makefiles
        - build unit tests
        - run unit tests
        - generate code coverage reports
    """

    def __init__(self,
                 make_program=None):
        """
        Constructor

        Keyword arguments:
        make_program - Make tool to use
        """

        self.make_program = make_program

        if self.make_program is None:
            self.make_program = get_make_tool()

    def create_makefiles(self,
                         path_to_src=None,
                         generator=None,
                         coverage_output_type=None,
                         debug=False):
        """
        Create Makefiles and prepare targets with CMake.

        Keyword arguments:
        path_to_src - Path to source directory
        generator - Type of Makefiles to generate
        coverage_output_type - Generate HTML, XML or both reports
        debug - Target debug or release build
        """

        if generator is None:
            generator = DEFAULT_CMAKE_GENERATORS.get(self.make_program,
                                                     "Unix Makefiles")

        cmake = get_cmake_tool()

        if cmake is None:
            logging.error(
                "No CMake found in Path. Install all the required tools.")
            sys.exit(1)

        args = [cmake,
                "-G",
                generator,
                "-DCMAKE_MAKE_PROGRAM=%s" % self.make_program,
                "-DCMAKE_CXX_COMPILER=%s" % get_cxx_tool(),
                "-DCMAKE_C_COMPILER=%s" % get_c_tool()]

        if debug:
            args.append("-DCMAKE_BUILD_TYPE=Debug")

        if coverage_output_type:
            args.append("-DCOVERAGE:STRING=%s" % coverage_output_type)

        if path_to_src is not None:
            args.append(path_to_src)

        execute_program(args,
                        "CMake failed to run successfully. See error message.")

    def build_tests(self):
        """
        Build unit tests and libraries to be tested.
        """

        args = [self.make_program]

        if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
            args.append("VERBOSE=1")

        execute_program(args,
                        "Building unit tests failed.",
                        "Unit tests built successfully.")

    def _get_coverage_script(self, coverage_type, excludes):
        args = [get_gcovr_program(),
                "--gcov-executable",
                get_gcov_program(),
                "-r",
                "../..",
                "."]

        if coverage_type == "html":
            args.extend(["--html",
                         "--html-detail",
                         "-o",
                         "./coverage/index.html"])
        elif coverage_type == "xml":
            args.extend(["-x",
                         "-o",
                         "./coverage.xml"])

        for path in excludes:
            args.extend(["-e", path.replace("\\", "/")])

        if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
            args.append("-v")

        return args

    def generate_coverage_report(self,
                                 coverage_output_type=None,
                                 excludes=None,
                                 build_path=None):
        """
        Run tests to generate coverage data, and generate coverage reports.
        """

        self.run_tests()

        if get_gcovr_program() is None:
            logging.error("No gcovr tool found in path. \
            Cannot generate coverage report.")
            return

        if build_path is None:
            build_path = os.getcwd()

        if coverage_output_type is None:
            logging.warning("No coverage output type give. \
                            Cannot generate coverage reports.")
            return

        if excludes is None:
            excludes = []

        if coverage_output_type == "html" or coverage_output_type == "both":
            # Create build directory if not exist.
            coverage_path = os.path.join(build_path, "coverage")
            if not os.path.exists(coverage_path):
                os.mkdir(coverage_path)

            args = self._get_coverage_script("html", excludes)

            execute_program(args,
                            "HTML code coverage report generation failed.",
                            "HTML code coverage report created.")

        if coverage_output_type == "xml" or coverage_output_type == "both":
            args = self._get_coverage_script("xml", excludes)

            execute_program(args,
                            "XML code coverage report generation failed.",
                            "XML code coverage report created.")

    def run_tests(self, filter_regex=None):
        """
        Run unit tests.

        Keyword arguments:
        filter_regex - Regular expression to select which tests to run
        """

        args = [self.make_program, "test"]

        if filter_regex:
            args.append("ARGS=-R %s -V -D ExperimentalTest" % filter_regex)
        else:
            args.append("ARGS=-V -D ExperimentalTest")

        if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
            args.append("VERBOSE=1")

        execute_program(args, "Unit test run failed.")

    def prepare_build_directory(self,
                                path_to_src=None,
                                build_path=None,
                                clean=False):
        """
        Create build directory if not exist and
        change current working directory to it

        Keyword arguments:
        path_to_src - Path to source directory
        build_path - Path to build directory
        clean - Clean build directory
        """

        if build_path is None:
            build_path = os.getcwd()

        # Clean CMake data if exists.
        if clean:
            self._clean_build(build_path)

        # Create build directory if not exist.
        if not os.path.exists(build_path):
            os.makedirs(build_path)
            filename = ".mbedignore"
            inputfile = os.path.join(path_to_src, filename)
            outputfile = os.path.join(build_path, filename)
            copy(inputfile, outputfile)

        # Change current working directory to build directory.
        os.chdir(build_path)

    def _clean_build(self, build_path=None):
        """
        Try clean build directory

        Keyword arguments:
        build_path - Path to build directory
        """

        logging.info("Cleaning build directory...")

        if os.path.exists(os.path.join(build_path, "CMakeCache.txt")):
            args = [self.make_program,
                    "-C",
                    build_path,
                    "--no-print-directory",
                    "clean"]

            # Remove coverage files
            for root, _, files in os.walk(build_path):
                for current_file in files:
                    if current_file.endswith((".gcno", ".gcda")):
                        os.remove(os.path.join(root, current_file))

            execute_program(args,
                            "Clean step failed.",
                            "Clean done.")

        else:
            logging.warning("%s does not exist or \
            does not contain previous build data.", build_path)
<|endoftext|>


<commit_before>#!/usr/bin/env python

from os.path import join
import requests
import argparse

class AssetHubClient(object):
    def __init__(self, url, application=None, component=None):
        self.url = url
        self._base_url = join(self.url, 'api', 'assets')
        self.application = application
        self.component = component
        self.tag = None

    def _get_url(self):
        if self.application is not None and self.component is not None:
            return join(self._base_url, self.application, self.component)
        elif self.application is not None:
            return join(self._base_url, self.application)
        else:
            return self._base_url

    def _get_params(self):
        params = {}
        if self.tag is not None:
            params['tag'] = self.tag
        if self.author is not None:
            params['author'] = author
        return params

    def list(self):
        r = requests.get(self._get_url(), params=self._get_params())
        r.raise_for_status()
        return r.json()

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description='AssetHub command-line client')
    parser.add_argument('-u', '--url', metavar='HTTP://SERVER.ORG/', help='AssetHub server URL')
    parser.add_argument('-a', '--application', metavar='APP', help='Application')
    parser.add_argument('-c', '--component', metavar='COMPONENT', help='Component')
    parser.add_argument('--author', metavar='USER', help='Asset author')
    parser.add_argument('-t', '--tag', metavar='TAG', help='Asset tag')
    args = parser.parse_args()

    if args.url:
        url = args.url
    else:
        url = 'http://assethub.iportnov.tech/'

    client = AssetHubClient(url, args.application, args.component)
    client.tag = args.tag
    client.author = args.author
    for asset in client.list():
        print(asset['title'])
<commit_msg>Add options to client.<commit_after>#!/usr/bin/env python

from os.path import join, basename
import requests
import argparse

class Asset(object):
    def __init__(self, json):
        self.dict = json

    def __getattr__(self, attr):
        return self.dict[attr]

    def json(self):
        return self.dict

    def __unicode__(self):
        return unicode(self.dict)

    def __str__(self):
        return str(self.dict)

    def __repr__(self):
        return repr(self.dict)

    def download_data(self, outpath):
        r = requests.get(self.data)
        r.raise_for_status()
        with open(outpath, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024):
                f.write(chunk)

    def get_filename(self):
        return basename(self.data)

class AssetHubClient(object):
    def __init__(self, url, application=None, component=None):
        self.url = url
        self._base_url = join(self.url, 'api', 'assets')
        self.application = application
        self.component = component
        self.tag = None

    def _get_url(self):
        if self.application is not None and self.component is not None:
            return join(self._base_url, self.application, self.component)
        elif self.application is not None:
            return join(self._base_url, self.application)
        else:
            return self._base_url

    def _get_params(self):
        params = {}
        if self.tag is not None:
            params['tag'] = self.tag
        if self.author is not None:
            params['author'] = author
        return params

    def list(self):
        r = requests.get(self._get_url(), params=self._get_params())
        r.raise_for_status()
        return [Asset(a) for a in r.json()]

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description='AssetHub command-line client')
    parser.add_argument('-u', '--url', metavar='HTTP://SERVER.ORG/', help='AssetHub server URL')
    parser.add_argument('-a', '--application', metavar='APP', help='Application')
    parser.add_argument('-c', '--component', metavar='COMPONENT', help='Component')
    parser.add_argument('--author', metavar='USER', help='Asset author')
    parser.add_argument('-t', '--tag', metavar='TAG', help='Asset tag')
    parser.add_argument('-J', '--json', action='store_true', help='Output data as JSON')
    parser.add_argument('-D', '--download', action='store_true', help='Download asset data')
    args = parser.parse_args()

    if args.url:
        url = args.url
    else:
        url = 'http://assethub.iportnov.tech/'

    client = AssetHubClient(url, args.application, args.component)
    client.tag = args.tag
    client.author = args.author
    for asset in client.list():
        if args.download:
            asset.download_data(asset.get_filename())
            print("Downloaded " + asset.get_filename())
        elif args.json:
            print(asset)
        else:
            print(asset.title)

<|endoftext|>


<commit_before>import itertools as it
import pandas as pd
from scipy import stats
import numpy as np
import collections as cl
from treelib import Node, Tree

class Node(object):
	def __init__(self, choices=None, members={}, split_variable=None, chi=0, p=0, terminal_indices=[], id=0, parent=None):
		self.choices = choices
		self.members = members
		self.split_variable = split_variable
		self.chi = chi
		self.p = p
		self.terminal_indices = terminal_indices
		self.id = id
		self.parent = parent

	def __hash__(self):
		return self.__dict__

	def __eq__(self, other):
		if isinstance(other, self.__class__):
		    return self.__dict__ == other.__dict__
		else:
		    return False

	def __repr__(self):
		return str((self.choices, self.members, self.split_variable, self.chi, self.p))

class Split(object):
	def __init__(self, index, splits, chi, p):
		self.index = index
		self.splits = splits
		self.chi = chi
		self.p = p

class CHAID(object):
	DEFAULT_CONDITIONS =  { 'alpha_merge': .05, 'max_depth': 2, 'min_sample': 30 }

	def from_pandas_df(self, ind_df, dep_series, conditions):
		new_conditions = self.DEFAULT_CONDITIONS.copy()
		new_conditions.update(conditions)
		params = {
			'independent_variable_names': ind_df.columns
		}
		ind_df.apply(lambda x: x.fillna('-1.0', inplace=True) if (x.dtype == object) else x.fillna(-1.0, inplace=True))
		ind_values = ind_df.values
		dep_values = dep_series.values
		self.data_size = dep_values.shape[0]
		self.tree_data, node_id =  self.node(params, np.arange(0, dep_values.shape[0] + 1, dtype=np.int), ind_values, dep_values, new_conditions)
		return self

	def node(self, params, rows, ind, dep, conditions, depth=0, tree_store=[], parent=None, node_id=0, parent_decisions=None):
		depth = depth + 1

		members = dict(np.transpose(np.unique(dep, return_counts=True)))

		if conditions['max_depth'] < depth:
			terminal_node = Node(choices=parent_decisions, members=members, id=node_id, parent=parent, terminal_indices=rows)
			tree_store.append(terminal_node)
			return tree_store, node_id + 1

		split = self.generate_best_split(ind, dep, conditions)

		node = Node(choices=parent_decisions, members=members, id=node_id, parent=parent, split_variable=split.index, chi=split.chi, p=split.p)
		tree_store.append(node)
		parent = node_id
		node_id = node_id + 1

		if split.index is None:
			return tree_store, node_id

		for choices in split.splits:
			correct_rows = np.in1d(ind[:, split.index], choices)
			dep_slice = dep[correct_rows]
			ind_slice = ind[correct_rows, :]
			row_slice = rows[correct_rows]
			if conditions['min_sample'] < len(dep_slice):
				tree_store, node_id = self.node(params, row_slice, ind_slice, dep_slice, conditions, depth, node_id=node_id, parent=parent, tree_store=tree_store, parent_decisions=choices)
			else:
				memebers = dict(np.transpose(np.unique(dep_slice, return_counts=True)))
				terminal_node = Node(choices=choices, members=memebers, id=node_id, parent=parent, terminal_indices=row_slice)
				tree_store.append(terminal_node)
				node_id = node_id + 1

		return tree_store, node_id

	def generate_best_split(self, ind, dep, conditions):
		split = Split(None, None, None, 1)
		for i in range(0, ind.shape[1]):
			index = np.array(ind[:,i])
			unique = set(index)

			mappings = {}

			while len(unique) > 1:
				size  = (len(unique) * (len(unique) - 1)) / 2
				sub_data = np.ndarray(shape=(size, 3), dtype=object, order='F')
				for j, comb in enumerate(it.combinations(unique, 2)):
					y = np.where(np.in1d(index, comb[0]))[0]
					g = np.where(np.in1d(index, comb[1]))[0]

					uni_y = dict(np.transpose(np.unique(dep[y], return_counts=True)))
					uni_g = dict(np.transpose(np.unique(dep[g], return_counts=True)))

					keys = set(uni_y.keys() + uni_g.keys())

					cr_table = [
						[ uni_y.get(k, 0) for k in keys ],
						[ uni_g.get(k, 0) for k in keys ]
					]

					chi = stats.chi2_contingency(np.array(cr_table))
					sub_data[j] = (comb, chi[0], chi[1])

				highest_p_split = np.sort(sub_data[:, 2])[-1]
				correct_row = np.where(np.in1d(sub_data[:, 2], highest_p_split))[0][0]

				if size == 1 or highest_p_split < conditions['alpha_merge']:
					if highest_p_split < split.p:
						split = Split(i, [ mappings.get(x, [x]) for x in unique ], sub_data[correct_row][1], highest_p_split)
					break

				choice = list(sub_data[correct_row, 0])

				if choice[1] in mappings:
					mappings[choice[0]] =  mappings.get(choice[0], [choice[0]]) + mappings[choice[1]]
					del mappings[choice[1]]
				else:
					mappings[choice[0]] = mappings.get(choice[0], [choice[0]]) + [choice[1]]

				index[ index == choice[1] ] = choice[0]
				unique.remove(choice[1])

		return split

	def to_tree(self):
		tree = Tree()
		for node in self.tree_data:
			tree.create_node(node, node.id, parent=node.parent)
		return tree

	def print_tree(self):
		self.to_tree().show()

	def predict(self):
		pred = np.zeros(self.data_size)
		for node in self.tree_data:
			pred[node.terminal_indices] = node.id
		return pred

	def __repr__(self):
		 return str(self.tree_data)

if __name__ == "__main__":
	import argparse
	parser = argparse.ArgumentParser(description='Run the chaid algorithm on a csv file.')
	parser.add_argument('file')
	parser.add_argument('dependent_variable', nargs=1)
	parser.add_argument('independent_variables', nargs='+')

	parser.add_argument('--max-depth', type=int, help='Max depth of generated tree')
	parser.add_argument('--min-samples', type=int, help='Minimum number of samples required to split node')
	parser.add_argument('--alpha-merge', type=float, help='Alpha Merge')
	nspace = parser.parse_args()

	df = pd.read_csv(nspace.file)
	dep_series = df[nspace.dependent_variable]
	ind_df = df[nspace.independent_variables]

	config = {}
	if nspace.max_depth:
		config['max_depth'] = nspace.max_depth
	if nspace.alpha_merge:
		config['alpha_merge'] = nspace.alpha_merge
	if nspace.min_samples:
		config['min_sample'] = nspace.min_samples
	print CHAID().from_pandas_df(ind_df, dep_series, config)
<commit_msg>Speed boost<commit_after>import itertools as it
import pandas as pd
from scipy import stats
import numpy as np
import collections as cl
from treelib import Node, Tree

class Node(object):
	def __init__(self, choices=None, members={}, split_variable=None, chi=0, p=0, terminal_indices=[], id=0, parent=None):
		self.choices = choices
		self.members = members
		self.split_variable = split_variable
		self.chi = chi
		self.p = p
		self.terminal_indices = terminal_indices
		self.id = id
		self.parent = parent

	def __hash__(self):
		return self.__dict__

	def __eq__(self, other):
		if isinstance(other, self.__class__):
		    return self.__dict__ == other.__dict__
		else:
		    return False

	def __repr__(self):
		return str((self.choices, self.members, self.split_variable, self.chi, self.p))

class Split(object):
	def __init__(self, index, splits, chi, p):
		self.index = index
		self.splits = splits
		self.chi = chi
		self.p = p

class CHAID(object):
	DEFAULT_CONDITIONS =  { 'alpha_merge': .05, 'max_depth': 2, 'min_sample': 30 }

	def from_pandas_df(self, ind_df, dep_series, conditions):
		new_conditions = self.DEFAULT_CONDITIONS.copy()
		new_conditions.update(conditions)
		params = {
			'independent_variable_names': ind_df.columns
		}
		ind_df.apply(lambda x: x.fillna('-1.0', inplace=True) if (x.dtype == object) else x.fillna(-1.0, inplace=True))
		ind_values = ind_df.values
		dep_values = dep_series.values
		self.data_size = dep_values.shape[0]
		self.tree_data, node_id =  self.node(params, np.arange(0, dep_values.shape[0] + 1, dtype=np.int), ind_values, dep_values, new_conditions)
		return self

	def node(self, params, rows, ind, dep, conditions, depth=0, tree_store=[], parent=None, node_id=0, parent_decisions=None):
		depth = depth + 1

		members = dict(np.transpose(np.unique(dep, return_counts=True)))

		if conditions['max_depth'] < depth:
			terminal_node = Node(choices=parent_decisions, members=members, id=node_id, parent=parent, terminal_indices=rows)
			tree_store.append(terminal_node)
			return tree_store, node_id + 1

		split = self.generate_best_split(ind, dep, conditions)

		node = Node(choices=parent_decisions, members=members, id=node_id, parent=parent, split_variable=split.index, chi=split.chi, p=split.p)
		tree_store.append(node)
		parent = node_id
		node_id = node_id + 1

		if split.index is None:
			return tree_store, node_id

		for choices in split.splits:
			correct_rows = np.in1d(ind[:, split.index], choices)
			dep_slice = dep[correct_rows]
			ind_slice = ind[correct_rows, :]
			row_slice = rows[correct_rows]
			if conditions['min_sample'] < len(dep_slice):
				tree_store, node_id = self.node(params, row_slice, ind_slice, dep_slice, conditions, depth, node_id=node_id, parent=parent, tree_store=tree_store, parent_decisions=choices)
			else:
				memebers = dict(np.transpose(np.unique(dep_slice, return_counts=True)))
				terminal_node = Node(choices=choices, members=memebers, id=node_id, parent=parent, terminal_indices=row_slice)
				tree_store.append(terminal_node)
				node_id = node_id + 1

		return tree_store, node_id

	def generate_best_split(self, ind, dep, conditions):
		split = Split(None, None, None, 1)
		for i in range(0, ind.shape[1]):
			index = np.array(ind[:,i])
			unique = set(index)

			mappings = {}
			frequincies = dict([ (col, dict(np.transpose(np.unique(dep[np.where(np.in1d(index, col))[0]], return_counts=True)))) for col in unique])

			while len(unique) > 1:
				size  = (len(unique) * (len(unique) - 1)) / 2
				sub_data = np.ndarray(shape=(size, 3), dtype=object, order='F')
				for j, comb in enumerate(it.combinations(unique, 2)):
					y = frequincies[comb[0]]
					g = frequincies[comb[1]]

					keys = set(y.keys() + g.keys())

					cr_table = [
						[ y.get(k, 0) for k in keys],
						[ g.get(k, 0) for k in keys]
					]

					chi = stats.chi2_contingency(np.array(cr_table))
					sub_data[j] = (comb, chi[0], chi[1])

				highest_p_split = np.sort(sub_data[:, 2])[-1]
				correct_row = np.where(np.in1d(sub_data[:, 2], highest_p_split))[0][0]

				if size == 1 or highest_p_split < conditions['alpha_merge']:
					if highest_p_split < split.p:
						split = Split(i, [ mappings.get(x, [x]) for x in unique ], sub_data[correct_row][1], highest_p_split)
					break

				choice = list(sub_data[correct_row, 0])

				if choice[1] in mappings:
					mappings[choice[0]] =  mappings.get(choice[0], [choice[0]]) + mappings[choice[1]]
					del mappings[choice[1]]
				else:
					mappings[choice[0]] = mappings.get(choice[0], [choice[0]]) + [choice[1]]

				index[ index == choice[1] ] = choice[0]
				unique.remove(choice[1])

				for val, count in frequincies[choice[1]].items():
					frequincies[choice[0]][val] = frequincies[choice[0]].get(val, 0) + count
				del frequincies[choice[1]]

		return split

	def to_tree(self):
		tree = Tree()
		for node in self.tree_data:
			tree.create_node(node, node.id, parent=node.parent)
		return tree

	def print_tree(self):
		self.to_tree().show()

	def predict(self):
		pred = np.zeros(self.data_size)
		for node in self.tree_data:
			pred[node.terminal_indices] = node.id
		return pred

	def __repr__(self):
		 return str(self.tree_data)

if __name__ == "__main__":
	import argparse
	parser = argparse.ArgumentParser(description='Run the chaid algorithm on a csv file.')
	parser.add_argument('file')
	parser.add_argument('dependent_variable', nargs=1)
	parser.add_argument('independent_variables', nargs='+')

	parser.add_argument('--max-depth', type=int, help='Max depth of generated tree')
	parser.add_argument('--min-samples', type=int, help='Minimum number of samples required to split node')
	parser.add_argument('--alpha-merge', type=float, help='Alpha Merge')
	nspace = parser.parse_args()

	df = pd.read_csv(nspace.file)
	dep_series = df[nspace.dependent_variable]
	ind_df = df[nspace.independent_variables]

	config = {}
	if nspace.max_depth:
		config['max_depth'] = nspace.max_depth
	if nspace.alpha_merge:
		config['alpha_merge'] = nspace.alpha_merge
	if nspace.min_samples:
		config['min_sample'] = nspace.min_samples
	print CHAID().from_pandas_df(ind_df, dep_series, config)
<|endoftext|>


<commit_before># Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import datetime
from app.exceptions import InvalidIdentifierException

from data.utils import populate_models
from data.models.base import Base, ma
from data.models.user import User
from sqlalchemy import Boolean, Column, Integer, String, Text, ForeignKey, Index, DateTime
from sqlalchemy.dialects.mysql import LONGTEXT
from sqlalchemy.orm import relationship
from sqlalchemy_fulltext import FullText
from lib.vcs_management import getVcsHandler

import marshmallow

class RevisionMixin:
  revision = Column(Integer, nullable=False, default=0)
  active = Column(Boolean, nullable=False, default=True)
  archived_at = Column(DateTime)

  def archive(self):
    self.active = False
    self.archived_at = datetime.datetime.utcnow()

class MarshmallowBase(ma.ModelSchema):

  __abstract__ = True

  class Meta:
    exclude = ('id', 'date_created', 'date_modified')

class VulnerabilityResources(Base):
  __table_args__ = {'schema': 'main'}
  link = Column(String(1000), nullable=False)
  vulnerability_details_id = Column(Integer,
                                    ForeignKey('main.vulnerability.id'))

class CreatorSchema(MarshmallowBase):

  class Meta:
    model = User

class RepositoryFileMarkers(RevisionMixin, Base):
  __tablename__ = 'repository_file_markers'
  row_from = Column(Integer)
  row_to = Column(Integer)
  column_from = Column(Integer)
  column_to = Column(Integer)
  marker_class = Column(String(100), nullable=False)
  repository_file_id = Column(Integer, ForeignKey('repository_files.id'))
  creator_id = Column(Integer, ForeignKey(User.id), nullable=True)
  creator = relationship(User)

class RepositoryFileMarkersSchema(MarshmallowBase):

  class Meta(MarshmallowBase.Meta):
    model = RepositoryFileMarkers

  creator = marshmallow.fields.Nested(CreatorSchema, only=['name'])

class RepositoryFileComments(RevisionMixin, Base):
  __tablename__ = 'repository_file_comments'
  row_from = Column(Integer)
  row_to = Column(Integer)
  text = Column(Text, nullable=False)
  sort_pos = Column(Integer)
  repository_file_id = Column(Integer, ForeignKey('repository_files.id'))
  creator_id = Column(Integer, ForeignKey(User.id), nullable=True)
  creator = relationship(User)

class RepositoryFileCommentsSchema(MarshmallowBase):

  class Meta(MarshmallowBase.Meta):
    model = RepositoryFileComments
    exclude = ['archived_at', 'active']

  creator = marshmallow.fields.Nested(CreatorSchema, only=['name'])

class RepositoryFiles(Base):
  __tablename__ = 'repository_files'
  file_name = Column(String(1000), nullable=False)
  file_path = Column(String(1000), nullable=False)
  file_hash = Column(String(1000), nullable=False)
  # A cached version of all file changes for the given commit.
  file_patch = Column(Text, nullable=False)

  markers = relationship(
      RepositoryFileMarkers,
      backref='repository_file',
      cascade='all, delete-orphan',
      primaryjoin='and_(RepositoryFiles.id==RepositoryFileMarkers.repository_file_id, RepositoryFileMarkers.active==True)',
      single_parent=True)

  comments = relationship(
      RepositoryFileComments,
      backref='repository_file',
      cascade='all, delete-orphan',
      primaryjoin='and_(RepositoryFiles.id==RepositoryFileComments.repository_file_id, RepositoryFileComments.active==True)',
      single_parent=True)
  commit_id = Column(Integer, ForeignKey('vulnerability_git_commits.id'))

class RepositoryFilesSchema(MarshmallowBase):
  # TODO: Add exlude=[] parameter here to skip redundant date and id fields.
  file_patch = ma.Method('get_patch')

  def get_patch(self, obj):
    return 'DEPRECATED'

  markers = ma.Nested(RepositoryFileMarkersSchema, many=True)
  comments = ma.Nested(RepositoryFileCommentsSchema, many=True)

  class Meta(MarshmallowBase.Meta):
    model = RepositoryFiles

class VulnerabilityGitCommits(Base):
  __tablename__ = 'vulnerability_git_commits'

  commit_hash = Column(String(1000), nullable=False)
  _commit_link = Column('commit_link', String(1000), nullable=False)
  repo_name = Column(String(1000), nullable=False)
  repo_owner = Column(String(1000))
  # URL to a *.git Git repository (if applicable).
  _repo_url = Column('repo_url', String(1000))
  vulnerability_details_id = Column(Integer,
                                    ForeignKey('main.vulnerability.id'))
  # Used to store/cache the repository tree files with hashes.
  tree_cache = Column(LONGTEXT())

  repository_files = relationship(
      RepositoryFiles,
      backref='commit',
      cascade='all, delete-orphan',
      single_parent=True,
  )
  # link to comments through RepositoryFiles
  comments = relationship(
      RepositoryFileComments,
      backref='commit',
      secondary=RepositoryFiles.__table__,
      primaryjoin='VulnerabilityGitCommits.id==RepositoryFiles.commit_id',
      secondaryjoin='and_(RepositoryFiles.id==RepositoryFileComments.repository_file_id, RepositoryFileComments.active==True)',
      viewonly=True,
  )
  # link to markers through RepositoryFiles
  markers = relationship(
      RepositoryFileMarkers,
      backref='commit',
      secondary=RepositoryFiles.__table__,
      primaryjoin='VulnerabilityGitCommits.id==RepositoryFiles.commit_id',
      secondaryjoin='and_(RepositoryFiles.id==RepositoryFileMarkers.repository_file_id, RepositoryFileMarkers.active==True)',
      viewonly=True,
  )

  @property
  def num_files(self):
    # TODO: This should be refactored as it is incredibly inefficient.
    #       We should use a count on the database side instead.
    return len(self.repository_files)

  @property
  def num_comments(self):
    # TODO: see comment regarding performance above.
    return len(self.comments)

  @property
  def num_markers(self):
    # TODO: see comment regarding performance above.
    return len(self.markers)

  @property
  def repo_url(self):
    if not self._repo_url:
      # TODO: Refactor this apporach of retrieving github.com urls.
      if self.commit_link and 'github.com' in self.commit_link:
        if self.repo_owner and self.repo_name:
          return 'https://github.com/' + self.repo_owner + '/' + self.repo_name
    return self._repo_url

  @repo_url.setter
  def repo_url(self, repo_url):
    self._repo_url = repo_url

  @property
  def commit_link(self):
    return self._commit_link

  @commit_link.setter
  def commit_link(self, commit_link):
    # TODO: Add commit link sanitization back here. We're currently skipping
    #  it as on object creation (populate) there might be no repo_url set
    #  and the commit_link might be just a VCS UI link to the patch.
    #  We should still always require a separate repository link and commit
    #  hash if it's not a simple Github entry.
    #if not self.repo_url and commit_link:
    # vcs_handler = getVcsHandler(None, commit_link)
    # if not vcs_handler:
    #   raise InvalidIdentifierException('Please provide a valid commit link.')
    if commit_link:
      if not commit_link.startswith('http'):
        raise InvalidIdentifierException('Please provide a valid commit link.')

    self._commit_link = commit_link

  def __init__(self,
               commit_link=None,
               repo_owner=None,
               repo_name=None,
               repo_url=None,
               commit_hash=None):
    self.repo_owner = repo_owner
    self.repo_name = repo_name
    if repo_url:
      vcs_handler = getVcsHandler(None, repo_url)
      if not vcs_handler:
        raise InvalidIdentifierException('Please provide a valid git repo URL.')
      self.repo_url = repo_url
    self.commit_link = commit_link
    self.commit_hash = commit_hash

Index('cl_hash_index', VulnerabilityGitCommits.commit_hash)

# TODO: adjust indices!

class Vulnerability(FullText, Base):
  __fulltext_columns__ = ('comment',)
  __tablename__ = 'vulnerability'
  __table_args__ = {'schema': 'main'}
  comment = Column(Text, nullable=False)
  exploit_exists = Column(Boolean, default=False)
  cve_id = Column(String(255), ForeignKey('cve.nvds.cve_id'), nullable=True)
  creator_id = Column(Integer, ForeignKey(User.id), nullable=True)
  creator = relationship(User)

  resource_links = relationship(
      VulnerabilityResources,
      backref='vulnerability',
      cascade='all, delete-orphan',
      single_parent=True)

  commits = relationship(
      VulnerabilityGitCommits,
      backref='vulnerability',
      cascade='all, delete-orphan',
      single_parent=True)

  nvd = relationship('Nvd', backref='vulnerability', single_parent=True)

  @property
  def master_commit(self):
    # TODO: refactor assumption that the first commit is the "master" one!
    if self.commits:
      return self.commits[0]
    return None

  def __repr__(self):
    return 'Vulnerability Info({:s})'.format(vars(self))

  @classmethod
  def get_by_cve_id(cls, cve_id):
    return cls.query.filter_by(cve_id=cve_id).first()

  @classmethod
  def get_by_commit_hash(cls, commit_hash):
    return cls.query.join(
        Vulnerability.commits,
        aliased=True).filter_by(commit_hash=commit_hash).first()

# must be set after all definitions
__all__ = populate_models(__name__)
<commit_msg>Update vulnerability.py<commit_after># Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import datetime
from app.exceptions import InvalidIdentifierException

from data.utils import populate_models
from data.models.base import Base, ma
from data.models.user import User
from sqlalchemy import Boolean, Column, Integer, String, Text, ForeignKey, Index, DateTime
from sqlalchemy.dialects.mysql import LONGTEXT
from sqlalchemy.orm import relationship
from sqlalchemy_fulltext import FullText
from lib.vcs_management import get_vcs_handler

import marshmallow

class RevisionMixin:
  revision = Column(Integer, nullable=False, default=0)
  active = Column(Boolean, nullable=False, default=True)
  archived_at = Column(DateTime)

  def archive(self):
    self.active = False
    self.archived_at = datetime.datetime.utcnow()

class MarshmallowBase(ma.ModelSchema):

  __abstract__ = True

  class Meta:
    exclude = ('id', 'date_created', 'date_modified')

class VulnerabilityResources(Base):
  __table_args__ = {'schema': 'main'}
  link = Column(String(1000), nullable=False)
  vulnerability_details_id = Column(Integer,
                                    ForeignKey('main.vulnerability.id'))

class CreatorSchema(MarshmallowBase):

  class Meta:
    model = User

class RepositoryFileMarkers(RevisionMixin, Base):
  __tablename__ = 'repository_file_markers'
  row_from = Column(Integer)
  row_to = Column(Integer)
  column_from = Column(Integer)
  column_to = Column(Integer)
  marker_class = Column(String(100), nullable=False)
  repository_file_id = Column(Integer, ForeignKey('repository_files.id'))
  creator_id = Column(Integer, ForeignKey(User.id), nullable=True)
  creator = relationship(User)

class RepositoryFileMarkersSchema(MarshmallowBase):

  class Meta(MarshmallowBase.Meta):
    model = RepositoryFileMarkers

  creator = marshmallow.fields.Nested(CreatorSchema, only=['name'])

class RepositoryFileComments(RevisionMixin, Base):
  __tablename__ = 'repository_file_comments'
  row_from = Column(Integer)
  row_to = Column(Integer)
  text = Column(Text, nullable=False)
  sort_pos = Column(Integer)
  repository_file_id = Column(Integer, ForeignKey('repository_files.id'))
  creator_id = Column(Integer, ForeignKey(User.id), nullable=True)
  creator = relationship(User)

class RepositoryFileCommentsSchema(MarshmallowBase):

  class Meta(MarshmallowBase.Meta):
    model = RepositoryFileComments
    exclude = ['archived_at', 'active']

  creator = marshmallow.fields.Nested(CreatorSchema, only=['name'])

class RepositoryFiles(Base):
  __tablename__ = 'repository_files'
  file_name = Column(String(1000), nullable=False)
  file_path = Column(String(1000), nullable=False)
  file_hash = Column(String(1000), nullable=False)
  # A cached version of all file changes for the given commit.
  file_patch = Column(Text, nullable=False)

  markers = relationship(
      RepositoryFileMarkers,
      backref='repository_file',
      cascade='all, delete-orphan',
      primaryjoin='and_(RepositoryFiles.id==RepositoryFileMarkers.repository_file_id, RepositoryFileMarkers.active==True)',
      single_parent=True)

  comments = relationship(
      RepositoryFileComments,
      backref='repository_file',
      cascade='all, delete-orphan',
      primaryjoin='and_(RepositoryFiles.id==RepositoryFileComments.repository_file_id, RepositoryFileComments.active==True)',
      single_parent=True)
  commit_id = Column(Integer, ForeignKey('vulnerability_git_commits.id'))

class RepositoryFilesSchema(MarshmallowBase):
  # TODO: Add exlude=[] parameter here to skip redundant date and id fields.
  file_patch = ma.Method('get_patch')

  def get_patch(self, obj):
    return 'DEPRECATED'

  markers = ma.Nested(RepositoryFileMarkersSchema, many=True)
  comments = ma.Nested(RepositoryFileCommentsSchema, many=True)

  class Meta(MarshmallowBase.Meta):
    model = RepositoryFiles

class VulnerabilityGitCommits(Base):
  __tablename__ = 'vulnerability_git_commits'

  commit_hash = Column(String(1000), nullable=False)
  _commit_link = Column('commit_link', String(1000), nullable=False)
  repo_name = Column(String(1000), nullable=False)
  repo_owner = Column(String(1000))
  # URL to a *.git Git repository (if applicable).
  _repo_url = Column('repo_url', String(1000))
  vulnerability_details_id = Column(Integer,
                                    ForeignKey('main.vulnerability.id'))
  # Used to store/cache the repository tree files with hashes.
  tree_cache = Column(LONGTEXT())

  repository_files = relationship(
      RepositoryFiles,
      backref='commit',
      cascade='all, delete-orphan',
      single_parent=True,
  )
  # link to comments through RepositoryFiles
  comments = relationship(
      RepositoryFileComments,
      backref='commit',
      secondary=RepositoryFiles.__table__,
      primaryjoin='VulnerabilityGitCommits.id==RepositoryFiles.commit_id',
      secondaryjoin='and_(RepositoryFiles.id==RepositoryFileComments.repository_file_id, RepositoryFileComments.active==True)',
      viewonly=True,
  )
  # link to markers through RepositoryFiles
  markers = relationship(
      RepositoryFileMarkers,
      backref='commit',
      secondary=RepositoryFiles.__table__,
      primaryjoin='VulnerabilityGitCommits.id==RepositoryFiles.commit_id',
      secondaryjoin='and_(RepositoryFiles.id==RepositoryFileMarkers.repository_file_id, RepositoryFileMarkers.active==True)',
      viewonly=True,
  )

  @property
  def num_files(self):
    # TODO: This should be refactored as it is incredibly inefficient.
    #       We should use a count on the database side instead.
    return len(self.repository_files)

  @property
  def num_comments(self):
    # TODO: see comment regarding performance above.
    return len(self.comments)

  @property
  def num_markers(self):
    # TODO: see comment regarding performance above.
    return len(self.markers)

  @property
  def repo_url(self):
    if not self._repo_url:
      # TODO: Refactor this apporach of retrieving github.com urls.
      if self.commit_link and 'github.com' in self.commit_link:
        if self.repo_owner and self.repo_name:
          return 'https://github.com/' + self.repo_owner + '/' + self.repo_name
    return self._repo_url

  @repo_url.setter
  def repo_url(self, repo_url):
    self._repo_url = repo_url

  @property
  def commit_link(self):
    return self._commit_link

  @commit_link.setter
  def commit_link(self, commit_link):
    # TODO: Add commit link sanitization back here. We're currently skipping
    #  it as on object creation (populate) there might be no repo_url set
    #  and the commit_link might be just a VCS UI link to the patch.
    #  We should still always require a separate repository link and commit
    #  hash if it's not a simple Github entry.
    #if not self.repo_url and commit_link:
    # vcs_handler = get_vcs_handler(None, commit_link)
    # if not vcs_handler:
    #   raise InvalidIdentifierException('Please provide a valid commit link.')
    if commit_link:
      if not commit_link.startswith('http'):
        raise InvalidIdentifierException('Please provide a valid commit link.')

    self._commit_link = commit_link

  def __init__(self,
               commit_link=None,
               repo_owner=None,
               repo_name=None,
               repo_url=None,
               commit_hash=None):
    self.repo_owner = repo_owner
    self.repo_name = repo_name
    if repo_url:
      vcs_handler = get_vcs_handler(None, repo_url)
      if not vcs_handler:
        raise InvalidIdentifierException('Please provide a valid git repo URL.')
      self.repo_url = repo_url
    self.commit_link = commit_link
    self.commit_hash = commit_hash

Index('cl_hash_index', VulnerabilityGitCommits.commit_hash)

# TODO: adjust indices!

class Vulnerability(FullText, Base):
  __fulltext_columns__ = ('comment',)
  __tablename__ = 'vulnerability'
  __table_args__ = {'schema': 'main'}
  comment = Column(Text, nullable=False)
  exploit_exists = Column(Boolean, default=False)
  cve_id = Column(String(255), ForeignKey('cve.nvds.cve_id'), nullable=True)
  creator_id = Column(Integer, ForeignKey(User.id), nullable=True)
  creator = relationship(User)

  resource_links = relationship(
      VulnerabilityResources,
      backref='vulnerability',
      cascade='all, delete-orphan',
      single_parent=True)

  commits = relationship(
      VulnerabilityGitCommits,
      backref='vulnerability',
      cascade='all, delete-orphan',
      single_parent=True)

  nvd = relationship('Nvd', backref='vulnerability', single_parent=True)

  @property
  def master_commit(self):
    # TODO: refactor assumption that the first commit is the "master" one!
    if self.commits:
      return self.commits[0]
    return None

  def __repr__(self):
    return 'Vulnerability Info({:s})'.format(vars(self))

  @classmethod
  def get_by_cve_id(cls, cve_id):
    return cls.query.filter_by(cve_id=cve_id).first()

  @classmethod
  def get_by_commit_hash(cls, commit_hash):
    return cls.query.join(
        Vulnerability.commits,
        aliased=True).filter_by(commit_hash=commit_hash).first()

# must be set after all definitions
__all__ = populate_models(__name__)
<|endoftext|>


<commit_before>from django.contrib import admin
from lingcod.simplefaq.models import Faq, FaqGroup

admin.site.register(Faq)
admin.site.register(FaqGroup)<commit_msg>usable admin for simplefaq<commit_after>from django.contrib import admin
from lingcod.simplefaq.models import Faq, FaqGroup

class FaqAdmin(admin.ModelAdmin):
    search_fields = ('question', 'answer')
    list_display = ('question','importance','faq_group')

admin.site.register(Faq,FaqAdmin)
admin.site.register(FaqGroup)<|endoftext|>


<commit_before>import elasticsearch
import json
import zeit.cms.interfaces
import zeit.cms.tagging.interfaces
import zeit.retresco.interfaces
import zope.interface

@zope.interface.implementer(zeit.retresco.interfaces.IElasticsearch)
class Elasticsearch(object):
    """Search via Elasticsearch."""

    def __init__(self, url):
        self.client = elasticsearch.Elasticsearch([url])

    def search(self, query, sort_order, start=0, rows=25):
        """Search using `query` and sort by `sort_order`."""
        query = query.copy()
        query['_source'] = 'url'
        __traceback_info__ = query
        response = self.client.search(
            # FIXME index must be configured via product config
            # FIXME doctype = documents
            index='zeit_pool', body=json.dumps(query), sort=sort_order,
            from_=start, size=rows)
        result = zeit.cms.tagging.interfaces.Result(
            {'uniqueId': self._path_to_url(x['_source']['url'])}
            for x in response['hits']['hits'])
        result.hits = response['hits']['total']
        return result

    def _path_to_url(self, path):
        return path.replace('/', zeit.cms.interfaces.ID_NAMESPACE, 1)

@zope.interface.implementer(zeit.retresco.interfaces.IElasticsearch)
def from_product_config():
    """Get the utility configured with data from the product config."""
    config = zope.app.appsetup.product.getProductConfiguration('zeit.retresco')
    return Elasticsearch(config['elasticsearch-url'])
<commit_msg>ZON-3228: Restrict query to documents only.<commit_after>import elasticsearch
import json
import zeit.cms.interfaces
import zeit.cms.tagging.interfaces
import zeit.retresco.interfaces
import zope.interface

@zope.interface.implementer(zeit.retresco.interfaces.IElasticsearch)
class Elasticsearch(object):
    """Search via Elasticsearch."""

    def __init__(self, url):
        self.client = elasticsearch.Elasticsearch([url])

    def search(self, query, sort_order, start=0, rows=25):
        """Search using `query` and sort by `sort_order`."""
        query = query.copy()
        query['_source'] = 'url'
        __traceback_info__ = query
        response = self.client.search(
            # FIXME index must be configured via product config
            index='zeit_pool', body=json.dumps(query), sort=sort_order,
            from_=start, size=rows, doc_type='documents')
        result = zeit.cms.tagging.interfaces.Result(
            {'uniqueId': self._path_to_url(x['_source']['url'])}
            for x in response['hits']['hits'])
        result.hits = response['hits']['total']
        return result

    def _path_to_url(self, path):
        return path.replace('/', zeit.cms.interfaces.ID_NAMESPACE, 1)

@zope.interface.implementer(zeit.retresco.interfaces.IElasticsearch)
def from_product_config():
    """Get the utility configured with data from the product config."""
    config = zope.app.appsetup.product.getProductConfiguration('zeit.retresco')
    return Elasticsearch(config['elasticsearch-url'])
<|endoftext|>


<commit_before>#!/usr/bin/env python2
"""
Drives an Arduino to control a fermenter.
"""

import time
from datetime import datetime
from array import array
import numpy as np
from Arduino import Arduino
import threading
from threading import Thread
import signal
import sys

###############################################################################
# PARAMETERS
###############################################################################
# Pins
SENSOR_PINS = {
    "button": 13,
    "phototransistor": 14,
    "thermometer": 19,
}
ACTUATOR_PINS = {
    "impeller motor": 10,
    "heater fan": 3,
    "heater": 5,
    "red led": 9,
    "green led": 7,
}

# Light sensing
LOW_PASS_FILTER_TAU = 0.016 # (s): the RC constant of the low pass filter
STEADY_STATE_TAUS = 30 # number of taus to wait to reach steady state
FILTER_STEADY_STATE_TIME = LOW_PASS_FILTER_TAU * STEADY_STATE_TAUS
LIGHT_MEASUREMENT_INTERVAL = 20 # (sec): time to wait between measurements

# Light noise filtering
LIGHT_SAMPLES_PER_ACQUISITION = 10
LIGHT_SAMPLE_INTERVAL = 0.1 # (sec): time to wait between each sampling
LIGHT_OUTLIER_THRESHOLD = 50 # maximum allowed deviation from median
LIGHT_ACQUISITIONS_PER_MEASUREMENT = 10 # robustness to ambient light variation

# Temperature noise filtering
TEMP_SAMPLES_PER_ACQUISITION = 10
TEMP_SAMPLE_INTERVAL = 0.1 # (sec): time to wait between each sampling
TEMP_OUTLIER_THRESHOLD = 0.5 # (deg C): maximum allowed deviation from median

# Temperature control
HEAT_LOSS = 9.3 # (Watts): rate of heat loss at 37 deg C
MAX_HEATING = 14.9 # (Watts): rate at which TEC supply heat
SETPOINT = 37.5 # (deg C): target temperature to maintain
HEATER_SETPOINT_DUTY = HEAT_LOSS / MAX_HEATING # stable duty cycle at setpoint
GAIN = HEATER_SETPOINT_DUTY - 1 # proportional gain
TEMP_MEASUREMENT_INTERVAL = 5 # (sec): time to wait between measurements

# Button pressing
BUTTON_CHECK_INTERVAL = 0.5 # (sec): time to wait between checks of button

# Impeller
IMPELLER_DEFAULT_DUTY = 0.2 # default duty cycle of the impeller

# Constants
SERIAL_RATE = "115200" # (baud) rate of USB communication
PWM_MAX = 255
ARDUINO_PORT = "/dev/ttyACM1"

###############################################################################
# STATELESS FUNCTIONS
###############################################################################
def duty_cycle_to_pin_val(duty_cycle):
    """Return the pin value corresponding to a given duty cycle"""
    return round(duty_cycle * PWM_MAX)
def pin_val_to_temp(pin_value):
    """Return the deg C temperature represented by the input pin value."""
    return 0.174 * pin_value + 0.764
def temp_to_heating_control_effort(temp):
    """Return the duty cycle needed to reach the setpoint temperature."""
    raw_duty = HEATER_SETPOINT_DUTY + GAIN * (temp - SETPOINT)
    return max(0, min(1, raw_duty))
def discard_temp_outliers(samples):
    """Filter outliers out from a set of temperature samples.
    Assumes that the set of temperature samples is acquired when fluid and
    sensor are at quasi-steady state, and thus wild fluctuations in temperature
    are physically nonsensical.

    Arguments:
        data: a Numpy array of temperature measurements, in deg C.
        threshold: the maximum difference in temperature from the median for a
            given sample to be considered a valid sample.
    """
    distances = np.abs(samples - np.median(samples))
    return samples[distances <= TEMP_OUTLIER_THRESHOLD]
def discard_light_outliers(samples):
    """Filter outliers out from a set of light samples.
    Assumes that the set of light samples is acquired when fluid and
    phototransistor are at quasi-steady state, and thus wild fluctuations in
    light correspond to ambient noise < 10 Hz.

    Arguments:
        data: a Numpy array of temperature measurements, in deg C.
        threshold: the maximum difference in light from the median for a
            given sample to be considered a valid sample.
    """
    distances = np.abs(samples - np.median(samples))
    return samples[distances <= LIGHT_OUTLIER_THRESHOLD]
def transmittance_to_absorbance(transmittance, full_transmittance):
    """Normalizes a transmittance against a calibration value."""
    return float(full_transmittance - transmittance) / full_transmittance

###############################################################################
# ARDUINO SUBROUTINES
###############################################################################
def connect():
    """Initializes a connection to the Arduino"""
    return Arduino(SERIAL_RATE, port=ARDUINO_PORT)
def set_pin_modes(a):
    """Initializes pin modes for all pins"""
    for pin in SENSOR_PINS.values():
        a.pinMode(pin, "INPUT")
    for pin in ACTUATOR_PINS.values():
        a.pinMode(pin, "OUTPUT")
def turn_off_actuators(a):
    """Turns off all actuators"""
    for pin in ACTUATOR_PINS.values():
        a.digitalWrite(pin, "LOW")
def turn_off_leds(a):
    """Turns off all LEDs"""
    a.digitalWrite(ACTUATOR_PINS["red led"], "LOW")
    a.digitalWrite(ACTUATOR_PINS["green led"], "LOW")
def initialize_default_actuators(a):
    """Turns on open-loop actuators to default states"""
    a.digitalWrite(ACTUATOR_PINS["heater fan"], "HIGH")
    a.analogWrite(ACTUATOR_PINS["impeller motor"],
            duty_cycle_to_pin_val(IMPELLER_DEFAULT_DUTY))

###############################################################################
# DATA ACQUISITION & PROCESSING
###############################################################################
def acquire_pin(a, pin, nsamples, sample_interval):
    """Acquires a pin value as sampled over a time interval.
    Returns as a Numpy array.

    Arguments:
        pin: the pin from which to read
        nsamples: the number of samples to acquire
        sample_interval: the time to wait between samples
    """
    samples = array('I')
    for i in range(nsamples):
        if i != 0:
            time.sleep(sample_interval)
        samples.append(a.analogRead(pin))
    return np.array(samples)
def acquire_temp(a):
    """Returns the temperature as sampled over a short time interval.
    Discards outliers and returns the mean of the remaining samples.
    Temperature is returned as a pin value.
    """
    samples = acquire_pin(a, SENSOR_PINS["thermometer"],
            TEMP_SAMPLES_PER_ACQUISITION, TEMP_SAMPLE_INTERVAL)
    return np.mean(discard_temp_outliers(samples))
def acquire_light(a, color):
    """Returns the light intensity as sampled over a short time interval.
    Light intensity is returned as an absolute pin value.
    Turns off the green LED and turns on the LED and waits for filter response
    before sampling.
    Discards outliers and returns the mean of the remaining samples.

    Arguments:
        color: should be either "red", "green", or "ambient"
    """
    turn_off_leds(a)
    if color == "red":
        a.digitalWrite(ACTUATOR_PINS["red led"], "HIGH")
    elif color == "green":
        a.digitalWrite(ACTUATOR_PINS["green led"], "HIGH")
    time.sleep(FILTER_STEADY_STATE_TIME)
    samples = acquire_pin(a, SENSOR_PINS["phototransistor"],
            LIGHT_SAMPLES_PER_ACQUISITION, LIGHT_SAMPLE_INTERVAL)
    turn_off_leds(a)
    return np.mean(discard_light_outliers(samples))
def measure_temp(a):
    """Returns the temperature as measured over a short time.
    Temperature is returned in deg C.
    """
    return pin_val_to_temp(acquire_temp(a))
def measure_transmittances(a):
    """Returns normalized light intensities as acquired over an extended time.
    Light intensity is normalized to the ambient light.
    Acquisition of different colors is time multiplexed.
    Returns as a tuple of ambient light, red transmittance, and green
    transmittance.
    """
    acquisitions = {
        "red": array('f'),
        "ambient": array('f'),
        "green": array('f'),
    }
    for _ in range(LIGHT_ACQUISITIONS_PER_MEASUREMENT):
        for color in acquisitions.keys():
            acquisitions[color].append(acquire_light(a, color))
    ambient = np.mean(discard_light_outliers(acquisitions["ambient"]))
    red = np.mean(discard_light_outliers(acquisitions["red"]))
    green = np.mean(discard_light_outliers(acquisitions["green"]))
    return (ambient, ambient - red, ambient - green)

###############################################################################
# DATA LOGGING
###############################################################################
def record_heat_control(a):
    """Returns a heat control record.
    Also adjusts the heating control effort and records that.
    """
    temp = measure_temp(a)
    heater_duty_cycle = temp_to_heating_control_effort(temp)
    end_time = datetime.now()
    return (end_time, temp, heater_duty_cycle)
def record_transmittances(a):
    """Returns a transmittances record."""
    (ambient, red, green) = measure_transmittances(a)
    end_time = datetime.now()
    return (end_time, ambient, red, green)
def construct_records():
    """Returns an empty records dictionary."""
    records = {
        "start": datetime.now(),
        "stop": None,
        "temp": [],
        "heater": [],
        "impeller": [],
        "optics": {
            "calibration": {
                "red": None,
                "green": None,
            },
            "calibrations": [],
            "ambient": [],
            "red": [],
            "green": [],
        },
    }
    return records
def reinitialize_records(records):
    """Clears everything in records.
    Sets the start time of to be the current time.
    """
    records["start"] = datetime.now()
    records["stop"] = None
    records["temp"][:] = []
    records["heater"][:] = []
    records["impeller"][:] = []
    records["optics"]["calibration"]["red"] = None
    records["optics"]["calibration"]["green"] = None
    records["optics"]["calibrations"][:] = []
    records["optics"]["ambient"][:] = []
    records["optics"]["red"][:] = []

###############################################################################
# THREADS
###############################################################################
def construct_locks():
    """Returns an initial locks dictionary."""
    locks = {
        "records": threading.Lock(),
        "impeller motor": threading.Lock(),
        "heater": threading.Lock(),
        "leds": threading.Lock(),
    }
    return locks
def construct_events():
    """Returns an initial events dictionary with events in initial states."""
    events = {
        "fermenter idle": threading.Event(),
        "button pressed": threading.Event(),
        "calibrate": threading.Event(),
    }
    events["fermenter idle"].set()
    events["calibrate"].set()
    return events
def start_fermenter(a, records, locks, idle_event):
    """Starts fermenter operation."""
    idle_event.clear()
    with locks["records"]:
        reinitialize_records(records)
        with locks["impeller motor"]:
            records["impeller"].append((datetime.now(), IMPELLER_DEFAULT_DUTY))
            initialize_default_actuators(a)
def stop_fermenter(a, records, locks, idle_event):
    """Stops fermenter operation."""
    idle_event.set()
    with locks["leds"]:
        turn_off_leds(a)
    with locks["records"]:
        records["impeller"].append((datetime.now(),
                                    records["impeller"][-1][1]))
        records["heater"].append((datetime.now(), records["heater"][-1][1]))
        with locks["impeller motor"] and locks["heater"]:
            turn_off_actuators(a)
            records["impeller"].append((datetime.now(), 0))
            records["heater"].append((datetime.now(), 0))
        records["stop"] = datetime.now()
def monitor_temp(a, records, locks, idle_event):
    """Continuously monitor and record fluid temperature and heater state.
    Also adjust heater based on temperature control information.
    """
    while True:
        if not idle_event.is_set():
            record = record_heat_control(a)
            with locks["heater"]:
                a.analogWrite(ACTUATOR_PINS["heater"],
                              duty_cycle_to_pin_val(record[2]))
            with locks["records"]:
                records["temp"].append((record[0], record[1]))
                records["heater"].append((record[0], record[2]))
            idle_event.wait(TEMP_MEASUREMENT_INTERVAL)
        else:
            time.sleep(BUTTON_CHECK_INTERVAL)
def monitor_optics(a, records, locks, calibrate_event, idle_event):
    """Continuously monitor and record fluid optical properties"""
    while True:
        if not idle_event.is_set():
            with locks["leds"]:
                record = record_transmittances(a)
            with locks["records"]:
                if calibrate_event.is_set():
                    records["calibration"].append((record[0], record[2],
                                                   record[3]))
                    calibrate_event.clear()
                records["ambient"].append((record[0], record[1]))
                records["red"].append((record[0], record[2]))
                records["green"].append((record[0], record[3]))
            idle_event.wait(LIGHT_MEASUREMENT_INTERVAL)
        else:
            time.sleep(BUTTON_CHECK_INTERVAL)
def monitor_button(a, records, locks, idle_event, pressed_event):
    """Continuously monitor power button state"""
    while True:
        button_state = a.digitalRead(SENSOR_PINS["button"])
        if button_state:
            if not pressed_event.is_set():
                if not idle_event.is_set():
                    stop_fermenter(a, records, locks, idle_event)
                    idle_event.set()
                else:
                    start_fermenter(a, records, locks, idle_event)
                    idle_event.clear()
                pressed_event.set()
        else:
            if pressed_event.is_set():
                pressed_event.clear()
        time.sleep(BUTTON_CHECK_INTERVAL)

###############################################################################
# MAIN
###############################################################################
def interrupt_handler(signal_num, _):
    sys.exit(signal_num)
def run_fermenter():
    a = connect()
    set_pin_modes(a)
    signal.signal(signal.SIGINT, interrupt_handler)
    turn_off_actuators(a)
    turn_off_leds(a)
    records = construct_records()
    locks = construct_locks()
    events = construct_events()
    button_monitor = Thread(target=monitor_button, name="button",
                            args=(a, records, locks, events["fermenter idle"],
                                  events["button pressed"]))
    temp_monitor = Thread(target=monitor_temp, name="temp",
                          args=(a, records, locks, events["fermenter idle"]))
    optics_monitor = Thread(target=monitor_optics, name="optics",
                            args=(a, records, locks, events["calibrate"],
                                  events["fermenter idle"]))
    threads = {
        "button": button_monitor,
        "temp": temp_monitor,
        "optics": optics_monitor,
    }
    threads["temp"].daemon = True
    threads["optics"].daemon = True
    threads["button"].daemon = True
    threads["temp"].start()
    threads["optics"].start()
    threads["button"].start()
    return (records, locks, events, threads)

if __name__ == "__main__":
    (records, locks, events, threads) = run_fermenter()
    for thread in threads.values():
        signal.pause()
<commit_msg>Fixed some bugs.<commit_after>#!/usr/bin/env python2
"""
Drives an Arduino to control a fermenter.
"""

import time
from datetime import datetime
from array import array
import numpy as np
from Arduino import Arduino
import threading
from threading import Thread
import signal
import sys

###############################################################################
# PARAMETERS
###############################################################################
# Pins
SENSOR_PINS = {
    "button": 13,
    "phototransistor": 14,
    "thermometer": 19,
}
ACTUATOR_PINS = {
    "impeller motor": 10,
    "heater fan": 3,
    "heater": 5,
    "red led": 9,
    "green led": 7,
}

# Light sensing
LOW_PASS_FILTER_TAU = 0.016 # (s): the RC constant of the low pass filter
STEADY_STATE_TAUS = 30 # number of taus to wait to reach steady state
FILTER_STEADY_STATE_TIME = LOW_PASS_FILTER_TAU * STEADY_STATE_TAUS
LIGHT_MEASUREMENT_INTERVAL = 20 # (sec): time to wait between measurements

# Light noise filtering
LIGHT_SAMPLES_PER_ACQUISITION = 10
LIGHT_SAMPLE_INTERVAL = 0.1 # (sec): time to wait between each sampling
LIGHT_OUTLIER_THRESHOLD = 50 # maximum allowed deviation from median
LIGHT_ACQUISITIONS_PER_MEASUREMENT = 10 # robustness to ambient light variation

# Temperature noise filtering
TEMP_SAMPLES_PER_ACQUISITION = 10
TEMP_SAMPLE_INTERVAL = 0.5 # (sec): time to wait between each sampling
TEMP_OUTLIER_THRESHOLD = 20 # (deg C): maximum allowed deviation from median

# Temperature control
HEAT_LOSS = 9.3 # (Watts): rate of heat loss at 37 deg C
MAX_HEATING = 14.9 # (Watts): rate at which TEC supply heat
SETPOINT = 37.5 # (deg C): target temperature to maintain
HEATER_SETPOINT_DUTY = HEAT_LOSS / MAX_HEATING # stable duty cycle at setpoint
GAIN = HEATER_SETPOINT_DUTY - 1 # proportional gain
TEMP_MEASUREMENT_INTERVAL = 1 # (sec): time to wait between measurements

# Button pressing
BUTTON_CHECK_INTERVAL = 0.5 # (sec): time to wait between checks of button

# Impeller
IMPELLER_DEFAULT_DUTY = 0.2 # default duty cycle of the impeller

# Constants
SERIAL_RATE = "115200" # (baud) rate of USB communication
PWM_MAX = 255
ARDUINO_PORT = "/dev/ttyACM0"

###############################################################################
# STATELESS FUNCTIONS
###############################################################################
def duty_cycle_to_pin_val(duty_cycle):
    """Return the pin value corresponding to a given duty cycle"""
    return round(duty_cycle * PWM_MAX)
def pin_val_to_temp(pin_value):
    """Return the deg C temperature represented by the input pin value."""
    return 0.174 * pin_value + 0.764
def temp_to_heating_control_effort(temp):
    """Return the duty cycle needed to reach the setpoint temperature."""
    raw_duty = HEATER_SETPOINT_DUTY + GAIN * (temp - SETPOINT)
    return max(0, min(1, raw_duty))
def discard_temp_outliers(samples):
    """Filter outliers out from a set of temperature samples.
    Assumes that the set of temperature samples is acquired when fluid and
    sensor are at quasi-steady state, and thus wild fluctuations in temperature
    are physically nonsensical.

    Arguments:
        data: a Numpy array of temperature measurements, in deg C.
        threshold: the maximum difference in temperature from the median for a
            given sample to be considered a valid sample.
    """
    distances = np.abs(samples - np.median(samples))
    return samples[distances <= TEMP_OUTLIER_THRESHOLD]
def discard_light_outliers(samples):
    """Filter outliers out from a set of light samples.
    Assumes that the set of light samples is acquired when fluid and
    phototransistor are at quasi-steady state, and thus wild fluctuations in
    light correspond to ambient noise < 10 Hz.

    Arguments:
        data: a Numpy array of temperature measurements, in deg C.
        threshold: the maximum difference in light from the median for a
            given sample to be considered a valid sample.
    """
    distances = np.abs(samples - np.median(samples))
    return samples[distances <= LIGHT_OUTLIER_THRESHOLD]
def transmittance_to_absorbance(transmittance, full_transmittance):
    """Normalizes a transmittance against a calibration value."""
    return float(full_transmittance - transmittance) / full_transmittance

###############################################################################
# ARDUINO SUBROUTINES
###############################################################################
def connect():
    """Initializes a connection to the Arduino"""
    return Arduino(SERIAL_RATE, port=ARDUINO_PORT)
def set_pin_modes(a):
    """Initializes pin modes for all pins"""
    for pin in SENSOR_PINS.values():
        a.pinMode(pin, "INPUT")
    for pin in ACTUATOR_PINS.values():
        a.pinMode(pin, "OUTPUT")
def turn_off_actuators(a):
    """Turns off all actuators"""
    for pin in ACTUATOR_PINS.values():
        a.digitalWrite(pin, "LOW")
def turn_off_leds(a):
    """Turns off all LEDs"""
    a.digitalWrite(ACTUATOR_PINS["red led"], "LOW")
    a.digitalWrite(ACTUATOR_PINS["green led"], "LOW")
def initialize_default_actuators(a):
    """Turns on open-loop actuators to default states"""
    a.digitalWrite(ACTUATOR_PINS["heater fan"], "HIGH")
    a.analogWrite(ACTUATOR_PINS["impeller motor"],
            duty_cycle_to_pin_val(IMPELLER_DEFAULT_DUTY))

###############################################################################
# DATA ACQUISITION & PROCESSING
###############################################################################
def acquire_pin(a, pin, nsamples, sample_interval):
    """Acquires a pin value as sampled over a time interval.
    Returns as a Numpy array.

    Arguments:
        pin: the pin from which to read
        nsamples: the number of samples to acquire
        sample_interval: the time to wait between samples
    """
    samples = array('i')
    for i in range(nsamples):
        if i != 0:
            time.sleep(sample_interval)
        samples.append(a.analogRead(pin))
    return np.array(samples)
def acquire_temp(a):
    """Returns the temperature as sampled over a short time interval.
    Discards outliers and returns the mean of the remaining samples.
    Temperature is returned as a pin value.
    """
    samples = acquire_pin(a, SENSOR_PINS["thermometer"],
            TEMP_SAMPLES_PER_ACQUISITION, TEMP_SAMPLE_INTERVAL)
    return np.mean(discard_temp_outliers(samples))
def acquire_light(a, color):
    """Returns the light intensity as sampled over a short time interval.
    Light intensity is returned as an absolute pin value.
    Turns off the green LED and turns on the LED and waits for filter response
    before sampling.
    Discards outliers and returns the mean of the remaining samples.

    Arguments:
        color: should be either "red", "green", or "ambient"
    """
    turn_off_leds(a)
    if color == "red":
        a.digitalWrite(ACTUATOR_PINS["red led"], "HIGH")
    elif color == "green":
        a.digitalWrite(ACTUATOR_PINS["green led"], "HIGH")
    time.sleep(FILTER_STEADY_STATE_TIME)
    samples = acquire_pin(a, SENSOR_PINS["phototransistor"],
            LIGHT_SAMPLES_PER_ACQUISITION, LIGHT_SAMPLE_INTERVAL)
    turn_off_leds(a)
    return np.mean(discard_light_outliers(samples))
def measure_temp(a):
    """Returns the temperature as measured over a short time.
    Temperature is returned in deg C.
    """
    return pin_val_to_temp(acquire_temp(a))
def measure_transmittances(a):
    """Returns normalized light intensities as acquired over an extended time.
    Light intensity is normalized to the ambient light.
    Acquisition of different colors is time multiplexed.
    Returns as a tuple of ambient light, red transmittance, and green
    transmittance.
    """
    acquisitions = {
        "red": array('i'),
        "ambient": array('i'),
        "green": array('i'),
    }
    for _ in range(LIGHT_ACQUISITIONS_PER_MEASUREMENT):
        for color in acquisitions.keys():
            acquisitions[color].append(int(acquire_light(a, color))
    ambient = np.mean(discard_light_outliers(np.array(acquisitions["ambient"])))
    red = np.mean(discard_light_outliers(np.array(acquisitions["red"])))
    green = np.mean(discard_light_outliers(np.array(acquisitions["green"])))
    return (ambient, ambient - red, ambient - green)

###############################################################################
# DATA LOGGING
###############################################################################
def record_heat_control(a):
    """Returns a heat control record.
    Also adjusts the heating control effort and records that.
    """
    temp = measure_temp(a)
    heater_duty_cycle = temp_to_heating_control_effort(temp)
    end_time = datetime.now()
    return (end_time, temp, heater_duty_cycle)
def record_transmittances(a):
    """Returns a transmittances record."""
    (ambient, red, green) = measure_transmittances(a)
    end_time = datetime.now()
    return (end_time, ambient, red, green)
def construct_records():
    """Returns an empty records dictionary."""
    records = {
        "start": datetime.now(),
        "stop": None,
        "temp": [],
        "heater": [],
        "impeller": [],
        "optics": {
            "calibration": {
                "red": None,
                "green": None,
            },
            "calibrations": [],
            "ambient": [],
            "red": [],
            "green": [],
        },
    }
    return records
def reinitialize_records(records):
    """Clears everything in records.
    Sets the start time of to be the current time.
    """
    records["start"] = datetime.now()
    records["stop"] = None
    records["temp"][:] = []
    records["heater"][:] = []
    records["impeller"][:] = []
    records["optics"]["calibration"]["red"] = None
    records["optics"]["calibration"]["green"] = None
    records["optics"]["calibrations"][:] = []
    records["optics"]["ambient"][:] = []
    records["optics"]["red"][:] = []

###############################################################################
# THREADS
###############################################################################
def construct_locks():
    """Returns an initial locks dictionary."""
    locks = {
        "records": threading.Lock(),
        "impeller motor": threading.Lock(),
        "heater": threading.Lock(),
        "leds": threading.Lock(),
    }
    return locks
def construct_events():
    """Returns an initial events dictionary with events in initial states."""
    events = {
        "fermenter idle": threading.Event(),
        "button pressed": threading.Event(),
        "calibrate": threading.Event(),
    }
    events["fermenter idle"].set()
    events["calibrate"].set()
    return events
def start_fermenter(a, records, locks, idle_event):
    """Starts fermenter operation."""
    idle_event.clear()
    with locks["records"]:
        reinitialize_records(records)
        with locks["impeller motor"]:
            records["impeller"].append((datetime.now(), IMPELLER_DEFAULT_DUTY))
            initialize_default_actuators(a)
def stop_fermenter(a, records, locks, idle_event):
    """Stops fermenter operation."""
    idle_event.set()
    with locks["leds"]:
        turn_off_leds(a)
    with locks["records"]:
        records["impeller"].append((datetime.now(),
                                    records["impeller"][-1][1]))
        records["heater"].append((datetime.now(), records["heater"][-1][1]))
        with locks["impeller motor"] and locks["heater"]:
            turn_off_actuators(a)
            records["impeller"].append((datetime.now(), 0))
            records["heater"].append((datetime.now(), 0))
        records["stop"] = datetime.now()
def monitor_temp(a, records, locks, idle_event):
    """Continuously monitor and record fluid temperature and heater state.
    Also adjust heater based on temperature control information.
    """
    while True:
        if not idle_event.is_set():
            record = record_heat_control(a)
            print(record)
            with locks["heater"]:
                a.analogWrite(ACTUATOR_PINS["heater"],
                              duty_cycle_to_pin_val(record[2]))
            with locks["records"]:
                records["temp"].append((record[0], record[1]))
                records["heater"].append((record[0], record[2]))
            idle_event.wait(TEMP_MEASUREMENT_INTERVAL)
        else:
            time.sleep(BUTTON_CHECK_INTERVAL)
def monitor_optics(a, records, locks, calibrate_event, idle_event):
    """Continuously monitor and record fluid optical properties"""
    while True:
        if not idle_event.is_set():
            with locks["leds"]:
                print(record)
                record = record_transmittances(a)
            with locks["records"]:
                if calibrate_event.is_set():
                    records["optics"]["calibrations"].append((record[0],
                                                              record[2],
                                                              record[3]))
                    records["optics"]["calibration"]["red"] = record[2]
                    records["optics"]["calibration"]["green"] = record[3]
                    calibrate_event.clear()
                records["optics"]["ambient"].append((record[0], record[1]))
                records["optics"]["red"].append((record[0], record[2]))
                records["optics"]["green"].append((record[0], record[3]))
            idle_event.wait(LIGHT_MEASUREMENT_INTERVAL)
        else:
            time.sleep(BUTTON_CHECK_INTERVAL)
def monitor_button(a, records, locks, idle_event, pressed_event):
    """Continuously monitor power button state"""
    while True:
        button_state = a.digitalRead(SENSOR_PINS["button"])
        if button_state:
            if not pressed_event.is_set():
                if not idle_event.is_set():
                    stop_fermenter(a, records, locks, idle_event)
                    idle_event.set()
                else:
                    start_fermenter(a, records, locks, idle_event)
                    idle_event.clear()
                pressed_event.set()
        else:
            if pressed_event.is_set():
                pressed_event.clear()
        time.sleep(BUTTON_CHECK_INTERVAL)

###############################################################################
# MAIN
###############################################################################
def interrupt_handler(signal_num, _):
    sys.exit(signal_num)
def run_fermenter():
    a = connect()
    set_pin_modes(a)
    signal.signal(signal.SIGINT, interrupt_handler)
    turn_off_actuators(a)
    turn_off_leds(a)
    records = construct_records()
    locks = construct_locks()
    events = construct_events()
    button_monitor = Thread(target=monitor_button, name="button",
                            args=(a, records, locks, events["fermenter idle"],
                                  events["button pressed"]))
    temp_monitor = Thread(target=monitor_temp, name="temp",
                          args=(a, records, locks, events["fermenter idle"]))
    optics_monitor = Thread(target=monitor_optics, name="optics",
                            args=(a, records, locks, events["calibrate"],
                                  events["fermenter idle"]))
    threads = {
        "button": button_monitor,
        "temp": temp_monitor,
        "optics": optics_monitor,
    }
    threads["temp"].daemon = True
    threads["optics"].daemon = True
    threads["button"].daemon = True
    threads["temp"].start()
    threads["optics"].start()
    threads["button"].start()
    return (records, locks, events, threads)

if __name__ == "__main__":
    (records, locks, events, threads) = run_fermenter()
    for thread in threads.values():
        signal.pause()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2017 KMEE
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from datetime import datetime
from openerp import api, models, fields, _
from openerp.addons.l10n_br_hr_payroll.models.hr_payslip import TIPO_DE_FOLHA,\
    MES_DO_ANO

class PayslipPaymentCreateOrder(models.Model):
    _name = "payslip.payment.order.create"

    tipo_de_folha = fields.Selection(
        selection=TIPO_DE_FOLHA,
        string=u'Tipo de folha',
        default='normal',
    )

    mes_do_ano = fields.Selection(
        selection=MES_DO_ANO,
        string=u'Mês',
        required=True,
        default=datetime.now().month,
    )

    ano = fields.Integer(
        strin="Ano",
        default=datetime.now().year,
        required=True,
    )

    entries = fields.Many2many(
        comodel_name='hr.payslip.line',
        rel='payorder_line_payslip_line_rel',
        column1='pay_id',
        column2='line_id',
        string='Entries'
    )

    @api.model
    def default_get(self, field_list):
        res = super(PayslipPaymentCreateOrder, self).default_get(field_list)
        context = self.env.context
        if ('entries' in field_list and context.get('lines_id') and
                context.get('populate_results')):
            res.update({'entries': context['lines_id']})
        return res

    @api.multi
    def buscar_linhas_holerites(self):
        payslip_obj = self.env['hr.payslip']
        payslip_line_obj = self.env['hr.payslip.line']
        payslip_ids = payslip_obj.search(
            [
                ('tipo_de_folha', '=', self.tipo_de_folha),
                ('mes_do_ano', '=', self.mes_do_ano),
                ('ano', '=', self.ano),
                ('state', '=', 'done')
            ]
        )
        rubricas_obj = self.env['hr.salary.rule']
        rubricas_pagaveis = rubricas_obj.search(
            [
                ('eh_pagavel', '=', True)
            ]
        )
        payslip_line_ids = payslip_line_obj.search(
            [
                ('slip_id', 'in', payslip_ids.ids),
                ('salary_rule_id', 'in', rubricas_pagaveis.ids)
            ]
        )
        context = self.env.context.copy()
        context['lines_id'] = payslip_line_ids.ids
        context['populate_results'] = True
        model_data_obj = self.env['ir.model.data']
        model_datas = model_data_obj.search(
            [('model', '=', 'ir.ui.view'),
             ('name', '=', 'payslip_payment_lines_create_order_view')])
        return {'name': _('Entry Lines'),
                'context': context,
                'view_type': 'form',
                'view_mode': 'form',
                'res_model': 'payslip.payment.order.create',
                'views': [(model_datas[0].res_id, 'form')],
                'type': 'ir.actions.act_window',
                'target': 'new',
                }

    @api.multi
    def _preparar_linha_do_holerite(self, payment, line):
        self.ensure_one()
        _today = fields.Date.context_today(self)
        date_to_pay = False  # no payment date => immediate payment
        state = 'normal'
        communication = 'Holerite: ' + line.display_name or '-'
        amount_currency = line.total
        res = {
            'amount_currency': amount_currency,
            'bank_id':
                line.contract_id.employee_id.bank_account_id.id,
            'order_id': payment.id,
            'partner_id': line.partner_id and line.partner_id.id or False,
            # account banking
            'communication': communication,
            'state': state,
            # end account banking
            'date': date_to_pay,
            'payslip_id': line.slip_id.id,
        }
        return res

    @api.multi
    def create_payment(self):
        if not self.entries:
            return {'type': 'ir.actions.act_window_close'}
        context = self.env.context
        payment_line_obj = self.env['payment.line']
        payment = self.env['payment.order'].browse(context['active_id'])
        # Populate the current payment with new lines:
        for line in self.entries:
            vals = self._preparar_linha_do_holerite(payment, line)
            payment_line_obj.create(vals)
        # Force reload of payment order view as a workaround for lp:1155525
        return {'name': _('Payment Orders'),
                'context': context,
                'view_type': 'form',
                'view_mode': 'form,tree',
                'res_model': 'payment.order',
                'res_id': context['active_id'],
                'type': 'ir.actions.act_window'}
<commit_msg>fixup! [ADD] Funcao para criar as ordens de pagamento<commit_after># -*- coding: utf-8 -*-
# Copyright 2017 KMEE
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from datetime import datetime

from openerp import api, models, fields, _
from openerp.addons.l10n_br_hr_payroll.models.hr_payslip import \
    TIPO_DE_FOLHA, MES_DO_ANO

class PayslipPaymentCreateOrder(models.Model):
    _name = "payslip.payment.order.create"

    tipo_de_folha = fields.Selection(
        selection=TIPO_DE_FOLHA,
        string=u'Tipo de folha',
        default='normal',
    )

    mes_do_ano = fields.Selection(
        selection=MES_DO_ANO,
        string=u'Mês',
        required=True,
        default=datetime.now().month,
    )

    ano = fields.Integer(
        strin="Ano",
        default=datetime.now().year,
        required=True,
    )

    entries = fields.Many2many(
        comodel_name='hr.payslip.line',
        rel='payorder_line_payslip_line_rel',
        column1='pay_id',
        column2='line_id',
        string='Entries'
    )

    @api.model
    def default_get(self, field_list):
        res = super(PayslipPaymentCreateOrder, self).default_get(field_list)
        context = self.env.context
        if ('entries' in field_list and context.get('lines_id') and
                context.get('populate_results')):
            res.update({'entries': context['lines_id']})
        return res

    @api.multi
    def buscar_linhas_holerites(self):
        payslip_obj = self.env['hr.payslip']
        payslip_line_obj = self.env['hr.payslip.line']
        payslip_ids = payslip_obj.search(
            [
                ('tipo_de_folha', '=', self.tipo_de_folha),
                ('mes_do_ano', '=', self.mes_do_ano),
                ('ano', '=', self.ano),
                ('state', '=', 'done')
            ]
        )
        rubricas_obj = self.env['hr.salary.rule']
        rubricas_pagaveis = rubricas_obj.search(
            [
                ('eh_pagavel', '=', True)
            ]
        )
        payslip_line_ids = payslip_line_obj.search(
            [
                ('slip_id', 'in', payslip_ids.ids),
                ('salary_rule_id', 'in', rubricas_pagaveis.ids)
            ]
        )
        context = self.env.context.copy()
        context['lines_id'] = payslip_line_ids.ids
        context['populate_results'] = True
        model_data_obj = self.env['ir.model.data']
        model_datas = model_data_obj.search(
            [('model', '=', 'ir.ui.view'),
             ('name', '=', 'payslip_payment_lines_create_order_view')])
        return {'name': _('Entry Lines'),
                'context': context,
                'view_type': 'form',
                'view_mode': 'form',
                'res_model': 'payslip.payment.order.create',
                'views': [(model_datas[0].res_id, 'form')],
                'type': 'ir.actions.act_window',
                'target': 'new',
                }

    @api.multi
    def _preparar_linha_do_holerite(self, payment, line):
        self.ensure_one()
        date_to_pay = False  # no payment date => immediate payment
        state = 'normal'
        communication = 'Holerite: ' + line.display_name or '-'
        amount_currency = line.total
        res = {
            'amount_currency': amount_currency,
            'bank_id':
                line.contract_id.employee_id.bank_account_id.id,
            'order_id': payment.id,
            'partner_id': line.partner_id and line.partner_id.id or False,
            # account banking
            'communication': communication,
            'state': state,
            # end account banking
            'date': date_to_pay,
            'payslip_id': line.slip_id.id,
        }
        return res

    @api.multi
    def create_payment(self):
        if not self.entries:
            return {'type': 'ir.actions.act_window_close'}
        context = self.env.context
        payment_line_obj = self.env['payment.line']
        payment = self.env['payment.order'].browse(context['active_id'])
        # Populate the current payment with new lines:
        for line in self.entries:
            vals = self._preparar_linha_do_holerite(payment, line)
            payment_line_obj.create(vals)
        # Force reload of payment order view as a workaround for lp:1155525
        return {'name': _('Payment Orders'),
                'context': context,
                'view_type': 'form',
                'view_mode': 'form,tree',
                'res_model': 'payment.order',
                'res_id': context['active_id'],
                'type': 'ir.actions.act_window'}
<|endoftext|>


<commit_before>import pytz
from datetime import time
from zipline.utils.calendars import TradingCalendar

class ForexCalendar(TradingCalendar):
    @property
    def name(self):
        return "forex"

    @property
    def tz(self):
        return pytz.UTC

    @property
    def open_time(self):
        return time(0, 0)

    @property
    def close_time(self):
        return time(23, 59)
<commit_msg>Add weekend close and open times<commit_after>import pytz
import pandas as pd
from datetime import time
from zipline.utils.calendars import TradingCalendar

class ForexCalendar(TradingCalendar):

    NYT_5PM = time(9)

    @property
    def name(self):
        return "forex"

    @property
    def tz(self):
        return pytz.UTC

    @property
    def open_time(self):
        return time(0, 0)

    @property
    def close_time(self):
        return time(23, 59)

    def special_opens_adhoc(self):
        return [
            (self.NYT_5PM, self._sunday_dates())
        ]

    def special_closes_adhoc(self):
        return [
            (self.NYT_5PM, self._friday_dates())
        ]

    def _friday_dates(self):
        return pd.date_range(start=self.schedule.index[0],
                             end=self.schedule.idnex[-1],
                             freq='W-FRI')

    def _sunday_dates(self):
        return pd.date_range(start=self.schedule.index[0],
                             end=self.schedule.idnex[-1],
                             freq='W-SUN')
<|endoftext|>