<commit_before># show_dicomdir.py
"""Example file to show use of read_dicomdir()
"""
from __future__ import print_function
# Copyright (c) 2013 Darcy Mason
# This file is part of pydicom, relased under an MIT-style license.
#    See the file license.txt included with this distribution, also
#    available at https://github.com/darcymason/pydicom
#

import sys
import pydicom
import os.path
from pprint import pprint
# pydicom.debug()

if __name__ == "__main__":
    print("------------------------------------------------------------")
    print("Example program showing DICOMDIR contents, assuming standard")
    print("Patient -> Study -> Series -> Images hierarchy")
    print("------------------------------------------------------------")
    print()
    if len(sys.argv) > 1:
        filepath = sys.argv[1]
        if os.path.isdir(filepath):  # only gave directory, add standard name
            filepath = os.path.join(filepath, "DICOMDIR")
        dcmdir = pydicom.read_dicomdir(filepath)
        base_dir = os.path.dirname(filepath)
    else:
        # Read standard "DICOMDIR" filename from current directory
        dcmdir = pydicom.read_dicomdir()
        base_dir = "."

    for patrec in dcmdir.patient_records:
        print("Patient: {0.PatientID}: {0.PatientsName}".format(patrec))
        studies = patrec.children
        for study in studies:
            print("    Study {0.StudyID}: {0.StudyDate}:"
                  " {0.StudyDescription}".format(study))
            all_series = study.children
            for series in all_series:
                image_count = len(series.children)
                plural = ('', 's')[image_count > 1]

                # Write basic series info and image count

                # Put N/A in if no Series Description
                if 'SeriesDescription' not in series:
                    series.SeriesDescription = "N/A"
                print(" " * 8 + "Series {0.SeriesNumber}:  {0.Modality}: {0.SeriesDescription}"
                      " ({1} image{2})".format(series, image_count, plural))

                # Open and read something from each image, for demonstration purposes
                # For simple quick overview of DICOMDIR, leave the following out
                print(" " * 12 + "Reading images...")
                image_records = series.children
                image_filenames = [os.path.join(base_dir, *image_rec.ReferencedFileID)
                                   for image_rec in image_records]

                # slice_locations = [pydicom.read_file(image_filename).SliceLocation
                #                   for image_filename in image_filenames]

                datasets = [pydicom.read_file(image_filename)
                            for image_filename in image_filenames]

                patient_names = set(ds.PatientName for ds in datasets)
                patient_IDs = set(ds.PatientID for ds in datasets)

                # List the image filenames
                print("\n" + " " * 12 + "Image filenames:")
                print(" " * 12, end=' ')
                pprint(image_filenames, indent=12)

                # Expect all images to have same patient name, id
                # Show the set of all names, IDs found (should each have one)
                # In python3.4+, must make conversion to str explicit
                print(" " * 12 + "Patient Names in images..: "
                      "{0:s}".format(str(patient_names)))
                print(" " * 12 + "Patient IDs in images..:"
                      "{0:s}".format(str(patient_IDs)))

                # print (" " * 12 + "Slice Locations from "
                #       "{0} to {1}".format(min(slice_locations), max(slice_locations)))
<commit_msg>Fix usage of read_dicomdir<commit_after># show_dicomdir.py
"""Example file to show use of read_dicomdir()
"""
from __future__ import print_function
# Copyright (c) 2013 Darcy Mason
# This file is part of pydicom, relased under an MIT-style license.
#    See the file license.txt included with this distribution, also
#    available at https://github.com/darcymason/pydicom
#

import sys
import os.path
from pprint import pprint

import pydicom
from pydicom.filereader import read_dicomdir
# pydicom.debug()

if __name__ == "__main__":
    print("------------------------------------------------------------")
    print("Example program showing DICOMDIR contents, assuming standard")
    print("Patient -> Study -> Series -> Images hierarchy")
    print("------------------------------------------------------------")
    print()
    if len(sys.argv) > 1:
        filepath = sys.argv[1]
        if os.path.isdir(filepath):  # only gave directory, add standard name
            filepath = os.path.join(filepath, "DICOMDIR")
        dcmdir = read_dicomdir(filepath)
        base_dir = os.path.dirname(filepath)
    else:
        # Read standard "DICOMDIR" filename from current directory
        dcmdir = pydicom.read_dicomdir()
        base_dir = "."

    for patrec in dcmdir.patient_records:
        print("Patient: {0.PatientID}: {0.PatientsName}".format(patrec))
        studies = patrec.children
        for study in studies:
            print("    Study {0.StudyID}: {0.StudyDate}:"
                  " {0.StudyDescription}".format(study))
            all_series = study.children
            for series in all_series:
                image_count = len(series.children)
                plural = ('', 's')[image_count > 1]

                # Write basic series info and image count

                # Put N/A in if no Series Description
                if 'SeriesDescription' not in series:
                    series.SeriesDescription = "N/A"
                print(" " * 8 + "Series {0.SeriesNumber}:  {0.Modality}: {0.SeriesDescription}"
                      " ({1} image{2})".format(series, image_count, plural))

                # Open and read something from each image, for demonstration purposes
                # For simple quick overview of DICOMDIR, leave the following out
                print(" " * 12 + "Reading images...")
                image_records = series.children
                image_filenames = [os.path.join(base_dir, *image_rec.ReferencedFileID)
                                   for image_rec in image_records]

                # slice_locations = [pydicom.read_file(image_filename).SliceLocation
                #                   for image_filename in image_filenames]

                datasets = [pydicom.read_file(image_filename)
                            for image_filename in image_filenames]

                patient_names = set(ds.PatientName for ds in datasets)
                patient_IDs = set(ds.PatientID for ds in datasets)

                # List the image filenames
                print("\n" + " " * 12 + "Image filenames:")
                print(" " * 12, end=' ')
                pprint(image_filenames, indent=12)

                # Expect all images to have same patient name, id
                # Show the set of all names, IDs found (should each have one)
                # In python3.4+, must make conversion to str explicit
                print(" " * 12 + "Patient Names in images..: "
                      "{0:s}".format(str(patient_names)))
                print(" " * 12 + "Patient IDs in images..:"
                      "{0:s}".format(str(patient_IDs)))

                # print (" " * 12 + "Slice Locations from "
                #       "{0} to {1}".format(min(slice_locations), max(slice_locations)))
<|endoftext|>


<commit_before>#!/usr/bin/env python

from math import log as ln

import numpy as np
from scipy.sparse import csr_matrix
from scipy.linalg import pinv,inv
from scipy.sparse.linalg import inv as sparse_inv

def logdet(M):
    """ Returns the log determinant of a matrix. """
    sign,logdet = np.linalg.slogdet(M)
    return logdet
def restricted_loglikelihood(y,V,X):
    """
    Returns a value proportional to the restricted loglikelihood for mixed model estimation.

    References:

    Harville. 'Maximum Likelihood Approaches to Variance Component Estimation and to Related Problems'
    Journal of the American Statistical Association. (1977) (72):258

    Lange, Westlake, & Spence. Extensions to pedigree analysis III. Variance components by the scoring method.
    Ann Hum Genet. (1976). 39:4,485-491 DOI: 10.1111/j.1469-1809.1976.tb00156.x

    SAS documentation for PROC MIXED
    """

    Vinverse = csr_matrix(inv(V.todense()))
    r = y - X * pinv(X.transpose() * Vinverse * X) * X.transpose() * Vinverse * y
    # This value is only proportional to the restricted likelihood. I'm saving some needless expense by
    # skipping some terms that are constant across all formulations of the variance components.
    # To get the actual estimate of the likelihood, calculate -0.5(w + (n-p)*ln(2pi)), where:
    #   n is the number of rows of X
    #   p is the rank of X.
    #
    # I've left these terms out for two main reasons:
    # 1) They don't change across different values of V. For optimization purposes, they're irrelevant.
    # 2) The term (n-p)ln(2pi) requires numpy.linalg.matrix_rank, which is not found in older
    #    versions of numpy. At some point I might require newer versions of numpy but, because of (1)
    #    I don't think i'll bother.
    llik_restricted = logdet(V.todense()) + logdet(X.transpose() * Vinverse * X) + r.transpose() * Vinverse * r
    return scipy.matrix.item(llik_restricted)
<commit_msg>reml returns scalar now<commit_after>#!/usr/bin/env python

from math import log as ln

import numpy as np
from scipy.sparse import csr_matrix
from scipy.linalg import pinv,inv
from scipy.sparse.linalg import inv as sparse_inv
from scipy import matrix

def logdet(M):
    """ Returns the log determinant of a matrix. """
    sign,logdet = np.linalg.slogdet(M)
    return logdet

def restricted_loglikelihood(y,V,X):
    """
    Returns a value proportional to the restricted loglikelihood for mixed model estimation.

    References:

    Harville. 'Maximum Likelihood Approaches to Variance Component Estimation and to Related Problems'
    Journal of the American Statistical Association. (1977) (72):258

    Lange, Westlake, & Spence. Extensions to pedigree analysis III. Variance components by the scoring method.
    Ann Hum Genet. (1976). 39:4,485-491 DOI: 10.1111/j.1469-1809.1976.tb00156.x

    SAS documentation for PROC MIXED
    """

    Vinverse = csr_matrix(inv(V.todense()))
    r = y - X * pinv(X.transpose() * Vinverse * X) * X.transpose() * Vinverse * y
    # This value is only proportional to the restricted likelihood. I'm saving some needless expense by
    # skipping some terms that are constant across all formulations of the variance components.
    # To get the actual estimate of the likelihood, calculate -0.5(w + (n-p)*ln(2pi)), where:
    #   n is the number of rows of X
    #   p is the rank of X.
    #
    # I've left these terms out for two main reasons:
    # 1) They don't change across different values of V. For optimization purposes, they're irrelevant.
    # 2) The term (n-p)ln(2pi) requires numpy.linalg.matrix_rank, which is not found in older
    #    versions of numpy. At some point I might require newer versions of numpy but, because of (1)
    #    I don't think i'll bother.
    llik_restricted = logdet(V.todense()) + logdet(X.transpose() * Vinverse * X) + r.transpose() * Vinverse * r
    return matrix.item(llik_restricted)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    pyfire.tests.stream.test_zmqforwarder
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    Unittests for ZMQ Forwarder/router

    :copyright: 2011 by the pyfire Team, see AUTHORS for more details.
    :license: BSD, see LICENSE for more details.
"""

import xml.etree.ElementTree as ET

from pyfire.tests import PyfireTestCase

from pyfire import zmq_forwarder
from pyfire.stream.errors import InternalServerError
from pyfire.stream.stanzas.errors import ServiceUnavailableError
import zmq
import thread
import time

class TestZMQForwarder(PyfireTestCase):

    def setUp(self):
        self.forwarder_url = "tcp://127.0.0.1:42050"
        self.forwarder = zmq_forwarder.ZMQForwarder(self.forwarder_url)
        thread.start_new_thread(self.forwarder.start, ())

        self.ctx = zmq.Context()

    def tearDown(self):
        self.forwarder.loop.stop()
        self.forwarder.pull_sock.close()

    def test_register_peer(self):
        reg_cmd = zmq_forwarder.ZMQForwarder_message('REGISTER')
        reg_cmd.attributes = ('change_me', 'tcp://127.0.0.1:1234', ['localhost', ])
        self.forwarder.handle_forwarder_message(reg_cmd)
        self.assertEqual(len(self.forwarder.peers), 1)

    def test_register_peer_authfail(self):
        reg_cmd = zmq_forwarder.ZMQForwarder_message('REGISTER')
        reg_cmd.attributes = ('', 'tcp://127.0.0.1:1234', ['localhost', ])
        peers_temp = len(self.forwarder.peers)
        self.forwarder.handle_forwarder_message(reg_cmd)
        # Registered peer count may no change here as auth fails..
        self.assertEqual(len(self.forwarder.peers), peers_temp)

    def test_unknown_command(self):
        reg_cmd = zmq_forwarder.ZMQForwarder_message('NOT_IMPLEMENTED')
        reg_cmd.attributes = ('change_me', 'tcp://127.0.0.1:1234', ['localhost', ])
        with self.assertRaises(InternalServerError):
            self.forwarder.handle_forwarder_message(reg_cmd)

    def test_route_message(self):
        forwarder = self.ctx.socket(zmq.PUSH)
        forwarder.connect(self.forwarder_url)

        pull_socket = self.ctx.socket(zmq.PULL)
        test_port = pull_socket.bind_to_random_port('tcp://127.0.0.1')

        reg_cmd = zmq_forwarder.ZMQForwarder_message('REGISTER')
        reg_cmd.attributes = ('change_me', 'tcp://127.0.0.1:' + str(test_port), ['testhost', ])
        forwarder.send_pyobj(reg_cmd)

        test_stanza = ET.Element('Iq')
        test_stanza.set('id', '1')
        test_stanza.set('from', 'testhost')
        test_stanza.set('to', 'testhost')
        forwarder.send_pyobj(test_stanza)

        received_stanza = pull_socket.recv_pyobj()
        self.assertEqual(ET.tostring(test_stanza), ET.tostring(received_stanza))

    def test_route_unknownrecipient(self):
        forwarder = self.ctx.socket(zmq.PUSH)
        forwarder.connect(self.forwarder_url)

        pull_socket = self.ctx.socket(zmq.PULL)
        test_port = pull_socket.bind_to_random_port('tcp://127.0.0.1')

        reg_cmd = zmq_forwarder.ZMQForwarder_message('REGISTER')
        reg_cmd.attributes = ('change_me', 'tcp://127.0.0.1:' + str(test_port), ['testsender', ])
        forwarder.send_pyobj(reg_cmd)

        test_stanza = ET.Element('Iq')
        test_stanza.set('id', '1')
        test_stanza.set('from', 'testsender')
        test_stanza.set('to', 'unknown')
        expected_stanza = ServiceUnavailableError(test_stanza).element

        forwarder.send_pyobj(test_stanza)
        received_stanza = pull_socket.recv_pyobj()

        self.assertEqual(ET.tostring(expected_stanza), ET.tostring(received_stanza))
<commit_msg>Give ZMQ some time to shut down sockets<commit_after># -*- coding: utf-8 -*-
"""
    pyfire.tests.stream.test_zmqforwarder
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    Unittests for ZMQ Forwarder/router

    :copyright: 2011 by the pyfire Team, see AUTHORS for more details.
    :license: BSD, see LICENSE for more details.
"""

import xml.etree.ElementTree as ET

from pyfire.tests import PyfireTestCase

from pyfire import zmq_forwarder
from pyfire.stream.errors import InternalServerError
from pyfire.stream.stanzas.errors import ServiceUnavailableError
import zmq
import thread
import time

class TestZMQForwarder(PyfireTestCase):

    def setUp(self):
        self.forwarder_url = "tcp://127.0.0.1:42050"
        self.forwarder = zmq_forwarder.ZMQForwarder(self.forwarder_url)
        thread.start_new_thread(self.forwarder.start, ())

        self.ctx = zmq.Context()

    def tearDown(self):
        self.forwarder.loop.stop()
        self.forwarder.pull_sock.close()
        time.sleep(0.2)

    def test_register_peer(self):
        reg_cmd = zmq_forwarder.ZMQForwarder_message('REGISTER')
        reg_cmd.attributes = ('change_me', 'tcp://127.0.0.1:1234', ['localhost', ])
        self.forwarder.handle_forwarder_message(reg_cmd)
        self.assertEqual(len(self.forwarder.peers), 1)

    def test_register_peer_authfail(self):
        reg_cmd = zmq_forwarder.ZMQForwarder_message('REGISTER')
        reg_cmd.attributes = ('', 'tcp://127.0.0.1:1234', ['localhost', ])
        peers_temp = len(self.forwarder.peers)
        self.forwarder.handle_forwarder_message(reg_cmd)
        # Registered peer count may no change here as auth fails..
        self.assertEqual(len(self.forwarder.peers), peers_temp)

    def test_unknown_command(self):
        reg_cmd = zmq_forwarder.ZMQForwarder_message('NOT_IMPLEMENTED')
        reg_cmd.attributes = ('change_me', 'tcp://127.0.0.1:1234', ['localhost', ])
        with self.assertRaises(InternalServerError):
            self.forwarder.handle_forwarder_message(reg_cmd)

    def test_route_message(self):
        forwarder = self.ctx.socket(zmq.PUSH)
        forwarder.connect(self.forwarder_url)

        pull_socket = self.ctx.socket(zmq.PULL)
        test_port = pull_socket.bind_to_random_port('tcp://127.0.0.1')

        reg_cmd = zmq_forwarder.ZMQForwarder_message('REGISTER')
        reg_cmd.attributes = ('change_me', 'tcp://127.0.0.1:' + str(test_port), ['testhost', ])
        forwarder.send_pyobj(reg_cmd)

        test_stanza = ET.Element('Iq')
        test_stanza.set('id', '1')
        test_stanza.set('from', 'testhost')
        test_stanza.set('to', 'testhost')
        forwarder.send_pyobj(test_stanza)

        received_stanza = pull_socket.recv_pyobj()
        self.assertEqual(ET.tostring(test_stanza), ET.tostring(received_stanza))

    def test_route_unknownrecipient(self):
        forwarder = self.ctx.socket(zmq.PUSH)
        forwarder.connect(self.forwarder_url)

        pull_socket = self.ctx.socket(zmq.PULL)
        test_port = pull_socket.bind_to_random_port('tcp://127.0.0.1')

        reg_cmd = zmq_forwarder.ZMQForwarder_message('REGISTER')
        reg_cmd.attributes = ('change_me', 'tcp://127.0.0.1:' + str(test_port), ['testsender', ])
        forwarder.send_pyobj(reg_cmd)

        test_stanza = ET.Element('Iq')
        test_stanza.set('id', '1')
        test_stanza.set('from', 'testsender')
        test_stanza.set('to', 'unknown')
        expected_stanza = ServiceUnavailableError(test_stanza).element

        forwarder.send_pyobj(test_stanza)
        received_stanza = pull_socket.recv_pyobj()

        self.assertEqual(ET.tostring(expected_stanza), ET.tostring(received_stanza))
<|endoftext|>


<commit_before>import operators as ops
import operator
import random
import wm
import evaluation
import fis
import pprint

def eval_examples(examples):

    TP = 0.0
    TN = 0.0
    FP = 0.0
    FN = 0.0
    NONE = 0.0
    NUM_POSITIVES = 0.0
    NUM_NEGATIVES = 0.0

    for e in examples:
        if e['predicted_class']:
            if (e['known_class'] == 1.0 or e['known_class'] == 'positive') and e['predicted_class'].name == 'positive':
                TP = TP + 1.0
                NUM_POSITIVES += 1
            elif (e['known_class'] == -1.0 or e['known_class'] == 'negative') and e['predicted_class'].name == 'negative':
                TN = TN + 1.0
                NUM_NEGATIVES += 1
            elif (e['known_class'] == 1.0 or e['known_class'] == 'positive') and e['predicted_class'].name == 'negative':
                FN = FN + 1.0
                NUM_POSITIVES += 1
            elif (e['known_class'] == -1.0 or e['known_class'] == 'negative') and e['predicted_class'].name == 'positive':
                FP = FP + 1.0
                NUM_NEGATIVES += 1
        else:
            NONE += 1

    return {'precision': evaluation.precision(TP, FP),
            'recall': evaluation.recall(TP, FN),
            'f1': evaluation.f1(TP, FP, TN, FN),
            'accuracy': evaluation.accuracy(TP, FP, TN, FN),
            'non classified': NONE,
            'TP': TP,
            'TN': TN,
            'FP': FP,
            'FN': FN,
            'NUM_POSITIVES': NUM_POSITIVES,
            'NUM_NEGATIVES': NUM_NEGATIVES}

def __rule_compatibility(example, rule):
    """
    Calculates the compatibility degree between example and rule

    example - numeric array with input values
    rule - sets.Rule object
    """

    compatibility = 1
    for ei, input_example in enumerate(example):
        # checks if the antecedent fuzzy set is the highest set
        temp_input_example = input_example
        antecedent_fuzzy_set = rule.antecedents[ei].fuzzy_set
        if (antecedent_fuzzy_set.params[len(antecedent_fuzzy_set.params) - 1] == antecedent_fuzzy_set.range[len(antecedent_fuzzy_set.range) - 1]) and \
                (antecedent_fuzzy_set.params[len(antecedent_fuzzy_set.params) - 1] == antecedent_fuzzy_set.params[len(antecedent_fuzzy_set.params) - 2]):
            if temp_input_example > antecedent_fuzzy_set.params[len(antecedent_fuzzy_set.params) - 1]:
                temp_input_example = float(
                    antecedent_fuzzy_set.params[len(antecedent_fuzzy_set.params) - 1])

        # checks if the antecedent fuzzy set is the lowest set
        if (antecedent_fuzzy_set.params[0] == antecedent_fuzzy_set.range[0]) and \
                (antecedent_fuzzy_set.params[0] == antecedent_fuzzy_set.params[1]):
            if temp_input_example < antecedent_fuzzy_set.params[0]:
                temp_input_example = float(antecedent_fuzzy_set.params[0])

        compatibility = ops.t_norm(
            compatibility, rule.antecedents[ei].fuzzy_set.degree(temp_input_example))
    return compatibility

def cfrm(examples, rules, verbose=False, weights=None):
    """
    Classifies examples using the rule that has the highest compatibility degree with the example. The function fills 'predicted_class' field in each example with the predicted FuzzySet class.

    examples -- list of dicts with the following structure:
                            {'inputs': [x1, x2, x3, ..., xn],
                            'known_class': 'positive',
                            'predicted_class:' None}
    rules -- list of Rule objects to classify examples
    """

    for ei, e in enumerate(examples):
        if verbose:
            print str(ei), '/', str(len(examples))
        e_rule = {}
        for r in rules:
            if not weights:
                e_rule[r] = __rule_compatibility(e['inputs'], r)
            else:
                e_rule[r] = __rule_compatibility(e['inputs'], r)
                for rule_weight in weights:
                    if r in rule_weight:
                        e_rule[r] = e_rule[r] * rule_weight[r]
                        break
        rule_compatibility = max(
            e_rule.iteritems(), key=operator.itemgetter(1))
        max_rule = rule_compatibility[0]
        if rule_compatibility[1] == 0:
            e['predicted_class'] = None
        else:
            e['predicted_class'] = max_rule.outputs[0].fuzzy_set

def gfrm(examples, rules, verbose=False):
    """
    Classifies examples using the class that has the highest compatibility degree with the example.
    The function fills 'predicted_class' field in each example with the predicted FuzzySet class.

            **Binary classfication only
            ** It is a messy code. PLEASE REFACTOR!!!!!

    examples -- list of dicts with the following structure:
                            {'inputs': [x1, x2, x3, ..., xn],
                            'known_class': 'positive',
                            'predicted_class:' None}
    rules -- list of Rule objects to classify examples
    """

    for ei, example in enumerate(examples):
        if verbose:
            print str(ei), '/', str(len(examples))
        example_rule = {}
        for rule in rules:
            rule_compat = __rule_compatibility(example['inputs'], rule)
            for output in rule.outputs:
                if output.fuzzy_set.name not in example_rule:
                    example_rule[output.fuzzy_set.name] = []

                degrees = example_rule[output.fuzzy_set.name]
                degrees.append(rule_compat)
                example_rule[output.fuzzy_set.name] = degrees

        # print 'class compatibilities', example_rule
        # print '--------------------------------'

        for (key, value) in example_rule.iteritems():
            example_rule[key] = float(sum(value)) / len(value)

        if verbose:
            print 'class compatibilities', example_rule

        rule_compatibility = max(
            example_rule.iteritems(), key=operator.itemgetter(1))
        max_class_name = rule_compatibility[0]
        if verbose:
            print 'max_class_name:', max_class_name
        max_class = None
        for r in rules:
            for o in r.outputs:
                if max_class_name == o.fuzzy_set.name:
                    max_class = o.fuzzy_set
                    break
            else:
                continue
            break

        if verbose:
            print 'known_class: ', example['known_class']
        if rule_compatibility[1] == 0:
            example['predicted_class'] = None
            if verbose:
                print 'predicted_class', None
        else:
            if verbose:
                print 'predicted_class', max_class.name
            example['predicted_class'] = max_class
        if verbose:
            print '--------------------------------'

def _k_folds(_list, k):

    fold_size = len(_list) / k
    folds = []
    start = 0
    end = fold_size
    for ki in xrange(1, k + 1):
        folds.append(_list[start:end])
        start = end
        end = fold_size * (ki + 1)

    return folds

def _create_examples(inputs, discrete_class_beginning):

    examples = []
    for input in inputs:
        y = input[0] if discrete_class_beginning else input[len(input) - 1]
        xs = input[1:] if discrete_class_beginning else input[:len(input) - 1]
        examples.append(
            {'inputs': xs, 'known_class': y, 'predicted_class': None})

    return examples

def cfrm_cross_validation(inputs, inputs_names, inputs_regions, output_name, outputs_regions, discrete_class_beginning=True, verbose=False, k=10, scramble=True):

    if len(inputs) % k != 0:
        raise ValueError("Cannot split inputs in " + str(k) + " equal parts.")

    if k < 2:
        raise ValueError(
            "Cannot make k-fold cross validation with k less than 2")

    if scramble:
        random.shuffle(inputs)

    folds = _k_folds(inputs, k)

    accuracies = []
    for fi, fold in enumerate(folds):
        if verbose:
            print '===>', "fold ", fi, '<==='
        fold_rules = []
        for i_fi, i_fold in enumerate(folds):
            if fi != i_fi:
                i_fold_rules = wm.wang_mendel(
                    i_fold, inputs_names, inputs_regions, output_name, outputs_regions, discrete_class_beginning)
                fold_rules = fold_rules + i_fold_rules.values()

        examples = _create_examples(fold, discrete_class_beginning)
        cfrm(examples, fold_rules, verbose=False)
        fi_accuracy = eval_examples(examples)['accuracy']
        if verbose:
            print '--->', "fold_accuracy", fi, fi_accuracy, '<---'
        accuracies.append(fi_accuracy)

    return sum(accuracies) * 1.0 / len(accuracies)

def average(examples):
    for example in examples:
        avg = reduce(
            lambda x, y: x + y, example['inputs']) / len(example['inputs'])
        if avg > 0:
            example['predicted_class'] = fis.FuzzySet('positive', [], [])
        elif avg < 0:
            example['predicted_class'] = fis.FuzzySet('negative', [], [])
        else:
            example['predicted_class'] = None

def rules_weights(combined_fuzzy_rule_base_list, list_of_modified_fold_test_data):

    combined_fuzzy_rule_base_weights = []
    for fold_rules_idx, fold_rules in enumerate(combined_fuzzy_rule_base_list):
        fold_rules_weights = []
        for rule_key in fold_rules:
            rule = fold_rules[rule_key]
            pos_beta = 0
            neg_beta = 0
            examples_data = _create_examples(
                list_of_modified_fold_test_data[fold_rules_idx], False)
            for example in examples_data:
                if example['known_class'] == 1.0:
                    pos_beta += __rule_compatibility(example['inputs'], rule)
                elif example['known_class'] == -1.0:
                    neg_beta += __rule_compatibility(example['inputs'], rule)

            if pos_beta == neg_beta:
                cf = 0
            else:
                cf = abs(pos_beta - neg_beta) / (pos_beta + neg_beta)

            fold_rules_weights.append({rule: cf})
        combined_fuzzy_rule_base_weights.append(fold_rules_weights)
    return combined_fuzzy_rule_base_weights
<commit_msg>added weights to GFRM<commit_after>import operators as ops
import operator
import random
import wm
import evaluation
import fis
import pprint

def eval_examples(examples):

    TP = 0.0
    TN = 0.0
    FP = 0.0
    FN = 0.0
    NONE = 0.0
    NUM_POSITIVES = 0.0
    NUM_NEGATIVES = 0.0

    for e in examples:
        if e['predicted_class']:
            if (e['known_class'] == 1.0 or e['known_class'] == 'positive') and e['predicted_class'].name == 'positive':
                TP = TP + 1.0
                NUM_POSITIVES += 1
            elif (e['known_class'] == -1.0 or e['known_class'] == 'negative') and e['predicted_class'].name == 'negative':
                TN = TN + 1.0
                NUM_NEGATIVES += 1
            elif (e['known_class'] == 1.0 or e['known_class'] == 'positive') and e['predicted_class'].name == 'negative':
                FN = FN + 1.0
                NUM_POSITIVES += 1
            elif (e['known_class'] == -1.0 or e['known_class'] == 'negative') and e['predicted_class'].name == 'positive':
                FP = FP + 1.0
                NUM_NEGATIVES += 1
        else:
            NONE += 1

    return {'precision': evaluation.precision(TP, FP),
            'recall': evaluation.recall(TP, FN),
            'f1': evaluation.f1(TP, FP, TN, FN),
            'accuracy': evaluation.accuracy(TP, FP, TN, FN),
            'non classified': NONE,
            'TP': TP,
            'TN': TN,
            'FP': FP,
            'FN': FN,
            'NUM_POSITIVES': NUM_POSITIVES,
            'NUM_NEGATIVES': NUM_NEGATIVES}

def __rule_compatibility(example, rule):
    """
    Calculates the compatibility degree between example and rule

    example - numeric array with input values
    rule - sets.Rule object
    """

    compatibility = 1
    for ei, input_example in enumerate(example):
        # checks if the antecedent fuzzy set is the highest set
        temp_input_example = input_example
        antecedent_fuzzy_set = rule.antecedents[ei].fuzzy_set
        if (antecedent_fuzzy_set.params[len(antecedent_fuzzy_set.params) - 1] == antecedent_fuzzy_set.range[len(antecedent_fuzzy_set.range) - 1]) and \
                (antecedent_fuzzy_set.params[len(antecedent_fuzzy_set.params) - 1] == antecedent_fuzzy_set.params[len(antecedent_fuzzy_set.params) - 2]):
            if temp_input_example > antecedent_fuzzy_set.params[len(antecedent_fuzzy_set.params) - 1]:
                temp_input_example = float(
                    antecedent_fuzzy_set.params[len(antecedent_fuzzy_set.params) - 1])

        # checks if the antecedent fuzzy set is the lowest set
        if (antecedent_fuzzy_set.params[0] == antecedent_fuzzy_set.range[0]) and \
                (antecedent_fuzzy_set.params[0] == antecedent_fuzzy_set.params[1]):
            if temp_input_example < antecedent_fuzzy_set.params[0]:
                temp_input_example = float(antecedent_fuzzy_set.params[0])

        compatibility = ops.t_norm(
            compatibility, rule.antecedents[ei].fuzzy_set.degree(temp_input_example))
    return compatibility

def cfrm(examples, rules, verbose=False, weights=None):
    """
    Classifies examples using the rule that has the highest compatibility degree with the example. The function fills 'predicted_class' field in each example with the predicted FuzzySet class.

    examples -- list of dicts with the following structure:
                            {'inputs': [x1, x2, x3, ..., xn],
                            'known_class': 'positive',
                            'predicted_class:' None}
    rules -- list of Rule objects to classify examples
    """

    for ei, e in enumerate(examples):
        if verbose:
            print str(ei), '/', str(len(examples))
        e_rule = {}
        for r in rules:
            if not weights:
                e_rule[r] = __rule_compatibility(e['inputs'], r)
            else:
                e_rule[r] = __rule_compatibility(e['inputs'], r)
                for rule_weight in weights:
                    if r in rule_weight:
                        e_rule[r] = e_rule[r] * rule_weight[r]
                        break
        rule_compatibility = max(
            e_rule.iteritems(), key=operator.itemgetter(1))
        max_rule = rule_compatibility[0]
        if rule_compatibility[1] == 0:
            e['predicted_class'] = None
        else:
            e['predicted_class'] = max_rule.outputs[0].fuzzy_set

def gfrm(examples, rules, verbose=False, weights=None):
    """
    Classifies examples using the class that has the highest compatibility degree with the example.
    The function fills 'predicted_class' field in each example with the predicted FuzzySet class.

            **Binary classfication only
            ** It is a messy code. PLEASE REFACTOR!!!!!

    examples -- list of dicts with the following structure:
                            {'inputs': [x1, x2, x3, ..., xn],
                            'known_class': 'positive',
                            'predicted_class:' None}
    rules -- list of Rule objects to classify examples
    """

    for ei, example in enumerate(examples):
        if verbose:
            print str(ei), '/', str(len(examples))
        example_rule = {}
        for rule in rules:
            rule_compat = __rule_compatibility(example['inputs'], rule)
            if weights:
                for rule_weight in weights:
                    if rule in rule_weight:
                        rule_compat = rule_compat * rule_weight[rule]
                        break

            for output in rule.outputs:
                if output.fuzzy_set.name not in example_rule:
                    example_rule[output.fuzzy_set.name] = []

                degrees = example_rule[output.fuzzy_set.name]
                degrees.append(rule_compat)
                example_rule[output.fuzzy_set.name] = degrees

        # print 'class compatibilities', example_rule
        # print '--------------------------------'

        for (key, value) in example_rule.iteritems():
            example_rule[key] = float(sum(value)) / len(value)

        if verbose:
            print 'class compatibilities', example_rule

        rule_compatibility = max(
            example_rule.iteritems(), key=operator.itemgetter(1))
        max_class_name = rule_compatibility[0]
        if verbose:
            print 'max_class_name:', max_class_name
        max_class = None
        for r in rules:
            for o in r.outputs:
                if max_class_name == o.fuzzy_set.name:
                    max_class = o.fuzzy_set
                    break
            else:
                continue
            break

        if verbose:
            print 'known_class: ', example['known_class']
        if rule_compatibility[1] == 0:
            example['predicted_class'] = None
            if verbose:
                print 'predicted_class', None
        else:
            if verbose:
                print 'predicted_class', max_class.name
            example['predicted_class'] = max_class
        if verbose:
            print '--------------------------------'

def _k_folds(_list, k):

    fold_size = len(_list) / k
    folds = []
    start = 0
    end = fold_size
    for ki in xrange(1, k + 1):
        folds.append(_list[start:end])
        start = end
        end = fold_size * (ki + 1)

    return folds

def _create_examples(inputs, discrete_class_beginning):

    examples = []
    for input in inputs:
        y = input[0] if discrete_class_beginning else input[len(input) - 1]
        xs = input[1:] if discrete_class_beginning else input[:len(input) - 1]
        examples.append(
            {'inputs': xs, 'known_class': y, 'predicted_class': None})

    return examples

def cfrm_cross_validation(inputs, inputs_names, inputs_regions, output_name, outputs_regions, discrete_class_beginning=True, verbose=False, k=10, scramble=True):

    if len(inputs) % k != 0:
        raise ValueError("Cannot split inputs in " + str(k) + " equal parts.")

    if k < 2:
        raise ValueError(
            "Cannot make k-fold cross validation with k less than 2")

    if scramble:
        random.shuffle(inputs)

    folds = _k_folds(inputs, k)

    accuracies = []
    for fi, fold in enumerate(folds):
        if verbose:
            print '===>', "fold ", fi, '<==='
        fold_rules = []
        for i_fi, i_fold in enumerate(folds):
            if fi != i_fi:
                i_fold_rules = wm.wang_mendel(
                    i_fold, inputs_names, inputs_regions, output_name, outputs_regions, discrete_class_beginning)
                fold_rules = fold_rules + i_fold_rules.values()

        examples = _create_examples(fold, discrete_class_beginning)
        cfrm(examples, fold_rules, verbose=False)
        fi_accuracy = eval_examples(examples)['accuracy']
        if verbose:
            print '--->', "fold_accuracy", fi, fi_accuracy, '<---'
        accuracies.append(fi_accuracy)

    return sum(accuracies) * 1.0 / len(accuracies)

def average(examples):
    for example in examples:
        avg = reduce(
            lambda x, y: x + y, example['inputs']) / len(example['inputs'])
        if avg > 0:
            example['predicted_class'] = fis.FuzzySet('positive', [], [])
        elif avg < 0:
            example['predicted_class'] = fis.FuzzySet('negative', [], [])
        else:
            example['predicted_class'] = None

def rules_weights(combined_fuzzy_rule_base_list, list_of_modified_fold_test_data):

    combined_fuzzy_rule_base_weights = []
    for fold_rules_idx, fold_rules in enumerate(combined_fuzzy_rule_base_list):
        fold_rules_weights = []
        for rule_key in fold_rules:
            rule = fold_rules[rule_key]
            pos_beta = 0
            neg_beta = 0
            examples_data = _create_examples(
                list_of_modified_fold_test_data[fold_rules_idx], False)
            for example in examples_data:
                if example['known_class'] == 1.0:
                    pos_beta += __rule_compatibility(example['inputs'], rule)
                elif example['known_class'] == -1.0:
                    neg_beta += __rule_compatibility(example['inputs'], rule)

            if pos_beta == neg_beta:
                cf = 0
            else:
                cf = abs(pos_beta - neg_beta) / (pos_beta + neg_beta)

            fold_rules_weights.append({rule: cf})
        combined_fuzzy_rule_base_weights.append(fold_rules_weights)
    return combined_fuzzy_rule_base_weights
<|endoftext|>


<commit_before>from pyparsing import Word, alphas, Suppress, Combine, nums, string, Optional, Regex

class LogGrammar(object):
    """A class to define the format (grammar) of a log file.

    We heavily rely on pyparsing in this case. The code for auth.log grammar is derived from syslog parser
    by L. Silva [Silva2012]_.

    References
    ----------
    .. [Silva2012] L. Silva, Parsing syslog files with Python and PyParsing, 2012.
                   https://gist.github.com/leandrosilva/3651640
    """
    def __init__(self, log_type=None):
        """The constructor of LogGrammar.
        """
        self.log_type = log_type
        self.authlog_grammar = self.__get_authlog_grammar()

    @staticmethod
    def __get_authlog_grammar():
        """The definition of auth.log grammar.
        """
        ints = Word(nums)

        # timestamp
        month = Word(string.uppercase, string.lowercase, exact=3)
        day = ints
        hour = Combine(ints + ":" + ints + ":" + ints)
        timestamp = month + day + hour

        # hostname, service name, message
        hostname_or_ip = Word(alphas + nums + "_" + "-" + ".")
        appname = Word(alphas + "/" + "-" + "_" + ".") + Optional(Suppress("[") + ints + Suppress("]")) + Suppress(":")
        message = Regex(".*")

        # auth log grammar
        authlog_grammar = timestamp + hostname_or_ip + appname + message
        return authlog_grammar

    def parse_authlog(self, log_line):
        """Parse auth.log based on defined grammar.

        Returns
        -------
        parsed  : dict[str, str]
            A parsed auth.log containing these elements: timestamp, hostname, service, pid, and message.
        """
        parsed_authlog = self.authlog_grammar.parseString(log_line)

        # get parsed auth.log
        parsed = dict()
        parsed['timestamp'] = parsed_authlog[0] + ' ' + parsed_authlog[1] + ' ' + parsed_authlog[2]
        if self.log_type == 'Hofstede2014':
            parsed['ip_address'] = parsed_authlog[3]
        else:
            parsed['hostname'] = parsed_authlog[3]

        parsed['service'] = parsed_authlog[4]

        if len(parsed_authlog) < 7:
            parsed['message'] = parsed_authlog[5]
        else:
            parsed['pid'] = parsed_authlog[5]
            parsed['message'] = parsed_authlog[6]

        return parsed
<commit_msg>Add kippo honeypot log grammar<commit_after>from pyparsing import Word, alphas, Suppress, Combine, nums, string, Optional, Regex

class LogGrammar(object):
    """A class to define the format (grammar) of a log file.

    We heavily rely on pyparsing in this case. The code for auth.log grammar is derived from syslog parser
    by L. Silva [Silva2012]_.

    References
    ----------
    .. [Silva2012] L. Silva, Parsing syslog files with Python and PyParsing, 2012.
                   https://gist.github.com/leandrosilva/3651640
    """
    def __init__(self, log_type=None):
        """The constructor of LogGrammar.
        """
        self.log_type = log_type
        self.authlog_grammar = self.__get_authlog_grammar()

    @staticmethod
    def __get_authlog_grammar():
        """The definition of auth.log grammar.
        """
        ints = Word(nums)

        # timestamp
        month = Word(string.uppercase, string.lowercase, exact=3)
        day = ints
        hour = Combine(ints + ":" + ints + ":" + ints)
        timestamp = month + day + hour

        # hostname, service name, message
        hostname_or_ip = Word(alphas + nums + "_" + "-" + ".")
        appname = Word(alphas + "/" + "-" + "_" + ".") + Optional(Suppress("[") + ints + Suppress("]")) + Suppress(":")
        message = Regex(".*")

        # auth log grammar
        authlog_grammar = timestamp + hostname_or_ip + appname + message
        return authlog_grammar

    def parse_authlog(self, log_line):
        """Parse auth.log based on defined grammar.

        Returns
        -------
        parsed  : dict[str, str]
            A parsed auth.log containing these elements: timestamp, hostname, service, pid, and message.
        """
        parsed_authlog = self.authlog_grammar.parseString(log_line)

        # get parsed auth.log
        parsed = dict()
        parsed['timestamp'] = parsed_authlog[0] + ' ' + parsed_authlog[1] + ' ' + parsed_authlog[2]
        if self.log_type == 'Hofstede2014':
            parsed['ip_address'] = parsed_authlog[3]
        else:
            parsed['hostname'] = parsed_authlog[3]

        parsed['service'] = parsed_authlog[4]

        if len(parsed_authlog) < 7:
            parsed['message'] = parsed_authlog[5]
        else:
            parsed['pid'] = parsed_authlog[5]
            parsed['message'] = parsed_authlog[6]

        return parsed

    @staticmethod
    def __get_kippolog_grammar():
        ints = Word(nums)

        # date and time
        date = Combine(ints + '-' + ints + '-' + ints)
        time = Combine(ints + ':' + ints + ':' + ints + '+0000')
        datetime = date + time

        # 2017-02-23 14:58:33+0000 [-] unauthorized login:
        # 2017-02-23 14:58:34+0000 [SSHService ssh-userauth on HoneyPotTransport,13943,113.195.145.21]
        # root trying auth password

        # service = activity, port, and ip address
        activity = Word(alphas + nums + '_' + '-' + '.') + Optional(',') + Optional(ints) + \
            Optional(',') + Optional(nums + '.')
        service = Suppress('[') + activity + Suppress(']')

        # message
        message = Regex(".*")

        # kippo honeypot log grammar
        kippolog_grammar = datetime + service + message
        return kippolog_grammar
<|endoftext|>


<commit_before>"""
This an example script inserting a pylearn2 yaml code into a jobman database.

The code below defines a yaml template string in state.yaml_template,
and the values of its hyper-parameters in state.hyper_parameters, and
run the code that is located in state.extract_results on this model
using jobman.

Actually, we add the job here and it can be launched later as usual
(please check how to start jobs using jobman from the jobman tutorial
website)

"""

from jobman.tools import DD, flatten
from jobman import api0, sql

from pylearn2.scripts.jobman import experiment

def result_extractor(train_obj):
    """
    This is a user specific function, that is used by jobman to extract results

    The returned dictionary will be saved in state.results
    """
    channels = train_obj.model.monitor.channels['sgd_cost(SGD[X])']
    # This function returns the reconstruction_error and the bach numbers for
    # Contractive Auto-Encoder
    return dict(
            reconstruction_error=channels.val_record,
            batch_num=channels.batch_record)

if __name__ == '__main__':
    TABLE_NAME = 'test_jobman_pylearn2'
    db = api0.open_db(
            'postgres://almouslh@gershwin.iro.umontreal.ca/almouslh_db?table='
            + TABLE_NAME)

    state = DD()

    state.yaml_template = '''
        !obj:pylearn2.scripts.train.Train {
        "dataset": !obj:pylearn2.datasets.npy_npz.NpyDataset &dataset {
            "file" : "%(file)s" # Should be an N x 300 matrix on disk.
        },
        "model": !obj:pylearn2.autoencoder.ContractiveAutoencoder {
            "nvis" : %(nvis)d,
            "nhid" : %(nhid)d,
            "irange" : 0.05,
            "act_enc": "sigmoid", #for some reason only sigmoid function works
            "act_dec": "sigmoid",
        },
        "algorithm": !obj:pylearn2.training_algorithms.sgd.SGD {
            "learning_rate" : %(learning_rate)f,
            "batch_size" : %(batch_size)d,
            "monitoring_batches" : 5,
            "monitoring_dataset" : *dataset,
            "cost" : [
                !obj:pylearn2.costs.autoencoder.MeanSquaredReconstructionError {},
                !obj:pylearn2.costs.autoencoder.ScaleBy {
                    cost: !obj:pylearn2.costs.autoencoder.ModelMethodPenalty {
                        method_name: contraction_penalty
                    },
                    coefficient: %(coefficient)f
                }
            ],
            "termination_criterion" : %(term_crit)s,
        }
    }
    '''

    state.hyper_parameters = {
            "file": "/u/almouslh/Documents/UTLCChallenge/Version23Feb/ift6266h12/ift6266h12/experiments/sylvester/cae/layers/npy/sylvester_train_pca8.npy",
            "nvis": 8,
            "nhid": 6,
            "learning_rate": 0.1,
            "batch_size": 10,
            "coefficient": 0.5,
            "term_crit": {
                "__builder__": "pylearn2.training_algorithms.sgd.EpochCounter",
                "max_epochs": 2
                }
            }

    state.extract_results = "pylearn2.scripts.jobman.tester.result_extractor"

    sql.insert_job(
            experiment.train_experiment,
            flatten(state),
            db,
            force_dup=True)
<commit_msg>Final fixes to tester.py<commit_after>"""
This an example script inserting a pylearn2 yaml code into a jobman database.

The code below defines a yaml template string in state.yaml_template,
and the values of its hyper-parameters in state.hyper_parameters, and
run the code that is located in state.extract_results on this model
using jobman.

Actually, we add the job here and it can be launched later as usual
(please check how to start jobs using jobman from the jobman tutorial
website)

"""

from jobman.tools import DD, flatten
from jobman import api0, sql

from pylearn2.scripts.jobman import experiment

def result_extractor(train_obj):
    """
    This is a user specific function, that is used by jobman to extract results

    The returned dictionary will be saved in state.results
    """
    import numpy

    channels = train_obj.model.monitor.channels
    train_cost = channels['sgd_cost(ExhaustiveSGD[X])']
    best_epoch = numpy.argmin(train_cost.val_record)
    best_rec_error = train_cost.val_record[best_epoch]
    batch_num = train_cost.batch_record[best_epoch]
    return dict(
            best_epoch=best_epoch,
            train_rec_error=best_rec_error,
            batch_num=batch_num)

if __name__ == '__main__':
    db = api0.open_db('sqlite:///test.db?table=test_jobman_pylearn2')

    state = DD()

    state.yaml_template = '''
        !obj:pylearn2.scripts.train.Train {
        "dataset": !obj:pylearn2.datasets.npy_npz.NpyDataset &dataset {
            "file" : "%(file)s"
        },
        "model": !obj:pylearn2.autoencoder.ContractiveAutoencoder {
            "nvis" : %(nvis)d,
            "nhid" : %(nhid)d,
            "irange" : 0.05,
            "act_enc": "sigmoid", #for some reason only sigmoid function works
            "act_dec": "sigmoid",
        },
        "algorithm": !obj:pylearn2.training_algorithms.sgd.ExhaustiveSGD {
            "learning_rate" : %(learning_rate)f,
            "batch_size" : %(batch_size)d,
            "monitoring_batches" : 5,
            "monitoring_dataset" : *dataset,
            "cost" : [
                !obj:pylearn2.costs.autoencoder.MeanSquaredReconstructionError {},
                !obj:pylearn2.costs.autoencoder.ScaleBy {
                    cost: !obj:pylearn2.costs.autoencoder.ModelMethodPenalty {
                        method_name: contraction_penalty
                    },
                    coefficient: %(coefficient)f
                }
            ],
            "termination_criterion" : %(term_crit)s,
        }
    }
    '''

    state.hyper_parameters = {
            "file": "${PYLEARN2_DATA_PATH}/UTLC/pca/sylvester_train_x_pca32.npy",
            "nvis": 32,
            "nhid": 6,
            "learning_rate": 0.1,
            "batch_size": 10,
            "coefficient": 0.5,
            "term_crit": {
                "__builder__": "pylearn2.training_algorithms.sgd.EpochCounter",
                "max_epochs": 2
                }
            }

    state.extract_results = "pylearn2.scripts.jobman.tester.result_extractor"

    sql.insert_job(
            experiment.train_experiment,
            flatten(state),
            db,
            force_dup=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
This module implements classes to perform bond valence analyses.
"""

from __future__ import division

__author__ = "Shyue Ping Ong"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "shyue@mit.edu"
__date__ = "Oct 26, 2012"

import collections
import json
import itertools
import os
from math import exp, sqrt, floor

from pymatgen.core.periodic_table import Element, Specie
from pymatgen.symmetry.finder import SymmetryFinder
from pymatgen.core.structure_modifier import StructureEditor

#Let's initialize some module level properties.

#List of electronegative elements specified in M. O'Keefe, & N. Brese,
#JACS, 1991, 113(9), 3226-3229. doi:10.1021/ja00009a002.
ELECTRONEG = [Element(sym) for sym in ["H",
                                       "B", "C", "Si",
                                       "N", "P", "As", "Sb",
                                       "O", "S", "Se", "Te",
                                       "F", "Cl", "Br", "I"]]

module_dir = os.path.dirname(os.path.abspath(__file__))

#Read in BV parameters.
BV_PARAMS = {}
with open(os.path.join(module_dir, "bvparam_1991.json"), "r") as f:
    for k, v in json.load(f).items():
        BV_PARAMS[Element(k)] = v

#Read in json containing data-mined ICSD BV data.
with open(os.path.join(module_dir, "icsd_bv.json"), "r") as f:
    all_data = json.load(f)
    ICSD_BV_DATA = {Specie.from_string(sp): data
                    for sp, data in all_data["bvsum"].items()}
    PRIOR_PROB = {Specie.from_string(sp): data
                  for sp, data in all_data["occurrence"].items()}

def calculate_bv_sum(site, nn_list, anion_el, scale_factor=1):
    """
    Calculates the BV sum of a site.

    Args:
        site:
            The site
        nn_list:
            List of nearest neighbors in the format [(nn_site, dist), ...].
        anion_el:
            The most electronegative element in the structure.
        scale_factor:
            A scale factor to be applied. This is useful for scaling distance,
            esp in the case of calculation-relaxed structures which may tend
            to under (GGA) or over bind (LDA).
    """
    el1 = Element(site.specie.symbol)
    bvsum = 0
    for (nn, dist) in nn_list:
        el2 = Element(nn.specie.symbol)
        if (el1 == anion_el or el2 == anion_el) and el1 != el2:
            r1 = BV_PARAMS[el1]["r"]
            r2 = BV_PARAMS[el2]["r"]
            c1 = BV_PARAMS[el1]["c"]
            c2 = BV_PARAMS[el2]["c"]
            R = r1 + r2 - r1 * r2 * (sqrt(c1) - sqrt(c2)) ** 2 / \
                (c1 * r1 + c2 * r2)
            vij = exp((R - dist * scale_factor) / 0.31)
            bvsum += vij * (1 if el1.X < el2.X else -1)
    return bvsum

class BVAnalyzer(object):
    """
    This class implements a maximum a posteriori (MAP) estimation method to
    determine oxidation states in a structure. The algorithm is as follows:
    1) The bond valence sum of all symmetrically distinct sites in a structure
    is calculated using the element-based parameters in M. O'Keefe, & N. Brese,
    JACS, 1991, 113(9), 3226-3229. doi:10.1021/ja00009a002.
    2) The posterior probabilities of all oxidation states is then calculated
    using: P(oxi_state/BV) = K * P(BV/oxi_state) * P(oxi_state), where K is
    a constant factor for each element. P(BV/oxi_state) is calculated as a
    Gaussian with mean and std deviation determined from an analysis of
    the ICSD. The posterior P(oxi_state) is determined from a frequency
    analysis of the ICSD.
    3) The oxidation states are then ranked in order of decreasing probability
    and the oxidation state combination that result in a charge neutral cell
    is selected.
    """

    def __init__(self, symm_tol=0.1, max_radius=4, max_permutations=1000000):
        """
        Args:
            symm_tol:
                Symmetry tolerance used to determine which sites are
                symmetrically equivalent. Set to 0 to turn off symmetry.
            max_radius:
                Maximum radius in Angstrom used to find nearest neighbors.
            max_permutations:
                The maximum number of permutations of oxidation states to test.
        """
        self.symm_tol = symm_tol
        self.max_radius = max_radius
        self.max_permutations = max_permutations

    def get_valences(self, structure):
        """
        Returns a list of valences for the structure. This currently works only
        for ordered structures only.

        Args:
            structure:
                Structure to analyze

        Returns:
            A list of valences for each site in the structure,
            e.g., [1, 1, -2].

        Raises:
            A ValueError is the valences cannot be determined.
        """
        els = [Element(el.symbol) for el in structure.composition.elements]

        if not set(els).issubset(set(BV_PARAMS.keys())):
            raise ValueError("Valences cannot be assigned!")

        #Perform symmetry determination and get sites grouped by symmetry.
        if self.symm_tol:
            finder = SymmetryFinder(structure, self.symm_tol)
            symm_structure = finder.get_symmetrized_structure()
            equi_sites = symm_structure.equivalent_sites
        else:
            symm_structure = structure
            equi_sites = [[site] for site in structure]

        equi_sites = sorted(equi_sites, key=lambda sites:-sites[0].specie.X)

        all_nn = structure.get_all_neighbors(self.max_radius)

        #The anion element is the most electronegative element.
        anion_el = Element(equi_sites[0][0].specie.symbol)

        def get_equi_index(site):
            for j, sites in enumerate(equi_sites):
                if site in sites:
                    return j
            raise ValueError("Can't find equi-index!")

        #Get a list of valences and probabilities for each symmetrically
        #distinct site.
        valences = []
        all_prob = []
        for sites in equi_sites:
            test_site = sites[0]
            el = test_site.specie.symbol
            ind = structure.sites.index(test_site)
            bv_sum = calculate_bv_sum(test_site, all_nn[ind], anion_el,
                                      scale_factor=1.015)
            prob = {}
            for sp, data in ICSD_BV_DATA.items():
                if sp.symbol == el and sp.oxi_state != 0 and data["std"] > 0:
                    u = data["mean"]
                    sigma = data["std"]
                    #Calculate posterior probability. Note that constant
                    #factors are ignored. They have no effect on the results.
                    prob[sp.oxi_state] = exp(-(bv_sum - u) ** 2 /
                                             2 / (sigma ** 2)) \
                                             / sigma * PRIOR_PROB[sp]
            #Normalize the probabilities
            prob = {k: v / sum(prob.values()) for k, v in prob.items()}

            all_prob.append(prob)
            val = list(prob.keys())
            #Sort valences in order of decreasing probability.
            val = sorted(val, key=lambda v:-prob[v])
            valences.append(val)

        #Based on the max allowed permutations, determine the number of
        #candidates per site.
        ncand_per_site = int(floor(self.max_permutations **
                                   (1 / len(valences))))
        ncand_per_site = max(1, ncand_per_site)
        valences = [v[0:min(ncand_per_site, len(v))] for v in valences]
        found = None
        scores = {}
        #Find valid valence combinations and score them by total probability.
        for v_set in itertools.product(*valences):
            total = 0
            el_oxi = collections.defaultdict(list)
            for i, sites in enumerate(equi_sites):
                total += v_set[i] * len(sites)
                el_oxi[sites[0].specie.symbol].append(v_set[i])

            #Calculate the maximum range in oxidation states for each element.
            max_diff = max([max(v) - min(v) for v in el_oxi.values()])

            if total == 0 and max_diff <= 1:
                #Cell has to be charge neutral. And the maximum difference in
                #oxidation state for each element cannot exceed 1.
                found = tuple(v_set)
                score = 1
                for i, v in enumerate(v_set):
                    score *= all_prob[i][v]
                scores[found] = score

        if found:
            best = sorted(scores.keys(), key=lambda k: scores[k])[-1]
            all_oxi = []
            for site in structure:
                all_oxi.append(int(best[get_equi_index(site)]))
            return all_oxi
        else:
            raise ValueError("Valences cannot be assigned!")

    def get_oxi_state_decorated_structure(self, structure):
        """
        Get an oxidation state decorated structure. This currently works only
        for ordered structures only.

        Args:
            structure:
                Structure to analyze

        Returns:
            A modified structure that is oxidation state decorated.

        Raises:
            A ValueError is the valences cannot be determined.
        """
        valences = self.get_valences(structure)
        editor = StructureEditor(structure)
        editor.add_oxidation_state_by_site(valences)
        return editor.modified_structure
<commit_msg>Considerable speed up of bond valence code.<commit_after>#!/usr/bin/env python

"""
This module implements classes to perform bond valence analyses.
"""

from __future__ import division

__author__ = "Shyue Ping Ong"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "shyue@mit.edu"
__date__ = "Oct 26, 2012"

import collections
import json
import itertools
import os
from math import exp, sqrt, floor

from pymatgen.core.periodic_table import Element, Specie
from pymatgen.symmetry.finder import SymmetryFinder
from pymatgen.core.structure_modifier import StructureEditor

#Let's initialize some module level properties.

#List of electronegative elements specified in M. O'Keefe, & N. Brese,
#JACS, 1991, 113(9), 3226-3229. doi:10.1021/ja00009a002.
ELECTRONEG = [Element(sym) for sym in ["H",
                                       "B", "C", "Si",
                                       "N", "P", "As", "Sb",
                                       "O", "S", "Se", "Te",
                                       "F", "Cl", "Br", "I"]]

module_dir = os.path.dirname(os.path.abspath(__file__))

#Read in BV parameters.
BV_PARAMS = {}
with open(os.path.join(module_dir, "bvparam_1991.json"), "r") as f:
    for k, v in json.load(f).items():
        BV_PARAMS[Element(k)] = v

#Read in json containing data-mined ICSD BV data.
with open(os.path.join(module_dir, "icsd_bv.json"), "r") as f:
    all_data = json.load(f)
    ICSD_BV_DATA = {Specie.from_string(sp): data
                    for sp, data in all_data["bvsum"].items()}
    PRIOR_PROB = {Specie.from_string(sp): data
                  for sp, data in all_data["occurrence"].items()}

def calculate_bv_sum(site, nn_list, anion_el, scale_factor=1):
    """
    Calculates the BV sum of a site.

    Args:
        site:
            The site
        nn_list:
            List of nearest neighbors in the format [(nn_site, dist), ...].
        anion_el:
            The most electronegative element in the structure.
        scale_factor:
            A scale factor to be applied. This is useful for scaling distance,
            esp in the case of calculation-relaxed structures which may tend
            to under (GGA) or over bind (LDA).
    """
    el1 = Element(site.specie.symbol)
    bvsum = 0
    for (nn, dist) in nn_list:
        el2 = Element(nn.specie.symbol)
        if (el1 == anion_el or el2 == anion_el) and el1 != el2:
            r1 = BV_PARAMS[el1]["r"]
            r2 = BV_PARAMS[el2]["r"]
            c1 = BV_PARAMS[el1]["c"]
            c2 = BV_PARAMS[el2]["c"]
            R = r1 + r2 - r1 * r2 * (sqrt(c1) - sqrt(c2)) ** 2 / \
                (c1 * r1 + c2 * r2)
            vij = exp((R - dist * scale_factor) / 0.31)
            bvsum += vij * (1 if el1.X < el2.X else -1)
    return bvsum

class BVAnalyzer(object):
    """
    This class implements a maximum a posteriori (MAP) estimation method to
    determine oxidation states in a structure. The algorithm is as follows:
    1) The bond valence sum of all symmetrically distinct sites in a structure
    is calculated using the element-based parameters in M. O'Keefe, & N. Brese,
    JACS, 1991, 113(9), 3226-3229. doi:10.1021/ja00009a002.
    2) The posterior probabilities of all oxidation states is then calculated
    using: P(oxi_state/BV) = K * P(BV/oxi_state) * P(oxi_state), where K is
    a constant factor for each element. P(BV/oxi_state) is calculated as a
    Gaussian with mean and std deviation determined from an analysis of
    the ICSD. The posterior P(oxi_state) is determined from a frequency
    analysis of the ICSD.
    3) The oxidation states are then ranked in order of decreasing probability
    and the oxidation state combination that result in a charge neutral cell
    is selected.
    """

    def __init__(self, symm_tol=0.1, max_radius=4, max_permutations=1000000):
        """
        Args:
            symm_tol:
                Symmetry tolerance used to determine which sites are
                symmetrically equivalent. Set to 0 to turn off symmetry.
            max_radius:
                Maximum radius in Angstrom used to find nearest neighbors.
            max_permutations:
                The maximum number of permutations of oxidation states to test.
        """
        self.symm_tol = symm_tol
        self.max_radius = max_radius
        self.max_permutations = max_permutations

    def _calc_site_probabilities(self, site, nn, anion_el):
        el = site.specie.symbol
        bv_sum = calculate_bv_sum(site, nn, anion_el, scale_factor=1.015)
        prob = {}
        for sp, data in ICSD_BV_DATA.items():
            if sp.symbol == el and sp.oxi_state != 0 and data["std"] > 0:
                u = data["mean"]
                sigma = data["std"]
                #Calculate posterior probability. Note that constant
                #factors are ignored. They have no effect on the results.
                prob[sp.oxi_state] = exp(-(bv_sum - u) ** 2 /
                                         2 / (sigma ** 2)) \
                                         / sigma * PRIOR_PROB[sp]
        #Normalize the probabilities
        prob = {k: v / sum(prob.values()) for k, v in prob.items()}
        return prob

    def get_valences(self, structure):
        """
        Returns a list of valences for the structure. This currently works only
        for ordered structures only.

        Args:
            structure:
                Structure to analyze

        Returns:
            A list of valences for each site in the structure,
            e.g., [1, 1, -2].

        Raises:
            A ValueError is the valences cannot be determined.
        """
        els = [Element(el.symbol) for el in structure.composition.elements]

        if not set(els).issubset(set(BV_PARAMS.keys())):
            raise ValueError("Valences cannot be assigned!")

        #Perform symmetry determination and get sites grouped by symmetry.
        if self.symm_tol:
            finder = SymmetryFinder(structure, self.symm_tol)
            symm_structure = finder.get_symmetrized_structure()
            equi_sites = symm_structure.equivalent_sites
        else:
            symm_structure = structure
            equi_sites = [[site] for site in structure]

        #Sort the equivalent sites by decreasing electronegativity.
        equi_sites = sorted(equi_sites, key=lambda sites:-sites[0].specie.X)

        #all_nn = structure.get_all_neighbors(self.max_radius)

        #The anion element is the most electronegative element.
        anion_el = Element(equi_sites[0][0].specie.symbol)

        #Get a list of valences and probabilities for each symmetrically
        #distinct site.
        valences = []
        all_prob = []
        for sites in equi_sites:
            test_site = sites[0]
            nn = structure.get_neighbors(test_site, self.max_radius)
            prob = self._calc_site_probabilities(test_site, nn,
                                                 anion_el)
            all_prob.append(prob)
            val = list(prob.keys())
            #Sort valences in order of decreasing probability.
            val = sorted(val, key=lambda v:-prob[v])
            valences.append(val)

        #Based on the max allowed permutations, determine the number of
        #candidates per site.
        ncand_per_site = int(floor(self.max_permutations **
                                   (1 / len(valences))))
        ncand_per_site = max(1, ncand_per_site)
        valences = [v[0:min(ncand_per_site, len(v))] for v in valences]
        found = None
        scores = {}
        #Find valid valence combinations and score them by total probability.
        for v_set in itertools.product(*valences):
            total = 0
            el_oxi = collections.defaultdict(list)
            for i, sites in enumerate(equi_sites):
                total += v_set[i] * len(sites)
                el_oxi[sites[0].specie.symbol].append(v_set[i])

            #Calculate the maximum range in oxidation states for each element.
            max_diff = max([max(v) - min(v) for v in el_oxi.values()])

            if total == 0 and max_diff <= 1:
                #Cell has to be charge neutral. And the maximum difference in
                #oxidation state for each element cannot exceed 1.
                found = tuple(v_set)
                score = 1
                for i, v in enumerate(v_set):
                    score *= all_prob[i][v]
                scores[found] = score

        if found:
            best = sorted(scores.keys(), key=lambda k: scores[k])[-1]

            def get_equi_index(site):
                for j, sites in enumerate(equi_sites):
                    if site in sites:
                        return j
                raise ValueError("Can't find equi-index!")

            return [int(best[get_equi_index(site)]) for site in structure]
        else:
            raise ValueError("Valences cannot be assigned!")

    def get_oxi_state_decorated_structure(self, structure):
        """
        Get an oxidation state decorated structure. This currently works only
        for ordered structures only.

        Args:
            structure:
                Structure to analyze

        Returns:
            A modified structure that is oxidation state decorated.

        Raises:
            A ValueError is the valences cannot be determined.
        """
        valences = self.get_valences(structure)
        editor = StructureEditor(structure)
        editor.add_oxidation_state_by_site(valences)
        return editor.modified_structure
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Script process the GW output generated by GWsetup using VASP or ABINIT.
"""

from __future__ import division

__author__ = "Michiel van Setten"
__copyright__ = " "
__version__ = "0.9"
__maintainer__ = "Michiel van Setten"
__email__ = "mjvansetten@gmail.com"
__date__ = "May 2014"

import os
import os.path

from pymatgen.io.gwwrapper.GWsetup import get_spec

MODULE_DIR = os.path.dirname(os.path.abspath(__file__))

if __name__ == "__main__":
    if os.path.isfile('plots'):
        os.remove('plots')
    spec = get_spec('GW')
    spec.read_from_file('spec.in')
    print 'Found setup for ', spec.get_code()
    spec.loop_structures('o')<commit_msg>delta factor script<commit_after>#!/usr/bin/env python

"""
Script process the GW output generated by GWsetup using VASP or ABINIT.
"""

from __future__ import division

__author__ = "Michiel van Setten"
__copyright__ = " "
__version__ = "0.9"
__maintainer__ = "Michiel van Setten"
__email__ = "mjvansetten@gmail.com"
__date__ = "May 2014"

import os
import os.path

from pymatgen.io.gwwrapper.datastructures import get_spec

MODULE_DIR = os.path.dirname(os.path.abspath(__file__))

if __name__ == "__main__":
    if os.path.isfile('plots'):
        os.remove('plots')
    spec = get_spec('GW')
    spec.read_from_file('spec.in')
    print 'Found setup for ', spec.get_code()
    spec.loop_structures('o')<|endoftext|>


<commit_before>import numpy as np
import pytest
import pymbar
from pymbar.utils_for_testing import (
    assert_array_almost_equal,
    oscillators,
    exponentials,
)
from pymbar.tests.test_mbar import z_scale_factor

@pytest.fixture(scope="module")
def base_oscillator():
    name, u_kn, N_k, s_n, test = oscillators(50, 100, provide_test=True)
    return {"name": name, "u_kn": u_kn, "N_k": N_k, "s_n": s_n, "test": test}

@pytest.mark.flaky(max_runs=2)  # Uses flaky plugin for pytest
@pytest.mark.parametrize(
    "statesa, statesb, test_system",
    [(100, 100, oscillators), (200, 50, oscillators), (200, 50, exponentials)],
)
def test_solvers(statesa, statesb, test_system):
    name, U, N_k, s_n, _ = test_system(statesa, statesb, provide_test=True)
    print(name)
    mbar = pymbar.MBAR(U, N_k)
    assert_array_almost_equal(
        pymbar.mbar_solvers.mbar_gradient(U, N_k, mbar.f_k), np.zeros(N_k.shape), decimal=8
    )
    assert_array_almost_equal(np.exp(mbar.Log_W_nk).sum(0), np.ones(len(N_k)), decimal=10)
    assert_array_almost_equal(np.exp(mbar.Log_W_nk).dot(N_k), np.ones(U.shape[1]), decimal=10)
    assert_array_almost_equal(
        pymbar.mbar_solvers.self_consistent_update(U, N_k, mbar.f_k), mbar.f_k, decimal=10
    )

@pytest.mark.parametrize(
    "protocol",
    [
        "adaptive",
        "hybr",
        "lm",
        "L-BFGS-B",
        "dogleg",
        "CG",
        "BFGS",
        "Newton-CG",
        "TNC",
        "trust-ncg",
        "trust-krylov",
        pytest.param(
            "trust-exact", marks=pytest.mark.flaky(max_runs=2)
        ),  # Can rarely NaN on Windows
        "SLSQP",
    ],
)
def test_protocols(base_oscillator, protocol):
    """
    Test that free energy is moderately equal to analytical solution, independent of solver protocols
    """

    # Importing the hacky fix to asert that free energies are moderately correct

    test = base_oscillator["test"]
    u_kn = base_oscillator["u_kn"]
    N_k = base_oscillator["N_k"]
    fa = test.analytical_free_energies()
    fa = fa[1:] - fa[0]
    # Solve MBAR with zeros for initial weights
    mbar = pymbar.MBAR(u_kn, N_k, solver_protocol=({"method": protocol},))
    # Solve MBAR with the correct f_k used for the inital weights
    mbar = pymbar.MBAR(u_kn, N_k, initial_f_k=mbar.f_k, solver_protocol=({"method": protocol},))
    results = mbar.compute_free_energy_differences()
    fe = results["Delta_f"][0, 1:]
    fe_sigma = results["dDelta_f"][0, 1:]
    z = (fe - fa) / fe_sigma
    assert_array_almost_equal(z / z_scale_factor, np.zeros(len(z)), decimal=0)
<commit_msg>Make the test_protocols more robust by providing a better set of samples on first failure. Removed `flaky` flag in favor of doing 1 re-test with more samples and then raising if need be.<commit_after>import numpy as np
import pytest
import pymbar
from pymbar.utils_for_testing import (
    assert_array_almost_equal,
    oscillators,
    exponentials,
)
from pymbar.tests.test_mbar import z_scale_factor

@pytest.fixture(scope="module")
def base_oscillator():
    name, u_kn, N_k, s_n, test = oscillators(50, 100, provide_test=True)
    return {"name": name, "u_kn": u_kn, "N_k": N_k, "s_n": s_n, "test": test}

@pytest.fixture(scope="module")
def more_oscillators():
    name, u_kn, N_k, s_n, test = oscillators(50, 500, provide_test=True)
    return {"name": name, "u_kn": u_kn, "N_k": N_k, "s_n": s_n, "test": test}

@pytest.mark.flaky(max_runs=2)  # Uses flaky plugin for pytest
@pytest.mark.parametrize(
    "statesa, statesb, test_system",
    [(100, 100, oscillators), (200, 50, oscillators), (200, 50, exponentials)],
)
def test_solvers(statesa, statesb, test_system):
    name, U, N_k, s_n, _ = test_system(statesa, statesb, provide_test=True)
    print(name)
    mbar = pymbar.MBAR(U, N_k)
    assert_array_almost_equal(
        pymbar.mbar_solvers.mbar_gradient(U, N_k, mbar.f_k), np.zeros(N_k.shape), decimal=8
    )
    assert_array_almost_equal(np.exp(mbar.Log_W_nk).sum(0), np.ones(len(N_k)), decimal=10)
    assert_array_almost_equal(np.exp(mbar.Log_W_nk).dot(N_k), np.ones(U.shape[1]), decimal=10)
    assert_array_almost_equal(
        pymbar.mbar_solvers.self_consistent_update(U, N_k, mbar.f_k), mbar.f_k, decimal=10
    )

def run_mbar_protocol(oscillator_bundle, protocol):
    test = oscillator_bundle["test"]
    u_kn = oscillator_bundle["u_kn"]
    N_k = oscillator_bundle["N_k"]
    fa = test.analytical_free_energies()
    fa = fa[1:] - fa[0]
    # Solve MBAR with zeros for initial weights
    mbar = pymbar.MBAR(u_kn, N_k, solver_protocol=({"method": protocol},))
    # Solve MBAR with the correct f_k used for the initial weights
    mbar = pymbar.MBAR(u_kn, N_k, initial_f_k=mbar.f_k, solver_protocol=({"method": protocol},))
    return mbar, fa

@pytest.mark.parametrize(
    "protocol",
    [
        "adaptive",
        "hybr",
        "lm",
        "L-BFGS-B",
        "dogleg",
        "CG",
        "BFGS",
        "Newton-CG",
        "TNC",
        "trust-ncg",
        "trust-krylov",
        "trust-exact",
        "SLSQP",
    ],
)
def test_protocols(base_oscillator, more_oscillators, protocol):
    """
    Test that free energy is moderately equal to analytical solution, independent of solver protocols
    """

    # Importing the hacky fix to asert that free energies are moderately correct

    try:
        mbar, fa = run_mbar_protocol(base_oscillator, protocol)
    except Exception as e:
        print(f"Caught error in initial oscillator test, trying with more samples. Error:\n\n{e}")
        mbar, fa = run_mbar_protocol(more_oscillators, protocol)
    results = mbar.compute_free_energy_differences()
    fe = results["Delta_f"][0, 1:]
    fe_sigma = results["dDelta_f"][0, 1:]
    z = (fe - fa) / fe_sigma
    assert_array_almost_equal(z / z_scale_factor, np.zeros(len(z)), decimal=0)
<|endoftext|>


<commit_before>from contextlib import contextmanager, ExitStack

from ..errors import *

from .state import State
from .core import Context, Object, BoundObject, Scope, Type
from .util import inScope
from .links import Link

# A dependent object is a collector for behaviour
# Initially the object is used like any other, creating dependencies
# Later a object can be chosen to "replace" the dependent object,
# Which is then checked for all the dependencies.
# This is similar to templating, but much more powerful and less context
# dependent.
class DependentObject(Type, BoundObject):
    dependent = True

    # The target object which which to replace the dependent object
    target = None
    # The scope in which the dependent object is mutable
    scope = None

    # Child Dependencies
    _context = None
    _instance_context = None
    resolved_type = None
    resolved_calls = None
    resolved_instance_calls = None

    # Object Dependencies
    target_switch = None
    target_switch_determiner = None
    compatible_types = None
    compatible_type_switch = None
    switches = None

    # Hack for dependent functions
    _return_type = None

    def __init__(self, scope:Scope, name:str = None, tokens = None):
        super().__init__(name or "", tokens)
        self.scope = scope

        self.resolved_calls = dict()
        self.resolved_instance_calls = dict()
        self.compatible_types = set()
        self.switches = []

    @classmethod
    def switch(self, scope:Scope, targets:[Object], determiner:(lambda Object: bool)):
        out = DependentObject(scope)
        out.target_switch = targets
        out.target_switch_determiner = determiner
        return out

    @property
    def locked(self):
        return not inScope(self.scope, State.scope)

    def verify(self):
        if self.target: self.target.verify()

    # alias to the target if it exists
    def resolveValue(self):
        return self.target or self

    # Check whether an object matches the dependencies of this object
    @contextmanager
    def targetAt(self, target):
        with ExitStack() as stack:
            #TODO: Handle errors

            # Local checks
            if self.target_switch is not None:
                assert target.resolveValue() in self.target_switch

            if not self.checkLockedCompatibility(target):
                raise TypeError("TODO: Write this")

            # Set target
            previous_target = self.target
            self.target = target

            # Pass on dependency checks
            if self._context is not None:
                stack.enter_context(self.context.targetAt(target.context))

            if self._instance_context is not None:
                stack.enter_context(self.instance_context.targetAt(target.instance_context))

            if self.resolved_type is not None:
                stack.enter_context(self.resolved_type.targetAt(target.resolveType()))

            for call, obj in self.resolved_calls.items():
                stack.enter_context(obj.targetAt(target.resolveCall(call)))

            for call, obj in self.resolved_instance_calls.items():
                stack.enter_context(obj.targetAt(target.resolveInstanceCall(call)))

            for switch in self.switches:
                stack.enter_context(switch.resolveTarget())

            #TOOD: Return type

            yield
            self.target = previous_target

    # Same as targetAt but for switches
    @contextmanager
    def resolveTarget(self):
        matches = []
        for possibility in self.target_switch:
            if self.target_switch_determiner(possibility):
                matches.append(possibility)

        if len(matches) < 1:
            raise InternalError("Target switch has impossible value")
        elif len(matches) > 1:
            raise TypeError("TODO: Write this")
        target = matches[0]

        previous_target = self.target
        self.target = target
        yield
        self.target = previous_target

    # Create and cache dependencies for standard object functionality

    @property
    def context(self):
        self._context = self._context or DependentContext(self)
        return self._context

    @property
    def instance_context(self):
        self._instance_context = self._instance_context or DependentContext(self)
        return self._instance_context

    def resolveType(self):
        self.resolved_type = self.resolved_type or DependentObject(self.scope)
        return self.resolved_type

    def resolveCall(self, call):
        return self.resolved_calls.setdefault(call, DependentObject(self.scope))

    # Hack!
    @property
    def return_type(self):
        self._return_type = self._return_type or DependentObject(self.scope)
        return self._return_type

    def resolveInstanceCall(self, call):
        return self.dependent_instance_calls.setdefault(call, DependentObject(self.scope))

    def checkCompatibility(self, other:Type):
        if self.target is not None: return self.target.checkCompatibility(other)
        if self.locked: return self.checkLockedCompatibility(other)
        if State.type_switching:
            self.compatible_type_switch = self.compatible_type_switch or []
            self.compatible_type_switch.append(other)
            State.type_switch_cleanups.append(self.typeSwitchCleanup)
        else:
            self.compatible_types.add(other)
        return True

    def checkLockedCompatibility(self, other:Type):
        for types in self.compatible_types:
            matches = []
            for type in types:
                if other.checkCompatibility(type):
                    matches.append(type)

            if len(matches) != 1:
                print(other, matches, self.compatible_types)
                return False
        return True

    def typeSwitchCleanup(self):
        if self.compatible_type_switch is None: return
        self.compatible_types.add(tuple(self.compatible_type_switch))
        self.compatible_type_switch = None

    def __repr__(self):
        if self.target is None:
            return "{}".format(self.__class__.__name__)
        return "{} as {}".format(self.__class__.__name__, self.target)

class DependentTarget(Link):
    dependencies = None

    def __init__(self, value:Object, dependencies:[(DependentObject, Object)], tokens = None):
        super().__init__(value, tokens)
        self.dependencies = dependencies

    @contextmanager
    def target(self):
        with ExitStack() as stack:
            for object, target in self.dependencies:
                stack.enter_context(object.targetAt(target))
            yield

    def verify(self):
        with self.target():
            super().verify()

# The dependent context compliments the dependent object with the creation
# of dependencies for contexts.
class DependentContext(Context):
    dependent = True

    def __init__(self, scope:BoundObject):
        super().__init__(scope, [])

    def __contains__(self, name:str):
        if self.scope.locked: return super().__contains__(name)
        return True

    def __getitem__(self, name:str):
        if not self.scope.locked and name not in self.children:
            self.addChild(DependentObject(self.scope.scope, name))
        return self.children[name]

    def __setitem__(self, name:str, value:BoundObject):
        raise InternalError("Not Implemented.")

    @contextmanager
    def targetAt(self, target):
        with ExitStack() as stack:
            for name in self.children:
                if name not in target.children:
                    raise DependencyError("Dependent target context does not have attribute {}".format(name), target.scope.tokens)
            stack.enter_context(self[name].targetAt(target[name]))
            yield
<commit_msg>Implemented return type handling for dependent objects<commit_after>from contextlib import contextmanager, ExitStack

from ..errors import *

from .state import State
from .core import Context, Object, BoundObject, Scope, Type
from .util import inScope
from .links import Link

# A dependent object is a collector for behaviour
# Initially the object is used like any other, creating dependencies
# Later a object can be chosen to "replace" the dependent object,
# Which is then checked for all the dependencies.
# This is similar to templating, but much more powerful and less context
# dependent.
class DependentObject(Type, BoundObject):
    dependent = True

    # The target object which which to replace the dependent object
    target = None
    # The scope in which the dependent object is mutable
    scope = None

    # Child Dependencies
    _context = None
    _instance_context = None
    resolved_type = None
    resolved_calls = None
    resolved_instance_calls = None

    # Object Dependencies
    target_switch = None
    target_switch_determiner = None
    compatible_types = None
    compatible_type_switch = None
    switches = None

    # Hack for dependent functions
    _return_type = None

    def __init__(self, scope:Scope, name:str = None, tokens = None):
        super().__init__(name or "", tokens)
        self.scope = scope

        self.resolved_calls = dict()
        self.resolved_instance_calls = dict()
        self.compatible_types = set()
        self.switches = []

    @classmethod
    def switch(self, scope:Scope, targets:[Object], determiner:(lambda Object: bool)):
        out = DependentObject(scope)
        out.target_switch = targets
        out.target_switch_determiner = determiner
        return out

    @property
    def locked(self):
        return not inScope(self.scope, State.scope)

    def verify(self):
        if self.target: self.target.verify()

    # alias to the target if it exists
    def resolveValue(self):
        return self.target or self

    # Check whether an object matches the dependencies of this object
    @contextmanager
    def targetAt(self, target):
        with ExitStack() as stack:
            #TODO: Handle errors

            # Local checks
            if self.target_switch is not None:
                assert target.resolveValue() in self.target_switch

            if not self.checkLockedCompatibility(target):
                raise TypeError("TODO: Write this")

            # Set target
            previous_target = self.target
            self.target = target

            # Pass on dependency checks
            if self._context is not None:
                stack.enter_context(self.context.targetAt(target.context))

            if self._instance_context is not None:
                stack.enter_context(self.instance_context.targetAt(target.instance_context))

            if self.resolved_type is not None:
                stack.enter_context(self.resolved_type.targetAt(target.resolveType()))

            for call, obj in self.resolved_calls.items():
                stack.enter_context(obj.targetAt(target.resolveCall(call)))

            for call, obj in self.resolved_instance_calls.items():
                stack.enter_context(obj.targetAt(target.resolveInstanceCall(call)))

            for switch in self.switches:
                stack.enter_context(switch.resolveTarget())

            if self._return_type is not None:
                if not hasattr(target, "return_type"):
                    raise TypeError("TODO: Write this")
                stack.enter_context(self._return_type.targetAt(target.return_type))

            yield
            self.target = previous_target

    # Same as targetAt but for switches
    @contextmanager
    def resolveTarget(self):
        matches = []
        for possibility in self.target_switch:
            if self.target_switch_determiner(possibility):
                matches.append(possibility)

        if len(matches) < 1:
            raise InternalError("Target switch has impossible value")
        elif len(matches) > 1:
            raise TypeError("TODO: Write this")
        target = matches[0]

        previous_target = self.target
        self.target = target
        yield
        self.target = previous_target

    # Create and cache dependencies for standard object functionality

    @property
    def context(self):
        self._context = self._context or DependentContext(self)
        return self._context

    @property
    def instance_context(self):
        self._instance_context = self._instance_context or DependentContext(self)
        return self._instance_context

    def resolveType(self):
        self.resolved_type = self.resolved_type or DependentObject(self.scope)
        return self.resolved_type

    def resolveCall(self, call):
        return self.resolved_calls.setdefault(call, DependentObject(self.scope))

    # Hack!
    @property
    def return_type(self):
        self._return_type = self._return_type or DependentObject(self.scope)
        return self._return_type

    def resolveInstanceCall(self, call):
        return self.dependent_instance_calls.setdefault(call, DependentObject(self.scope))

    def checkCompatibility(self, other:Type):
        if self.target is not None: return self.target.checkCompatibility(other)
        if self.locked: return self.checkLockedCompatibility(other)
        if State.type_switching:
            self.compatible_type_switch = self.compatible_type_switch or []
            self.compatible_type_switch.append(other)
            State.type_switch_cleanups.append(self.typeSwitchCleanup)
        else:
            self.compatible_types.add(other)
        return True

    def checkLockedCompatibility(self, other:Type):
        for types in self.compatible_types:
            matches = []
            for type in types:
                if other.checkCompatibility(type):
                    matches.append(type)

            if len(matches) != 1:
                print(other, matches, self.compatible_types)
                return False
        return True

    def typeSwitchCleanup(self):
        if self.compatible_type_switch is None: return
        self.compatible_types.add(tuple(self.compatible_type_switch))
        self.compatible_type_switch = None

    def __repr__(self):
        if self.target is None:
            return "{}".format(self.__class__.__name__)
        return "{} as {}".format(self.__class__.__name__, self.target)

class DependentTarget(Link):
    dependencies = None

    def __init__(self, value:Object, dependencies:[(DependentObject, Object)], tokens = None):
        super().__init__(value, tokens)
        self.dependencies = dependencies

    @contextmanager
    def target(self):
        with ExitStack() as stack:
            for object, target in self.dependencies:
                stack.enter_context(object.targetAt(target))
            yield

    def verify(self):
        with self.target():
            super().verify()

# The dependent context compliments the dependent object with the creation
# of dependencies for contexts.
class DependentContext(Context):
    dependent = True

    def __init__(self, scope:BoundObject):
        super().__init__(scope, [])

    def __contains__(self, name:str):
        if self.scope.locked: return super().__contains__(name)
        return True

    def __getitem__(self, name:str):
        if not self.scope.locked and name not in self.children:
            self.addChild(DependentObject(self.scope.scope, name))
        return self.children[name]

    def __setitem__(self, name:str, value:BoundObject):
        raise InternalError("Not Implemented.")

    @contextmanager
    def targetAt(self, target):
        with ExitStack() as stack:
            for name in self.children:
                if name not in target.children:
                    raise DependencyError("Dependent target context does not have attribute {}".format(name), target.scope.tokens)
            stack.enter_context(self[name].targetAt(target[name]))
            yield
<|endoftext|>