<commit_before>from __future__ import with_statement
import sys
from warnings import warn  # TODO - catch known warnings
from unittest import TestCase, TestLoader, TextTestRunner
from tempfile import gettempdir
from os import unlink
from copy import deepcopy
from os.path import join
try:  #skipIf is not in python 2.6 / 2.5, so use unittest2
    from unittest import skipIf
except:
    from unittest2 import skipIf

# deal with absolute imports by adding the appropriate directory to the path
if __name__ == "__main__":
    sys.path.insert(0, "../..")
    from cobra.test import data_directory, create_test_model
    from cobra.test import salmonella_sbml as test_sbml_file
    from cobra.test import salmonella_pickle as test_pickle
    from cobra import Object, Model, Metabolite, Reaction, io, DictList
    sys.path.pop(0)
    #assert 0
else:
    from . import data_directory, create_test_model
    from . import salmonella_sbml as test_sbml_file
    from . import salmonella_pickle as test_pickle
    from .. import Object, Model, Metabolite, Reaction, io, DictList

# libraries which may or may not be installed
libraries = ["glpk", "gurobipy", "cplex"]
for library in libraries:
    try:
        exec("import %s" % library)
    except ImportError:
        exec("%s = None" % library)

class TestDictList(TestCase):
    def setUp(self):
        self.obj = Object("test1")
        self.list = DictList()
        self.list.append(self.obj)

    def testAppend(self):
        obj2 = Object("test2")
        self.list.append(obj2)
        self.assertRaises(ValueError, self.list.append,
            Object("test1"))
        self.assertEqual(self.list.index(obj2), 1)
        self.assertEqual(self.list[1], obj2)
        self.assertEqual(self.list.get_by_id("test2"), obj2)
        self.assertEqual(len(self.list), 2)

    def testExtend(self):
        obj_list = [Object("test%d" % (i)) for i in range(2, 10)]
        self.list.extend(obj_list)
        self.assertEqual(self.list[1].id, "test2")
        self.assertEqual(self.list.get_by_id("test2"), obj_list[0])
        self.assertEqual(self.list[8].id, "test9")
        self.assertEqual(len(self.list), 9)

    def testIadd(self):
        obj_list = [Object("test%d" % (i)) for i in range(2, 10)]
        self.list += obj_list
        self.assertEqual(self.list[1].id, "test2")
        self.assertEqual(self.list.get_by_id("test2"), obj_list[0])
        self.assertEqual(self.list[8].id, "test9")
        self.assertEqual(len(self.list), 9)

    def testAdd(self):
        obj_list = [Object("test%d" % (i)) for i in range(2, 10)]
        sum = self.list + obj_list
        self.assertEqual(self.list[0].id, "test1")
        self.assertEqual(sum[1].id, "test2")
        self.assertEqual(sum.get_by_id("test2"), obj_list[0])
        self.assertEqual(sum[8].id, "test9")
        self.assertEqual(len(self.list), 1)
        self.assertEqual(len(sum), 9)

    def testDeepcopy(self):
        from copy import deepcopy
        copied = deepcopy(self.list)
        for i, v in enumerate(self.list):
            assert self.list[i].id == copied[i].id
            assert self.list[i] is not copied[i]

    def testQuery(self):
        obj2 = Object("test2")
        self.list.append(obj2)
        result = self.list.query("test1")  # matches only test1
        self.assertEqual(len(result), 1)
        self.assertEqual(result[0], self.obj)
        result = self.list.query("test")  # matches test1 and test2
        self.assertEqual(len(result), 2)

class CobraTestCase(TestCase):
    def setUp(self):
        self.model = create_test_model(test_pickle)

class TestReactions(TestCase):
    def testGPR(self):
        model = Model()
        reaction = Reaction("test")
        # set a gpr to  reaction not in a model
        reaction.gene_reaction_rule = "(g1 or g2) and g3"
        self.assertEqual(reaction.gene_reaction_rule, "(g1 or g2) and g3")
        self.assertEqual(len(reaction.genes), 3)
        # adding reaction with a GPR propagates to the model
        model.add_reaction(reaction)
        self.assertEqual(len(model.genes), 3)
        # ensure the reaction objects are the same
        # TODO implement and make pass
        # modify gpr of reaction already in the model
        # TODO implement and make pass

class TestCobraCore(CobraTestCase):
    """test core cobra functions"""

    def test_add_reaction(self):
        old_reaction_count = len(self.model.reactions)
        old_metabolite_count = len(self.model.metabolites)
        dummy_metabolite_1 = Metabolite("test_foo_1")
        dummy_metabolite_2 = Metabolite("test_foo_2")
        actual_metabolite = self.model.metabolites[0]
        copy_metabolite = self.model.metabolites[1].copy()
        dummy_reaction = Reaction("test_foo_reaction")
        dummy_reaction.add_metabolites({dummy_metabolite_1: -1,
                                        dummy_metabolite_2: 1,
                                        copy_metabolite: -2,
                                        actual_metabolite: 1})
        self.model.add_reaction(dummy_reaction)
        self.assertEqual(self.model.reactions.get_by_id(dummy_reaction.id),
                         dummy_reaction)
        for x in [dummy_metabolite_1, dummy_metabolite_2]:
            self.assertEqual(self.model.metabolites.get_by_id(x.id), x)
        # should have added 1 reaction and 2 metabolites
        self.assertEqual(len(self.model.reactions), old_reaction_count + 1)
        self.assertEqual(len(self.model.metabolites), old_metabolite_count + 2)
        # tests on theadded reaction
        reaction_in_model = self.model.reactions.get_by_id(dummy_reaction.id)
        self.assertIs(type(reaction_in_model), Reaction)
        self.assertIs(reaction_in_model, dummy_reaction)
        self.assertEqual(len(reaction_in_model._metabolites), 4)
        for i in reaction_in_model._metabolites:
            self.assertEqual(type(i), Metabolite)
        # tests on the added metabolites
        met1_in_model = self.model.metabolites.get_by_id(dummy_metabolite_1.id)
        self.assertIs(met1_in_model, dummy_metabolite_1)
        #assertIsNot is not in python 2.6
        copy_in_model = self.model.metabolites.get_by_id(copy_metabolite.id)
        self.assertTrue(copy_metabolite is not copy_in_model)
        self.assertIs(type(copy_in_model), Metabolite)
        self.assertTrue(dummy_reaction in actual_metabolite._reaction)

    def test_delete_reaction(self):
        old_reaction_count = len(self.model.reactions)
        self.model.remove_reactions([self.model.reactions.get_by_id("PGI")])
        self.assertEqual(len(self.model.reactions), old_reaction_count - 1)
        with self.assertRaises(KeyError):
            self.model.reactions.get_by_id("PGI")
        # TODO - delete by id - will this be supported?
        # TODO - delete orphan metabolites - will this be expected behavior?

    def test_copy(self):
        """modifying copy should not modify the original"""
        # test that deleting reactions in the copy does not change the
        # number of reactions in the original model
        model_copy = self.model.copy()
        old_reaction_count = len(self.model.reactions)
        self.assertEqual(len(self.model.reactions), len(model_copy.reactions))
        self.assertEqual(len(self.model.metabolites),
            len(model_copy.metabolites))
        model_copy.remove_reactions(model_copy.reactions[0:5])
        self.assertEqual(old_reaction_count, len(self.model.reactions))
        self.assertNotEqual(len(self.model.reactions),
            len(model_copy.reactions))

    def test_deepcopy(self):
        """Verify that reference structures are maintained when deepcopying.

        """
        model_copy = deepcopy(self.model)
        for gene, gene_copy in zip(self.model.genes, model_copy.genes):
            self.assertEqual(gene.id, gene_copy.id)
            reactions = list(gene.get_reaction())
            reactions.sort()
            reactions_copy = list(gene_copy.get_reaction())
            reactions_copy.sort()
            self.assertEqual(reactions, reactions_copy)
        for reaction, reaction_copy in zip(self.model.reactions, model_copy.reactions):
            self.assertEqual(reaction.id, reaction_copy.id)
            metabolites = reaction._metabolites.keys()
            metabolites.sort()
            metabolites_copy = reaction_copy._metabolites.keys()
            metabolites_copy.sort()
            self.assertEqual(metabolites, metabolites_copy)

class TestCobraIO(CobraTestCase):
    try:
        from cobra.io import sbml
        __test_sbml = True
    except:
        __test_sbml = False
    try:
        from scipy.io import loadmat
        __test_matlab = True
    except:
        __test_matlab = False

    @skipIf(not __test_sbml, "libsbml required")
    def test_sbml_read(self):
        from warnings import catch_warnings
        with catch_warnings(record=True) as w:
            model = io.read_sbml_model(test_sbml_file)
        self.assertEqual(len(model.reactions), len(self.model.reactions))
        self.assertEqual(len(model.metabolites), len(self.model.metabolites))
        # make sure that an error is raised when given a nonexistent file
        self.assertRaises(IOError, io.read_sbml_model,
                          "fake_file_which_does_not_exist")

    @skipIf(not __test_sbml, "libsbml required")
    def test_sbml_write(self):
        test_output_filename = join(gettempdir(), 'test_sbml_write.xml')
        io.write_sbml_model(self.model, test_output_filename)
        #cleanup the test file
        unlink(test_output_filename)

    @skipIf(not __test_matlab, "scipy.io.loadmat required")
    def test_mat_read_write(self):
        from warnings import catch_warnings
        test_output_filename = join(gettempdir(), "test_mat_write.mat")
        io.save_matlab_model(self.model, test_output_filename)
        with catch_warnings(record=True) as w:
            reread = io.load_matlab_model(test_output_filename)
        self.assertEqual(len(self.model.reactions), len(reread.reactions))
        self.assertEqual(len(self.model.metabolites), len(reread.metabolites))
        for i in range(len(self.model.reactions)):
            self.assertEqual(len(self.model.reactions[i]._metabolites), \
                len(reread.reactions[i]._metabolites))
            self.assertEqual(self.model.reactions[i].id, reread.reactions[i].id)
        unlink(test_output_filename)

# make a test suite to run all of the tests
loader = TestLoader()
suite = loader.loadTestsFromModule(sys.modules[__name__])

def test_all():
    TextTestRunner(verbosity=2).run(suite)

if __name__ == "__main__":
    test_all()
<commit_msg>one more unit test for reaction gpr<commit_after>from __future__ import with_statement
import sys
from warnings import warn  # TODO - catch known warnings
from unittest import TestCase, TestLoader, TextTestRunner
from tempfile import gettempdir
from os import unlink
from copy import deepcopy
from os.path import join
try:  #skipIf is not in python 2.6 / 2.5, so use unittest2
    from unittest import skipIf
except:
    from unittest2 import skipIf

# deal with absolute imports by adding the appropriate directory to the path
if __name__ == "__main__":
    sys.path.insert(0, "../..")
    from cobra.test import data_directory, create_test_model
    from cobra.test import salmonella_sbml as test_sbml_file
    from cobra.test import salmonella_pickle as test_pickle
    from cobra import Object, Model, Metabolite, Reaction, io, DictList
    sys.path.pop(0)
    #assert 0
else:
    from . import data_directory, create_test_model
    from . import salmonella_sbml as test_sbml_file
    from . import salmonella_pickle as test_pickle
    from .. import Object, Model, Metabolite, Reaction, io, DictList

# libraries which may or may not be installed
libraries = ["glpk", "gurobipy", "cplex"]
for library in libraries:
    try:
        exec("import %s" % library)
    except ImportError:
        exec("%s = None" % library)

class TestDictList(TestCase):
    def setUp(self):
        self.obj = Object("test1")
        self.list = DictList()
        self.list.append(self.obj)

    def testAppend(self):
        obj2 = Object("test2")
        self.list.append(obj2)
        self.assertRaises(ValueError, self.list.append,
            Object("test1"))
        self.assertEqual(self.list.index(obj2), 1)
        self.assertEqual(self.list[1], obj2)
        self.assertEqual(self.list.get_by_id("test2"), obj2)
        self.assertEqual(len(self.list), 2)

    def testExtend(self):
        obj_list = [Object("test%d" % (i)) for i in range(2, 10)]
        self.list.extend(obj_list)
        self.assertEqual(self.list[1].id, "test2")
        self.assertEqual(self.list.get_by_id("test2"), obj_list[0])
        self.assertEqual(self.list[8].id, "test9")
        self.assertEqual(len(self.list), 9)

    def testIadd(self):
        obj_list = [Object("test%d" % (i)) for i in range(2, 10)]
        self.list += obj_list
        self.assertEqual(self.list[1].id, "test2")
        self.assertEqual(self.list.get_by_id("test2"), obj_list[0])
        self.assertEqual(self.list[8].id, "test9")
        self.assertEqual(len(self.list), 9)

    def testAdd(self):
        obj_list = [Object("test%d" % (i)) for i in range(2, 10)]
        sum = self.list + obj_list
        self.assertEqual(self.list[0].id, "test1")
        self.assertEqual(sum[1].id, "test2")
        self.assertEqual(sum.get_by_id("test2"), obj_list[0])
        self.assertEqual(sum[8].id, "test9")
        self.assertEqual(len(self.list), 1)
        self.assertEqual(len(sum), 9)

    def testDeepcopy(self):
        from copy import deepcopy
        copied = deepcopy(self.list)
        for i, v in enumerate(self.list):
            assert self.list[i].id == copied[i].id
            assert self.list[i] is not copied[i]

    def testQuery(self):
        obj2 = Object("test2")
        self.list.append(obj2)
        result = self.list.query("test1")  # matches only test1
        self.assertEqual(len(result), 1)
        self.assertEqual(result[0], self.obj)
        result = self.list.query("test")  # matches test1 and test2
        self.assertEqual(len(result), 2)

class CobraTestCase(TestCase):
    def setUp(self):
        self.model = create_test_model(test_pickle)

class TestReactions(TestCase):
    def testGPR(self):
        model = Model()
        reaction = Reaction("test")
        # set a gpr to  reaction not in a model
        reaction.gene_reaction_rule = "(g1 or g2) and g3"
        self.assertEqual(reaction.gene_reaction_rule, "(g1 or g2) and g3")
        self.assertEqual(len(reaction.genes), 3)
        # adding reaction with a GPR propagates to the model
        model.add_reaction(reaction)
        self.assertEqual(len(model.genes), 3)
        # ensure the gene objects are the same in the model and reaction
        reaction_gene = list(reaction.genes)[0]
        model_gene = model.genes.get_by_id(reaction_gene.id)
        self.assertIs(reaction_gene, model_gene)
        # modify gpr of reaction already in the model
        # TODO implement and make pass

class TestCobraCore(CobraTestCase):
    """test core cobra functions"""

    def test_add_reaction(self):
        old_reaction_count = len(self.model.reactions)
        old_metabolite_count = len(self.model.metabolites)
        dummy_metabolite_1 = Metabolite("test_foo_1")
        dummy_metabolite_2 = Metabolite("test_foo_2")
        actual_metabolite = self.model.metabolites[0]
        copy_metabolite = self.model.metabolites[1].copy()
        dummy_reaction = Reaction("test_foo_reaction")
        dummy_reaction.add_metabolites({dummy_metabolite_1: -1,
                                        dummy_metabolite_2: 1,
                                        copy_metabolite: -2,
                                        actual_metabolite: 1})
        self.model.add_reaction(dummy_reaction)
        self.assertEqual(self.model.reactions.get_by_id(dummy_reaction.id),
                         dummy_reaction)
        for x in [dummy_metabolite_1, dummy_metabolite_2]:
            self.assertEqual(self.model.metabolites.get_by_id(x.id), x)
        # should have added 1 reaction and 2 metabolites
        self.assertEqual(len(self.model.reactions), old_reaction_count + 1)
        self.assertEqual(len(self.model.metabolites), old_metabolite_count + 2)
        # tests on theadded reaction
        reaction_in_model = self.model.reactions.get_by_id(dummy_reaction.id)
        self.assertIs(type(reaction_in_model), Reaction)
        self.assertIs(reaction_in_model, dummy_reaction)
        self.assertEqual(len(reaction_in_model._metabolites), 4)
        for i in reaction_in_model._metabolites:
            self.assertEqual(type(i), Metabolite)
        # tests on the added metabolites
        met1_in_model = self.model.metabolites.get_by_id(dummy_metabolite_1.id)
        self.assertIs(met1_in_model, dummy_metabolite_1)
        #assertIsNot is not in python 2.6
        copy_in_model = self.model.metabolites.get_by_id(copy_metabolite.id)
        self.assertTrue(copy_metabolite is not copy_in_model)
        self.assertIs(type(copy_in_model), Metabolite)
        self.assertTrue(dummy_reaction in actual_metabolite._reaction)

    def test_delete_reaction(self):
        old_reaction_count = len(self.model.reactions)
        self.model.remove_reactions([self.model.reactions.get_by_id("PGI")])
        self.assertEqual(len(self.model.reactions), old_reaction_count - 1)
        with self.assertRaises(KeyError):
            self.model.reactions.get_by_id("PGI")
        # TODO - delete by id - will this be supported?
        # TODO - delete orphan metabolites - will this be expected behavior?

    def test_copy(self):
        """modifying copy should not modify the original"""
        # test that deleting reactions in the copy does not change the
        # number of reactions in the original model
        model_copy = self.model.copy()
        old_reaction_count = len(self.model.reactions)
        self.assertEqual(len(self.model.reactions), len(model_copy.reactions))
        self.assertEqual(len(self.model.metabolites),
            len(model_copy.metabolites))
        model_copy.remove_reactions(model_copy.reactions[0:5])
        self.assertEqual(old_reaction_count, len(self.model.reactions))
        self.assertNotEqual(len(self.model.reactions),
            len(model_copy.reactions))

    def test_deepcopy(self):
        """Verify that reference structures are maintained when deepcopying.

        """
        model_copy = deepcopy(self.model)
        for gene, gene_copy in zip(self.model.genes, model_copy.genes):
            self.assertEqual(gene.id, gene_copy.id)
            reactions = list(gene.get_reaction())
            reactions.sort()
            reactions_copy = list(gene_copy.get_reaction())
            reactions_copy.sort()
            self.assertEqual(reactions, reactions_copy)
        for reaction, reaction_copy in zip(self.model.reactions, model_copy.reactions):
            self.assertEqual(reaction.id, reaction_copy.id)
            metabolites = reaction._metabolites.keys()
            metabolites.sort()
            metabolites_copy = reaction_copy._metabolites.keys()
            metabolites_copy.sort()
            self.assertEqual(metabolites, metabolites_copy)

class TestCobraIO(CobraTestCase):
    try:
        from cobra.io import sbml
        __test_sbml = True
    except:
        __test_sbml = False
    try:
        from scipy.io import loadmat
        __test_matlab = True
    except:
        __test_matlab = False

    @skipIf(not __test_sbml, "libsbml required")
    def test_sbml_read(self):
        from warnings import catch_warnings
        with catch_warnings(record=True) as w:
            model = io.read_sbml_model(test_sbml_file)
        self.assertEqual(len(model.reactions), len(self.model.reactions))
        self.assertEqual(len(model.metabolites), len(self.model.metabolites))
        # make sure that an error is raised when given a nonexistent file
        self.assertRaises(IOError, io.read_sbml_model,
                          "fake_file_which_does_not_exist")

    @skipIf(not __test_sbml, "libsbml required")
    def test_sbml_write(self):
        test_output_filename = join(gettempdir(), 'test_sbml_write.xml')
        io.write_sbml_model(self.model, test_output_filename)
        #cleanup the test file
        unlink(test_output_filename)

    @skipIf(not __test_matlab, "scipy.io.loadmat required")
    def test_mat_read_write(self):
        from warnings import catch_warnings
        test_output_filename = join(gettempdir(), "test_mat_write.mat")
        io.save_matlab_model(self.model, test_output_filename)
        with catch_warnings(record=True) as w:
            reread = io.load_matlab_model(test_output_filename)
        self.assertEqual(len(self.model.reactions), len(reread.reactions))
        self.assertEqual(len(self.model.metabolites), len(reread.metabolites))
        for i in range(len(self.model.reactions)):
            self.assertEqual(len(self.model.reactions[i]._metabolites), \
                len(reread.reactions[i]._metabolites))
            self.assertEqual(self.model.reactions[i].id, reread.reactions[i].id)
        unlink(test_output_filename)

# make a test suite to run all of the tests
loader = TestLoader()
suite = loader.loadTestsFromModule(sys.modules[__name__])

def test_all():
    TextTestRunner(verbosity=2).run(suite)

if __name__ == "__main__":
    test_all()
<|endoftext|>


<commit_before>from datetime import datetime, date, timedelta
import csv
import mirqi
import ReadXML
import os

def get_output_directory():
        return os.path.join(os.path.abspath(mirqi.__path__[0]), 'output')

def get_data_directory():
        return os.path.join(os.path.abspath(mirqi.__path__[0]),'Data')

_dir = os.path.abspath(mirqi.__path__[0])

BJH_SYNGO_FILES = [os.path.join(_dir,'Data/BJH/April_Output_Org.xls'), os.path.join(_dir,'Data/BJH/May_Output_Org.xls')]
BJH_XML_FILE = os.path.join(_dir,'Data/BJH/all bjh.xml')
SLCH_SYNGO_FILES = [os.path.join(_dir,'Data/SLCH/April_IR_Output_Org.xls'), os.path.join(_dir,'Data/SLCH/May_IR_Output_Org.xls')]
SLCH_XML_FILE = os.path.join(_dir,'Data/SLCH/all slch.xml')
TEST_XML_FILE = os.path.join(_dir,'Data/sample.xml')
TEST_SYNGO_FILES = [os.path.join(_dir,'Data/sample_syngo.xls')]

def care_datetime_to_python_datetime(care_date):
        care_date = str(care_date)
        return datetime(int(care_date[:4]),int(care_date[4:6]),int(care_date[6:8]),int(care_date[8:10]),int(care_date[10:12]), int(care_date[12:14]))

def care_date_to_python_date(care_date):
        care_date = str(care_date)
        return date(int(care_date[:4]),int(care_date[4:6]),int(care_date[6:8]))

def write_csv(table, file_name = 'output.csv'):
        writer = csv.writer(open(file_name,'wb'))
        writer.writerows(table)

def total_seconds(time_delta):
        td = time_delta
        return (td.microseconds + (td.seconds + td.days * 24 * 3600.0) * 10**6) / 10**6

def multiply_timedelta(td, x):
        return timedelta(days = td.days *x, seconds = td.seconds*x, microseconds = td.microseconds *int(x))

def transposed(lists):
   if not lists: return []
   return map(lambda *row: list(row), *lists)

def get_procs(group = 'all'):
        if group == 'bjh':
                procs = ReadXML.process_file(BJH_XML_FILE, BJH_SYNGO_FILES)
        elif group == 'slch':
                procs = ReadXML.process_file(SLCH_XML_FILE, SLCH_SYNGO_FILES)
        elif group == 'all':
                procs = ReadXML.process_file(BJH_XML_FILE, BJH_SYNGO_FILES)
                procs = procs + ReadXML.process_file(SLCH_XML_FILE, SLCH_SYNGO_FILES)
        elif group == 'test':
                procs = ReadXML.process_file(TEST_XML_FILE, TEST_SYNGO_FILES)
        else:
                raise ValueError("Invalid group")
        return procs

def average_fps(events):
        """Gets the average FPS weighted by event duration"""
        if len(events) == 0:
                raise ValueError("Cannot take average of empyt list")
        total_fluoro_time = sum([e.get_duration() for e in events],timedelta(0) )
        total_fluoro_seconds = total_seconds(total_fluoro_time)
        mean_numerator = sum([multiply_timedelta(e.get_duration(),e.Pulse_Rate) for e in events], timedelta(0))
        return total_seconds(mean_numerator)/total_fluoro_seconds

def is_subset(list1, list2):
    """Returns true if list 1 is a subset of list 2
    (assumes neither list has any repeats)
    """
    for item in list1:
        if not item in list2:
            return False
    return True

def same_contents(list1,list2):
    """Returns true if list 1 has the exact same
    contents as list 2. (assumes neither list has
    any repeats)
    """
    if not len(list1) == len(list2):
        return False
    return is_subset(list1,list2)

def matches(list1,list2):
    """Returns is_subset or same_contents
    depending on whether or not the last
    item in list1 is -99
    """
    if list1[-1] == -99:
        return same_contents(list1[:-1],list2)
    else:
        return is_subset(list1,list2)

import numbers
def standard_cpt(cpt):
        """Given a cpt code as an integer, float
        or string, convert it to a string in a
        standard format.
        * no leading or trailing whitespace
        * no '.0' at the end of integers
        * letters are all capitalized
        """
        if isinstance(cpt, numbers.Number):
                return str(int(cpt))
        elif isinstance(cpt,basestring):
                out = cpt.split('.')[0]
                out = out.strip()
                out = out.upper()
                return out

def organize(iterable, key):
        """Put all of the elements in `iterable` into
        a dictionary which maps possible return values
        of key onto lists of items of iterable

        iterable - any iterable object (e.g. a list, or tuple)
        key - a function that takes items in interable as inputs

        Example:
        organize([1,2,3],lambda x: x==2)
        {True:[1,3],False:[2]}
        """
        out = {}
        for item in iterable:
                k = key(item)
                if not k in out:
                        out[k] = []
                out[k].append(item)
        return out

import xlrd

def coerce_human_date(d, datemode = None):
        """Attempt to coerce d into a Python datetime.date

        Generally d will have been retrieved from an excel
        spreadsheet, so I expect it to be a string or an
        excel date.

        Args:
                d - object to be coerced
                datemode - optional xlrd datemode object
        """
        if d is None:
                raise ValueError("Cannot coerce None into date.")
        if isinstance(d, date):
                return d
        if isinstance(d, basestring):
                raise NotImplementedError("Can't coerce string into a date... yet")
        date_tuple = xlrd.xldate_as_tuple(d,datemode)
        return date(year = date_tuple[0],
                             month=date_tuple[1],
                             day=date_tuple[2])

_ARB_DATE = date(2000,1,1) # an arbitrary date
def subtract_times(t1, t2):
        """returns a datetime.timedelta object representing t1-t2
        """
        t1 = datetime.combine(_ARB_DATE, t1)
        t2 = datetime.combine(_ARB_DATE, t2)
        return t1-t2

import pkgutil
import mirqi.inquiries
def get_inquiries():
        """Get a list of inquiry classes
        """
        pkgpath = os.path.dirname(mirqi.inquiries.__file__)
        inq_module_names = [name for _, name, _ in pkgutil.iter_modules([pkgpath])]
        print inq_module_names
        inquiry_modules = [__import__('mirqi.inquiries.'+n, globals(), locals(), [],-1) for n in inq_module_names]
        #left off here
<commit_msg>finished my_utils.get_inquiries function<commit_after>from datetime import datetime, date, timedelta
import csv
import mirqi
import ReadXML
import os

def get_output_directory():
        return os.path.join(os.path.abspath(mirqi.__path__[0]), 'output')

def get_data_directory():
        return os.path.join(os.path.abspath(mirqi.__path__[0]),'Data')

_dir = os.path.abspath(mirqi.__path__[0])

BJH_SYNGO_FILES = [os.path.join(_dir,'Data/BJH/April_Output_Org.xls'), os.path.join(_dir,'Data/BJH/May_Output_Org.xls')]
BJH_XML_FILE = os.path.join(_dir,'Data/BJH/all bjh.xml')
SLCH_SYNGO_FILES = [os.path.join(_dir,'Data/SLCH/April_IR_Output_Org.xls'), os.path.join(_dir,'Data/SLCH/May_IR_Output_Org.xls')]
SLCH_XML_FILE = os.path.join(_dir,'Data/SLCH/all slch.xml')
TEST_XML_FILE = os.path.join(_dir,'Data/sample.xml')
TEST_SYNGO_FILES = [os.path.join(_dir,'Data/sample_syngo.xls')]

def care_datetime_to_python_datetime(care_date):
        care_date = str(care_date)
        return datetime(int(care_date[:4]),int(care_date[4:6]),int(care_date[6:8]),int(care_date[8:10]),int(care_date[10:12]), int(care_date[12:14]))

def care_date_to_python_date(care_date):
        care_date = str(care_date)
        return date(int(care_date[:4]),int(care_date[4:6]),int(care_date[6:8]))

def write_csv(table, file_name = 'output.csv'):
        writer = csv.writer(open(file_name,'wb'))
        writer.writerows(table)

def total_seconds(time_delta):
        td = time_delta
        return (td.microseconds + (td.seconds + td.days * 24 * 3600.0) * 10**6) / 10**6

def multiply_timedelta(td, x):
        return timedelta(days = td.days *x, seconds = td.seconds*x, microseconds = td.microseconds *int(x))

def transposed(lists):
   if not lists: return []
   return map(lambda *row: list(row), *lists)

def get_procs(group = 'all'):
        if group == 'bjh':
                procs = ReadXML.process_file(BJH_XML_FILE, BJH_SYNGO_FILES)
        elif group == 'slch':
                procs = ReadXML.process_file(SLCH_XML_FILE, SLCH_SYNGO_FILES)
        elif group == 'all':
                procs = ReadXML.process_file(BJH_XML_FILE, BJH_SYNGO_FILES)
                procs = procs + ReadXML.process_file(SLCH_XML_FILE, SLCH_SYNGO_FILES)
        elif group == 'test':
                procs = ReadXML.process_file(TEST_XML_FILE, TEST_SYNGO_FILES)
        else:
                raise ValueError("Invalid group")
        return procs

def average_fps(events):
        """Gets the average FPS weighted by event duration"""
        if len(events) == 0:
                raise ValueError("Cannot take average of empyt list")
        total_fluoro_time = sum([e.get_duration() for e in events],timedelta(0) )
        total_fluoro_seconds = total_seconds(total_fluoro_time)
        mean_numerator = sum([multiply_timedelta(e.get_duration(),e.Pulse_Rate) for e in events], timedelta(0))
        return total_seconds(mean_numerator)/total_fluoro_seconds

def is_subset(list1, list2):
    """Returns true if list 1 is a subset of list 2
    (assumes neither list has any repeats)
    """
    for item in list1:
        if not item in list2:
            return False
    return True

def same_contents(list1,list2):
    """Returns true if list 1 has the exact same
    contents as list 2. (assumes neither list has
    any repeats)
    """
    if not len(list1) == len(list2):
        return False
    return is_subset(list1,list2)

def matches(list1,list2):
    """Returns is_subset or same_contents
    depending on whether or not the last
    item in list1 is -99
    """
    if list1[-1] == -99:
        return same_contents(list1[:-1],list2)
    else:
        return is_subset(list1,list2)

import numbers
def standard_cpt(cpt):
        """Given a cpt code as an integer, float
        or string, convert it to a string in a
        standard format.
        * no leading or trailing whitespace
        * no '.0' at the end of integers
        * letters are all capitalized
        """
        if isinstance(cpt, numbers.Number):
                return str(int(cpt))
        elif isinstance(cpt,basestring):
                out = cpt.split('.')[0]
                out = out.strip()
                out = out.upper()
                return out

def organize(iterable, key):
        """Put all of the elements in `iterable` into
        a dictionary which maps possible return values
        of key onto lists of items of iterable

        iterable - any iterable object (e.g. a list, or tuple)
        key - a function that takes items in interable as inputs

        Example:
        organize([1,2,3],lambda x: x==2)
        {True:[1,3],False:[2]}
        """
        out = {}
        for item in iterable:
                k = key(item)
                if not k in out:
                        out[k] = []
                out[k].append(item)
        return out

import xlrd

def coerce_human_date(d, datemode = None):
        """Attempt to coerce d into a Python datetime.date

        Generally d will have been retrieved from an excel
        spreadsheet, so I expect it to be a string or an
        excel date.

        Args:
                d - object to be coerced
                datemode - optional xlrd datemode object
        """
        if d is None:
                raise ValueError("Cannot coerce None into date.")
        if isinstance(d, date):
                return d
        if isinstance(d, basestring):
                raise NotImplementedError("Can't coerce string into a date... yet")
        date_tuple = xlrd.xldate_as_tuple(d,datemode)
        return date(year = date_tuple[0],
                             month=date_tuple[1],
                             day=date_tuple[2])

_ARB_DATE = date(2000,1,1) # an arbitrary date
def subtract_times(t1, t2):
        """returns a datetime.timedelta object representing t1-t2
        """
        t1 = datetime.combine(_ARB_DATE, t1)
        t2 = datetime.combine(_ARB_DATE, t2)
        return t1-t2

import pkgutil
import mirqi.inquiries
def get_inquiries():
        """Get a list of inquiry classes
        """
        pkgpath = os.path.dirname(mirqi.inquiries.__file__)
        inq_module_names = [name for _, name, _ in pkgutil.iter_modules([pkgpath])]
        temp = __import__('mirqi.inquiries', globals(), locals(), inq_module_names,-1)
        inq_modules = [getattr(temp, name) for name in inq_module_names]
        inq_classes = [getattr(module,dir(module)[0]) for module in inq_modules]
        assert inq_module_names == [inq_class.__name__.lower() for inq_class in inq_classes]
        return inq_classes
<|endoftext|>


<commit_before>#simplified implementation of collabarative filtering

import numpy as np
from sklearn.neighbors import NearestNeighbors
from sklearn.metrics.pairwise import cosine_similarity
from scipy.spatial import distance
import pdb
import warnings
from scipy import sparse
import cPickle as pickle
from sklearn.naive_bayes import MultinomialNB
import collections

def loadData():
	print "loading data"
	useritem_sparse = pickle.load(open('../features/useritemmatrix_normalized.dat', 'rb'))
	valData = []
	question_feats = {}
	trainData = []

	with open('../train_data/validate_nolabel.txt', 'r') as f1:
		header = f1.readline()
		for line in f1:
			valData.append(line.rstrip('\r\n').split(','))
	ques_keys = pickle.load(open('../train_data/question_info_keys.dat', 'rb'))
	user_keys = pickle.load(open('../train_data/user_info_keys.dat', 'rb'))
	for i in range(len(user_keys)):
		user_keys_map[user_keys[i]] = i
	for i in range(len(ques_keys)):
		ques_keys_map[ques_keys[i]] = i

	# tf = pickle.load(open('../features/ques_charid_tfidf.dat', 'rb'))
	# tfx = tf.toarray()
	# for i in range(len(tfx)):
	# 	question_feats[ques_keys[i]] = tfx[0].tolist()
	topics = []
	with open('../train_data/question_info.txt', 'r') as f1:
		for line in f1:
			topic = int(line.split()[1])
			topics.append(topic)
	for i in range(len(question_keys)):
		question_feats[question_keys[i]] = [1 if x == topics[i] else 0 for x in range(22)]
	with open('../train_data/invited_info_train.txt', 'r') as f1:
		for line in f1:
			line = line.rstrip('\n')
			sp = line.split()
			trainData.append((sp[0], sp[1], int(sp[2])))

	return useritem_sparse, valData, ques_keys, user_keys, trainData, question_feats, ques_keys_map, user_keys_map

def getModels(trainData, question_feats):
	print "getting models"
	userX = {}
	userY = {}
	for qid, uid, val in trainData:
		if uid not in userX:
			userX[uid] = []
			userY[uid] = []
		userX[uid].append(question_feats[qid])
		userY[uid].append(val)
	nbmodels = {}
	for user in userX:
		nbmodels[user] = MultinomialNB()
		nbmodels[user].fit(userX[user], userY[user])

	return nbmodels

def contentBoosting(user_keys, ques_keys, useritem, usermodels, question_feats):
	print "boosting"
	useritem = useritem.toarray()
	for i in range(0, len(user_keys)):
		for j in range(0, len(ques_keys)):
			if useritem[i][j] == 0:
				if user_keys[i] not in usermodels:
					continue
				prediction = usermodels[user_keys[i]].predict([question_feats[ques_keys[j]]])[0]
				if prediction == 1:
					useritem[i][j] = 1
				elif prediction == 0:
					useritem[i][j] = -0.125
				else:
					print prediction
	return useritem

def collabFilteringPredictions(useritem, sparse, k, valData, ques_keys_map, user_keys_map):
	print "getting predictions"
	#input: useritem matrix
	#sparese: whether useritem is sparse or not
	#k : k nearest neighbors to consider
	#returns list of predictions
	similarities = cosine_similarity(useritem)
	scores = []
	print similarities.shape
	useritemfull = useritem.toarray()
	for qid, uid in valData:
		score = 0
		for nbindex in similarities[user_keys_map[uid]].argsort()[(-k-1):]:
			if nbindex == user_keys_map[uid]: #exclude self
				continue
			score += useritemfull[nbindex][ques_keys_map[qid]]*similarities[user_keys_map[uid]][nbindex]
		scores.append(score)

	predictions = []

	#normalization
	maxscore = max(scores)
	minscore = min(scores)
	for score in scores:
		predictions.append((score-minscore)/float(maxscore-minscore))

	return predictions

k = 180

useritem_sparse, valData, ques_keys, user_keys, trainData, question_feats, ques_keys_map, user_keys_map = loadData()
usermodels = getModels(trainData, question_feats)
useritem = contentBoosting(user_keys, ques_keys, useritem_sparse, usermodels, question_feats)
predictions = collabFilteringPredictions(useritem, False, k, valData, ques_keys_map, user_keys_map)

with open('../validation/content_boosted_'+str(k)+'.csv', 'w') as f1:
	f1.write('qid,uid,label\n')
	for i in range(0, len(predictions)):
		f1.write(valData[i][0]+','+valData[i][1]+','+str(predictions[i])+'\n')

<commit_msg>Fixed bugs"<commit_after>#simplified implementation of collabarative filtering

import numpy as np
from sklearn.neighbors import NearestNeighbors
from sklearn.metrics.pairwise import cosine_similarity
from scipy.spatial import distance
import pdb
import warnings
from scipy import sparse
import cPickle as pickle
from sklearn.naive_bayes import MultinomialNB
import collections

def loadData():
	print "loading data"
	useritem_sparse = pickle.load(open('../features/useritemmatrix_normalized.dat', 'rb'))
	valData = []
	question_feats = {}
	trainData = []

	with open('../train_data/validate_nolabel.txt', 'r') as f1:
		header = f1.readline()
		for line in f1:
			valData.append(line.rstrip('\r\n').split(','))
	ques_keys = pickle.load(open('../train_data/question_info_keys.dat', 'rb'))
	user_keys = pickle.load(open('../train_data/user_info_keys.dat', 'rb'))
	for i in range(len(user_keys)):
		user_keys_map[user_keys[i]] = i
	for i in range(len(ques_keys)):
		ques_keys_map[ques_keys[i]] = i

	# tf = pickle.load(open('../features/ques_charid_tfidf.dat', 'rb'))
	# tfx = tf.toarray()
	# for i in range(len(tfx)):
	# 	question_feats[ques_keys[i]] = tfx[0].tolist()
	topics = []
	with open('../train_data/question_info.txt', 'r') as f1:
		for line in f1:
			topic = int(line.split()[1])
			topics.append(topic)
	for i in range(len(question_keys)):
		question_feats[question_keys[i]] = [1 if x == topics[i] else 0 for x in range(22)]
	with open('../train_data/invited_info_train.txt', 'r') as f1:
		for line in f1:
			line = line.rstrip('\n')
			sp = line.split()
			trainData.append((sp[0], sp[1], int(sp[2])))

	return useritem_sparse, valData, ques_keys, user_keys, trainData, question_feats, ques_keys_map, user_keys_map

def getModels(trainData, question_feats):
	print "getting models"
	userX = {}
	userY = {}
	for qid, uid, val in trainData:
		if uid not in userX:
			userX[uid] = []
			userY[uid] = []
		userX[uid].append(question_feats[qid])
		userY[uid].append(val)
	nbmodels = {}
	for user in userX:
		nbmodels[user] = MultinomialNB()
		nbmodels[user].fit(userX[user], userY[user])

	return nbmodels

def contentBoosting(user_keys, ques_keys, useritem, usermodels, question_feats):
	print "boosting"
	useritem = useritem.toarray()
	topredict = [question_feats[ques_keys[i]] for i in range(len(ques_keys))]
	for i in range(0, len(user_keys)):
		if user_keys[i] not in usermodels:
			continue
		predictions = usermodels[user_keys[i]].predict(topredict)
		for j in range(0, len(ques_keys)):
			if useritem[i][j] == 0:
				prediction = predictions[j]
				if prediction == 1:
					useritem[i][j] = 1
				elif prediction == 0:
					useritem[i][j] = -0.125
				else:
					print prediction
	return useritem

def collabFilteringPredictions(useritem, sparse, k, valData, ques_keys_map, user_keys_map):
	print "getting predictions"
	#input: useritem matrix
	#sparese: whether useritem is sparse or not
	#k : k nearest neighbors to consider
	#returns list of predictions
	similarities = cosine_similarity(useritem)
	scores = []
	print similarities.shape
	useritemfull = useritem
	for qid, uid in valData:
		score = 0
		for nbindex in similarities[user_keys_map[uid]].argsort()[(-k-1):]:
			if nbindex == user_keys_map[uid]: #exclude self
				continue
			score += useritemfull[nbindex][ques_keys_map[qid]]*similarities[user_keys_map[uid]][nbindex]
		scores.append(score)

	predictions = []

	#normalization
	maxscore = max(scores)
	minscore = min(scores)
	for score in scores:
		predictions.append((score-minscore)/float(maxscore-minscore))

	return predictions

k = 180

useritem_sparse, valData, ques_keys, user_keys, trainData, question_feats, ques_keys_map, user_keys_map = loadData()
usermodels = getModels(trainData, question_feats)
useritem = contentBoosting(user_keys, ques_keys, useritem_sparse, usermodels, question_feats)
predictions = collabFilteringPredictions(useritem, False, k, valData, ques_keys_map, user_keys_map)

with open('../validation/content_boosted_'+str(k)+'.csv', 'w') as f1:
	f1.write('qid,uid,label\n')
	for i in range(0, len(predictions)):
		f1.write(valData[i][0]+','+valData[i][1]+','+str(predictions[i])+'\n')

<|endoftext|>


<commit_before>"""
This will perform a gamma analysis
It will collect two spectra:
M = Measurement Spectrum
B = Background Spectrum
"""
from __future__ import print_function
from SpectrumFileBase import SpectrumFileBase
import SPEFile
import math
import numpy as np
import matplotlib.pyplot as plt
execfile("Isotope_identification.py")

EFFICIENCY_CAL_COEFFS = [5.1164, 161.65, 3952.3, 30908]

def absolute_efficiency(energy, coeffs=EFFICIENCY_CAL_COEFFS):
    """
    Returns absolute efficiencies for a given set of energies, based on a
    provided efficiency calibration.
    """
    efficiency = []
    for i in range(len(energy)):
        efficiency.append(math.exp(coeffs[3] *
                          (math.log(energy[i])/energy[i])**3 -
                          coeffs[2]*(math.log(energy[i])/energy[i])**2 +
                          coeffs[1]*(math.log(energy[i])/energy[i]) -
                          coeffs[0]))
    return efficiency

def emission_rate(net_area, efficiency, livetime):
    """
    this function returns the emission rate of gammas per second
    alongside its uncertainty.
    """
    emission_rate = [net_area[0]/(efficiency*livetime),
                     net_area[1]/(efficiency*livetime)]
    return emission_rate

def Isotope_Activity(Isotope, emission_rates, emission_uncertainty):
    """
    Isotope_Activity will determine the activity of a given radioactive isotope
    based on the emission rates given and the isotope properties. It takes an
    Isotope object and a given set of emission rates and outputs an activity
    estimate alongside its uncertainty.
    """
    Branching_ratio = Isotope.list_sig_g_b_r
    Activity = []
    Uncertainty = []
    for i in range(len(Branching_ratio)):
        Activity.append(emission_rates[i]/Branching_ratio[i])
        Uncertainty.append(emission_uncertainty[i]/Branching_ratio[i])
    Isotope_Activity = np.mean(Activity)
    Activity_uncertainty = np.mean(Uncertainty)
    results = [Isotope_Activity, Activity_uncertainty]
    return results

def peak_measurement(M, energy):
    """
    Takes in a measured spectra alongside a specific energy and returns the net
    area and uncertainty for that energy.
    """
    energy_axis = M[0]
    M_counts = M[1]

    # Rough estimate of FWHM.
    FWHM = 0.05*energy**0.5

    # Peak Gross Area

    start_peak = 0
    while energy_axis[start_peak] < (energy - FWHM):
        start_peak += 1

    end_peak = start_peak
    while energy_axis[end_peak] < (energy + FWHM):
        end_peak += 1

    gross_counts_peak = sum(M_counts[start_peak:end_peak])

    # Left Gross Area

    left_peak = energy - 2*FWHM

    left_start = 0
    while energy_axis[left_start] < (left_peak - FWHM):
        left_start += 1

    left_end = left_start
    while energy_axis[left_end] < (left_peak + FWHM):
        left_end += 1

    gross_counts_left = sum(M_counts[left_start:left_end])

    # Right Gross Area

    right_peak = energy + 2*FWHM

    right_start = 0
    while energy_axis[right_start] < (right_peak - FWHM):
        right_start += 1

    right_end = right_start
    while energy_axis[right_end] < (right_peak + FWHM):
        right_end += 1

    gross_counts_right = sum(M_counts[right_start:right_end])

    # Net Area

    net_area = gross_counts_peak - (gross_counts_left + gross_counts_right)/2

    # Uncertainty

    uncertainty = (gross_counts_peak +
                   (gross_counts_left + gross_counts_right) / 4) ** 0.5

    # Returning results

    results = [net_area, uncertainty]

    return results

def Background_Subtract(M, B):
    """
    Background_Subtract will subtract the background spectrum
    from a given measurement. This will return the energy bins using a given
    energy calibration as well as the background subtracted counts of a
    measurement. Since both spectra come from the same detector, this assumes
    the energy calibrations are the same.
    """

    M_Counts = M.data
    B_Counts = B.data

    M_Time = M.livetime
    B_Time = B.livetime

    M_Channels = M.channel
    E0 = M.energy_cal[0]
    Eslope = M.energy_cal[1]

    Energy_Axis = B.channel
    Energy_Axis = Energy_Axis.astype(float)
    Energy_Axis[:] = [E0+Eslope*x for x in Channel]

    B_Counts_M = [1.0]*len(M.channel)

    B_Counts_M[:] = [x*(M_Time/B_Time) for x in B_Counts]

    M_Sub_Back = [1.0]*len(M_Channels)

    M_Sub_Back = [M_Counts[x]-B_Counts_M[x] for x in M.channel]

    Sub_Spect = [Energy_Axis, M_Sub_Back]

    return Sub_Spect

def main():
    Measurement = SPEFile.SPEFile()
    Background = SPEFile.SPEFile()

    Measurement.read()
    Background.read()

    Sub_Measurement = Background_Subtract(M=Measurement, B=Background)

    Isotope_List = [Caesium_134, Caesium_137, Cobalt_60, Potassium_40,
                    Thallium_208, Actinium_228, Lead_212, Bismuth_214
                    Lead_214, Thorium_234, Lead_210]
    for i in range(len(Isotope_List)):
        Isotope_Efficiency = absolute_efficiency(Isotope_List[i].list_sig_g_e)
        Isotope_Energy = Isotope_List[i].list_sig_g_e
        Activity_Info = []
        for j in range(len(Isotope_Energy)):
            Net_Area = peak_measurement(Sub_Measurement, Isotope_Energy[j])
            Peak_emission = emission_rate(Net_Area, Isotope_Efficiency[j],
                                          Measurement.livetime)
            Actvity = Isotope_Activity(Isotope_List[i], Peak_emission[0],
                                       Peak_emission[1])
            Activity_Info.append(Activity)

if __name__ == '__main__':
    main()
<commit_msg>Added a Flag<commit_after>"""
This will perform a gamma analysis
It will collect two spectra:
M = Measurement Spectrum
B = Background Spectrum
"""
from __future__ import print_function
from SpectrumFileBase import SpectrumFileBase
import SPEFile
import math
import numpy as np
import matplotlib.pyplot as plt
execfile("Isotope_identification.py")

EFFICIENCY_CAL_COEFFS = [5.1164, 161.65, 3952.3, 30908]

def absolute_efficiency(energy, coeffs=EFFICIENCY_CAL_COEFFS):
    """
    Returns absolute efficiencies for a given set of energies, based on a
    provided efficiency calibration.
    """
    efficiency = []
    for i in range(len(energy)):
        efficiency.append(math.exp(coeffs[3] *
                          (math.log(energy[i])/energy[i])**3 -
                          coeffs[2]*(math.log(energy[i])/energy[i])**2 +
                          coeffs[1]*(math.log(energy[i])/energy[i]) -
                          coeffs[0]))
    return efficiency

def emission_rate(net_area, efficiency, livetime):
    """
    this function returns the emission rate of gammas per second
    alongside its uncertainty.
    """
    emission_rate = [net_area[0]/(efficiency*livetime),
                     net_area[1]/(efficiency*livetime)]
    return emission_rate

def Isotope_Activity(Isotope, emission_rates, emission_uncertainty):
    """
    Isotope_Activity will determine the activity of a given radioactive isotope
    based on the emission rates given and the isotope properties. It takes an
    Isotope object and a given set of emission rates and outputs an activity
    estimate alongside its uncertainty.
    """
    Branching_ratio = Isotope.list_sig_g_b_r
    Activity = []
    Uncertainty = []
    for i in range(len(Branching_ratio)):
        Activity.append(emission_rates[i]/Branching_ratio[i])
        Uncertainty.append(emission_uncertainty[i]/Branching_ratio[i])
    Isotope_Activity = np.mean(Activity)
    Activity_uncertainty = np.mean(Uncertainty)
    results = [Isotope_Activity, Activity_uncertainty]
    return results

def peak_measurement(M, energy):
    """
    Takes in a measured spectra alongside a specific energy and returns the net
    area and uncertainty for that energy.
    """
    energy_axis = M[0]
    M_counts = M[1]

    # Rough estimate of FWHM.
    FWHM = 0.05*energy**0.5

    # Peak Gross Area

    start_peak = 0
    while energy_axis[start_peak] < (energy - FWHM):
        start_peak += 1

    end_peak = start_peak
    while energy_axis[end_peak] < (energy + FWHM):
        end_peak += 1

    gross_counts_peak = sum(M_counts[start_peak:end_peak])

    # Left Gross Area

    left_peak = energy - 2*FWHM

    left_start = 0
    while energy_axis[left_start] < (left_peak - FWHM):
        left_start += 1

    left_end = left_start
    while energy_axis[left_end] < (left_peak + FWHM):
        left_end += 1

    gross_counts_left = sum(M_counts[left_start:left_end])

    # Right Gross Area

    right_peak = energy + 2*FWHM

    right_start = 0
    while energy_axis[right_start] < (right_peak - FWHM):
        right_start += 1

    right_end = right_start
    while energy_axis[right_end] < (right_peak + FWHM):
        right_end += 1

    gross_counts_right = sum(M_counts[right_start:right_end])

    # Net Area

    net_area = gross_counts_peak - (gross_counts_left + gross_counts_right)/2

    # Uncertainty

    uncertainty = (gross_counts_peak +
                   (gross_counts_left + gross_counts_right) / 4) ** 0.5

    # Returning results

    results = [net_area, uncertainty]

    energy_range = energy_axis[left_start:right_end]
    for i in energy_range:
        turns = 0
        while turns <= len(energy_range):
            if abs(M_counts[i]) > 2*uncertainty:
                    print("Error: There is a bias in the measurement at %s KeV"
                          % energy_axis[i])
            turns += 1

    return results

def Background_Subtract(M, B):
    """
    Background_Subtract will subtract the background spectrum
    from a given measurement. This will return the energy bins using a given
    energy calibration as well as the background subtracted counts of a
    measurement. Since both spectra come from the same detector, this assumes
    the energy calibrations are the same.
    """

    M_Counts = M.data
    B_Counts = B.data

    M_Time = M.livetime
    B_Time = B.livetime

    M_Channels = M.channel
    E0 = M.energy_cal[0]
    Eslope = M.energy_cal[1]

    Energy_Axis = B.channel
    Energy_Axis = Energy_Axis.astype(float)
    Energy_Axis[:] = [E0+Eslope*x for x in Channel]

    B_Counts_M = [1.0]*len(M.channel)

    B_Counts_M[:] = [x*(M_Time/B_Time) for x in B_Counts]

    M_Sub_Back = [1.0]*len(M_Channels)

    M_Sub_Back = [M_Counts[x]-B_Counts_M[x] for x in M.channel]

    Sub_Spect = [Energy_Axis, M_Sub_Back]

    return Sub_Spect

def main():
    Measurement = SPEFile.SPEFile()
    Background = SPEFile.SPEFile()

    Measurement.read()
    Background.read()

    Sub_Measurement = Background_Subtract(M=Measurement, B=Background)

    Isotope_List = [Caesium_134, Caesium_137, Cobalt_60, Potassium_40,
                    Thallium_208, Actinium_228, Lead_212, Bismuth_214
                    Lead_214, Thorium_234, Lead_210]
    for i in range(len(Isotope_List)):
        Isotope_Efficiency = absolute_efficiency(Isotope_List[i].list_sig_g_e)
        Isotope_Energy = Isotope_List[i].list_sig_g_e
        Activity_Info = []
        for j in range(len(Isotope_Energy)):
            Net_Area = peak_measurement(Sub_Measurement, Isotope_Energy[j])
            Peak_emission = emission_rate(Net_Area, Isotope_Efficiency[j],
                                          Measurement.livetime)
            Actvity = Isotope_Activity(Isotope_List[i], Peak_emission[0],
                                       Peak_emission[1])
            Activity_Info.append(Activity)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import numpy as np
import pandas as pd
from numpy.polynomial import Legendre
import logging
from sklearn import linear_model

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('__GridRegression__')

#Online regression classes
class model():
    __coefs__ = None

    def coefs(self):
        if self.__coefs__ is None:
            self.setCoefs()
        return self.__coefs__

    def setCoefs(self,coefs):
        pass
    def partial_fit(self,X,y):
        pass
    def predict(self,X):
        pass

class onlineLinearRegression(model):

    __R__ = None
    __cov__ = None

    def partial_fit(self,X,y):
        self.__coefs__ = None
        self.__cov__ += np.dot(X,X.T)
        self.__R__ += np.dot(X,y.values)

    def setCoefs(self,coefs=None):
        if coefs is None:
            self.__coefs__ = np.dot(np.invert(self.__cov__),self.__R__)
        else:
            self.__coefs__ = coefs

    def predict(self,X):
        if self.__coefs__ is None:
            self.setCoefs()
        return np.dot(self.__coefs__,X)

class onlineSGDRegression(model):

    __model__ = None
    __ntargets__ = None
    def partial_fit(self,X,y):
        self.__ntargets__ = len(y.columns)
        self.__coefs__ = None

        if self.__model__ is None:
            self.__model__ = [linear_model.SGDRegressor() for i in range(self.__ntargets__)] #Model for each target variable
        for i in range(self.__ntargets__):
            self.__model__[i].partial_fit(X,y.values[:,i])

    def setCoefs(self,coefs=None):
        if coefs is None:
            self.__coefs__ = np.array([reglin.coef_ for reglin in self.__model__])
        else:
            self.__coefs__ = coefs

    def predict(self,X):
        if self.__coefs__ is None:
            self.setCoefs()
        coefs = self.coefs()
        for i in range(self.__ntargets__):
                model = self.__model__[i]
                model.coef_ = coefs[i,:]

class GridRegression:
    #Help with regression over grid data
    #Note that this class does not represent the design matrix explicitly but will instead calculate
    #it additively based on single events.

    #Private
    __noise__ = None
    __nlags__ = None
    __grid__ = None
    __noiseOrders__ = None
    __coefs__ = None
    __events__ = None
    __longestEvent__ = None
    __encoding__ = None
    __lambda__ = None
    __nClasses__ = None
    __nPoints__ = None
    __normFunc__ = None
    __model__ = None

    def __init__(self,grid,nlags=0,noiseOrders=None,encoding=None):
        self.setNumLags(nlags)
        self.setGrid(grid)
        self.setNoiseOrders(noiseOrders)
        self.setEncoding(encoding)
        self.setnClasses(encoding.numClasses())
        self.setnPoints(self.nlags()*self.nClasses()+self.nClasses()+len(self.noise().columns)) #Number of columns in design matrix

    #Getters
    def grid(self):
        return self.__grid__
    def noise(self):
        return self.__noise__
    def nClasses(self):
        return self.__nClasses__
    def nPoints(self):
        return self.__nPoints__
    def noiseOrders(self):
        return self.__noiseOrders__
    def nlags(self):
        return self.__nlags__
    def coefs(self):
        #Inferred parameters
        return self.__model__.coefs()
    def events(self):
        #Current events in use
        return self.__events__
    def longestEvent(self):
        #Longest event in samples
        return self.__longestEvent__
    def encoding(self):
        #Encoding matrix for gesign
        return self.__encoding__
    def alphaH(self):
        return self.coefs()[:,:-self.noise().shape[1]]
    def betaH(self):
        return self.coefs()[:,self.noise().shape[1]:]
    #Setters
    def setEvents(self,events):
        logger.info( 'Setting events')
        self.__events__ = events
    def setNoise(self,noise):
        logger.info( 'Setting noise matrix')
        #Noise should be a matrix of length=grid.times() describing noise in the signal over time
        self.__noise__ = noise
    def setnClasses(self,nclasses):
        self.__nClasses__ = nclasses
    def setnPoints(self,npoints):
        self.__nPoints__ = npoints
    def setCoefs(self,coefs):
        logger.info( 'Setting coefficients')
        self.__model__.setCoefs(coefs)
    def setGrid(self,grid):
        logger.info( 'Setting grid')
        #Change grid and update noise matrix if necessary
        self.__grid__ = grid
    def setNumLags(self,numLags):
        logger.info( 'Setting number of lags')
        self.__nlags__ = numLags
    def setLambda(self,lmbda):
        self.__lambda__ = lmbda
    def setNoiseOrders(self,noiseOrders):
        logger.info( 'Setting polynomial orders for noise matrix')
        #Set maximum order of noise and update the matrix
        self.__noiseOrders__= noiseOrders
        self.setNoise(self.genNoise(self.grid(), self.noiseOrders()))

    def setLongestEvent(self,longestEvent):
        logger.info( 'setting longest event time')
        self.__longestEvent__ = int(longestEvent*self.grid().fs())

    def setEncoding(self,encoding):
        logger.info( 'Setting encoding to be used in design matrix')
        self.__encoding__ = encoding

    #ML functions
    def genNoise(self,grid,maxNoiseOrder):
        #Noise is a matrix of Legendre polynomials of 0<order<maxNoiseOrder
        #Additionally 60Hz sine and cosine waves are added to account for the DC component of EEG

        if self.grid() is not None and self.noiseOrders() is not None:
            logger.info( 'Generating noise matrix')
            legpoly = np.array([Legendre.basis(i)(np.arange(len(grid.times()))) for i in self.noiseOrders()]).T #Polynomials
            sw = np.sin(60 * np.arange(len(grid.times())) * 2 * np.pi / float(grid.fs())) # Sine for AC component
            cw = np.cos(60 * np.arange(len(grid.times())) * 2 * np.pi / float(grid.fs())) # Cosine for AC component
            legpoly = np.column_stack((legpoly, sw, cw))
            return pd.DataFrame(legpoly,index=self.grid().times())

    def event_design(self,event,encoding=None,prevCode=None,longest=None):
        #Generates the design matrix for an event
        #encoding - class that returns the encoding of a given event
        #prevCode - is design is dependent on a previous state then pass it here
        #longest - longest event in seconds

        if encoding is None:
            encoding = self.encoding()

        if longest is not None:
            self.setLongestEvent(longest)

        longest=self.longestEvent()

        nClasses = encoding.numClasses()
        logger.debug('nClasses : %d'%(nClasses))

        npoints = self.nlags()*nClasses+nClasses #Number of columns in design matrix
        logger.debug('npoints : %d'%(npoints))

        if prevCode is None:
            prevCode = np.zeros([npoints])
        classCols = np.arange(nClasses)*(self.nlags()+1)
        lagCols= np.array([ix for ix in np.arange(npoints) if ix%(self.nlags()+1)])
        grid = self.grid()

        times = grid.event_times(event)[:longest] #Get the event times (limited to the maximum event length)
        logger.debug('Event size : %d'%(len(times)))
        noise = self.noise().ix[times]
        design = np.zeros([len(times),npoints+noise.shape[1]])
        eventEncoding = encoding[event]

        for i,time in enumerate(times.values):
            design[i,:] = np.hstack((np.zeros([npoints]),noise.ix[time]))
            design[i,classCols] = eventEncoding
            design[i,lagCols] = prevCode[lagCols-1]
            design = design
            prevCode = design[i,:]
        return pd.DataFrame(design,index=times)

    def genX(self,events):
        #Generates the event matrix for events (in memory)
        from tempfile import mkdtemp
        import os.path as path

        #Set longest event in samples
        longest = self.longestEvent()
        allNumPoints = self.grid().eventsTimes(events,limit=longest)
        self.setEvents(events)
        npoints = self.nPoints()
        logger.info('Preparing design matrix')
        filename = path.join(mkdtemp(), 'tempX.dat')
        X = np.memmap(filename, dtype='float32', mode='w+', shape=(len(allNumPoints),npoints))
        X.times = allNumPoints

        #Build X
        prevCode = np.zeros([npoints])
        pos = 0
        for i in range(len(events)):
            logger.info('Generating event %d/%d design'%(i+1,len(events)))
            design = self.event_design(events.iloc[i],prevCode=prevCode) #Get design matrix for event
            t = design.index #times for the event
            design = design.values
            X[pos:pos+len(t),:] = design
            #
            pos+=len(t)
            prevCode = design[-1,:]

        return X

    def __iterX__(self,events):
        #Generator for event matrix blocks

        encoding = self.encoding()
        longest=self.longestEvent()
        nClasses = encoding.numClasses()
        npoints = self.nlags()*nClasses+nClasses #Number of columns in design matrix

        prevCode = np.zeros([npoints])
        classCols = np.arange(nClasses)*(self.nlags()+1)
        lagCols= np.array([ix for ix in np.arange(npoints) if ix%(self.nlags()+1)])
        grid = self.grid()

        for i in range(len(events)):
            event = events.iloc[i]
            times = grid.event_times(event)[:longest] #Get the event times (limited to the maximum event length)
            logger.debug('Event size : %d'%(len(times)))
            noise = self.noise().ix[times]
            design = np.zeros([len(times),npoints+noise.shape[1]])
            eventEncoding = encoding[event]
            for i,time in enumerate(times.values):
                design[i,:] = np.hstack((np.zeros([npoints]),noise.ix[time]))
                design[i,classCols] = eventEncoding
                design[i,lagCols] = prevCode[lagCols-1]
                prevCode = design[i,:]
            yield design,times

    def meanDesign(self,events):
        #Calculates the mean on columns of the full events matrix

        logger.info('Calculating design mean')
        designMean = np.zeros(self.nPoints())
        design = self.__iterX__(events)
        N = 0
        for X,times in design:
            designMean += np.sum(X[:self.__longestEvent__,:],axis=0)
            N += len(times)
        return designMean / N

    def l1Norm(self,events):
        #Calculates the l1 normalisation parameter on columns of the design matrix

        logger.info('Calculating l1Norm')
        l1 = np.zeros(self.nPoints())
        design = self.__iterX__(events)
        for X,_ in design:
            l1 += np.sum(np.abs(X[:self.__longestEvent__,:]),axis=0)
        return np.sqrt(l1)

    def l2Norm(self,events):
        #L2 norm for real valued event matrix
        logger.info('Calculating l2Norm')
        l2 = np.zeros(self.nPoints())
        design = self.__iterX__(events)
        for X,_ in design:
            l2 += np.sum(X[:self.__longestEvent__,:]**2,axis=0)
        return np.sqrt(l2)

    def varDesign(self,events,mean=None):
        #Calculates variance on columns of design matrix

        if mean is None:
            mean = self.meanDesign(events)

        logger.info('Calculating design variance')
        designVar = np.zeros(self.nPoints())
        design = self.__iterX__(events)
        N = 0
        for X,times in design:
            designVar += np.sum((X[:self.__longestEvent__]-mean)**2,axis=0)
            N += len(times)
        return designVar / (N-1)

    def online_train(self,events,y,encoding=None,longest=None,normalise=None,model='SGD'):
        #Online training of regression over events
        #Longest - the maximum event time in seconds (useful for limiting feedback periods ect.)
        #Encoding - dictionary that has row encodings for a given label(not including lags)
        #Normalisation - Normalisation coefficients to apply to columns of the design {zscore,l1,l2}
        #Model - type of model to use {linear,SGD}
        if longest is not None:
            self.setLongestEvent(longest)
        if encoding is not None:
            self.setEncoding(encoding)
        if model == 'linear':
            self.__model__ = onlineLinearRegression()
        elif model == 'SGD':
            self.__model__ = onlineSGDRegression()
        if normalise is not None:
            designMean = self.meanDesign(events)
            if normalise == 'zscore':
                designVar = self.varDesign(events,self.__designMean__)
                designSTD = designVar**0.5
                self.__normFunc__ = lambda x: (x-designMean)/designSTD
            elif normalise == 'l2':
                l2Norm = self.l2Norm(events)
                self.__normFunc__ = lambda x: (x-designMean)/l2Norm
            elif normalise == 'l1':
                l1Norm = self.l1Norm(events)
                self.__normFunc__ = lambda x : (x-designMean)/l1Norm
            else:
                normalise = None
        self.setEvents(events)

        #Predict
        logger.info('Training events')
        design = self.__iterX__(events)

        t= np.array([])

        for j,(X,times) in enumerate(design):
            logger.info('Training event %d'%(j+1))
            X_block = self.__normFunc__(X) if normalise else X
            X_block[np.isnan(X_block)] = 0
            self.__model__.partial_fit(X_block,y.ix[times])
            t = np.concatenate((t,times))
        return self.__model__.coefs()

    def online_predict(self,events,coefs=None):

        if coefs is None:
            coefs= self.coefs()

        logger.info('Predicting events')
        design = self.__iterX__(events)
        ntargets = len(self.grid().wc())
        grid = self.grid()
        predictTimes = grid.eventsTimes(events,limit=self.longestEvent()/grid.fs())
        prediction = pd.DataFrame(np.zeros([len(predictTimes),ntargets]),index=predictTimes,columns=grid.wc())

        for j,(X,times) in enumerate(design):
            logger.info('Predicting event %d'%(j+1))
            X_block = self.__normFunc__(X) if self.__normFunc__ else X
            X_block[np.isnan(X_block)] = 0
            prediction = self.__model__.predict(X_block)
            prediction[times] = model.predict(X_block)

        return prediction
<commit_msg>Update GridRegression.py<commit_after>import numpy as np
import pandas as pd
from numpy.polynomial import Legendre
import logging
from sklearn import linear_model

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('__GridRegression__')

#Online regression classes
class model():
    __coefs__ = None

    def coefs(self):
        if self.__coefs__ is None:
            self.setCoefs()
        return self.__coefs__

    def setCoefs(self,coefs):
        pass
    def partial_fit(self,X,y):
        pass
    def predict(self,X):
        pass

class onlineLinearRegression(model):

    __R__ = None
    __cov__ = None
    __lambda__ = None #Regularisation term

    def partial_fit(self,X,y):
        self.__coefs__ = None
        self.__cov__ += np.dot(X,X.T)
        self.__R__ += np.dot(X,y.values)

    def setCoefs(self,coefs=None):
        if coefs is None:
            self.__coefs__ = np.dot(np.invert(self.__cov__),self.__R__)
        else:
            self.__coefs__ = coefs

    def predict(self,X):
        if self.__coefs__ is None:
            self.setCoefs()
        return np.dot(self.__coefs__,X)

    def __calcLambda__(self):
        #Define range for lambda (lambdaRange)
        #Define train and test set (xTrain, XTest)
        pass

class onlineSGDRegression(model):

    __model__ = None
    __ntargets__ = None
    def partial_fit(self,X,y):
        self.__ntargets__ = len(y.columns)
        self.__coefs__ = None

        if self.__model__ is None:
            self.__model__ = [linear_model.SGDRegressor() for i in range(self.__ntargets__)] #Model for each target variable
        for i in range(self.__ntargets__):
            self.__model__[i].partial_fit(X,y.values[:,i])

    def setCoefs(self,coefs=None):
        if coefs is None:
            self.__coefs__ = np.array([reglin.coef_ for reglin in self.__model__])
        else:
            self.__coefs__ = coefs

    def predict(self,X):
        if self.__coefs__ is None:
            self.setCoefs()
        coefs = self.coefs()
        for i in range(self.__ntargets__):
                model = self.__model__[i]
                model.coef_ = coefs[i,:]

class GridRegression:
    #Help with regression over grid data
    #Note that this class does not represent the design matrix explicitly but will instead calculate
    #it additively based on single events.

    #Private
    __noise__ = None
    __nlags__ = None
    __grid__ = None
    __noiseOrders__ = None
    __coefs__ = None
    __events__ = None
    __longestEvent__ = None
    __encoding__ = None
    __lambda__ = None
    __nClasses__ = None
    __nPoints__ = None
    __normFunc__ = None
    __model__ = None

    def __init__(self,grid,nlags=0,noiseOrders=None,encoding=None):
        self.setNumLags(nlags)
        self.setGrid(grid)
        self.setNoiseOrders(noiseOrders)
        self.setEncoding(encoding)
        self.setnClasses(encoding.numClasses())
        self.setnPoints(self.nlags()*self.nClasses()+self.nClasses()+len(self.noise().columns)) #Number of columns in design matrix

    #Getters
    def grid(self):
        return self.__grid__
    def noise(self):
        return self.__noise__
    def nClasses(self):
        return self.__nClasses__
    def nPoints(self):
        return self.__nPoints__
    def noiseOrders(self):
        return self.__noiseOrders__
    def nlags(self):
        return self.__nlags__
    def coefs(self):
        #Inferred parameters
        return self.__model__.coefs()
    def events(self):
        #Current events in use
        return self.__events__
    def longestEvent(self):
        #Longest event in samples
        return self.__longestEvent__
    def encoding(self):
        #Encoding matrix for gesign
        return self.__encoding__
    def alphaH(self):
        return self.coefs()[:,:-self.noise().shape[1]]
    def betaH(self):
        return self.coefs()[:,self.noise().shape[1]:]
    #Setters
    def setEvents(self,events):
        logger.info( 'Setting events')
        self.__events__ = events
    def setNoise(self,noise):
        logger.info( 'Setting noise matrix')
        #Noise should be a matrix of length=grid.times() describing noise in the signal over time
        self.__noise__ = noise
    def setnClasses(self,nclasses):
        self.__nClasses__ = nclasses
    def setnPoints(self,npoints):
        self.__nPoints__ = npoints
    def setCoefs(self,coefs):
        logger.info( 'Setting coefficients')
        self.__model__.setCoefs(coefs)
    def setGrid(self,grid):
        logger.info( 'Setting grid')
        #Change grid and update noise matrix if necessary
        self.__grid__ = grid
    def setNumLags(self,numLags):
        logger.info( 'Setting number of lags')
        self.__nlags__ = numLags
    def setLambda(self,lmbda):
        self.__lambda__ = lmbda
    def setNoiseOrders(self,noiseOrders):
        logger.info( 'Setting polynomial orders for noise matrix')
        #Set maximum order of noise and update the matrix
        self.__noiseOrders__= noiseOrders
        self.setNoise(self.genNoise(self.grid(), self.noiseOrders()))

    def setLongestEvent(self,longestEvent):
        logger.info( 'setting longest event time')
        self.__longestEvent__ = int(longestEvent*self.grid().fs())

    def setEncoding(self,encoding):
        logger.info( 'Setting encoding to be used in design matrix')
        self.__encoding__ = encoding

    #ML functions
    def genNoise(self,grid,maxNoiseOrder):
        #Noise is a matrix of Legendre polynomials of 0<order<maxNoiseOrder
        #Additionally 60Hz sine and cosine waves are added to account for the DC component of EEG

        if self.grid() is not None and self.noiseOrders() is not None:
            logger.info( 'Generating noise matrix')
            legpoly = np.array([Legendre.basis(i)(np.arange(len(grid.times()))) for i in self.noiseOrders()]).T #Polynomials
            sw = np.sin(60 * np.arange(len(grid.times())) * 2 * np.pi / float(grid.fs())) # Sine for AC component
            cw = np.cos(60 * np.arange(len(grid.times())) * 2 * np.pi / float(grid.fs())) # Cosine for AC component
            legpoly = np.column_stack((legpoly, sw, cw))
            return pd.DataFrame(legpoly,index=self.grid().times())

    def event_design(self,event,encoding=None,prevCode=None,longest=None):
        #Generates the design matrix for an event
        #encoding - class that returns the encoding of a given event
        #prevCode - is design is dependent on a previous state then pass it here
        #longest - longest event in seconds

        if encoding is None:
            encoding = self.encoding()

        if longest is not None:
            self.setLongestEvent(longest)

        longest=self.longestEvent()

        nClasses = encoding.numClasses()
        logger.debug('nClasses : %d'%(nClasses))

        npoints = self.nlags()*nClasses+nClasses #Number of columns in design matrix
        logger.debug('npoints : %d'%(npoints))

        if prevCode is None:
            prevCode = np.zeros([npoints])
        classCols = np.arange(nClasses)*(self.nlags()+1)
        lagCols= np.array([ix for ix in np.arange(npoints) if ix%(self.nlags()+1)])
        grid = self.grid()

        times = grid.event_times(event)[:longest] #Get the event times (limited to the maximum event length)
        logger.debug('Event size : %d'%(len(times)))
        noise = self.noise().ix[times]
        design = np.zeros([len(times),npoints+noise.shape[1]])
        eventEncoding = encoding[event]

        for i,time in enumerate(times.values):
            design[i,:] = np.hstack((np.zeros([npoints]),noise.ix[time]))
            design[i,classCols] = eventEncoding
            design[i,lagCols] = prevCode[lagCols-1]
            design = design
            prevCode = design[i,:]
        return pd.DataFrame(design,index=times)

    def genX(self,events):
        #Generates the event matrix for events (in memory)
        from tempfile import mkdtemp
        import os.path as path

        #Set longest event in samples
        longest = self.longestEvent()
        allNumPoints = self.grid().eventsTimes(events,limit=longest)
        self.setEvents(events)
        npoints = self.nPoints()
        logger.info('Preparing design matrix')
        filename = path.join(mkdtemp(), 'tempX.dat')
        X = np.memmap(filename, dtype='float32', mode='w+', shape=(len(allNumPoints),npoints))
        X.times = allNumPoints

        #Build X
        prevCode = np.zeros([npoints])
        pos = 0
        for i in range(len(events)):
            logger.info('Generating event %d/%d design'%(i+1,len(events)))
            design = self.event_design(events.iloc[i],prevCode=prevCode) #Get design matrix for event
            t = design.index #times for the event
            design = design.values
            X[pos:pos+len(t),:] = design
            #
            pos+=len(t)
            prevCode = design[-1,:]

        return X

    def __iterX__(self,events):
        #Generator for event matrix blocks

        encoding = self.encoding()
        longest=self.longestEvent()
        nClasses = encoding.numClasses()
        npoints = self.nlags()*nClasses+nClasses #Number of columns in design matrix

        prevCode = np.zeros([npoints])
        classCols = np.arange(nClasses)*(self.nlags()+1)
        lagCols= np.array([ix for ix in np.arange(npoints) if ix%(self.nlags()+1)])
        grid = self.grid()

        for i in range(len(events)):
            event = events.iloc[i]
            times = grid.event_times(event)[:longest] #Get the event times (limited to the maximum event length)
            logger.debug('Event size : %d'%(len(times)))
            noise = self.noise().ix[times]
            design = np.zeros([len(times),npoints+noise.shape[1]])
            eventEncoding = encoding[event]
            for i,time in enumerate(times.values):
                design[i,:] = np.hstack((np.zeros([npoints]),noise.ix[time]))
                design[i,classCols] = eventEncoding
                design[i,lagCols] = prevCode[lagCols-1]
                prevCode = design[i,:]
            yield design,times

    def meanDesign(self,events):
        #Calculates the mean on columns of the full events matrix

        logger.info('Calculating design mean')
        designMean = np.zeros(self.nPoints())
        design = self.__iterX__(events)
        N = 0
        for X,times in design:
            designMean += np.sum(X[:self.__longestEvent__,:],axis=0)
            N += len(times)
        return designMean / N

    def l1Norm(self,events):
        #Calculates the l1 normalisation parameter on columns of the design matrix

        logger.info('Calculating l1Norm')
        l1 = np.zeros(self.nPoints())
        design = self.__iterX__(events)
        for X,_ in design:
            l1 += np.sum(np.abs(X[:self.__longestEvent__,:]),axis=0)
        return np.sqrt(l1)

    def l2Norm(self,events):
        #L2 norm for real valued event matrix
        logger.info('Calculating l2Norm')
        l2 = np.zeros(self.nPoints())
        design = self.__iterX__(events)
        for X,_ in design:
            l2 += np.sum(X[:self.__longestEvent__,:]**2,axis=0)
        return np.sqrt(l2)

    def varDesign(self,events,mean=None):
        #Calculates variance on columns of design matrix

        if mean is None:
            mean = self.meanDesign(events)

        logger.info('Calculating design variance')
        designVar = np.zeros(self.nPoints())
        design = self.__iterX__(events)
        N = 0
        for X,times in design:
            designVar += np.sum((X[:self.__longestEvent__]-mean)**2,axis=0)
            N += len(times)
        return designVar / (N-1)

    def online_train(self,events,y,encoding=None,longest=None,normalise=None,model='SGD'):
        #Online training of regression over events
        #Longest - the maximum event time in seconds (useful for limiting feedback periods ect.)
        #Encoding - dictionary that has row encodings for a given label(not including lags)
        #Normalisation - Normalisation coefficients to apply to columns of the design {zscore,l1,l2}
        #Model - type of model to use {linear,SGD}
        if longest is not None:
            self.setLongestEvent(longest)
        if encoding is not None:
            self.setEncoding(encoding)
        if model == 'linear':
            self.__model__ = onlineLinearRegression()
        elif model == 'SGD':
            self.__model__ = onlineSGDRegression()
        if normalise is not None:
            designMean = self.meanDesign(events)
            if normalise == 'zscore':
                designVar = self.varDesign(events,self.__designMean__)
                designSTD = designVar**0.5
                self.__normFunc__ = lambda x: (x-designMean)/designSTD
            elif normalise == 'l2':
                l2Norm = self.l2Norm(events)
                self.__normFunc__ = lambda x: (x-designMean)/l2Norm
            elif normalise == 'l1':
                l1Norm = self.l1Norm(events)
                self.__normFunc__ = lambda x : (x-designMean)/l1Norm
            else:
                normalise = None
        self.setEvents(events)

        #Predict
        logger.info('Training events')
        design = self.__iterX__(events)

        t= np.array([])

        for j,(X,times) in enumerate(design):
            logger.info('Training event %d'%(j+1))
            X_block = self.__normFunc__(X) if normalise else X
            X_block[np.isnan(X_block)] = 0
            self.__model__.partial_fit(X_block,y.ix[times])
            t = np.concatenate((t,times))
        return self.__model__.coefs()

    def online_predict(self,events,coefs=None):

        if coefs is None:
            coefs= self.coefs()

        logger.info('Predicting events')
        design = self.__iterX__(events)
        ntargets = len(self.grid().wc())
        grid = self.grid()
        predictTimes = grid.eventsTimes(events,limit=self.longestEvent()/grid.fs())
        prediction = pd.DataFrame(np.zeros([len(predictTimes),ntargets]),index=predictTimes,columns=grid.wc())

        for j,(X,times) in enumerate(design):
            logger.info('Predicting event %d'%(j+1))
            X_block = self.__normFunc__(X) if self.__normFunc__ else X
            X_block[np.isnan(X_block)] = 0
            prediction = self.__model__.predict(X_block)
            prediction[times] = model.predict(X_block)

        return prediction
<|endoftext|>


<commit_before>import os
import sys
import urllib.request
import datetime
import time
import re
import itertools

def main(date):
  dishes = []
  pdf = "/tmp/MA-aktuell.pdf"
  url = "http://personalkantine.personalabteilung.tu-berlin.de/pdf/MA-aktuell.pdf"
  menu = "/tmp/MA-aktuell.txt"

  urllib.request.urlretrieve(url, pdf)
  os.system("pdftotext -layout -nopgbrk " + pdf + " " + menu)
  prepare_menu(menu)

  # find current date, offset and header line numbers
  with open(menu) as f:
    in_date_range = False
    wrapped = False
    startline = -1
    endline = -1

    # jump to start of file
    f.seek(0, 0)
    for i, line in enumerate(f):
      dishes = []
      line = line.rstrip()

      # check for not price finished line (occurs after friday)
      if in_date_range:
        if not line.endswith("€"):
          if not wrapped:
            wrapped = True
          else:
            endline = i-1
            in_date_range = False
        else:
          if wrapped:
            wrapped = False

      # check for next date
      if not wrapped and in_date_range:
        exp = re.compile('^\d\d\.\d\d\.\d\d ')
        if exp.match(line):
          endline = i-1
          in_date_range = False

      # find start line (MUST be after checking for next date)
      if line.startswith( date ):
        startline = i-1
        in_date_range = True

    f.seek(0, 0)

    if startline == -1 or endline == -1:
      return ["Date not found"]

    lines = itertools.islice(f, startline, endline)
    for line in lines:
      line = line.rstrip()

      # remove date tag
      exp = re.compile('^\d\d\.\d\d\.\d\d ')
      line = exp.sub('', line)

      # remove day tag
      exp = re.compile('^(Montag|Dienstag|Mittwoch|Donnerstag|Freitag), ')
      line = exp.sub('', line)

      # replace "vegetarisch" with "(v)"
      exp = re.compile('^vegetarisch')
      line = exp.sub('(v)', line)

      # use common price tag design
      exp = re.compile(' +(\d,\d+) +€$')
      line = exp.sub(': \g<1>€', line)

      dishes.append(line)

  # cleanup
  os.system("rm " + pdf + " " + menu)
  return dishes

def get_menue(date):
  dishes = main(date)
  menue = ""
  for dish in dishes:
    menue = menue + dish + "\n"
  menue = menue.rstrip()
  return menue

def prepare_menu(f):
  new_menue = []
  with open(f, 'r') as menu:
    for line in menu:
      # remove indregend hints (e.g. '(Sch)')
      exp = re.compile('\([\w +]+\)')
      line = exp.sub('', line)

      # reduce whitespace
      exp = re.compile('[ \t]+')
      line = exp.sub(' ', line)

      # remove leading whitespace
      exp = re.compile('^ ')
      line = exp.sub('', line)

      # remove empty lines
      if line.strip():
        new_menue.append(line)

  with open(f, 'w') as menu:
    menu.writelines(new_menue)

if __name__ == '__main__':
  print( get_menue( datetime.date.today().strftime("%d.%m.%y")))

# vim:set ft et sw=2 sts=2:
<commit_msg>rewrite for the new layout (html instead of pdf)<commit_after>import os
import sys
import datetime
import time
import re
import urllib.request
from bs4 import BeautifulSoup

def main(date):
  dishes = []
  url = "http://personalkantine.personalabteilung.tu-berlin.de/"
  html = urllib.request.urlopen(url).read()
  menu = BeautifulSoup( html, 'html.parser' ).find("ul", class_="Menu__accordion")

  for day in menu.children:
    if day.name and date in day.find("h2").string:
      for dishlist in day.children:
        if dishlist.name == 'ul':
          for dish in dishlist.find_all('li'):
            this_dish = ""
            for string in dish.stripped_strings:
              this_dish = this_dish + " " + string
            this_dish = _format( this_dish )
            dishes.append( this_dish )

  return dishes

def _format(line):
  line = line.strip()

  # remove indregended hints
  exp = re.compile('\([\w\s+]+\)')
  line = exp.sub('', line)

  # use common price tag design
  exp = re.compile('\s+(\d,\d+)\s+€')
  line = exp.sub(': \g<1>€', line)

  return line

def get_menue(date):
  dishes = main(date)
  menue = ""
  for dish in dishes:
    menue = menue + dish + "\n"
  menue = menue.rstrip()
  return menue

if __name__ == '__main__':
  print( get_menue( datetime.date.today().strftime("%d.%m.%Y")))

# vim:set ft et sw=2 sts=2:
<|endoftext|>


<commit_before># ======================================================================
# matscipy - Python materials science tools
# https://github.com/libAtoms/matscipy
#
# Copyright (2014) James Kermode, King's College London
#                  Lars Pastewka, Karlsruhe Institute of Technology
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
# ======================================================================

import re

import numpy as np

###

def savetbl(fn, **kwargs):
    """
    Save tabulated data and write column header strings.

    Example:
        savetbl('file.dat', time=time, strain=strain, energy=energy)

    Parameters
    ----------
    fn : str
        Name of file to write to.
    kwargs : dict
        Keyword argument pass data and column header names.
    """
    sorted_kwargs = sorted(kwargs.items(), key=lambda t: t[0])
    header = ''
    for i, (x, y) in enumerate(sorted_kwargs):
        header = '{0} {1}:{2}'.format(header, i+1, x)
    print(header)
    data = np.transpose([y for x, y in sorted_kwargs])
    np.savetxt(fn, data, header=header)

def loadtbl(fn, usecols=None, fromfile=False):
    """
    Load tabulated data from column header strings.

    Example data file:
        # time strain energy
        1.0 0.01 283
        2.0 0.02 398
        ...

        strain, energy = loadtbl('file.dat', usecols=['strain', 'energy'])

    Parameters
    ----------
    fn : str
        Name of file to load.
    usecols : list of strings
        List of column names.
    fromfile : bool
        Use numpy.fromfile instead of numpy.loadtxt if set to True. Can be
        faster in some circumstances.

    Returns
    -------
    data : tuple of arrays
        Return tuple of array with data for each colume in usecols if
        usecols specified. For usecols=None, return dictionary with header
        keys and arrays as data entries.
    """
    f = open(fn)
    line = f.readline()
    column_labels = None
    while line.startswith('#'):
        line = line[1:].strip()
        column_labels = [s.strip() for s in re.split('[\s,]+', line)]
        line = f.readline()
    f.close()
    if column_labels is None:
        raise RuntimeError("No header found in file '{}'".format(fn))

    sep_i = [x.find(':') for x in column_labels]
    column_labels = map(lambda s,i: s[i+1:] if i >= 0 else s, column_labels,
                        sep_i)

    if fromfile:
        f = open(fn)
        f.readline()
        data = np.fromfile(f, sep=' ')
        data.shape = (-1, len(column_labels))
        if usecols is None:
            return dict((s, d) for s, d in zip(column_labels, data.T))
        else:
            return [data[:, column_labels.index(s)] for s in usecols]
    else:
        if usecols is None:
            data = np.loadtxt(fn, unpack=True)
            return dict((s, d) for s, d in zip(column_labels, data))
        else:
            column_i = [ column_labels.index(s) for s in usecols ]
            return np.loadtxt(fn, usecols=column_i, unpack=True)
<commit_msg>MAINT: Removed debug output.<commit_after># ======================================================================
# matscipy - Python materials science tools
# https://github.com/libAtoms/matscipy
#
# Copyright (2014) James Kermode, King's College London
#                  Lars Pastewka, Karlsruhe Institute of Technology
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
# ======================================================================

import re

import numpy as np

###

def savetbl(fn, **kwargs):
    """
    Save tabulated data and write column header strings.

    Example:
        savetbl('file.dat', time=time, strain=strain, energy=energy)

    Parameters
    ----------
    fn : str
        Name of file to write to.
    kwargs : dict
        Keyword argument pass data and column header names.
    """
    sorted_kwargs = sorted(kwargs.items(), key=lambda t: t[0])
    header = ''
    for i, (x, y) in enumerate(sorted_kwargs):
        header = '{0} {1}:{2}'.format(header, i+1, x)
    data = np.transpose([y for x, y in sorted_kwargs])
    np.savetxt(fn, data, header=header)

def loadtbl(fn, usecols=None, fromfile=False):
    """
    Load tabulated data from column header strings.

    Example data file:
        # time strain energy
        1.0 0.01 283
        2.0 0.02 398
        ...

        strain, energy = loadtbl('file.dat', usecols=['strain', 'energy'])

    Parameters
    ----------
    fn : str
        Name of file to load.
    usecols : list of strings
        List of column names.
    fromfile : bool
        Use numpy.fromfile instead of numpy.loadtxt if set to True. Can be
        faster in some circumstances.

    Returns
    -------
    data : tuple of arrays
        Return tuple of array with data for each colume in usecols if
        usecols specified. For usecols=None, return dictionary with header
        keys and arrays as data entries.
    """
    f = open(fn)
    line = f.readline()
    column_labels = None
    while line.startswith('#'):
        line = line[1:].strip()
        column_labels = [s.strip() for s in re.split('[\s,]+', line)]
        line = f.readline()
    f.close()
    if column_labels is None:
        raise RuntimeError("No header found in file '{}'".format(fn))

    sep_i = [x.find(':') for x in column_labels]
    column_labels = map(lambda s,i: s[i+1:] if i >= 0 else s, column_labels,
                        sep_i)

    if fromfile:
        f = open(fn)
        f.readline()
        data = np.fromfile(f, sep=' ')
        data.shape = (-1, len(column_labels))
        if usecols is None:
            return dict((s, d) for s, d in zip(column_labels, data.T))
        else:
            return [data[:, column_labels.index(s)] for s in usecols]
    else:
        if usecols is None:
            data = np.loadtxt(fn, unpack=True)
            return dict((s, d) for s, d in zip(column_labels, data))
        else:
            column_i = [ column_labels.index(s) for s in usecols ]
            return np.loadtxt(fn, usecols=column_i, unpack=True)
<|endoftext|>


<commit_before>from cupyx.scipy.linalg.solve_triangular import solve_triangular  # NOQA
from cupyx.scipy.linalg.decomp_lu import lu, lu_factor, lu_solve  # NOQA
<commit_msg>add NOQA<commit_after># flake8: NOQA
from cupyx.scipy.linalg.special_matrices import (
    tri, tril, triu, toeplitz, circulant, hankel,
    hadamard, leslie, kron, block_diag, companion,
    helmert, hilbert, dft,
    fiedler, fiedler_companion, convolution_matrix
)
from cupyx.scipy.linalg.solve_triangular import solve_triangular  # NOQA
from cupyx.scipy.linalg.decomp_lu import lu, lu_factor, lu_solve  # NOQA
<|endoftext|>