<commit_before># Author: Michele Mattioni
# Fri Feb  4 15:03:02 GMT 2011

import sys
import os
import numpy as np

import matplotlib.pyplot as plt

from helpers.loader import Loader
try:
    import ecellControl.ecellManager as eC
except ImportError:
    pass

class StimulPlotter():
    """Helps to plot stimuli"""

    def __init__(self):
        pass

    def build_stimul_vec(self, time, stimul, height_in_the_graph):
        """Return a masked array which has `height_in_the_graph` value
        for each point recorded in stimul array.

        time - the time array of the simulation
        stimul - the synvecRef vectors with the synapstic inputs
        height_in_graph"""

        inputs_event = np.zeros_like(time)
        indxs = np.searchsorted(time, stimul)
        inputs_event[indxs] = height_in_the_graph
        masked = np.ma.masked_where(inputs_event == 0, inputs_event)
        return masked

    def plot_input(self, spine, manager, ax=None, style='k.',
                   height_in_the_graph=-88):
        """Plot the built stimuli on the current figure.
        If an active axes is passed, it will be used to plot.
        Handy on figure with two axes."""

        spine += '_psd'
        stimul = manager.get_vector(spine, 'stimul_ampa', 'SynVecRef')

        if stimul is None:
            print "Error - Stimul not Plotted. No stim found for %s." %spine
        else:
            time = manager.groups['t']
            print "Buiding stim array for %s" %spine
            m_array = self.build_stimul_vec(time, stimul, height_in_the_graph)
            if ax:
                ax.plot(time, m_array, 'k.', label = "stimul")
            else:
                plt.plot(time, m_array, 'k.', label = "stimul")
            return m_array

class EcellPlotter():

    def plot_timeCourses(self, timeCourses, save=False,
                        dir=None, name='test_spine', x_lims=None, tstop_bio=300):
         """Plot the default timecourses"""
         import matplotlib.pyplot as plt
         ca_tc = timeCourses['ca']
         plt.figure()
         time_ms = (ca_tc[:,0] - tstop_bio) * 1e3 #scaling to ms
         plt.plot(time_ms, ca_tc[:,1], label="Calcium")
         plt.xlabel("Time [s]")
         if x_lims:
             plt.xlim(x_lims)
         plt.legend(loc=0)

         if save :
             filename = name +"_caInput.png"
             plt.savefig(os.path.join(dir, filename))
             print "figure saved in: %s" % os.path.join(dir, filename)

         vars = ['PP2Bbar', 'CaMKIIbar', 'PP1abar', 'AMPAR', 'AMPAR_P']

         for var in vars:
             plt.figure()
             var_tc = timeCourses[var]
             lbl = name + '_' + var
             plt.plot(var_tc[:,0], var_tc[:,1], label=lbl)
             plt.xlabel("Time [s]")
             if x_lims:
                 plt.xlim(x_lims)
             plt.legend(loc=0)

             if save :
                 filename = name + "_" + var + ".png"
                 plt.savefig(os.path.join(dir, filename))
                 print "figure saved in: %s" % os.path.join(dir, filename)

    def plot_weight(self, timecourses, dir=None):
        import matplotlib.pyplot as plt
        scaled_CaMKII = []
        time = timecourses['AMPAR'][:,0] # time equal for everything

        plt.figure()
        plt.plot(time, timecourses['AMPAR_P'][:,1], label='AMPAR_P')
        #plt.plot(time, weight_baseline, label='w_b')
        title = "AMPA weight, calculated from E-Cell"
        plt.title(title)
        plt.legend(loc=0)
        if dir is not None:
            plt.savefig(os.path.join(dir, "weight.png"))
        plt.plot(time, timecourses['AMPAR'][:,1], label='AMPAR')
        plt.legend(loc=0)
        if dir is not None:
            plt.savefig(os.path.join(dir, "weight_both.png"))

    def plot_var(self, timeCourses, var):
        "Plot the variable in the current figure"
        var_tc = timeCourses[var]
        lbl = var
        plt.plot(var_tc[:,0], var_tc[:,1], label=lbl)
        plt.xlabel("Time [s]")
        plt.legend(loc=0)

class DoublePlotter():
    "plot both electrical and biochemical in one graph."
    def plot_double_axes(self, manager, spine, el_var, bio_var,
    	bio_group, tstop_bio=300):
        """
        Plot a double axes figure with the variables as argument,
        return a tuple with the two axes used.

        Param
        -----
        manager - neuronvisio manager
        spine - spine to plot
        el_var - electrical variable to plot
        bio_var - biochemical variable to plot
        bio_group - group where the Ref is part of (e.g VecRef,
                    SynVecRef, YourRef)

        Example:
		ax1, ax2 = dp.plot_double_axes(controls.manager, 'spine1', 'v',
		'AMPAR_P', bio_group='timeSeries_spine1')
        """
        ax1 = plt.subplot(111)
        sec_head = spine + '_head'
        vec = manager.get_vector(sec_head, el_var)
        t = manager.groups['t']
        label = sec_head + "_" + el_var
        l1, = ax1.plot(t, vec, label=label)

        # plotting the second one

        vec_bio = manager.get_vector(spine, bio_var, group=bio_group)

        t2 = manager.groups[bio_group]
        t2_ms = (t2.read()- tstop_bio) * 1e3 #scaling to ms
        label = spine + "_" + bio_var
        ax2 = ax1.twinx()
        l2, = ax2.plot(t2_ms, vec_bio, 'r-', label=label)

        # Setting the axes and colour.
        ax1.set_ylabel('v [mV]')
        ax2.set_ylabel('# molecule')

        ax1.yaxis.label.set_color(l1.get_color())
        ax2.yaxis.label.set_color(l2.get_color())
        tkw = dict(size=4, width=1.5)
    	ax1.tick_params(axis='y', colors=l1.get_color(), **tkw)
    	ax2.tick_params(axis='y', colors=l2.get_color(), **tkw)

        lines = [l1,l2]
        ax1.legend(lines, [l.get_label() for l in lines], loc='best')
        ax1.set_xlabel('Time [ms]')
        ax1.set_xlim(0, 20000)
        return (ax1, ax2)

    def plot_bio_and_ele_calcium(self, manager, spine, dir=None):
        """Plot Calcium from bio and ele with the same scale in
        two graphs.
        Param:
            spine: Spine which Calcium should be plotted.
            manager: classic Neuronvisio manager.
        Example:
            dp.plot_bio_and_ele_calcium(controls.manager, "spine1")"""
        timeSeries = "timeSeries_%s" %spine
        ecell_time_equilibrium = 300 #s
        t_bio = manager.groups[timeSeries]
        t_bio_ms = (t_bio.read() - ecell_time_equilibrium) * 1e3 #s_to_ms

        ca_conc = manager.get_vector(spine, 'ca_conc', group=timeSeries)
        label = "bio_ca_%s" %spine
        plt.plot(t_bio_ms, ca_conc.read()*1e6, #M_to_uM
                 label=label)
        plt.title("Biochemical")
        plt.xlabel('Time [ms]')
        plt.ylabel('Concentration [um]')

        neuron_time_equilibrium = 100 #ms
        spine_head = "%s_head" %spine
        cai = manager.get_vector(spine_head, 'cai')
        cali = manager.get_vector(spine_head, 'cali')
        t = manager.groups['t']
        plt.figure()
        label = "ele_ca_%s" %spine
        plt.plot((t.read()-neuron_time_equilibrium),
                 ((cali.read()+cai.read())*1e3), #mM_to_uM
                 label=label)
        plt.xlabel('Time [ms]')
        plt.ylabel('Concentration [um]')
        plt.title("Electrical")
        xlim = plt.xlim(0,) #starting from zero
        plt.legend()
        if dir is not None:
            for format in [".png", ".pdf"]:
                filename = spine + "_electrical_calcium" + format
                plt.savefig(os.path.join(dir, filename))
        plt.figure(1)
        plt.xlim(xlim) # Getting bio graph on the electric xlim.
        plt.legend()
        if dir is not None:
            for format in [".png", ".pdf"]:
                filename = spine + "_biochemical_calcium" + format
                plt.savefig(os.path.join(dir, filename))
        # calcium buffered by the Calmodulin

        moles_bound_ca_per_moles_cam = manager.get_vector('spine1', 'moles_bound_ca_per_moles_cam', group='timeSeries_spine1')
        camR = 1.45e-9
        camT = 3e-5
        totcam = camT + camR
        t_bio = manager.groups['timeSeries_spine1']
        conc_tot_calcium_bound_to_cam = moles_bound_ca_per_moles_cam.read() * totcam
        plt.figure()
        plt.plot(t_bio_ms, conc_tot_calcium_bound_to_cam * 1e6, label="buffered_ca")
        plt.title("Calcium buffered by Calmodulin")
        plt.xlim(xlim)
        plt.xlabel("Time [ms]")
        plt.ylabel("Concentration [uM]")
        plt.legend()
        if dir is not None:
            for format in [".png", ".pdf"]:
                filename = spine + "_biochemical_calcium_buffered_by_Calmodulin" + format
                plt.savefig(os.path.join(dir, filename))

        ca_conc = manager.get_vector('spine1', 'ca_conc', group='timeSeries_spine1')

        calcium_bio_and_calcium_calmodulin = (conc_tot_calcium_bound_to_cam + ca_conc.read()) * 1e6
        plt.figure()
        plt.plot(t_bio_ms, calcium_bio_and_calcium_calmodulin, label="buffered+free_ca")
        plt.title("Total Calcium in the biochemical ")
        plt.xlim(xlim)
        plt.xlabel("Time [ms]")
        plt.ylabel("Concentration [uM]")
        plt.legend()
        if dir is not None:
            for format in [".png", ".pdf"]:
                filename = spine + "_biochemical_free_and_buffered_calcium" + format
                plt.savefig(os.path.join(dir, filename))

        plt.figure()
        calcium_bio_and_calcium_calmodulin_buffered_subtracted = ((conc_tot_calcium_bound_to_cam - conc_tot_calcium_bound_to_cam[489]) + ca_conc.read()) * 1e6
        plt.plot(t_bio_ms, calcium_bio_and_calcium_calmodulin_buffered_subtracted,
                 label="bio calcium")
        plt.title("Total Calcium in the biochemical ")
        plt.xlim(xlim)
        plt.xlabel("Time [ms]")
        plt.ylabel("Concentration [uM]")
        plt.legend()
        plt.ylim(-0.1, 5)
        if dir is not None:
            for format in [".png", ".pdf"]:
                filename = spine + "_biochemical_free_and_buffered_calcium_subtracted" + format
                plt.savefig(os.path.join(dir, filename))

if __name__ == "__main__":

    if len(sys.argv) == 2:

        filename = sys.argv[1]
        l = Loader()
        ecp = EcellPlotter()
        timecourses = l.load(filename)
        ecp.plot_timeCourses(timecourses, name='test_spine')
        print "Time courses loaded in 'timecourses' object. Easy plotting \n\
        with 'ecp' object. Have fun."
        plt.show()

    else:
        print "Tell me which file with ecell Timeseries to load."
        print "visualizatorBiochemical.py path/to/timecourses\n"
<commit_msg>Fixed the variable name as variable<commit_after># Author: Michele Mattioni
# Fri Feb  4 15:03:02 GMT 2011

import sys
import os
import numpy as np

import matplotlib.pyplot as plt

from helpers.loader import Loader
try:
    import ecellControl.ecellManager as eC
except ImportError:
    pass

class StimulPlotter():
    """Helps to plot stimuli"""

    def __init__(self):
        pass

    def build_stimul_vec(self, time, stimul, height_in_the_graph):
        """Return a masked array which has `height_in_the_graph` value
        for each point recorded in stimul array.

        time - the time array of the simulation
        stimul - the synvecRef vectors with the synapstic inputs
        height_in_graph"""

        inputs_event = np.zeros_like(time)
        indxs = np.searchsorted(time, stimul)
        inputs_event[indxs] = height_in_the_graph
        masked = np.ma.masked_where(inputs_event == 0, inputs_event)
        return masked

    def plot_input(self, spine, manager, ax=None, style='k.',
                   height_in_the_graph=-88):
        """Plot the built stimuli on the current figure.
        If an active axes is passed, it will be used to plot.
        Handy on figure with two axes."""

        spine += '_psd'
        stimul = manager.get_vector(spine, 'stimul_ampa', 'SynVecRef')

        if stimul is None:
            print "Error - Stimul not Plotted. No stim found for %s." %spine
        else:
            time = manager.groups['t']
            print "Buiding stim array for %s" %spine
            m_array = self.build_stimul_vec(time, stimul, height_in_the_graph)
            if ax:
                ax.plot(time, m_array, 'k.', label = "stimul")
            else:
                plt.plot(time, m_array, 'k.', label = "stimul")
            return m_array

class EcellPlotter():

    def plot_timeCourses(self, timeCourses, save=False,
                        dir=None, name='test_spine', x_lims=None, tstop_bio=300):
         """Plot the default timecourses"""
         import matplotlib.pyplot as plt
         ca_tc = timeCourses['ca']
         plt.figure()
         time_ms = (ca_tc[:,0] - tstop_bio) * 1e3 #scaling to ms
         plt.plot(time_ms, ca_tc[:,1], label="Calcium")
         plt.xlabel("Time [s]")
         if x_lims:
             plt.xlim(x_lims)
         plt.legend(loc=0)

         if save :
             filename = name +"_caInput.png"
             plt.savefig(os.path.join(dir, filename))
             print "figure saved in: %s" % os.path.join(dir, filename)

         vars = ['PP2Bbar', 'CaMKIIbar', 'PP1abar', 'AMPAR', 'AMPAR_P']

         for var in vars:
             plt.figure()
             var_tc = timeCourses[var]
             lbl = name + '_' + var
             plt.plot(var_tc[:,0], var_tc[:,1], label=lbl)
             plt.xlabel("Time [s]")
             if x_lims:
                 plt.xlim(x_lims)
             plt.legend(loc=0)

             if save :
                 filename = name + "_" + var + ".png"
                 plt.savefig(os.path.join(dir, filename))
                 print "figure saved in: %s" % os.path.join(dir, filename)

    def plot_weight(self, timecourses, dir=None):
        import matplotlib.pyplot as plt
        scaled_CaMKII = []
        time = timecourses['AMPAR'][:,0] # time equal for everything

        plt.figure()
        plt.plot(time, timecourses['AMPAR_P'][:,1], label='AMPAR_P')
        #plt.plot(time, weight_baseline, label='w_b')
        title = "AMPA weight, calculated from E-Cell"
        plt.title(title)
        plt.legend(loc=0)
        if dir is not None:
            plt.savefig(os.path.join(dir, "weight.png"))
        plt.plot(time, timecourses['AMPAR'][:,1], label='AMPAR')
        plt.legend(loc=0)
        if dir is not None:
            plt.savefig(os.path.join(dir, "weight_both.png"))

    def plot_var(self, timeCourses, var):
        "Plot the variable in the current figure"
        var_tc = timeCourses[var]
        lbl = var
        plt.plot(var_tc[:,0], var_tc[:,1], label=lbl)
        plt.xlabel("Time [s]")
        plt.legend(loc=0)

class DoublePlotter():
    "plot both electrical and biochemical in one graph."
    def plot_double_axes(self, manager, spine, el_var, bio_var,
    	bio_group, tstop_bio=300):
        """
        Plot a double axes figure with the variables as argument,
        return a tuple with the two axes used.

        Param
        -----
        manager - neuronvisio manager
        spine - spine to plot
        el_var - electrical variable to plot
        bio_var - biochemical variable to plot
        bio_group - group where the Ref is part of (e.g VecRef,
                    SynVecRef, YourRef)

        Example:
		ax1, ax2 = dp.plot_double_axes(controls.manager, 'spine1', 'v',
		'AMPAR_P', bio_group='timeSeries_spine1')
        """
        ax1 = plt.subplot(111)
        sec_head = spine + '_head'
        vec = manager.get_vector(sec_head, el_var)
        t = manager.groups['t']
        label = sec_head + "_" + el_var
        l1, = ax1.plot(t, vec, label=label)

        # plotting the second one

        vec_bio = manager.get_vector(spine, bio_var, group=bio_group)

        t2 = manager.groups[bio_group]
        t2_ms = (t2.read()- tstop_bio) * 1e3 #scaling to ms
        label = spine + "_" + bio_var
        ax2 = ax1.twinx()
        l2, = ax2.plot(t2_ms, vec_bio, 'r-', label=label)

        # Setting the axes and colour.
        ax1.set_ylabel('v [mV]')
        ax2.set_ylabel('# molecule')

        ax1.yaxis.label.set_color(l1.get_color())
        ax2.yaxis.label.set_color(l2.get_color())
        tkw = dict(size=4, width=1.5)
    	ax1.tick_params(axis='y', colors=l1.get_color(), **tkw)
    	ax2.tick_params(axis='y', colors=l2.get_color(), **tkw)

        lines = [l1,l2]
        ax1.legend(lines, [l.get_label() for l in lines], loc='best')
        ax1.set_xlabel('Time [ms]')
        ax1.set_xlim(0, 20000)
        return (ax1, ax2)

    def plot_bio_and_ele_calcium(self, manager, spine, dir=None):
        """Plot Calcium from bio and ele with the same scale in
        two graphs.
        Param:
            spine: Spine which Calcium should be plotted.
            manager: classic Neuronvisio manager.
        Example:
            dp.plot_bio_and_ele_calcium(controls.manager, "spine1")"""
        timeSeries = "timeSeries_%s" %spine
        ecell_time_equilibrium = 300 #s
        t_bio = manager.groups[timeSeries]
        t_bio_ms = (t_bio.read() - ecell_time_equilibrium) * 1e3 #s_to_ms

        ca_conc = manager.get_vector(spine, 'ca_conc', group=timeSeries)
        label = "bio_ca_%s" %spine
        plt.plot(t_bio_ms, ca_conc.read()*1e6, #M_to_uM
                 label=label)
        plt.title("Biochemical")
        plt.xlabel('Time [ms]')
        plt.ylabel('Concentration [um]')

        neuron_time_equilibrium = 100 #ms
        spine_head = "%s_head" %spine
        cai = manager.get_vector(spine_head, 'cai')
        cali = manager.get_vector(spine_head, 'cali')
        t = manager.groups['t']
        plt.figure()
        label = "ele_ca_%s" %spine
        plt.plot((t.read()-neuron_time_equilibrium),
                 ((cali.read()+cai.read())*1e3), #mM_to_uM
                 label=label)
        plt.xlabel('Time [ms]')
        plt.ylabel('Concentration [um]')
        plt.title("Electrical")
        xlim = plt.xlim(0,) #starting from zero
        plt.legend()
        if dir is not None:
            for format in [".png", ".pdf"]:
                filename = spine + "_electrical_calcium" + format
                plt.savefig(os.path.join(dir, filename))
        plt.figure(1)
        plt.xlim(xlim) # Getting bio graph on the electric xlim.
        plt.legend()
        if dir is not None:
            for format in [".png", ".pdf"]:
                filename = spine + "_biochemical_calcium" + format
                plt.savefig(os.path.join(dir, filename))
        # calcium buffered by the Calmodulin

        moles_bound_ca_per_moles_cam = manager.get_vector(spine,
                                                          'moles_bound_ca_per_moles_cam',
                                                          group=timeSeries)
        camR = 1.45e-9
        camT = 3e-5
        totcam = camT + camR
        t_bio = manager.groups['timeSeries_spine1']
        conc_tot_calcium_bound_to_cam = moles_bound_ca_per_moles_cam.read() * totcam
        plt.figure()
        plt.plot(t_bio_ms, conc_tot_calcium_bound_to_cam * 1e6, label="buffered_ca")
        plt.title("Calcium buffered by Calmodulin")
        plt.xlim(xlim)
        plt.xlabel("Time [ms]")
        plt.ylabel("Concentration [uM]")
        plt.legend()
        if dir is not None:
            for format in [".png", ".pdf"]:
                filename = spine + "_biochemical_calcium_buffered_by_Calmodulin" + format
                plt.savefig(os.path.join(dir, filename))

        ca_conc = manager.get_vector('spine1', 'ca_conc', group='timeSeries_spine1')

        calcium_bio_and_calcium_calmodulin = (conc_tot_calcium_bound_to_cam + ca_conc.read()) * 1e6
        plt.figure()
        plt.plot(t_bio_ms, calcium_bio_and_calcium_calmodulin, label="buffered+free_ca")
        plt.title("Total Calcium in the biochemical ")
        plt.xlim(xlim)
        plt.xlabel("Time [ms]")
        plt.ylabel("Concentration [uM]")
        plt.legend()
        if dir is not None:
            for format in [".png", ".pdf"]:
                filename = spine + "_biochemical_free_and_buffered_calcium" + format
                plt.savefig(os.path.join(dir, filename))

        plt.figure()
        calcium_bio_and_calcium_calmodulin_buffered_subtracted = ((conc_tot_calcium_bound_to_cam - conc_tot_calcium_bound_to_cam[489]) + ca_conc.read()) * 1e6
        plt.plot(t_bio_ms, calcium_bio_and_calcium_calmodulin_buffered_subtracted,
                 label="bio calcium")
        plt.title("Total Calcium in the biochemical ")
        plt.xlim(xlim)
        plt.xlabel("Time [ms]")
        plt.ylabel("Concentration [uM]")
        plt.legend()
        plt.ylim(-0.1, 5)
        if dir is not None:
            for format in [".png", ".pdf"]:
                filename = spine + "_biochemical_free_and_buffered_calcium_subtracted" + format
                plt.savefig(os.path.join(dir, filename))

if __name__ == "__main__":

    if len(sys.argv) == 2:

        filename = sys.argv[1]
        l = Loader()
        ecp = EcellPlotter()
        timecourses = l.load(filename)
        ecp.plot_timeCourses(timecourses, name='test_spine')
        print "Time courses loaded in 'timecourses' object. Easy plotting \n\
        with 'ecp' object. Have fun."
        plt.show()

    else:
        print "Tell me which file with ecell Timeseries to load."
        print "visualizatorBiochemical.py path/to/timecourses\n"
<|endoftext|>


<commit_before>from datetime import date, datetime, time, timedelta
from django.core.cache import cache
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.db.models import Sum
from django.shortcuts import get_object_or_404, redirect
from django.views.generic.list_detail import object_list
from accounts import badges as account_badges
from accounts.models import UserProfile
from beta import badges as beta_badges
from blog.models import Entry
from race.models import Map, MapType, Run, BestRun, Server
from race import badges as race_badges
from race import tasks
from annoying.decorators import render_to

@render_to('home.html')
def homepage(request):
	try:
		latest_entries = Entry.objects.filter(status=Entry.PUBLISHED_STATUS) \
			.order_by('-created_at').select_related()[:2]

		if latest_entries.count() > 1:
			if not latest_entries[0].is_micro and not latest_entries[1].is_micro:
				latest_entries = latest_entries[:1]
	except Entry.DoesNotExist:
		latest_entries = None

	user_base = User.objects.exclude(is_active=False)
	try:
		latest_user = user_base.latest('id')
	except User.DoesNotExist:
		latest_user = None
	user_count = user_base.count()

	try:
		latest_map = Map.objects.latest('id')
	except Map.DoesNotExist:
		latest_map = None
	map_count = Map.objects.count()

	if request.user.is_authenticated():
		user_runs = Run.objects.filter(user=request.user) \
			.order_by('-created_at')[:5]
	else:
		user_runs = []

	today = date.today()
	runs_today = Run.objects.filter(created_at__range=
		(datetime.combine(today, time.min),
		datetime.combine(today, time.max)))

	runs_yesterday = cache.get('runs_yesterday')
	if runs_yesterday is None:
		tasks.update_yesterday_runs.apply()
		runs_yesterday = cache.get('runs_yesterday')

	total_runs = cache.get('total_runs')
	if total_runs is None:
		tasks.update_totals.apply()
		total_runs = cache.get('total_runs')

	total_playtime = cache.get('total_playtime')
	if total_playtime is None:
		tasks.update_totals.apply()
		total_playtime = cache.get('total_playtime')

	total_downloads = Map.objects.aggregate(
		Sum('download_count')
	)['download_count__sum']

	return {
		'latest_entries': latest_entries,
		'latest_user': latest_user,
		'user_count': user_count,
		'latest_map': latest_map,
		'map_count': map_count,
		'user_runs': user_runs,
		'run_count': total_runs,
		'runs_today': runs_today,
		'runs_yesterday': runs_yesterday,
		'total_playtime': total_playtime,
		'total_downloads': total_downloads,
	}

def ranks(request):
	# exclude banned users from rank
	# (that user shouldn't have any points anyway, just a precaution)
	users = UserProfile.objects.filter(points__gt=0) \
		.exclude(user__is_active=False).extra(
		select = {'sql_position':
			"SELECT COUNT(*)+1 FROM accounts_userprofile s "
			"WHERE s.points > accounts_userprofile.points"
		},
		order_by = ['sql_position']
	)
	total_playtime = Run.objects.aggregate(Sum('time'))['time__sum']
	total_runs = Run.objects.count()
	extra_context = {
		'total_playtime': total_playtime,
		'total_runs': total_runs,
	}
	return object_list(request, queryset=users, template_name='race/ranks.html',
		extra_context=extra_context)

def ranks_map_list(request):
	maps = Map.objects.all().select_related()
	return object_list(request, queryset=maps,
		template_name='race/ranks_map_list.html')

# actually, it's a list of records of a particular map
def ranks_map_detail(request, map_id):
	map_obj = get_object_or_404(Map.objects.select_related(), pk=map_id)
	best_runs = BestRun.objects.filter(map=map_obj) \
		.exclude(user__is_active=False).order_by('run__time').extra(
		select = {'position':
			"SELECT COUNT(*)+1 FROM race_bestrun s "
			"WHERE s.map_id = race_bestrun.map_id AND s.time < race_bestrun.time"
		},
	)
	extra_context = {
		'map': map_obj,
	}
	return object_list(request, queryset=best_runs,
		template_name='race/ranks_map_detail.html', extra_context=extra_context)

def map_list(request, map_type=None):
	maps = Map.objects.all().select_related()
	filtered_type = None
	if map_type:
		filtered_type = get_object_or_404(MapType, slug=map_type)
		maps = maps.filter(map_type__slug=map_type)
	maps = maps.order_by('name')
	map_types = MapType.objects.all()
	extra_context = {
		'map_types': map_types,
		'filtered_type': filtered_type,
	}
	return object_list(request, queryset=maps,
		extra_context=extra_context)

@render_to('race/map_detail.html')
def map_detail(request, map_id):
	map_obj = get_object_or_404(Map.objects.select_related(), pk=map_id)
	best_runs = BestRun.objects.filter(map=map_obj).order_by('run__time').extra(
		select = {'position':
			"SELECT COUNT(*)+1 FROM race_bestrun s "
			"WHERE s.map_id = race_bestrun.map_id AND s.time < race_bestrun.time"
		},
	)[:5]
	latest_runs = Run.objects.filter(map=map_obj).order_by('-created_at')[:5]
	if request.user.is_authenticated():
		user_runs = Run.objects.filter(user=request.user).filter(map=map_obj) \
			.order_by('-created_at')[:5]
	else:
		user_runs = None
	return {
		'map': map_obj,
		'best_runs': best_runs,
		'latest_runs': latest_runs,
		'user_runs': user_runs,
	}

@login_required
def user_activity(request):
	latest_runs = Run.objects.filter(user=request.user).order_by('-created_at')
	return object_list(request, queryset=latest_runs,
		template_name='race/user_activity.html')

@render_to('race/awards.html')
def awards(request):
	"""
	magic.

	yes, you will hate me for this one.
	"""
	badges = dir(race_badges) + dir(account_badges) + dir(beta_badges)
	award_names = [x for x in badges if x.endswith('Badge')]
	award_list = []
	for award in award_names:
		award_list.append(getattr(badges, award))
	return {
		'award_list': award_list
	}

def map_download(request, map_id):
	map_obj = get_object_or_404(Map, pk=map_id)
	map_obj.download_count += 1
	map_obj.save()
	return redirect(map_obj.get_download_url())

@render_to('race/servers.html')
def servers(request):
	servers_online = Server.objects.filter(
		last_connection_at__gte=(datetime.now()-timedelta(minutes=10))
	)
	players_online = UserProfile.objects.filter(
		last_connection_at__gte=(datetime.now()-timedelta(minutes=10))
	)
	# TODO graph?
	return {
		'servers_online': servers_online,
		'players_online': players_online,
	}
<commit_msg>fixed the last commit<commit_after>from datetime import date, datetime, time, timedelta
from django.core.cache import cache
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.db.models import Sum
from django.shortcuts import get_object_or_404, redirect
from django.views.generic.list_detail import object_list
from accounts import badges as account_badges
from accounts.models import UserProfile
from beta import badges as beta_badges
from blog.models import Entry
from race.models import Map, MapType, Run, BestRun, Server
from race import badges as race_badges
from race import tasks
from annoying.decorators import render_to

@render_to('home.html')
def homepage(request):
	try:
		latest_entries = Entry.objects.filter(status=Entry.PUBLISHED_STATUS) \
			.order_by('-created_at').select_related()[:2]

		if latest_entries.count() > 1:
			if not latest_entries[0].is_micro and not latest_entries[1].is_micro:
				latest_entries = latest_entries[:1]
	except Entry.DoesNotExist:
		latest_entries = None

	user_base = User.objects.exclude(is_active=False)
	try:
		latest_user = user_base.latest('id')
	except User.DoesNotExist:
		latest_user = None
	user_count = user_base.count()

	try:
		latest_map = Map.objects.latest('id')
	except Map.DoesNotExist:
		latest_map = None
	map_count = Map.objects.count()

	if request.user.is_authenticated():
		user_runs = Run.objects.filter(user=request.user) \
			.order_by('-created_at')[:5]
	else:
		user_runs = []

	today = date.today()
	runs_today = Run.objects.filter(created_at__range=
		(datetime.combine(today, time.min),
		datetime.combine(today, time.max)))

	runs_yesterday = cache.get('runs_yesterday')
	if runs_yesterday is None:
		tasks.update_yesterday_runs.apply()
		runs_yesterday = cache.get('runs_yesterday')

	total_runs = cache.get('total_runs')
	if total_runs is None:
		tasks.update_totals.apply()
		total_runs = cache.get('total_runs')

	total_playtime = cache.get('total_playtime')
	if total_playtime is None:
		tasks.update_totals.apply()
		total_playtime = cache.get('total_playtime')

	total_downloads = Map.objects.aggregate(
		Sum('download_count')
	)['download_count__sum']

	return {
		'latest_entries': latest_entries,
		'latest_user': latest_user,
		'user_count': user_count,
		'latest_map': latest_map,
		'map_count': map_count,
		'user_runs': user_runs,
		'run_count': total_runs,
		'runs_today': runs_today,
		'runs_yesterday': runs_yesterday,
		'total_playtime': total_playtime,
		'total_downloads': total_downloads,
	}

def ranks(request):
	# exclude banned users from rank
	# (that user shouldn't have any points anyway, just a precaution)
	users = UserProfile.objects.filter(points__gt=0) \
		.exclude(user__is_active=False).extra(
		select = {'sql_position':
			"SELECT COUNT(*)+1 FROM accounts_userprofile s "
			"WHERE s.points > accounts_userprofile.points"
		},
		order_by = ['sql_position']
	)
	total_playtime = Run.objects.aggregate(Sum('time'))['time__sum']
	total_runs = Run.objects.count()
	extra_context = {
		'total_playtime': total_playtime,
		'total_runs': total_runs,
	}
	return object_list(request, queryset=users, template_name='race/ranks.html',
		extra_context=extra_context)

def ranks_map_list(request):
	maps = Map.objects.all().select_related()
	return object_list(request, queryset=maps,
		template_name='race/ranks_map_list.html')

# actually, it's a list of records of a particular map
def ranks_map_detail(request, map_id):
	map_obj = get_object_or_404(Map.objects.select_related(), pk=map_id)
	best_runs = BestRun.objects.filter(map=map_obj) \
		.exclude(user__is_active=False).order_by('run__time').extra(
		select = {'position':
			"SELECT COUNT(*)+1 FROM race_bestrun s "
			"WHERE s.map_id = race_bestrun.map_id AND s.time < race_bestrun.time"
		},
	)
	extra_context = {
		'map': map_obj,
	}
	return object_list(request, queryset=best_runs,
		template_name='race/ranks_map_detail.html', extra_context=extra_context)

def map_list(request, map_type=None):
	maps = Map.objects.all().select_related()
	filtered_type = None
	if map_type:
		filtered_type = get_object_or_404(MapType, slug=map_type)
		maps = maps.filter(map_type__slug=map_type)
	maps = maps.order_by('name')
	map_types = MapType.objects.all()
	extra_context = {
		'map_types': map_types,
		'filtered_type': filtered_type,
	}
	return object_list(request, queryset=maps,
		extra_context=extra_context)

@render_to('race/map_detail.html')
def map_detail(request, map_id):
	map_obj = get_object_or_404(Map.objects.select_related(), pk=map_id)
	best_runs = BestRun.objects.filter(map=map_obj).order_by('run__time').extra(
		select = {'position':
			"SELECT COUNT(*)+1 FROM race_bestrun s "
			"WHERE s.map_id = race_bestrun.map_id AND s.time < race_bestrun.time"
		},
	)[:5]
	latest_runs = Run.objects.filter(map=map_obj).order_by('-created_at')[:5]
	if request.user.is_authenticated():
		user_runs = Run.objects.filter(user=request.user).filter(map=map_obj) \
			.order_by('-created_at')[:5]
	else:
		user_runs = None
	return {
		'map': map_obj,
		'best_runs': best_runs,
		'latest_runs': latest_runs,
		'user_runs': user_runs,
	}

@login_required
def user_activity(request):
	latest_runs = Run.objects.filter(user=request.user).order_by('-created_at')
	return object_list(request, queryset=latest_runs,
		template_name='race/user_activity.html')

@render_to('race/awards.html')
def awards(request):
	"""
	magic.

	yes, you will hate me for this one.
	"""
	award_list = []
	for badges in [race_badges, account_badges, beta_badges]:
		award_names = [x for x in dir(badges) if x.endswith('Badge')]
		for award in award_names:
			award_list.append(getattr(badges, award))
	return {
		'award_list': award_list
	}

def map_download(request, map_id):
	map_obj = get_object_or_404(Map, pk=map_id)
	map_obj.download_count += 1
	map_obj.save()
	return redirect(map_obj.get_download_url())

@render_to('race/servers.html')
def servers(request):
	servers_online = Server.objects.filter(
		last_connection_at__gte=(datetime.now()-timedelta(minutes=10))
	)
	players_online = UserProfile.objects.filter(
		last_connection_at__gte=(datetime.now()-timedelta(minutes=10))
	)
	# TODO graph?
	return {
		'servers_online': servers_online,
		'players_online': players_online,
	}
<|endoftext|>


<commit_before>

class Dimensions(object):

    def __init__(self, **kwargs):

        object.__setattr__(self, "_Dimensions__dimensions", [])

        for key, value in kwargs.items():
            self.__dimensions.append(key)
            setattr(self, key, value)

    def update(self, **kwargs):
        for name, value in kwargs.items():
            self.__convert_dimension(value)
            if name not in self.__dimensions:
                setattr(self, name, value)
        for name, value in kwargs.items():
            setattr(self, name, value)

    def to_dict(self):
        return {name:object.__getattribute__(self, name) for name in self.__dimensions}

    def copy(self):
        return Dimensions(**self.to_dict())

    def __convert_dimension(self, value):
        if isinstance(value, int):
            value = float(value)
        elif isinstance(value, (float, type(None))):
            pass
        else:
            raise TypeError("Cannot set dimension to value of type %s" %type(value))
        return value

    def __setattr__(self, name, value):
        if name not in self.__dimensions:
            raise AttributeError("Cannot set attribute %s" %name)

        value = self.__convert_dimension(value)
        object.__setattr__(self, name, value)

    def __getattribute__(self, name):
        value = object.__getattribute__(self, name)

        if name.startswith("_Dimensions"):
            return value
        elif name in self.__dimensions and value is None:
            raise ValueError("Dimension '%s' is not set" %name)
        else:
            return value
<commit_msg>Added method Dimensions.__repr__<commit_after>

class Dimensions(object):

    def __init__(self, **kwargs):

        object.__setattr__(self, "_Dimensions__dimensions", [])

        for key, value in kwargs.items():
            self.__dimensions.append(key)
            setattr(self, key, value)

    def update(self, **kwargs):
        for name, value in kwargs.items():
            self.__convert_dimension(value)
            if name not in self.__dimensions:
                setattr(self, name, value)
        for name, value in kwargs.items():
            setattr(self, name, value)

    def to_dict(self):
        return {name:object.__getattribute__(self, name) for name in self.__dimensions}

    def copy(self):
        return Dimensions(**self.to_dict())

    def __convert_dimension(self, value):
        if isinstance(value, int):
            value = float(value)
        elif isinstance(value, (float, type(None))):
            pass
        else:
            raise TypeError("Cannot set dimension to value of type %s" %type(value))
        return value

    def __setattr__(self, name, value):
        if name not in self.__dimensions:
            raise AttributeError("Cannot set attribute %s" %name)

        value = self.__convert_dimension(value)
        object.__setattr__(self, name, value)

    def __getattribute__(self, name):
        value = object.__getattribute__(self, name)

        if name.startswith("_Dimensions"):
            return value
        elif name in self.__dimensions and value is None:
            raise ValueError("Dimension '%s' is not set" %name)
        else:
            return value

    def __repr__(self):
        return "Dimensions%s" %self.to_dict()
<|endoftext|>


<commit_before># telepathy-spec - Base classes defining the interfaces of the Telepathy framework
#
# Copyright (C) 2005,2006 Collabora Limited
# Copyright (C) 2005,2006 Nokia Corporation
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

version = (0, 13, 6)

from constants import *
from errors import *
from interfaces import *
<commit_msg>version 0.13.7<commit_after># telepathy-spec - Base classes defining the interfaces of the Telepathy framework
#
# Copyright (C) 2005,2006 Collabora Limited
# Copyright (C) 2005,2006 Nokia Corporation
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

version = (0, 13, 7)

from constants import *
from errors import *
from interfaces import *
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""

.. autosummary::
    :toctree: generated/

    mcnemar

"""

import numpy

def mcnemar(reference, estimated_a, estimated_b):
    """McNemar's test

    Wikipedia entry https://en.wikipedia.org/wiki/McNemar%27s_test

    Parameters
    ----------
    reference : list
        Reference value

    estimated_a : list
        System output A

    estimated_b : list
        System output B

    Returns
    -------
    float
        exact P-value

    """

    if len(reference) != len(estimated_a) or len(reference) != len(estimated_b):
        raise ValueError('Input arrays needs to be same length.')

    # Convert lists into numpy.array
    reference = numpy.array(reference)
    estimated_a = numpy.array(estimated_a)
    estimated_b = numpy.array(estimated_b)

    # Intermediate values
    correct_a = estimated_a == reference
    correct_b = estimated_b == reference

    incorrect_a = estimated_a != reference
    incorrect_b = estimated_b != reference

    # Contingency table values
    b = float( numpy.sum( numpy.logical_and(incorrect_a, correct_b) ) )
    c = float( numpy.sum( numpy.logical_and(correct_a, incorrect_b) ) )

    # Continuity corrected version of the McNemar test to approximate the binomial exact-P-value
    # Edwards, A (1948). "Note on the "correction for continuity" in testing the significance of the difference
    # between correlated proportions". Psychometrika. 13: 185–187
    if b + c > 0:
        return (numpy.abs(b - c) - 1)**2 / (b + c)
    else:
        return 0
<commit_msg>Minor edit<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""

.. autosummary::
    :toctree: generated/

    mcnemar

"""

import numpy

def mcnemar(reference, estimated_a, estimated_b):
    """McNemar's test

    Wikipedia entry https://en.wikipedia.org/wiki/McNemar%27s_test

    Parameters
    ----------
    reference : list
        Reference value

    estimated_a : list
        System output A

    estimated_b : list
        System output B

    Returns
    -------
    float
        exact P-value

    """

    if len(reference) != len(estimated_a) or len(reference) != len(estimated_b):
        raise ValueError('Input arrays needs to be same length.')

    # Convert lists into numpy.array
    reference = numpy.array(reference)
    estimated_a = numpy.array(estimated_a)
    estimated_b = numpy.array(estimated_b)

    # Intermediate values
    correct_a = estimated_a == reference
    correct_b = estimated_b == reference

    incorrect_a = estimated_a != reference
    incorrect_b = estimated_b != reference

    # Contingency table values
    b = float( numpy.sum( numpy.logical_and(incorrect_a, correct_b) ) )
    c = float( numpy.sum( numpy.logical_and(correct_a, incorrect_b) ) )

    # Continuity corrected version of the McNemar test to approximate the binomial exact-P-value
    # Edwards, A (1948). "Note on the "correction for continuity" in testing the significance of the difference
    # between correlated proportions". Psychometrika. 13: 185–187
    if b + c > 0:
        return (numpy.abs(b - c) - 1)**2 / (b + c)
    else:
        return 0
<|endoftext|>


<commit_before>import re
import os
import sys
import operator
import smdrreader
from collections import defaultdict

def debug_print(message, file=sys.stderr):
    if debug_mode is True:
        print(message, file=file)

def read_all_data(file_dict, date):
    all_data = []
    for dir,file in file_dict.items():
        if file is None:
            print('No data file found for {} in {}'.format(
                  date.strftime('%Y-%m-%d'),dir), file=sys.stderr)
            continue
        debug_print('{} lines read from file for {}'.format(len(file),
                    os.path.basename(dir)))
        for line in file:
            line = line.decode('UTF-8-SIG')
            try:
                event = smdrreader.SMDREvent(line)
                all_data.append(event)
            except smdrreader.InvalidInputException as e:
                if e.severity > 0:
                    print(str(e) + ': ' + line.rstrip(), file=sys.stderr)
                else:
                    debug_print(str(e) + ': ' + line.rstrip())
    debug_print('{} events processed from dict'.format(len(all_data)))
    return all_data

def add_event(call_id, event, events_by_id, ids_by_events):
    events_by_id[call_id].append(event)
    if id(events_by_id[call_id]) not in ids_by_events:
        ids_by_events[id(events_by_id[call_id])].append(call_id)

def merge_ids(id_one, id_two, events_by_id, ids_by_events):
    debug_print('Merging IDs {} and {}'.format(id_one, id_two))
    if events_by_id[id_one] is not events_by_id[id_two]:
        debug_print('Not already merged')
        events_one = events_by_id[id_one]
        events_two = events_by_id[id_two]
        events_references = []
        events_references.extend(ids_by_events.pop(id(events_one)))
        events_references.extend(ids_by_events.pop(id(events_two)))
        new_events = []
        new_events.extend(events_one)
        new_events.extend(events_two)
        for call_id in events_references:
            events_by_id[call_id] = new_events
        ids_by_events[id(new_events)] = events_references
    else:
        debug_print('Already merged')

def associate_new_id(existing_id, new_id, events_by_id, ids_by_events):
    debug_print('Creating association for new ID {} with existing ID {}'
              .format(new_id, existing_id))
    events_by_id[new_id] = events_by_id[existing_id]
    existing_events_obj_id = id(events_by_id[existing_id])
    ids_by_events[existing_events_obj_id].append(new_id)

def associate_ids(id_one, id_two, events_by_id, ids_by_events):
    if id_one in events_by_id and id_two in events_by_id:
        merge_ids(id_one, id_two, events_by_id, ids_by_events)
    elif id_one in events_by_id and id_two not in events_by_id:
        associate_new_id(id_one, id_two, events_by_id, ids_by_events)
    elif id_two in events_by_id and id_one not in events_by_id:
        associate_new_id(id_two, id_one, events_by_id, ids_by_events)
    else:
        debug_print('Warning: tried to associate two new IDs: {} and {}'
                    .format(id_one, id_two))

def group_calls_by_id(call_ids, all_data):
    events_by_id = defaultdict(list)
    ids_by_events = defaultdict(list)
    while len(call_ids) > 0:
        assoc_ids = set()
        my_data = all_data[:]
        for event in all_data:
            if event.call_id in call_ids:
                debug_print('Found call ID {} in call_ids'
                            .format(event.call_id))
                add_event(event.call_id, event, events_by_id, ids_by_events)
                my_data.remove(event)
                if event.associated_id != '':
                    associate_ids(event.associated_id, event.call_id,
                                  events_by_id, ids_by_events)
                    if event.associated_id not in call_ids:
                        assoc_ids.add(event.associated_id)
            elif event.associated_id in call_ids:
                debug_print('Found associated ID {} in call_ids'
                            .format(event.associated_id))
                associate_ids(event.call_id, event.associated_id,
                              events_by_id, ids_by_events)
                add_event(event.call_id, event, events_by_id, ids_by_events)
                my_data.remove(event)
                if event.call_id not in call_ids:
                    assoc_ids.add(event.call_id)
        call_ids = assoc_ids
        all_data = my_data
    return events_by_id

def group_all_calls(all_data):
    events_by_id = defaultdict(list)
    ids_by_events = defaultdict(list)
    for event in all_data:
        if event.call_id == '        ':
            continue
        add_event(event.call_id, event, events_by_id, ids_by_events)
        if event.associated_id != '':
            associate_ids(event.associated_id, event.call_id, events_by_id,
                          ids_by_events)
    return events_by_id

def get_unique_calls(events_by_id):
    printed_ids = set()
    unique_event_lists = []
    for list in events_by_id.values():
        if id(list) not in printed_ids:
            printed_ids.add(id(list))
            unique_event_lists.append(list)
    return unique_event_lists

def print_calls(unique_event_lists):
    unique_event_lists.sort(key=lambda x: x[0].time)
    unique_event_lists.sort(key=lambda x: x[0].date)
    for list in unique_event_lists:
            print()
            debug_print('Event list id {}:'.format(id(list)), file=sys.stdout)
            print('\n'.join([str(event) for event in list]))
    return len(unique_event_lists)

def print_anis(unique_event_lists):
    unique_event_lists.sort(key=lambda x: x[0].ani)
    for events in unique_event_lists:
        anis = set()
        for event in events:
            anis.add(event.ani)
        print('\n'.join(anis))

def sort_calls(call_event_lists):
    debug_print('Sorting calls')
    sorted_event_lists = []
    for event_list in call_event_lists:
        event_list.sort(key=operator.attrgetter('sequence_id'))
        event_list.sort(key=operator.attrgetter('call_id'))
        call_id_lists = {}
        last_call_ids = (None, None)
        for event in event_list:
            if event.associated_id not in [last_assoc_id, '', event.call_id]:
                call_id_lists[(event.call_id, event.associated_id)] = \
                        (last_assoc_id, [event])
                last_assoc_id = event.associated_id
            call_id_lists[(event.call_id, last_assoc_id)].append(event)
        sorted_list = recombine_call_events(call_id_lists)
        if sorted_list is None:
            print('Warning: failed to sort call', file=sys.stderr)
            sorted_event_lists.append(event_list)
        elif len(sorted_list) != len(event_list):
            print('Warning: call events lost during sorting, '
                  'falling back to unsorted', file=sys.stderr)
            sorted_event_lists.append(event_list)
        else:
            sorted_event_lists.append(sorted_list)
    return sorted_event_lists

def recombine_call_events(call_id_lists):
    first_id = None
    for call_id,assoc_id in call_id_lists:
        if assoc_id == '':
            first_id = (call_id, assoc_id)
            break
    if first_id is None:
        return None
    insertions_to_make = []
    for call_id,assoc_id in call_id_lists:
        if assoc_id != '':
            insertions_to_make.append((call_id,assoc_id))
    while len(insertions_to_make) > 0:
        ids_with_insertions = [insertion[1] for insertion
                               in insertions_to_make]
        insertions = 0
        for insertion in insertions_to_make:
            if insertion[0] not in ids_with_insertions:
                insert_events(call_id_lists, insertion)
                insertions_to_make.remove(insertion)
                insertions += 1
        if insertions == 0:
            return None
    return call_id_lists[first_id]

def insert_events(call_id_lists, insertion):
    insert_id = insertion[0]
    insert_into_id = insertion[1]
    insert_list = call_id_lists[insertion]
    insert_into_list = call_id_lists[insert_into_id]
    insert_index = 0
    while insert_index < len(insert_into_list):
        if insert_list[0].time < insert_into_list[insert_index].time\
           or insert_into_list[insert_index].associated_id == insert_id:
            break
        insert_index += 1
    insert_into_list[insert_index:insert_index] = insert_list

def get_call_ids_by_filter(all_data, filter_condition):
    debug_print('Processing filter "{}"'.format(filter_condition))
    call_ids = set()
    for event in all_data:
        try:
            result = eval(filter_condition)
        except:
            print('Failure evaluating filter condition "{}"'
                        .format(filter_condition), file=sys.stderr)
            break
        if result is True:
            if event.call_id == '        ':
                continue
            debug_print('Found call id {} matching filter'
                        .format(event.call_id))
            call_ids.add(event.call_id)
            if event.associated_id != '':
                call_ids.add(event.associated_id)
    return call_ids

def get_no_id_events_by_filter(all_data, filter_condition):
    event_lists = []
    for event in all_data:
        if event.call_id == '        ':
            try:
                result = eval(filter_condition)
            except:
                print('Failure evaluating filter condition "{}"'
                            .format(filter_condition), file=sys.stderr)
                break
            if result is True:
                event_lists.append([event])
    return event_lists

def process_days(reader, filter_conditions, call_ids):
    no_id_events = []
    unique_call_count = 0
    for file_dict in reader.date_reader():
        debug_print('Retrieved file dictionary for date {}'.format(
                    reader.current_date.strftime('%Y-%m-%d'))
                    + ' from reader containing {} files'.format(
                    len(file_dict)))
        all_data = read_all_data(file_dict, reader.current_date)

        current_call_ids = call_ids
        for condition in filter_conditions:
            current_call_ids = current_call_ids.union(get_call_ids_by_filter(
                                                      all_data, condition))
            no_id_events.extend(get_no_id_events_by_filter(all_data,
                                                           condition))

        debug_print('{} call IDs selected for {}:'.format(len(current_call_ids),
                    reader.current_date.strftime('%Y-%m-%d')))
        debug_print(current_call_ids)

        if len(current_call_ids) > 0:
            events = group_calls_by_id(current_call_ids, all_data)
        elif len(filter_conditions) == 0:
            events = group_all_calls(all_data)
        else:
            continue
        unique_events = get_unique_calls(events)
        unique_events.extend(no_id_events)
        unique_call_count += len(unique_events)
        sorted_calls = sort_calls(unique_events)
        print_calls(sorted_calls)
    print('\n{} unique calls processed'.format(unique_call_count), file=sys.stderr)

call_ids = set()
filter_conditions = set()
debug_mode = False

while '-v' in sys.argv:
    debug_mode = True
    sys.argv.remove('-v')

while '-f' in sys.argv:
    argindex = sys.argv.index('-f')
    parameter = sys.argv[argindex + 1]
    filter_conditions.add(parameter)
    sys.argv.remove('-f')
    sys.argv.remove(parameter)

while '-c' in sys.argv:
    argindex = sys.argv.index('-c')
    parameter = sys.argv[argindex + 1]
    if re.match('[A-Z][0-9]{7}', parameter) is not None:
        call_ids.add(parameter)
    else:
        raise smdrreader.InvalidInputException('Invalid call id: ' + parameter)
    sys.argv.remove('-c')
    sys.argv.remove(parameter)

start_date = sys.argv[1]
end_date = sys.argv[2]
data_dir = sys.argv[3]

try:
    smdr_reader = smdrreader.SMDRReader(data_dir, start_date, end_date)
    debug_print('SMDRReader created successfully')
except smdrreader.InvalidInputException as e:
    print('Error: ' + str(e), file=sys.stderr)
    sys.exit(1)

process_days(smdr_reader,filter_conditions,call_ids)
<commit_msg>Fixes for no-id filtering and broken sorting<commit_after>import re
import os
import sys
import operator
import smdrreader
from collections import defaultdict

def debug_print(message, file=sys.stderr):
    if debug_mode is True:
        print(message, file=file)

def read_all_data(file_dict, date):
    all_data = []
    for dir,file in file_dict.items():
        if file is None:
            print('No data file found for {} in {}'.format(
                  date.strftime('%Y-%m-%d'),dir), file=sys.stderr)
            continue
        debug_print('{} lines read from file for {}'.format(len(file),
                    os.path.basename(dir)))
        for line in file:
            line = line.decode('UTF-8-SIG')
            try:
                event = smdrreader.SMDREvent(line)
                all_data.append(event)
            except smdrreader.InvalidInputException as e:
                if e.severity > 0:
                    print(str(e) + ': ' + line.rstrip(), file=sys.stderr)
                else:
                    debug_print(str(e) + ': ' + line.rstrip())
    debug_print('{} events processed from dict'.format(len(all_data)))
    return all_data

def add_event(call_id, event, events_by_id, ids_by_events):
    events_by_id[call_id].append(event)
    if id(events_by_id[call_id]) not in ids_by_events:
        ids_by_events[id(events_by_id[call_id])].append(call_id)

def merge_ids(id_one, id_two, events_by_id, ids_by_events):
    debug_print('Merging IDs {} and {}'.format(id_one, id_two))
    if events_by_id[id_one] is not events_by_id[id_two]:
        debug_print('Not already merged')
        events_one = events_by_id[id_one]
        events_two = events_by_id[id_two]
        events_references = []
        events_references.extend(ids_by_events.pop(id(events_one)))
        events_references.extend(ids_by_events.pop(id(events_two)))
        new_events = []
        new_events.extend(events_one)
        new_events.extend(events_two)
        for call_id in events_references:
            events_by_id[call_id] = new_events
        ids_by_events[id(new_events)] = events_references
    else:
        debug_print('Already merged')

def associate_new_id(existing_id, new_id, events_by_id, ids_by_events):
    debug_print('Creating association for new ID {} with existing ID {}'
              .format(new_id, existing_id))
    events_by_id[new_id] = events_by_id[existing_id]
    existing_events_obj_id = id(events_by_id[existing_id])
    ids_by_events[existing_events_obj_id].append(new_id)

def associate_ids(id_one, id_two, events_by_id, ids_by_events):
    if id_one in events_by_id and id_two in events_by_id:
        merge_ids(id_one, id_two, events_by_id, ids_by_events)
    elif id_one in events_by_id and id_two not in events_by_id:
        associate_new_id(id_one, id_two, events_by_id, ids_by_events)
    elif id_two in events_by_id and id_one not in events_by_id:
        associate_new_id(id_two, id_one, events_by_id, ids_by_events)
    else:
        debug_print('Warning: tried to associate two new IDs: {} and {}'
                    .format(id_one, id_two))

def group_calls_by_id(call_ids, all_data):
    events_by_id = defaultdict(list)
    ids_by_events = defaultdict(list)
    while len(call_ids) > 0:
        assoc_ids = set()
        my_data = all_data[:]
        for event in all_data:
            if event.call_id in call_ids:
                debug_print('Found call ID {} in call_ids'
                            .format(event.call_id))
                add_event(event.call_id, event, events_by_id, ids_by_events)
                my_data.remove(event)
                if event.associated_id != '':
                    associate_ids(event.associated_id, event.call_id,
                                  events_by_id, ids_by_events)
                    if event.associated_id not in call_ids:
                        assoc_ids.add(event.associated_id)
            elif event.associated_id in call_ids:
                debug_print('Found associated ID {} in call_ids'
                            .format(event.associated_id))
                associate_ids(event.call_id, event.associated_id,
                              events_by_id, ids_by_events)
                add_event(event.call_id, event, events_by_id, ids_by_events)
                my_data.remove(event)
                if event.call_id not in call_ids:
                    assoc_ids.add(event.call_id)
        call_ids = assoc_ids
        all_data = my_data
    return events_by_id

def group_all_calls(all_data):
    events_by_id = defaultdict(list)
    ids_by_events = defaultdict(list)
    for event in all_data:
        if event.call_id == '        ':
            continue
        add_event(event.call_id, event, events_by_id, ids_by_events)
        if event.associated_id != '':
            associate_ids(event.associated_id, event.call_id, events_by_id,
                          ids_by_events)
    return events_by_id

def get_unique_calls(events_by_id):
    printed_ids = set()
    unique_event_lists = []
    for list in events_by_id.values():
        if id(list) not in printed_ids:
            printed_ids.add(id(list))
            unique_event_lists.append(list)
    return unique_event_lists

def print_calls(unique_event_lists):
    unique_event_lists.sort(key=lambda x: x[0].time)
    unique_event_lists.sort(key=lambda x: x[0].date)
    for list in unique_event_lists:
            print()
            debug_print('Event list id {}:'.format(id(list)), file=sys.stdout)
            print('\n'.join([str(event) for event in list]))
    return len(unique_event_lists)

def print_anis(unique_event_lists):
    unique_event_lists.sort(key=lambda x: x[0].ani)
    for events in unique_event_lists:
        anis = set()
        for event in events:
            anis.add(event.ani)
        print('\n'.join(anis))

def sort_calls(call_event_lists):
    debug_print('Sorting calls')
    sorted_event_lists = []
    for event_list in call_event_lists:
        event_list.sort(key=operator.attrgetter('sequence_id'))
        event_list.sort(key=operator.attrgetter('call_id'))
        call_id_lists = {}
        last_call_ids = (None, None)
        last_assoc_id = ''
        for event in event_list:
            if event.associated_id not in [last_assoc_id, '', event.call_id]:
                call_id_lists[(event.call_id, event.associated_id)] = \
                        (last_assoc_id, [event])
                last_assoc_id = event.associated_id
            call_id_lists[(event.call_id, last_assoc_id)].append(event)
        sorted_list = recombine_call_events(call_id_lists)
        if sorted_list is None:
            print('Warning: failed to sort call', file=sys.stderr)
            sorted_event_lists.append(event_list)
        elif len(sorted_list) != len(event_list):
            print('Warning: call events lost during sorting, '
                  'falling back to unsorted', file=sys.stderr)
            sorted_event_lists.append(event_list)
        else:
            sorted_event_lists.append(sorted_list)
    return sorted_event_lists

def recombine_call_events(call_id_lists):
    first_id = None
    for call_id,assoc_id in call_id_lists:
        if assoc_id == '':
            first_id = (call_id, assoc_id)
            break
    if first_id is None:
        return None
    insertions_to_make = []
    for call_id,assoc_id in call_id_lists:
        if assoc_id != '':
            insertions_to_make.append((call_id,assoc_id))
    while len(insertions_to_make) > 0:
        ids_with_insertions = [insertion[1] for insertion
                               in insertions_to_make]
        insertions = 0
        for insertion in insertions_to_make:
            if insertion[0] not in ids_with_insertions:
                insert_events(call_id_lists, insertion)
                insertions_to_make.remove(insertion)
                insertions += 1
        if insertions == 0:
            return None
    return call_id_lists[first_id]

def insert_events(call_id_lists, insertion):
    insert_id = insertion[0]
    insert_into_id = insertion[1]
    insert_list = call_id_lists[insertion]
    insert_into_list = call_id_lists[insert_into_id]
    insert_index = 0
    while insert_index < len(insert_into_list):
        if insert_list[0].time < insert_into_list[insert_index].time\
           or insert_into_list[insert_index].associated_id == insert_id:
            break
        insert_index += 1
    insert_into_list[insert_index:insert_index] = insert_list

def get_call_ids_by_filter(all_data, filter_condition):
    debug_print('Processing filter "{}"'.format(filter_condition))
    call_ids = set()
    for event in all_data:
        if event.call_id == '        ':
            continue
        try:
            result = eval(filter_condition)
        except:
            print('Failure evaluating filter condition "{}"'
                        .format(filter_condition), file=sys.stderr)
            break
        if result is True:
            debug_print('Found call id {} matching filter'
                        .format(event.call_id))
            call_ids.add(event.call_id)
            if event.associated_id != '':
                call_ids.add(event.associated_id)
    return call_ids

def get_no_id_events_by_filter(all_data, filter_condition):
    event_lists = []
    for event in all_data:
        if event.call_id == '        ':
            try:
                result = eval(filter_condition)
            except:
                print('Failure evaluating filter condition "{}"'
                            .format(filter_condition), file=sys.stderr)
                break
            if result is True:
                event_lists.append([event])
    return event_lists

def process_days(reader, filter_conditions, call_ids):
    unique_call_count = 0
    for file_dict in reader.date_reader():
        no_id_events = []
        debug_print('Retrieved file dictionary for date {}'.format(
                    reader.current_date.strftime('%Y-%m-%d'))
                    + ' from reader containing {} files'.format(
                    len(file_dict)))
        all_data = read_all_data(file_dict, reader.current_date)

        current_call_ids = call_ids
        for condition in filter_conditions:
            current_call_ids = current_call_ids.union(get_call_ids_by_filter(
                                                      all_data, condition))
            no_id_events.extend(get_no_id_events_by_filter(all_data,
                                                           condition))

        debug_print('{} call IDs selected for {}:'.format(len(current_call_ids),
                    reader.current_date.strftime('%Y-%m-%d')))
        debug_print(current_call_ids)

        if len(current_call_ids) > 0:
            events = group_calls_by_id(current_call_ids, all_data)
        elif len(filter_conditions) == 0:
            events = group_all_calls(all_data)
        else:
            events = {}
        unique_events = get_unique_calls(events)
        unique_events.extend(no_id_events)
        unique_call_count += len(unique_events)
        sorted_calls = sort_calls(unique_events)
        print_calls(sorted_calls)
    print('\n{} unique calls processed'.format(unique_call_count), file=sys.stderr)

call_ids = set()
filter_conditions = set()
debug_mode = False

while '-v' in sys.argv:
    debug_mode = True
    sys.argv.remove('-v')

while '-f' in sys.argv:
    argindex = sys.argv.index('-f')
    parameter = sys.argv[argindex + 1]
    filter_conditions.add(parameter)
    sys.argv.remove('-f')
    sys.argv.remove(parameter)

while '-c' in sys.argv:
    argindex = sys.argv.index('-c')
    parameter = sys.argv[argindex + 1]
    if re.match('[A-Z][0-9]{7}', parameter) is not None:
        call_ids.add(parameter)
    else:
        raise smdrreader.InvalidInputException('Invalid call id: ' + parameter)
    sys.argv.remove('-c')
    sys.argv.remove(parameter)

start_date = sys.argv[1]
end_date = sys.argv[2]
data_dir = sys.argv[3]

try:
    smdr_reader = smdrreader.SMDRReader(data_dir, start_date, end_date)
    debug_print('SMDRReader created successfully')
except smdrreader.InvalidInputException as e:
    print('Error: ' + str(e), file=sys.stderr)
    sys.exit(1)

process_days(smdr_reader,filter_conditions,call_ids)
<|endoftext|>


<commit_before>
import warnings

import numpy as np
from numpy.testing import assert_almost_equal, assert_equal, run_module_suite

from scipy.signal.ltisys import ss2tf, lsim2, impulse2, step2, lti, bode
from scipy.signal.filter_design import BadCoefficients
import scipy.linalg as linalg

class TestSS2TF:
    def tst_matrix_shapes(self, p, q, r):
        ss2tf(np.zeros((p, p)),
              np.zeros((p, q)),
              np.zeros((r, p)),
              np.zeros((r, q)), 0)

    def test_basic(self):
        for p, q, r in [
            (3, 3, 3),
            (1, 3, 3),
            (1, 1, 1)]:
            yield self.tst_matrix_shapes, p, q, r

class Test_lsim2(object):

    def test_01(self):
        t = np.linspace(0,10,1001)
        u = np.zeros_like(t)
        # First order system: x'(t) + x(t) = u(t), x(0) = 1.
        # Exact solution is x(t) = exp(-t).
        system = ([1.0],[1.0,1.0])
        tout, y, x = lsim2(system, u, t, X0=[1.0])
        expected_x = np.exp(-tout)
        assert_almost_equal(x[:,0], expected_x)

    def test_02(self):
        t = np.array([0.0, 1.0, 1.0, 3.0])
        u = np.array([0.0, 0.0, 1.0, 1.0])
        # Simple integrator: x'(t) = u(t)
        system = ([1.0],[1.0,0.0])
        tout, y, x = lsim2(system, u, t, X0=[1.0])
        expected_x = np.maximum(1.0, tout)
        assert_almost_equal(x[:,0], expected_x)

    def test_03(self):
        t = np.array([0.0, 1.0, 1.0, 1.1, 1.1, 2.0])
        u = np.array([0.0, 0.0, 1.0, 1.0, 0.0, 0.0])
        # Simple integrator:  x'(t) = u(t)
        system = ([1.0],[1.0, 0.0])
        tout, y, x = lsim2(system, u, t, hmax=0.01)
        expected_x = np.array([0.0, 0.0, 0.0, 0.1, 0.1, 0.1])
        assert_almost_equal(x[:,0], expected_x)

    def test_04(self):
        t = np.linspace(0, 10, 1001)
        u = np.zeros_like(t)
        # Second order system with a repeated root: x''(t) + 2*x(t) + x(t) = 0.
        # With initial conditions x(0)=1.0 and x'(t)=0.0, the exact solution
        # is (1-t)*exp(-t).
        system = ([1.0], [1.0, 2.0, 1.0])
        tout, y, x = lsim2(system, u, t, X0=[1.0, 0.0])
        expected_x = (1.0 - tout) * np.exp(-tout)
        assert_almost_equal(x[:,0], expected_x)

    def test_05(self):
        # The call to lsim2 triggers a "BadCoefficients" warning from
        # scipy.signal.filter_design, but the test passes.  I think the warning
        # is related to the incomplete handling of multi-input systems in
        # scipy.signal.

        # A system with two state variables, two inputs, and one output.
        A = np.array([[-1.0, 0.0], [0.0, -2.0]])
        B = np.array([[1.0, 0.0], [0.0, 1.0]])
        C = np.array([1.0, 0.0])
        D = np.zeros((1,2))

        t = np.linspace(0, 10.0, 101)
        warnings.simplefilter("ignore", BadCoefficients)
        try:
            tout, y, x = lsim2((A,B,C,D), T=t, X0=[1.0, 1.0])
        finally:
            del warnings.filters[0]
        expected_y = np.exp(-tout)
        expected_x0 = np.exp(-tout)
        expected_x1 = np.exp(-2.0*tout)
        assert_almost_equal(y, expected_y)
        assert_almost_equal(x[:,0], expected_x0)
        assert_almost_equal(x[:,1], expected_x1)

    def test_06(self):
        """Test use of the default values of the arguments `T` and `U`."""
        # Second order system with a repeated root: x''(t) + 2*x(t) + x(t) = 0.
        # With initial conditions x(0)=1.0 and x'(t)=0.0, the exact solution
        # is (1-t)*exp(-t).
        system = ([1.0], [1.0, 2.0, 1.0])
        tout, y, x = lsim2(system, X0=[1.0, 0.0])
        expected_x = (1.0 - tout) * np.exp(-tout)
        assert_almost_equal(x[:,0], expected_x)

class Test_impulse2(object):

    def test_01(self):
        # First order system: x'(t) + x(t) = u(t)
        # Exact impulse response is x(t) = exp(-t).
        system = ([1.0],[1.0,1.0])
        tout, y = impulse2(system)
        expected_y = np.exp(-tout)
        assert_almost_equal(y, expected_y)

    def test_02(self):
        """Specify the desired time values for the output."""

        # First order system: x'(t) + x(t) = u(t)
        # Exact impulse response is x(t) = exp(-t).
        system = ([1.0],[1.0,1.0])
        n = 21
        t = np.linspace(0, 2.0, n)
        tout, y = impulse2(system, T=t)
        assert_equal(tout.shape, (n,))
        assert_almost_equal(tout, t)
        expected_y = np.exp(-t)
        assert_almost_equal(y, expected_y)

    def test_03(self):
        """Specify an initial condition as a scalar."""

        # First order system: x'(t) + x(t) = u(t), x(0)=3.0
        # Exact impulse response is x(t) = 4*exp(-t).
        system = ([1.0],[1.0,1.0])
        tout, y = impulse2(system, X0=3.0)
        expected_y = 4.0*np.exp(-tout)
        assert_almost_equal(y, expected_y)

    def test_04(self):
        """Specify an initial condition as a list."""

        # First order system: x'(t) + x(t) = u(t), x(0)=3.0
        # Exact impulse response is x(t) = 4*exp(-t).
        system = ([1.0],[1.0,1.0])
        tout, y = impulse2(system, X0=[3.0])
        expected_y = 4.0*np.exp(-tout)
        assert_almost_equal(y, expected_y)

    def test_05(self):
        # Simple integrator: x'(t) = u(t)
        system = ([1.0],[1.0,0.0])
        tout, y = impulse2(system)
        expected_y = np.ones_like(tout)
        assert_almost_equal(y, expected_y)

    def test_06(self):
        # Second order system with a repeated root:
        #     x''(t) + 2*x(t) + x(t) = u(t)
        # The exact impulse response is t*exp(-t).
        system = ([1.0], [1.0, 2.0, 1.0])
        tout, y = impulse2(system)
        expected_y = tout * np.exp(-tout)
        assert_almost_equal(y, expected_y)

class Test_step2(object):

    def test_01(self):
        # First order system: x'(t) + x(t) = u(t)
        # Exact step response is x(t) = 1 - exp(-t).
        system = ([1.0],[1.0,1.0])
        tout, y = step2(system)
        expected_y = 1.0 - np.exp(-tout)
        assert_almost_equal(y, expected_y)

    def test_02(self):
        """Specify the desired time values for the output."""

        # First order system: x'(t) + x(t) = u(t)
        # Exact step response is x(t) = 1 - exp(-t).
        system = ([1.0],[1.0,1.0])
        n = 21
        t = np.linspace(0, 2.0, n)
        tout, y = step2(system, T=t)
        assert_equal(tout.shape, (n,))
        assert_almost_equal(tout, t)
        expected_y = 1 - np.exp(-t)
        assert_almost_equal(y, expected_y)

    def test_03(self):
        """Specify an initial condition as a scalar."""

        # First order system: x'(t) + x(t) = u(t), x(0)=3.0
        # Exact step response is x(t) = 1 + 2*exp(-t).
        system = ([1.0],[1.0,1.0])
        tout, y = step2(system, X0=3.0)
        expected_y = 1 + 2.0*np.exp(-tout)
        assert_almost_equal(y, expected_y)

    def test_04(self):
        """Specify an initial condition as a list."""

        # First order system: x'(t) + x(t) = u(t), x(0)=3.0
        # Exact step response is x(t) = 1 + 2*exp(-t).
        system = ([1.0],[1.0,1.0])
        tout, y = step2(system, X0=[3.0])
        expected_y = 1 + 2.0*np.exp(-tout)
        assert_almost_equal(y, expected_y)

    def test_05(self):
        # Simple integrator: x'(t) = u(t)
        # Exact step response is x(t) = t.
        system = ([1.0],[1.0,0.0])
        tout, y = step2(system, atol=1e-10, rtol=1e-8)
        expected_y = tout
        assert_almost_equal(y, expected_y)

    def test_06(self):
        # Second order system with a repeated root:
        #     x''(t) + 2*x(t) + x(t) = u(t)
        # The exact step response is 1 - (1 + t)*exp(-t).
        system = ([1.0], [1.0, 2.0, 1.0])
        tout, y = step2(system, atol=1e-10, rtol=1e-8)
        expected_y = 1 - (1 + tout) * np.exp(-tout)
        assert_almost_equal(y, expected_y)

def test_lti_instantiation():
    # Test that lti can be instantiated with sequences, scalars.  See PR-225.
    s = lti([1], [-1])
    s = lti(np.array([]), np.array([-1]), 1)
    s = lti([], [-1], 1)
    s = lti([1], [-1], 1, 3)

class Test_bode(object):

    def test_01(self):
        """Test bode() magnitude calculation (manual sanity check)."""
        # 1st order low-pass filter: H(s) = 1 / (s + 1),
        # cutoff: 1 rad/s, slope: -20 dB/decade
        #   H(s=0.1) ~= 0 dB
        #   H(s=1) ~= -3 dB
        #   H(s=10) ~= -20 dB
        #   H(s=100) ~= -40 dB
        system = lti([1], [1, 1])
        w = [0.1, 1, 10, 100]
        w, mag, phase = bode(system, w=w)
        expected_mag = [0, -3, -20, -40]
        assert_almost_equal(mag, expected_mag, decimal=1)

    def test_02(self):
        """Test bode() phase calculation (manual sanity check)."""
        # 1st order low-pass filter: H(s) = 1 / (s + 1),
        #   angle(H(s=0.1)) ~= -5.7 deg
        #   angle(H(s=1)) ~= -45 deg
        #   angle(H(s=10)) ~= -84.3 deg
        system = lti([1], [1, 1])
        w = [0.1, 1, 10]
        w, mag, phase = bode(system, w=w)
        expected_phase = [-5.7, -45, -84.3]
        assert_almost_equal(phase, expected_phase, decimal=1)

    def test_03(self):
        """Test bode() magnitude calculation."""
        # 1st order low-pass filter: H(s) = 1 / (s + 1)
        system = lti([1], [1, 1])
        w = [0.1, 1, 10, 100]
        w, mag, phase = bode(system, w=w)
        jw = w * 1j
        y = np.polyval(system.num, jw) / np.polyval(system.den, jw)
        expected_mag = 20.0 * np.log10(abs(y))
        assert_almost_equal(mag, expected_mag)

    def test_04(self):
        """Test bode() phase calculation."""
        # 1st order low-pass filter: H(s) = 1 / (s + 1)
        system = lti([1], [1, 1])
        w = [0.1, 1, 10, 100]
        w, mag, phase = bode(system, w=w)
        jw = w * 1j
        y = np.polyval(system.num, jw) / np.polyval(system.den, jw)
        expected_phase = np.arctan2(y.imag, y.real) * 180.0 / np.pi
        assert_almost_equal(phase, expected_phase)

    def test_05(self):
        """Test that bode() finds a reasonable frequency range.

        A reasonable frequency range is two orders of magnitude before the
        minimum (slowest) pole and two orders of magnitude after the maximum
        (fastest) pole.
        """
        # 1st order low-pass filter: H(s) = 1 / (s + 1)
        system = lti([1], [1, 1])
        vals = linalg.eigvals(system.A)
        minpole = min(abs(np.real(vals)))
        maxpole = max(abs(np.real(vals)))
        n = 10;
        expected_w = np.logspace(np.log10(minpole) - 2, np.log10(maxpole) + 2, n)
        w, mag, phase = bode(system, n=n)
        assert_almost_equal(w, expected_w)

if __name__ == "__main__":
    run_module_suite()
<commit_msg>TST: ltisys: test that bode() doesn't fail on systems where pole=0<commit_after>
import warnings

import numpy as np
from numpy.testing import assert_almost_equal, assert_equal, run_module_suite

from scipy.signal.ltisys import ss2tf, lsim2, impulse2, step2, lti, bode
from scipy.signal.filter_design import BadCoefficients
import scipy.linalg as linalg

class TestSS2TF:
    def tst_matrix_shapes(self, p, q, r):
        ss2tf(np.zeros((p, p)),
              np.zeros((p, q)),
              np.zeros((r, p)),
              np.zeros((r, q)), 0)

    def test_basic(self):
        for p, q, r in [
            (3, 3, 3),
            (1, 3, 3),
            (1, 1, 1)]:
            yield self.tst_matrix_shapes, p, q, r

class Test_lsim2(object):

    def test_01(self):
        t = np.linspace(0,10,1001)
        u = np.zeros_like(t)
        # First order system: x'(t) + x(t) = u(t), x(0) = 1.
        # Exact solution is x(t) = exp(-t).
        system = ([1.0],[1.0,1.0])
        tout, y, x = lsim2(system, u, t, X0=[1.0])
        expected_x = np.exp(-tout)
        assert_almost_equal(x[:,0], expected_x)

    def test_02(self):
        t = np.array([0.0, 1.0, 1.0, 3.0])
        u = np.array([0.0, 0.0, 1.0, 1.0])
        # Simple integrator: x'(t) = u(t)
        system = ([1.0],[1.0,0.0])
        tout, y, x = lsim2(system, u, t, X0=[1.0])
        expected_x = np.maximum(1.0, tout)
        assert_almost_equal(x[:,0], expected_x)

    def test_03(self):
        t = np.array([0.0, 1.0, 1.0, 1.1, 1.1, 2.0])
        u = np.array([0.0, 0.0, 1.0, 1.0, 0.0, 0.0])
        # Simple integrator:  x'(t) = u(t)
        system = ([1.0],[1.0, 0.0])
        tout, y, x = lsim2(system, u, t, hmax=0.01)
        expected_x = np.array([0.0, 0.0, 0.0, 0.1, 0.1, 0.1])
        assert_almost_equal(x[:,0], expected_x)

    def test_04(self):
        t = np.linspace(0, 10, 1001)
        u = np.zeros_like(t)
        # Second order system with a repeated root: x''(t) + 2*x(t) + x(t) = 0.
        # With initial conditions x(0)=1.0 and x'(t)=0.0, the exact solution
        # is (1-t)*exp(-t).
        system = ([1.0], [1.0, 2.0, 1.0])
        tout, y, x = lsim2(system, u, t, X0=[1.0, 0.0])
        expected_x = (1.0 - tout) * np.exp(-tout)
        assert_almost_equal(x[:,0], expected_x)

    def test_05(self):
        # The call to lsim2 triggers a "BadCoefficients" warning from
        # scipy.signal.filter_design, but the test passes.  I think the warning
        # is related to the incomplete handling of multi-input systems in
        # scipy.signal.

        # A system with two state variables, two inputs, and one output.
        A = np.array([[-1.0, 0.0], [0.0, -2.0]])
        B = np.array([[1.0, 0.0], [0.0, 1.0]])
        C = np.array([1.0, 0.0])
        D = np.zeros((1,2))

        t = np.linspace(0, 10.0, 101)
        warnings.simplefilter("ignore", BadCoefficients)
        try:
            tout, y, x = lsim2((A,B,C,D), T=t, X0=[1.0, 1.0])
        finally:
            del warnings.filters[0]
        expected_y = np.exp(-tout)
        expected_x0 = np.exp(-tout)
        expected_x1 = np.exp(-2.0*tout)
        assert_almost_equal(y, expected_y)
        assert_almost_equal(x[:,0], expected_x0)
        assert_almost_equal(x[:,1], expected_x1)

    def test_06(self):
        """Test use of the default values of the arguments `T` and `U`."""
        # Second order system with a repeated root: x''(t) + 2*x(t) + x(t) = 0.
        # With initial conditions x(0)=1.0 and x'(t)=0.0, the exact solution
        # is (1-t)*exp(-t).
        system = ([1.0], [1.0, 2.0, 1.0])
        tout, y, x = lsim2(system, X0=[1.0, 0.0])
        expected_x = (1.0 - tout) * np.exp(-tout)
        assert_almost_equal(x[:,0], expected_x)

class Test_impulse2(object):

    def test_01(self):
        # First order system: x'(t) + x(t) = u(t)
        # Exact impulse response is x(t) = exp(-t).
        system = ([1.0],[1.0,1.0])
        tout, y = impulse2(system)
        expected_y = np.exp(-tout)
        assert_almost_equal(y, expected_y)

    def test_02(self):
        """Specify the desired time values for the output."""

        # First order system: x'(t) + x(t) = u(t)
        # Exact impulse response is x(t) = exp(-t).
        system = ([1.0],[1.0,1.0])
        n = 21
        t = np.linspace(0, 2.0, n)
        tout, y = impulse2(system, T=t)
        assert_equal(tout.shape, (n,))
        assert_almost_equal(tout, t)
        expected_y = np.exp(-t)
        assert_almost_equal(y, expected_y)

    def test_03(self):
        """Specify an initial condition as a scalar."""

        # First order system: x'(t) + x(t) = u(t), x(0)=3.0
        # Exact impulse response is x(t) = 4*exp(-t).
        system = ([1.0],[1.0,1.0])
        tout, y = impulse2(system, X0=3.0)
        expected_y = 4.0*np.exp(-tout)
        assert_almost_equal(y, expected_y)

    def test_04(self):
        """Specify an initial condition as a list."""

        # First order system: x'(t) + x(t) = u(t), x(0)=3.0
        # Exact impulse response is x(t) = 4*exp(-t).
        system = ([1.0],[1.0,1.0])
        tout, y = impulse2(system, X0=[3.0])
        expected_y = 4.0*np.exp(-tout)
        assert_almost_equal(y, expected_y)

    def test_05(self):
        # Simple integrator: x'(t) = u(t)
        system = ([1.0],[1.0,0.0])
        tout, y = impulse2(system)
        expected_y = np.ones_like(tout)
        assert_almost_equal(y, expected_y)

    def test_06(self):
        # Second order system with a repeated root:
        #     x''(t) + 2*x(t) + x(t) = u(t)
        # The exact impulse response is t*exp(-t).
        system = ([1.0], [1.0, 2.0, 1.0])
        tout, y = impulse2(system)
        expected_y = tout * np.exp(-tout)
        assert_almost_equal(y, expected_y)

class Test_step2(object):

    def test_01(self):
        # First order system: x'(t) + x(t) = u(t)
        # Exact step response is x(t) = 1 - exp(-t).
        system = ([1.0],[1.0,1.0])
        tout, y = step2(system)
        expected_y = 1.0 - np.exp(-tout)
        assert_almost_equal(y, expected_y)

    def test_02(self):
        """Specify the desired time values for the output."""

        # First order system: x'(t) + x(t) = u(t)
        # Exact step response is x(t) = 1 - exp(-t).
        system = ([1.0],[1.0,1.0])
        n = 21
        t = np.linspace(0, 2.0, n)
        tout, y = step2(system, T=t)
        assert_equal(tout.shape, (n,))
        assert_almost_equal(tout, t)
        expected_y = 1 - np.exp(-t)
        assert_almost_equal(y, expected_y)

    def test_03(self):
        """Specify an initial condition as a scalar."""

        # First order system: x'(t) + x(t) = u(t), x(0)=3.0
        # Exact step response is x(t) = 1 + 2*exp(-t).
        system = ([1.0],[1.0,1.0])
        tout, y = step2(system, X0=3.0)
        expected_y = 1 + 2.0*np.exp(-tout)
        assert_almost_equal(y, expected_y)

    def test_04(self):
        """Specify an initial condition as a list."""

        # First order system: x'(t) + x(t) = u(t), x(0)=3.0
        # Exact step response is x(t) = 1 + 2*exp(-t).
        system = ([1.0],[1.0,1.0])
        tout, y = step2(system, X0=[3.0])
        expected_y = 1 + 2.0*np.exp(-tout)
        assert_almost_equal(y, expected_y)

    def test_05(self):
        # Simple integrator: x'(t) = u(t)
        # Exact step response is x(t) = t.
        system = ([1.0],[1.0,0.0])
        tout, y = step2(system, atol=1e-10, rtol=1e-8)
        expected_y = tout
        assert_almost_equal(y, expected_y)

    def test_06(self):
        # Second order system with a repeated root:
        #     x''(t) + 2*x(t) + x(t) = u(t)
        # The exact step response is 1 - (1 + t)*exp(-t).
        system = ([1.0], [1.0, 2.0, 1.0])
        tout, y = step2(system, atol=1e-10, rtol=1e-8)
        expected_y = 1 - (1 + tout) * np.exp(-tout)
        assert_almost_equal(y, expected_y)

def test_lti_instantiation():
    # Test that lti can be instantiated with sequences, scalars.  See PR-225.
    s = lti([1], [-1])
    s = lti(np.array([]), np.array([-1]), 1)
    s = lti([], [-1], 1)
    s = lti([1], [-1], 1, 3)

class Test_bode(object):

    def test_01(self):
        """Test bode() magnitude calculation (manual sanity check)."""
        # 1st order low-pass filter: H(s) = 1 / (s + 1),
        # cutoff: 1 rad/s, slope: -20 dB/decade
        #   H(s=0.1) ~= 0 dB
        #   H(s=1) ~= -3 dB
        #   H(s=10) ~= -20 dB
        #   H(s=100) ~= -40 dB
        system = lti([1], [1, 1])
        w = [0.1, 1, 10, 100]
        w, mag, phase = bode(system, w=w)
        expected_mag = [0, -3, -20, -40]
        assert_almost_equal(mag, expected_mag, decimal=1)

    def test_02(self):
        """Test bode() phase calculation (manual sanity check)."""
        # 1st order low-pass filter: H(s) = 1 / (s + 1),
        #   angle(H(s=0.1)) ~= -5.7 deg
        #   angle(H(s=1)) ~= -45 deg
        #   angle(H(s=10)) ~= -84.3 deg
        system = lti([1], [1, 1])
        w = [0.1, 1, 10]
        w, mag, phase = bode(system, w=w)
        expected_phase = [-5.7, -45, -84.3]
        assert_almost_equal(phase, expected_phase, decimal=1)

    def test_03(self):
        """Test bode() magnitude calculation."""
        # 1st order low-pass filter: H(s) = 1 / (s + 1)
        system = lti([1], [1, 1])
        w = [0.1, 1, 10, 100]
        w, mag, phase = bode(system, w=w)
        jw = w * 1j
        y = np.polyval(system.num, jw) / np.polyval(system.den, jw)
        expected_mag = 20.0 * np.log10(abs(y))
        assert_almost_equal(mag, expected_mag)

    def test_04(self):
        """Test bode() phase calculation."""
        # 1st order low-pass filter: H(s) = 1 / (s + 1)
        system = lti([1], [1, 1])
        w = [0.1, 1, 10, 100]
        w, mag, phase = bode(system, w=w)
        jw = w * 1j
        y = np.polyval(system.num, jw) / np.polyval(system.den, jw)
        expected_phase = np.arctan2(y.imag, y.real) * 180.0 / np.pi
        assert_almost_equal(phase, expected_phase)

    def test_05(self):
        """Test that bode() finds a reasonable frequency range.

        A reasonable frequency range is two orders of magnitude before the
        minimum (slowest) pole and two orders of magnitude after the maximum
        (fastest) pole.
        """
        # 1st order low-pass filter: H(s) = 1 / (s + 1)
        system = lti([1], [1, 1])
        vals = linalg.eigvals(system.A)
        minpole = min(abs(np.real(vals)))
        maxpole = max(abs(np.real(vals)))
        n = 10;
        expected_w = np.logspace(np.log10(minpole) - 2, np.log10(maxpole) + 2, n)
        w, mag, phase = bode(system, n=n)
        assert_almost_equal(w, expected_w)

    def test_06(self):
        """Test that bode() doesn't fail on a system with a pole at 0."""
        # integrator, pole at zero: H(s) = 1 / s
        system = lti([1], [1, 0])
        w, mag, phase = bode(system, n=2)
        assert_equal(w[0], 0.01)  # a fail would give not-a-number

if __name__ == "__main__":
    run_module_suite()
<|endoftext|>


<commit_before>from __future__ import division, print_function, absolute_import

import numpy as np
import numpy.testing as npt

def check_normalization(distfn, args, distname):
    norm_moment = distfn.moment(0, *args)
    npt.assert_allclose(norm_moment, 1.0)

    # this is a temporary plug: either ncf or expect is problematic;
	# best be marked as a knownfail, but I've no clue how to do it.
    if distname == "ncf":
        atol, rtol = 1e-5, 0
    else:
        atol, rtol = 1e-7, 1e-7

    normalization_expect = distfn.expect(lambda x: 1, args=args)
    npt.assert_allclose(normaztion_expect, 1.0, atol=atol, rtol=rtol,
            err_msg=distname, verbose=True)

    normalization_cdf = distfn.cdf(distfn.b, *args)
    npt.assert_allclose(normalization_cdf, 1.0)

def check_moment(distfn, arg, m, v, msg):
    m1 = distfn.moment(1, *arg)
    m2 = distfn.moment(2, *arg)
    if not np.isinf(m):
        npt.assert_almost_equal(m1, m, decimal=10, err_msg=msg +
                            ' - 1st moment')
    else:                     # or np.isnan(m1),
        npt.assert_(np.isinf(m1),
               msg + ' - 1st moment -infinite, m1=%s' % str(m1))

    if not np.isinf(v):
        npt.assert_almost_equal(m2 - m1 * m1, v, decimal=10, err_msg=msg +
                            ' - 2ndt moment')
    else:                     # or np.isnan(m2),
        npt.assert_(np.isinf(m2),
               msg + ' - 2nd moment -infinite, m2=%s' % str(m2))

def check_mean_expect(distfn, arg, m, msg):
    if np.isfinite(m):
        m1 = distfn.expect(lambda x: x, arg)
        npt.assert_almost_equal(m1, m, decimal=5, err_msg=msg +
                            ' - 1st moment (expect)')

def check_var_expect(distfn, arg, m, v, msg):
    if np.isfinite(v):
        m2 = distfn.expect(lambda x: x*x, arg)
        npt.assert_almost_equal(m2, v + m*m, decimal=5, err_msg=msg +
                            ' - 2st moment (expect)')

def check_skew_expect(distfn, arg, m, v, s, msg):
    if np.isfinite(s):
        m3e = distfn.expect(lambda x: np.power(x-m, 3), arg)
        npt.assert_almost_equal(m3e, s * np.power(v, 1.5),
                decimal=5, err_msg=msg + ' - skew')
    else:
        npt.assert_(np.isnan(s))

def check_kurt_expect(distfn, arg, m, v, k, msg):
    if np.isfinite(k):
        m4e = distfn.expect(lambda x: np.power(x-m, 4), arg)
        npt.assert_almost_equal(m4e, (k + 3.) * np.power(v, 2),
                decimal=5, err_msg=msg + ' - kurtosis')
    else:
        npt.assert_(np.isnan(k))

<commit_msg>MAINT: typos, fixed<commit_after>from __future__ import division, print_function, absolute_import

import numpy as np
import numpy.testing as npt

def check_normalization(distfn, args, distname):
    norm_moment = distfn.moment(0, *args)
    npt.assert_allclose(norm_moment, 1.0)

    # this is a temporary plug: either ncf or expect is problematic;
	# best be marked as a knownfail, but I've no clue how to do it.
    if distname == "ncf":
        atol, rtol = 1e-5, 0
    else:
        atol, rtol = 1e-7, 1e-7

    normalization_expect = distfn.expect(lambda x: 1, args=args)
    npt.assert_allclose(normalization_expect, 1.0, atol=atol, rtol=rtol,
            err_msg=distname, verbose=True)

    normalization_cdf = distfn.cdf(distfn.b, *args)
    npt.assert_allclose(normalization_cdf, 1.0)

def check_moment(distfn, arg, m, v, msg):
    m1 = distfn.moment(1, *arg)
    m2 = distfn.moment(2, *arg)
    if not np.isinf(m):
        npt.assert_almost_equal(m1, m, decimal=10, err_msg=msg +
                            ' - 1st moment')
    else:                     # or np.isnan(m1),
        npt.assert_(np.isinf(m1),
               msg + ' - 1st moment -infinite, m1=%s' % str(m1))

    if not np.isinf(v):
        npt.assert_almost_equal(m2 - m1 * m1, v, decimal=10, err_msg=msg +
                            ' - 2ndt moment')
    else:                     # or np.isnan(m2),
        npt.assert_(np.isinf(m2),
               msg + ' - 2nd moment -infinite, m2=%s' % str(m2))

def check_mean_expect(distfn, arg, m, msg):
    if np.isfinite(m):
        m1 = distfn.expect(lambda x: x, arg)
        npt.assert_almost_equal(m1, m, decimal=5, err_msg=msg +
                            ' - 1st moment (expect)')

def check_var_expect(distfn, arg, m, v, msg):
    if np.isfinite(v):
        m2 = distfn.expect(lambda x: x*x, arg)
        npt.assert_almost_equal(m2, v + m*m, decimal=5, err_msg=msg +
                            ' - 2st moment (expect)')

def check_skew_expect(distfn, arg, m, v, s, msg):
    if np.isfinite(s):
        m3e = distfn.expect(lambda x: np.power(x-m, 3), arg)
        npt.assert_almost_equal(m3e, s * np.power(v, 1.5),
                decimal=5, err_msg=msg + ' - skew')
    else:
        npt.assert_(np.isnan(s))

def check_kurt_expect(distfn, arg, m, v, k, msg):
    if np.isfinite(k):
        m4e = distfn.expect(lambda x: np.power(x-m, 4), arg)
        npt.assert_almost_equal(m4e, (k + 3.) * np.power(v, 2),
                decimal=5, err_msg=msg + ' - kurtosis')
    else:
        npt.assert_(np.isnan(k))

<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys
import subprocess
import shutil

def execute(command):
    try:
        response = subprocess.call(command, shell=True)
        if response == 0:
            return True
        else:
            return False
    except subprocess.CalledProcessError as cpe:
        try:
            sys.stderr.write(cpe.output)
        except TypeError as te:
            sys.stderr.write(str(cpe.output))

versions_dir = "/.DocumentRevisions-V100"

if os.path.isdir(versions_dir):
    print("Versions directory size before cleaning:")
    execute_command = "du -sh " + versions_dir
    execute(execute_command)
    print(" ")

    try:
        shutil.rmtree(versions_dir)
    except Exception as e:
        print("Error in the attempt to remove the files. Error message: " + str(e))
        sys.exit(1)

    if not os.path.isdir(versions_dir):
        print("The versions directory was deleted.")

        if sys.version_info[0] == 2:
            user_response = raw_input("Reboot to complete cleanup? (y/n): ")
        else:
            user_response = input("Reboot to complete cleanup? (y/n): ")

        if user_response.lower() == "y":
            print("Rebooting...")
            execute("reboot")
        else:
            print("Skipping system reboot.  Please reboot your system manually to begin using document versioning again.")
else:
    print("Unable to locate a versions directory at the path '" + versions_dir + "'. It appears that it was already cleaned.")
    print("Complete the cleanup process by rebooting your system.")

<commit_msg>PEP8 formatting changes, added license to script<commit_after>#!/usr/bin/env python

# ------------------------------------------
#
# Mac OSX Versions Cleaner
# version_cleaner.py
# Copyright 2015 Christopher Simpkins
# MIT License
#
# ------------------------------------------

import os
import sys
import subprocess
import shutil

def execute(command):
    try:
        response = subprocess.call(command, shell=True)
        if response == 0:
            return True
        else:
            return False
    except subprocess.CalledProcessError as cpe:
        try:
            sys.stderr.write(cpe.output)
        except TypeError:
            sys.stderr.write(str(cpe.output))

versions_dir = "/.DocumentRevisions-V100"

if os.path.isdir(versions_dir):
    print("Versions directory size before cleaning:")
    execute_command = "du -sh " + versions_dir
    execute(execute_command)
    print(" ")

    try:
        shutil.rmtree(versions_dir)
    except Exception as e:
        print("Error in the attempt to remove the files. Error message: " + str(e))
        sys.exit(1)

    if not os.path.isdir(versions_dir):
        print("The versions directory was deleted.")

        if sys.version_info[0] == 2:
            user_response = raw_input("Reboot to complete cleanup? (y/n): ")
        else:
            user_response = input("Reboot to complete cleanup? (y/n): ")

        if user_response.lower() == "y":
            print("Rebooting...")
            execute("reboot")
        else:
            print("Skipping system reboot.  Please reboot your system manually to begin using document versioning again.")
else:
    print("Unable to locate a versions directory at the path '" + versions_dir + "'. It appears that it was already cleaned.")
    print("Complete the cleanup process by rebooting your system.")
<|endoftext|>