<commit_before>#- Provided by Data Systems to be called by DOS

import math
import time
from PyAstronomy import pyasl
from astropy import coordinates
from astropy.time import Time
from astropy.coordinates import SkyCoord
from astropy.coordinates import ICRS, FK5, AltAz, EarthLocation
from astropy.coordinates import Angle, Latitude, Longitude
from astropy.utils.data import download_file
from astropy.utils import iers
import astropy.units as u

def get_next_field(dateobs, skylevel, seeing, transparency, previoustiles,
    programname=None):
    """
    Returns structure with information about next field to observe.

    Args:
        dateobs (float): start time of observation in UTC (TAI).
            Could be past, present, or future.
        skylevel: current sky level [counts/s/cm^2/arcsec^2]
        seeing: current astmospheric seeing PSF FWHM [arcsec]
        transparency: current atmospheric transparency
        previoustiles: list of tile IDs previously observed.
        programname (string, optional): if given, the output result will be for
            that program.  Otherwise, next_field_selector() chooses the
            program based upon the current conditions.

    Returns dictionary with keys
        tileid: tile ID [integer]
            --> DOS should just add this to the raw data header
        programname: DESI (or other) program name, e.g. "Dark Time Survey",
            "Bright Galaxy Survey", etc.
            --> DOS should just add this to the raw data header
        telera, teledec: telescope central pointing RA, dec [J2000 degrees]
        exptime: expected exposure time [seconds]
        maxtime: maximum allowable exposure time [seconds]
        fibers: dictionary with the following keys, each of which contains
            a list of 5000 values for each of the positioners
            - ra: RA for each fiber [J2000 degrees]
            - dec: dec for each fiber [J2000 degrees]
            - lambdaref: wavelength to optimize each positioner [Angstrom]
        gfa: dictionary with the following keys, each of which contains
            a list of values for objects detectecable by the GFAs, including
            border regions in RA,dec to assist with acquisition
            - id : ID of GFA for this object
            - ra, dec : RA and dec for each object [J2000 degrees]
            - objtype : 'point', 'extended', 'sky'
                --> point sources with okguide=True can be used for guiding;
                    knowledge of the existence of extended sources may help
                    with acquisition; sky locations are large enough to be
                    used for estimating sky backgrounds.
            - okguide : True if good for guiding
            - mag : magnitude [SDSS r-band AB magnitude]
                --> or a flux instead?

        Additional keys may be present and should be ignored

    e.g. result['fibers']['ra'] gives the 5000 RA locations for the fibers

    Notes:
      * get_next_field() will calculate the LST and moon phase/location
        based upon the input datetime.
      * skylevel, seeing, and transparency are in the filter of the guider.
      * The contents of the returned dictionary should be *everything* needed
        as input to point the telescope and take an exposure.  If that isn't
        true, we need to add more.  An ancillary/test/commissioning program
        that defines all of these quantities (e.g. in a JSON file)
        should be sufficient to take observations.
      * previoustiles is a required input rather than having get_next_field()
        query ObsDB to get the history.  Two reasons:
        - Easier to test without requiring live database
        - Decouples code dependencies
      * result['fibers'] will be pre-calculated by fiber assignment;
        DOS shouldn't care as long as get_next_field is fast (<1 sec).
      * Current expectation is the ObsDB/DOS only tracks the past, i.e. what
        observations were taken, but not which observations we would like to
        take in the future.  As such, get_next_field() will need to look up
        the DESI tiling (currently in desimodel/data/footprint/desi-tiles.*)
        and a list of overrides for tiles that were observed by deemed bad
        and need to be redone (details TBD).
        DOS shouldn't care about those details.

    TBD:
      * Error handling: if the request is impossible (e.g. the sun is up),
        should this raise an exception?  Or return a default zenith answer
        with some calib programname?  Or?
    """

    """
        Below is an algorithm for calculating the local apparent sidereal time.
        astropy should be able to this (I think), but I keep getting an error when I try
        to use those functions (and I'm pretty sure I'm importing all the necessary
        components of astropy). A bunch of lines have been commented out which were
        used for testing purposes.
    """

    tobs = Time(dateobs, format='jd', scale='utc')
    kitt_peak = EarthLocation(lat=31.9634*u.deg, lon=-111.6003*u.deg, height=2120*u.m)
    iers.IERS.iers_table = iers.IERS_A.open(download_file(iers.IERS_A_URL, cache=True))

    #Find the Julian date of the previous midnight
    if (dateobs-math.floor(dateobs) >= 0.5):
        jd_0 = math.floor(dateobs)+0.5
    elif (dateobs-math.floor(dateobs) < 0.5):
        jd_0 = math.floor(dateobs)-0.5

    d_0 = jd_0-2451545.0 #Difference between last Julian midnight and J2000
    d = dateobs-2451545.0 #Difference between observation date and J2000
    t = d_0/36525 #Fraction of Julian century that's past since J2000

    #Calculate the sideral time in Greenwich for the last Julian midnight
    gmst_0 = 100.4606184+36000.77005361*t+0.00038793*t**2-2.6E-08*t**3

    #Add correction for the number of hours that have past since midnight
    gmst = gmst_0+0.25068447733746215*(dateobs-jd_0)*24*60

    #Calculate the equation of equinoxes
    l = 280.47+0.98565*d
    omega = 125.04-0.052954*d
    del_psi = -0.000319*math.sin(omega*math.pi/180)-0.00024*math.sin(2*l*math.pi/180)
    e = 23.4393-0.0000004*d

    eqeq = del_psi*math.cos(e*math.pi/180)

    #Correct with the equation of equinoxes to get the current apparent sidereal time
    gast = gmst+eqeq

    #Add the longitude of the observatory to get the local sidereal time
    last = gast-111.5984796

    #Shift the local sidereal time into the range of 0 to 360 degrees
    if last >= 360:
        n = math.floor(last/360)
        last = last-360*n

    #hour = math.floor(LAST/15)
    #minute = math.floor((LAST/15-hour)*60)
    #second = ((LAST/15-hour)*60-minute)*60

    #print( str(hour) + "h " + str(minute) + "m " + str(second) +"s")

    #print("dateobs = " + str(dateobs))
    #print("JD_0 = " + str(JD_0))
    #print("D_0 = " + str(D_0))
    #print("D = " + str(D))
    #print("T = " + str(T))
    #print("H = " + str(H))
    #print("GMST = " + str(GMST))
    #print("LAST = " + str(LAST))

    #Use astropy to calculate the position of the Sun.
    pos_sun = coordinates.get_sun(tobs)

    #Check to see if the Sun is up.
    ra_sun = pos_sun.ra.value
    dec_sun = pos_sun.dec.value

    ha_sun = last - ra_sun
    if (ha_sun < 0):
        ha_sun = ha_sun + 360
    if (ha_sun > 360):
        ha_sun = ha_sun - 360
    #print("ha_sun = " + str(ha_sun))

    #Calculate the altitude of the Sun to determine if it is up
    #alt_sun = (math.asin(math.sin(dec_sun*math.pi/180)
                         #*math.sin(31.9614929*math.pi/180)
                         #+math.cos(dec_sun*math.pi/180)
                         #*math.cos(31.9614929*math.pi/180)
                         #*math.cos(ha_sun*math.pi/180)))*(180/math.pi)

    #print("alt_sun = " + str(alt_sun))

    sun_altaz = pos_sun.transform_to(AltAz(obstime=tobs,location=kitt_peak))

    #print sun_altaz

    alt_sun = sun_altaz.alt.value

    print alt_sun

    #Print warning if the Sun is up. We may decide this should do more than just warn
    if (alt_sun >= 0):
        print("WARNING: The Sun is currently up!")
    if (alt_sun >=-30 and alt_sun < 0):
        print("WARNING: The Sun will soon be rising and dark sky conditions will be lost.")

    #Find the position of the Moon using PyAstronomy
    pos_moon = pyasl.moonpos(dateobs)

    ra_moon = float(pos_moon[0])
    dec_moon = float(pos_moon[1])

    print("ra_moon = " + str(ra_moon) + " dec_moon = " + str(dec_moon))

    #Find the phase of the moon using PyAstronomy
    moon_phase = pyasl.moonphase(dateobs)
    phase = float(moon_phase)

    print(phase)

    #Opens a file containing information about all of the DESI tiles. Not sure if
    #this is the best way to get that information?
    tiles_file = open("desi-tiles-full.par.txt","r")

    #Declare some variables to store data from the file
    tiles = []
    idnum = []
    ra = []
    dec = []
    passnum = []
    in_desi = []
    ebv_med = []
    airmass = []
    exposefac = []
    i = 0
    j = 0
    possibletiles = 0
    impossibletiles = 0
    mindec = 100.0
    nextfield = 0
    #Read the data from the file and find the next field
    for line in tiles_file:
        tiles.append(line)
        c = tiles[j].split(" ")
        if (c[0] == "STRUCT1"):
            idnum.append(int(c[1]))
            ra.append(float(c[2]))
            dec.append(float(c[3]))
            passnum.append(int(c[4]))
            in_desi.append(int(c[5]))
            ebv_med.append(float(c[6]))
            airmass.append(float(c[7]))
            exposefac.append(float(c[8]))
            ha = last-ra[i]
            if ha < 0:
                ha = ha + 360
            if ha > 360:
                ha = ha - 360
            alt = (math.asin(math.sin(dec[i]*math.pi/180)
                             *math.sin(31.9614929*math.pi/180)
                             +math.cos(dec[i]*math.pi/180)
                             *math.cos(31.9614929*math.pi/180)
                             *math.cos(ha*math.pi/180)))*(180/math.pi)
            if (alt >= 0):
                if (dec[i] < mindec and ra[i] >= last-1 and ra[i] <= last+1):
                    mindec = dec[i]
                    nextfield = idnum[i]
            i = i + 1
        j = j + 1

    tiles_file.close()

    #raise NotImplementedError

    #Create dictionary with information that is needed to point the telescope
    next_field = {'tileid':idnum[nextfield-1], 'programname':'DESI', 'telera':ra[nextfield-1], 'teledec':dec[nextfield-1], 'exptime':1800, 'maxtime':2000, 'fibers':{}, 'gfa':{}}

    #Return the dictionary
    return next_field

""" The lines below allow the function to be tested by itself with the user
inputting a Julian date of observation. They also calculate the execution time for
purposes of optimizing."""

dateobs = float(raw_input('Enter the date of observation: '))
skylevel = 0
transparency = 0
previoustiles = []
programname = 'DESI'
start_time = time.time()
next_field = get_next_field(dateobs, skylevel, transparency, previoustiles, programname)

print("Total execution time: %s seconds" % (time.time()-start_time))
print next_field
<commit_msg>Reverted to calculating most things without using astropy. Astropy currently only gives the position of the Sun. Position transformation in astropy are slow, and calculating sidereal time with astropy is slow. Current algorithms produce the same reseults and reduce runtime to ~0.3 seconds from ~1.3 seconds.<commit_after>#- Provided by Data Systems to be called by DOS

import math
import time
import warnings
from PyAstronomy import pyasl
from astropy import coordinates
from astropy.time import Time
from astropy.coordinates import SkyCoord
from astropy.coordinates import ICRS, FK5, AltAz, EarthLocation
from astropy.coordinates import Angle, Latitude, Longitude
from astropy.utils.data import download_file
from astropy.utils import iers
import astropy.units as u

def get_next_field(dateobs, skylevel, seeing, transparency, previoustiles,
    programname=None):
    """
    Returns structure with information about next field to observe.

    Args:
        dateobs (float): start time of observation in UTC (TAI).
            Could be past, present, or future.
        skylevel: current sky level [counts/s/cm^2/arcsec^2]
        seeing: current astmospheric seeing PSF FWHM [arcsec]
        transparency: current atmospheric transparency
        previoustiles: list of tile IDs previously observed.
        programname (string, optional): if given, the output result will be for
            that program.  Otherwise, next_field_selector() chooses the
            program based upon the current conditions.

    Returns dictionary with keys
        tileid: tile ID [integer]
            --> DOS should just add this to the raw data header
        programname: DESI (or other) program name, e.g. "Dark Time Survey",
            "Bright Galaxy Survey", etc.
            --> DOS should just add this to the raw data header
        telera, teledec: telescope central pointing RA, dec [J2000 degrees]
        exptime: expected exposure time [seconds]
        maxtime: maximum allowable exposure time [seconds]
        fibers: dictionary with the following keys, each of which contains
            a list of 5000 values for each of the positioners
            - ra: RA for each fiber [J2000 degrees]
            - dec: dec for each fiber [J2000 degrees]
            - lambdaref: wavelength to optimize each positioner [Angstrom]
        gfa: dictionary with the following keys, each of which contains
            a list of values for objects detectecable by the GFAs, including
            border regions in RA,dec to assist with acquisition
            - id : ID of GFA for this object
            - ra, dec : RA and dec for each object [J2000 degrees]
            - objtype : 'point', 'extended', 'sky'
                --> point sources with okguide=True can be used for guiding;
                    knowledge of the existence of extended sources may help
                    with acquisition; sky locations are large enough to be
                    used for estimating sky backgrounds.
            - okguide : True if good for guiding
            - mag : magnitude [SDSS r-band AB magnitude]
                --> or a flux instead?

        Additional keys may be present and should be ignored

    e.g. result['fibers']['ra'] gives the 5000 RA locations for the fibers

    Notes:
      * get_next_field() will calculate the LST and moon phase/location
        based upon the input datetime.
      * skylevel, seeing, and transparency are in the filter of the guider.
      * The contents of the returned dictionary should be *everything* needed
        as input to point the telescope and take an exposure.  If that isn't
        true, we need to add more.  An ancillary/test/commissioning program
        that defines all of these quantities (e.g. in a JSON file)
        should be sufficient to take observations.
      * previoustiles is a required input rather than having get_next_field()
        query ObsDB to get the history.  Two reasons:
        - Easier to test without requiring live database
        - Decouples code dependencies
      * result['fibers'] will be pre-calculated by fiber assignment;
        DOS shouldn't care as long as get_next_field is fast (<1 sec).
      * Current expectation is the ObsDB/DOS only tracks the past, i.e. what
        observations were taken, but not which observations we would like to
        take in the future.  As such, get_next_field() will need to look up
        the DESI tiling (currently in desimodel/data/footprint/desi-tiles.*)
        and a list of overrides for tiles that were observed by deemed bad
        and need to be redone (details TBD).
        DOS shouldn't care about those details.

    TBD:
      * Error handling: if the request is impossible (e.g. the sun is up),
        should this raise an exception?  Or return a default zenith answer
        with some calib programname?  Or?
    """

    """
        Below is an algorithm for calculating the local apparent sidereal time.
        astropy should be able to this (I think), but I keep getting an error when I try
        to use those functions (and I'm pretty sure I'm importing all the necessary
        components of astropy). A bunch of lines have been commented out which were
        used for testing purposes.
    """

    tobs = Time(dateobs, format='jd', scale='utc')
    kitt_peak = EarthLocation(lat=31.9634*u.deg, lon=-111.6003*u.deg, height=2120*u.m)
    kitt_peak_long = Longitude(-111.5984796*u.deg)
    #iers.IERS.iers_table = iers.IERS_A.open('finals2000A.all.txt')
    #iers.IERS.iers_table = iers.IERS_A.open(download_file(iers.IERS_A_URL, cache=True))

    #Find the Julian date of the previous midnight
    if (dateobs-math.floor(dateobs) >= 0.5):
        jd_0 = math.floor(dateobs)+0.5
    elif (dateobs-math.floor(dateobs) < 0.5):
        jd_0 = math.floor(dateobs)-0.5

    d_0 = jd_0-2451545.0 #Difference between last Julian midnight and J2000
    d = dateobs-2451545.0 #Difference between observation date and J2000
    t = d_0/36525 #Fraction of Julian century that's past since J2000

    #Calculate the sideral time in Greenwich for the last Julian midnight
    gmst_0 = 100.4606184+36000.77005361*t+0.00038793*t**2-2.6E-08*t**3

    #Add correction for the number of hours that have past since midnight
    gmst = gmst_0+0.25068447733746215*(dateobs-jd_0)*24*60

    #Calculate the equation of equinoxes
    l = 280.47+0.98565*d
    omega = 125.04-0.052954*d
    del_psi = -0.000319*math.sin(omega*math.pi/180)-0.00024*math.sin(2*l*math.pi/180)
    e = 23.4393-0.0000004*d

    eqeq = del_psi*math.cos(e*math.pi/180)

    #Correct with the equation of equinoxes to get the current apparent sidereal time
    gast = gmst+eqeq

    #gast_2 = tobs.sidereal_time('apparent', 'greenwich')

    #print gast
    #print gast_2

    #Add the longitude of the observatory to get the local sidereal time
    last = gast-111.5984796

    #Shift the local sidereal time into the range of 0 to 360 degrees
    if last >= 360:
        n = math.floor(last/360)
        last = last-360*n

    #hour = math.floor(last/15)
    #minute = math.floor((last/15-hour)*60)
    #second = ((last/15-hour)*60-minute)*60

    #print( str(int(hour)) + "h" + str(int(minute)) + "m" + str(second) +"s")

    #last_2 = gast_2+kitt_peak_long

    #last = last_2.deg

    #print("dateobs = " + str(dateobs))
    #print("JD_0 = " + str(JD_0))
    #print("D_0 = " + str(D_0))
    #print("D = " + str(D))
    #print("T = " + str(T))
    #print("H = " + str(H))
    #print("GMST = " + str(GMST))
    #print("LAST = " + str(LAST))

    #Use astropy to calculate the position of the Sun.
    pos_sun = coordinates.get_sun(tobs)

    #Check to see if the Sun is up.
    ra_sun = pos_sun.ra.value
    dec_sun = pos_sun.dec.value

    ha_sun = last - ra_sun
    if (ha_sun < 0):
        ha_sun = ha_sun + 360
    if (ha_sun > 360):
        ha_sun = ha_sun - 360
    #print("ha_sun = " + str(ha_sun))

    #Calculate the altitude of the Sun to determine if it is up
    alt_sun = (math.asin(math.sin(dec_sun*math.pi/180)
                         *math.sin(31.9614929*math.pi/180)
                         +math.cos(dec_sun*math.pi/180)
                         *math.cos(31.9614929*math.pi/180)
                         *math.cos(ha_sun*math.pi/180)))*(180/math.pi)+0.634000964

    #print("alt_sun = " + str(alt_sun))

    #sun_altaz = pos_sun.transform_to(AltAz(obstime=tobs,location=kitt_peak))

    #print sun_altaz

    #alt_sun = sun_altaz.alt.value

    print alt_sun

    #Print warning if the Sun is up. We may decide this should do more than just warn
    if (alt_sun >=-30):
        print("WARNING: Observation time is within two hours of sunrise.")

    #Find the position of the Moon using PyAstronomy
    pos_moon = pyasl.moonpos(dateobs)

    ra_moon = float(pos_moon[0])
    dec_moon = float(pos_moon[1])

    print("ra_moon = " + str(ra_moon) + " dec_moon = " + str(dec_moon))

    #Find the phase of the moon using PyAstronomy
    moon_phase = pyasl.moonphase(dateobs)
    phase = float(moon_phase)

    print(phase)

    #Opens a file containing information about all of the DESI tiles. Not sure if
    #this is the best way to get that information?
    tiles_file = open("desi-tiles-full.par.txt","r")

    #Declare some variables to store data from the file
    tiles = []
    idnum = []
    ra = []
    dec = []
    passnum = []
    in_desi = []
    ebv_med = []
    airmass = []
    exposefac = []
    i = 0
    j = 0
    mindec = 100.0
    nextfield = 0
    #Read the data from the file and find the next field
    for line in tiles_file:
        tiles.append(line)
        c = tiles[j].split(" ")
        if (c[0] == "STRUCT1"):
            idnum.append(int(c[1]))
            ra.append(float(c[2]))
            dec.append(float(c[3]))
            passnum.append(int(c[4]))
            in_desi.append(int(c[5]))
            ebv_med.append(float(c[6]))
            airmass.append(float(c[7]))
            exposefac.append(float(c[8]))
            if (ra[i] >= last-15 and ra[i] <= last+15):
                ha = last-ra[i]
                if ha < 0:
                    ha = ha + 360
                if ha > 360:
                    ha = ha - 360
                alt = (math.asin(math.sin(dec[i]*math.pi/180)
                                 *math.sin(31.9614929*math.pi/180)
                                 +math.cos(dec[i]*math.pi/180)
                                 *math.cos(31.9614929*math.pi/180)
                                 *math.cos(ha*math.pi/180)))*(180/math.pi)
                if (alt >= 0):
                    if (dec[i] < mindec):
                        mindec = dec[i]
                        nextfield = idnum[i]
            i = i + 1
        j = j + 1

    tiles_file.close()

    #raise NotImplementedError

    #Create dictionary with information that is needed to point the telescope
    next_field = {'tileid':idnum[nextfield-1], 'programname':'DESI', 'telera':ra[nextfield-1], 'teledec':dec[nextfield-1], 'exptime':1800, 'maxtime':2000, 'fibers':{}, 'gfa':{}}

    #Return the dictionary
    return next_field

""" The lines below allow the function to be tested by itself with the user
inputting a Julian date of observation. They also calculate the execution time for
purposes of optimizing."""

dateobs = float(raw_input('Enter the date of observation: '))
skylevel = 0
transparency = 0
previoustiles = []
programname = 'DESI'
start_time = time.time()
next_field = get_next_field(dateobs, skylevel, transparency, previoustiles, programname)

print("Total execution time: %s seconds" % (time.time()-start_time))
print next_field
<|endoftext|>


<commit_before>#!/usr/bin/env python
from siconos.tests_setup import working_dir
import os

# this test is taken almost ve@rbatim from RelayBiSimulation_OT2_noCplugin.py
def test_smc1():
    from siconos.kernel import FirstOrderLinearDS, Model, TimeDiscretisation, \
        TimeStepping, ZeroOrderHoldOSI, TD_EVENT
    from siconos.control.simulation import ControlManager
    from siconos.control.sensor import LinearSensor
    from siconos.control.controller import LinearSMCOT2
    from numpy import eye, empty, zeros
    import numpy as np
    from math import ceil, sin

    # Derive our own version of FirstOrderLinearDS
    class MyFOLDS(FirstOrderLinearDS):
        def computeb(self, time):
            t = sin(50*time)
            # XXX fix this !
            u = [t, -t]
            self.setb(u)

    # variable declaration
    ndof = 2   # Number of degrees of freedom of your system
    t0 = 0.0   # start time
    T = 1    # end time
    h = 1.0e-4  # time step for simulation
    hControl = 1.0e-2  # time step for control
    Xinit = 1.0  # initial position
    N = ceil((T-t0)/h + 10)  # number of time steps
    outputSize = 4  # number of variable to store at each time step

    # Matrix declaration
    A = zeros((ndof, ndof))
    x0 = [Xinit, -Xinit]
    Brel = np.array([[0], [1]])
    sensorC = eye(ndof)
    sensorD = zeros((ndof, ndof))
    Csurface = [[0, 1.0]]

    # Simple check
    if h > hControl:
        print("hControl must be bigger than h")
        exit(1)

    # Declaration of the Dynamical System
    processDS = MyFOLDS(x0, A)
    # XXX b is not automatically created ...
#    processDS.setb([0, 0])
    # Model
    process = Model(t0, T)
    process.nonSmoothDynamicalSystem().insertDynamicalSystem(processDS)
    # time discretization
    processTD = TimeDiscretisation(t0, h)
    tSensor = TimeDiscretisation(t0, hControl)
    tActuator = TimeDiscretisation(t0, hControl)
    # Creation of the Simulation
    processSimulation = TimeStepping(processTD, 0)
    processSimulation.setName("plant simulation")
    # Declaration of the integrator
    processIntegrator = ZeroOrderHoldOSI(processDS)
    processSimulation.insertIntegrator(processIntegrator)
    # Actuator, Sensor & ControlManager
    control = ControlManager(processSimulation)
    sens = LinearSensor(processDS, sensorC, sensorD)

    control.addSensorPtr(sens, tSensor)
    act = LinearSMCOT2(sens)
    act.setCsurface(Csurface)
    act.setB(Brel)
    control.addActuatorPtr(act, tActuator)

    # Initialization.
    process.initialize(processSimulation)
    control.initialize(process)
    # This is not working right now
    # eventsManager = s.eventsManager()

    # Matrix for data storage
    dataPlot = empty((3*(N+1), outputSize))
    dataPlot[0, 0] = t0
    dataPlot[0, 1] = processDS.x()[0]
    dataPlot[0, 2] = processDS.x()[1]
    dataPlot[0, 3] = act.u()[0]

    # Main loop
    k = 1
    while processSimulation.hasNextEvent():
        if processSimulation.eventsManager().nextEvent().getType() == TD_EVENT:
            processSimulation.computeOneStep()
        dataPlot[k, 0] = processSimulation.nextTime()
        dataPlot[k, 1] = processDS.x()[0]
        dataPlot[k, 2] = processDS.x()[1]
        dataPlot[k, 3] = act.u()[0]
        k += 1
        processSimulation.nextStep()
    #    print processSimulation.nextTime()
    # Resize matrix
    dataPlot.resize(k, outputSize)

# Same test, but with the simplified interface
def test_smc2():
    from siconos.kernel import FirstOrderLinearDS, getMatrix, SimpleMatrix
    from siconos.control.sensor import LinearSensor
    from siconos.control.controller import LinearSMCOT2
    from siconos.control.simulation import ControlZOHSimulation
    from numpy import eye, zeros
    import numpy as np
    from math import sin
    from numpy.linalg import norm

    # Derive our own version of FirstOrderLinearDS
    class MyFOLDS(FirstOrderLinearDS):
        def computeb(self, time):
            t = sin(50*time)
            u = [t, -t]
            self.setb(u)

    # variable declaration
    ndof = 2   # Number of degrees of freedom of your system
    t0 = 0.0   # start time
    T = 1    # end time
    h = 1.0e-4  # time step for simulation
    hControl = 1.0e-2  # time step for control
    Xinit = 1.0  # initial position

    # Matrix declaration
    A = zeros((ndof, ndof))
    x0 = [Xinit, -Xinit]
    sensorC = eye(ndof)
    sensorD = zeros((ndof, ndof))
    Brel = np.array([[0], [1]])
    Csurface = [[0, 1.0]]

    # Simple check
    if h > hControl:
        print("hControl must be bigger than h")
        exit(1)

    # Declaration of the Dynamical System
    processDS = MyFOLDS(x0, A)
    # XXX b is not automatically created ...
    processDS.setb([0, 0])
    sim = ControlZOHSimulation(t0, T, h)
    sim.addDynamicalSystem(processDS)
    # time discretisation
    # Actuator, Sensor & ControlManager
    sens = LinearSensor(processDS, sensorC, sensorD)
    sim.addSensor(sens, hControl)
    act = LinearSMCOT2(sens)
    act.setCsurface(Csurface)
    act.setB(Brel)
    sim.addActuator(act, hControl)

    # Run the simulation
    sim.initialize()
    sim.run()
    # get the results
    tmpData = sim.data()
    dataPlot = tmpData
    # compare with the reference
    ref = np.loadtxt(os.path.join(working_dir, "data/smc_2.ref.gz"))
    np.savetxt("smc_2.dat", dataPlot)
    print("%e" % norm(dataPlot - ref))
    if (norm(dataPlot - ref) > 5e-12):
        print(dataPlot - ref)
        print("ERROR: The result is rather different from the reference file.")
    assert norm(dataPlot - ref) < 5e-12
<commit_msg>[control] fix python tests<commit_after>#!/usr/bin/env python
from siconos.tests_setup import working_dir
import os

# this test is taken almost ve@rbatim from RelayBiSimulation_OT2_noCplugin.py
def test_smc1():
    from siconos.kernel import FirstOrderLinearDS, Model, TimeDiscretisation, \
        TimeStepping, ZeroOrderHoldOSI, TD_EVENT
    from siconos.control.simulation import ControlManager
    from siconos.control.sensor import LinearSensor
    from siconos.control.controller import LinearSMCOT2
    from numpy import eye, empty, zeros
    import numpy as np
    from math import ceil, sin

    # Derive our own version of FirstOrderLinearDS
    class MyFOLDS(FirstOrderLinearDS):
        def computeb(self, time):
            t = sin(50*time)
            # XXX fix this !
            u = [t, -t]
            self.setb(u)

    # variable declaration
    ndof = 2   # Number of degrees of freedom of your system
    t0 = 0.0   # start time
    T = 1    # end time
    h = 1.0e-4  # time step for simulation
    hControl = 1.0e-2  # time step for control
    Xinit = 1.0  # initial position
    N = ceil((T-t0)/h + 10)  # number of time steps
    outputSize = 4  # number of variable to store at each time step

    # Matrix declaration
    A = zeros((ndof, ndof))
    x0 = [Xinit, -Xinit]
    Brel = np.array([[0], [1]])
    sensorC = eye(ndof)
    sensorD = zeros((ndof, ndof))
    Csurface = [[0, 1.0]]

    # Simple check
    if h > hControl:
        print("hControl must be bigger than h")
        exit(1)

    # Declaration of the Dynamical System
    processDS = MyFOLDS(x0, A)
    # XXX b is not automatically created ...
#    processDS.setb([0, 0])
    # Model
    process = Model(t0, T)
    process.nonSmoothDynamicalSystem().insertDynamicalSystem(processDS)
    # time discretization
    processTD = TimeDiscretisation(t0, h)
    tSensor = TimeDiscretisation(t0, hControl)
    tActuator = TimeDiscretisation(t0, hControl)
    # Creation of the Simulation
    processSimulation = TimeStepping(processTD, 0)
    processSimulation.setName("plant simulation")
    # Declaration of the integrator
    processIntegrator = ZeroOrderHoldOSI()
    process.nonSmoothDynamicalSystem().setOSI(processDS, processIntegrator)
    processSimulation.insertIntegrator(processIntegrator)
    # Actuator, Sensor & ControlManager
    control = ControlManager(processSimulation)
    sens = LinearSensor(processDS, sensorC, sensorD)

    control.addSensorPtr(sens, tSensor)
    act = LinearSMCOT2(sens)
    act.setCsurface(Csurface)
    act.setB(Brel)
    control.addActuatorPtr(act, tActuator)

    # Initialization.
    process.initialize(processSimulation)
    control.initialize(process)
    # This is not working right now
    # eventsManager = s.eventsManager()

    # Matrix for data storage
    dataPlot = empty((3*(N+1), outputSize))
    dataPlot[0, 0] = t0
    dataPlot[0, 1] = processDS.x()[0]
    dataPlot[0, 2] = processDS.x()[1]
    dataPlot[0, 3] = act.u()[0]

    # Main loop
    k = 1
    while processSimulation.hasNextEvent():
        if processSimulation.eventsManager().nextEvent().getType() == TD_EVENT:
            processSimulation.computeOneStep()
        dataPlot[k, 0] = processSimulation.nextTime()
        dataPlot[k, 1] = processDS.x()[0]
        dataPlot[k, 2] = processDS.x()[1]
        dataPlot[k, 3] = act.u()[0]
        k += 1
        processSimulation.nextStep()
    #    print processSimulation.nextTime()
    # Resize matrix
    dataPlot.resize(k, outputSize)

# Same test, but with the simplified interface
def test_smc2():
    from siconos.kernel import FirstOrderLinearDS, getMatrix, SimpleMatrix
    from siconos.control.sensor import LinearSensor
    from siconos.control.controller import LinearSMCOT2
    from siconos.control.simulation import ControlZOHSimulation
    from numpy import eye, zeros
    import numpy as np
    from math import sin
    from numpy.linalg import norm

    # Derive our own version of FirstOrderLinearDS
    class MyFOLDS(FirstOrderLinearDS):
        def computeb(self, time):
            t = sin(50*time)
            u = [t, -t]
            self.setb(u)

    # variable declaration
    ndof = 2   # Number of degrees of freedom of your system
    t0 = 0.0   # start time
    T = 1    # end time
    h = 1.0e-4  # time step for simulation
    hControl = 1.0e-2  # time step for control
    Xinit = 1.0  # initial position

    # Matrix declaration
    A = zeros((ndof, ndof))
    x0 = [Xinit, -Xinit]
    sensorC = eye(ndof)
    sensorD = zeros((ndof, ndof))
    Brel = np.array([[0], [1]])
    Csurface = [[0, 1.0]]

    # Simple check
    if h > hControl:
        print("hControl must be bigger than h")
        exit(1)

    # Declaration of the Dynamical System
    processDS = MyFOLDS(x0, A)
    # XXX b is not automatically created ...
    processDS.setb([0, 0])
    sim = ControlZOHSimulation(t0, T, h)
    sim.addDynamicalSystem(processDS)
    # time discretisation
    # Actuator, Sensor & ControlManager
    sens = LinearSensor(processDS, sensorC, sensorD)
    sim.addSensor(sens, hControl)
    act = LinearSMCOT2(sens)
    act.setCsurface(Csurface)
    act.setB(Brel)
    sim.addActuator(act, hControl)

    # Run the simulation
    sim.initialize()
    sim.run()
    # get the results
    tmpData = sim.data()
    dataPlot = tmpData
    # compare with the reference
    ref = np.loadtxt(os.path.join(working_dir, "data/smc_2.ref.gz"))
    np.savetxt("smc_2.dat", dataPlot)
    print("%e" % norm(dataPlot - ref))
    if (norm(dataPlot - ref) > 5e-12):
        print(dataPlot - ref)
        print("ERROR: The result is rather different from the reference file.")
    assert norm(dataPlot - ref) < 5e-12
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
General documentation architecture:

Home
Index

- Getting started
    Getting started with the sequential model
    Getting started with the functional api
    FAQ

- Models
    About Keras models
        explain when one should use Sequential or functional API
        explain compilation step
        explain weight saving, weight loading
        explain serialization, deserialization
    Sequential
    Model (functional API)

- Layers
    About Keras layers
        explain common layer functions: get_weights, set_weights, get_config
        explain input_shape
        explain usage on non-Keras tensors
    Core Layers
    Convolutional Layers
    Pooling Layers
    Locally-connected Layers
    Recurrent Layers
    Embedding Layers
    Merge Layers
    Advanced Activations Layers
    Normalization Layers
    Noise Layers
    Layer Wrappers
    Writing your own Keras layers

- Preprocessing
    Sequence Preprocessing
    Text Preprocessing
    Image Preprocessing

Losses
Metrics
Optimizers
Activations
Callbacks
Datasets
Applications
Backend
Initializers
Regularizers
Constraints
Visualization
Scikit-learn API
Utils
Contributing

'''
from __future__ import print_function
from __future__ import unicode_literals

import re
import inspect
import os
import shutil
import sys
if sys.version[0] == '2':
    reload(sys)
    sys.setdefaultencoding('utf8')

import keras
from keras import utils
from keras import layers
from keras import initializers
from keras.layers import pooling
from keras.layers import local
from keras.layers import recurrent
from keras.layers import core
from keras.layers import noise
from keras.layers import normalization
from keras.layers import advanced_activations
from keras.layers import embeddings
from keras.layers import wrappers
from keras import optimizers
from keras import callbacks
from keras import models
from keras.engine import topology
from keras import losses
from keras import metrics
from keras import backend
from keras import constraints
from keras import activations
from keras import regularizers
from keras.utils import data_utils
from keras.utils import io_utils
from keras.utils import layer_utils
from keras.utils import np_utils
from keras.utils import generic_utils

EXCLUDE = {
    'Optimizer',
    'Wrapper',
    'get_session',
    'set_session',
    'CallbackList',
    'serialize',
    'deserialize',
    'get',
}

PAGES = [
    {
        'page': 'models/sequential.md',
        'functions': [
            models.Sequential.compile,
            models.Sequential.fit,
            models.Sequential.evaluate,
            models.Sequential.predict,
            models.Sequential.predict_classes,
            models.Sequential.predict_proba,
            models.Sequential.train_on_batch,
            models.Sequential.test_on_batch,
            models.Sequential.predict_on_batch,
            models.Sequential.fit_generator,
            models.Sequential.evaluate_generator,
            models.Sequential.predict_generator,
        ],
    },
    {
        'page': 'models/model.md',
        'functions': [
            models.Model.compile,
            models.Model.fit,
            models.Model.evaluate,
            models.Model.predict,
            models.Model.train_on_batch,
            models.Model.test_on_batch,
            models.Model.predict_on_batch,
            models.Model.fit_generator,
            models.Model.evaluate_generator,
            models.Model.predict_generator,
            models.Model.get_layer,
        ]
    },
    {
        'page': 'layers/core.md',
        'classes': [
            layers.Dense,
            layers.Activation,
            layers.Dropout,
            layers.Flatten,
            layers.Reshape,
            layers.Permute,
            layers.RepeatVector,
            layers.Lambda,
            layers.ActivityRegularization,
            layers.Masking,
        ],
    },
    {
        'page': 'layers/convolutional.md',
        'classes': [
            layers.Conv1D,
            layers.Conv2D,
            layers.SeparableConv2D,
            layers.Conv2DTranspose,
            layers.Conv3D,
            layers.Cropping1D,
            layers.Cropping2D,
            layers.Cropping3D,
            layers.UpSampling1D,
            layers.UpSampling2D,
            layers.UpSampling3D,
            layers.ZeroPadding1D,
            layers.ZeroPadding2D,
            layers.ZeroPadding3D,
        ],
    },
    {
        'page': 'layers/pooling.md',
        'classes': [
            pooling.MaxPooling1D,
            pooling.MaxPooling2D,
            pooling.MaxPooling3D,
            pooling.AveragePooling1D,
            pooling.AveragePooling2D,
            pooling.AveragePooling3D,
            pooling.GlobalMaxPooling1D,
            pooling.GlobalAveragePooling1D,
            pooling.GlobalMaxPooling2D,
            pooling.GlobalAveragePooling2D,
        ],
    },
    {
        'page': 'layers/local.md',
        'classes': [
            local.LocallyConnected1D,
            local.LocallyConnected2D,
        ],
    },
    {
        'page': 'layers/recurrent.md',
        'classes': [
            recurrent.Recurrent,
            recurrent.SimpleRNN,
            recurrent.GRU,
            recurrent.LSTM,
        ],
    },
    {
        'page': 'layers/embeddings.md',
        'classes': [
            embeddings.Embedding,
        ],
    },
    {
        'page': 'layers/normalization.md',
        'classes': [
            normalization.BatchNormalization,
        ],
    },
    {
        'page': 'layers/advanced-activations.md',
        'all_module_classes': [advanced_activations],
    },
    {
        'page': 'layers/noise.md',
        'all_module_classes': [noise],
    },
    {
        'page': 'layers/merge.md',
        'classes': [
            layers.Add,
            layers.Multiply,
            layers.Average,
            layers.Maximum,
            layers.Concatenate,
            layers.Dot,
        ],
        'functions': [
            layers.add,
            layers.multiply,
            layers.average,
            layers.maximum,
            layers.concatenate,
            layers.dot,
        ]
    },
    {
        'page': 'layers/wrappers.md',
        'all_module_classes': [wrappers],
    },
    {
        'page': 'metrics.md',
        'all_module_functions': [metrics],
    },
    {
        'page': 'losses.md',
        'all_module_functions': [losses],
    },
    {
        'page': 'initializers.md',
        'all_module_functions': [initializers],
        'all_module_classes': [initializers],
    },
    {
        'page': 'optimizers.md',
        'all_module_classes': [optimizers],
    },
    {
        'page': 'callbacks.md',
        'all_module_classes': [callbacks],
    },
    {
        'page': 'activations.md',
        'all_module_functions': [activations],
    },
    {
        'page': 'backend.md',
        'all_module_functions': [backend],
    },
    {
        'page': 'utils.md',
        'all_module_functions': [utils],
        'classes': [utils.CustomObjectScope,
                    utils.HDF5Matrix]
    },
]

ROOT = 'http://keras.io/'

def get_earliest_class_that_defined_member(member, cls):
    ancestors = get_classes_ancestors([cls])
    result = None
    for ancestor in ancestors:
        if member in dir(ancestor):
            result = ancestor
    if not result:
        return cls
    return result

def get_classes_ancestors(classes):
    ancestors = []
    for cls in classes:
        ancestors += cls.__bases__
    filtered_ancestors = []
    for ancestor in ancestors:
        if ancestor.__name__ in ['object']:
            continue
        filtered_ancestors.append(ancestor)
    if filtered_ancestors:
        return filtered_ancestors + get_classes_ancestors(filtered_ancestors)
    else:
        return filtered_ancestors

def get_function_signature(function, method=True):
    signature = getattr(function, '_legacy_support_signature', None)
    if signature is None:
        signature = inspect.getargspec(function)
    defaults = signature.defaults
    if method:
        args = signature.args[1:]
    else:
        args = signature.args
    if defaults:
        kwargs = zip(args[-len(defaults):], defaults)
        args = args[:-len(defaults)]
    else:
        kwargs = []
    st = '%s.%s(' % (function.__module__, function.__name__)
    for a in args:
        st += str(a) + ', '
    for a, v in kwargs:
        if isinstance(v, str):
            v = '\'' + v + '\''
        st += str(a) + '=' + str(v) + ', '
    if kwargs or args:
        return st[:-2] + ')'
    else:
        return st + ')'

def get_class_signature(cls):
    try:
        class_signature = get_function_signature(cls.__init__)
        class_signature = class_signature.replace('__init__', cls.__name__)
    except:
        # in case the class inherits from object and does not
        # define __init__
        class_signature = cls.__module__ + '.' + cls.__name__ + '()'
    return class_signature

def class_to_docs_link(cls):
    module_name = cls.__module__
    assert module_name[:6] == 'keras.'
    module_name = module_name[6:]
    link = ROOT + module_name.replace('.', '/') + '#' + cls.__name__.lower()
    return link

def class_to_source_link(cls):
    module_name = cls.__module__
    assert module_name[:6] == 'keras.'
    path = module_name.replace('.', '/')
    path += '.py'
    line = inspect.getsourcelines(cls)[-1]
    link = 'https://github.com/fchollet/keras/blob/master/' + path + '#L' + str(line)
    return '[[source]](' + link + ')'

def code_snippet(snippet):
    result = '```python\n'
    result += snippet + '\n'
    result += '```\n'
    return result

def process_class_docstring(docstring):
    docstring = re.sub(r'\n    # (.*)\n',
                       r'\n    __\1__\n\n',
                       docstring)

    docstring = re.sub(r'    ([^\s\\]+):(.*)\n',
                       r'    - __\1__:\2\n',
                       docstring)

    docstring = docstring.replace('    ' * 5, '\t\t')
    docstring = docstring.replace('    ' * 3, '\t')
    docstring = docstring.replace('    ', '')
    return docstring

def process_function_docstring(docstring):
    docstring = re.sub(r'\n    # (.*)\n',
                       r'\n    __\1__\n\n',
                       docstring)
    docstring = re.sub(r'\n        # (.*)\n',
                       r'\n        __\1__\n\n',
                       docstring)

    docstring = re.sub(r'    ([^\s\\]+):(.*)\n',
                       r'    - __\1__:\2\n',
                       docstring)

    docstring = docstring.replace('    ' * 6, '\t\t')
    docstring = docstring.replace('    ' * 4, '\t')
    docstring = docstring.replace('    ', '')
    return docstring

print('Cleaning up existing sources directory.')
if os.path.exists('sources'):
    shutil.rmtree('sources')

print('Populating sources directory with templates.')
for subdir, dirs, fnames in os.walk('templates'):
    for fname in fnames:
        new_subdir = subdir.replace('templates', 'sources')
        if not os.path.exists(new_subdir):
            os.makedirs(new_subdir)
        if fname[-3:] == '.md':
            fpath = os.path.join(subdir, fname)
            new_fpath = fpath.replace('templates', 'sources')
            shutil.copy(fpath, new_fpath)

# Take care of index page.
readme = open('../README.md').read()
index = open('templates/index.md').read()
index = index.replace('{{autogenerated}}', readme[readme.find('##'):])
f = open('sources/index.md', 'w')
f.write(index)
f.close()

print('Starting autogeneration.')
for page_data in PAGES:
    blocks = []
    classes = page_data.get('classes', [])
    for module in page_data.get('all_module_classes', []):
        module_classes = []
        for name in dir(module):
            if name[0] == '_' or name in EXCLUDE:
                continue
            module_member = getattr(module, name)
            if inspect.isclass(module_member):
                cls = module_member
                if cls.__module__ == module.__name__:
                    if cls not in module_classes:
                        module_classes.append(cls)
        module_classes.sort(key=lambda x: id(x))
        classes += module_classes

    for cls in classes:
        subblocks = []
        signature = get_class_signature(cls)
        subblocks.append('' + class_to_source_link(cls) + '')
        subblocks.append('### ' + cls.__name__ + '\n')
        subblocks.append(code_snippet(signature))
        docstring = cls.__doc__
        if docstring:
            subblocks.append(process_class_docstring(docstring))
        blocks.append('\n'.join(subblocks))

    functions = page_data.get('functions', [])
    for module in page_data.get('all_module_functions', []):
        module_functions = []
        for name in dir(module):
            if name[0] == '_' or name in EXCLUDE:
                continue
            module_member = getattr(module, name)
            if inspect.isfunction(module_member):
                function = module_member
                if module.__name__ in function.__module__:
                    if function not in module_functions:
                        module_functions.append(function)
        module_functions.sort(key=lambda x: id(x))
        functions += module_functions

    for function in functions:
        subblocks = []
        signature = get_function_signature(function, method=False)
        signature = signature.replace(function.__module__ + '.', '')
        subblocks.append('### ' + function.__name__ + '\n')
        subblocks.append(code_snippet(signature))
        docstring = function.__doc__
        if docstring:
            subblocks.append(process_function_docstring(docstring))
        blocks.append('\n\n'.join(subblocks))

    if not blocks:
        raise RuntimeError('Found no content for page ' +
                           page_data['page'])

    mkdown = '\n----\n\n'.join(blocks)
    # save module page.
    # Either insert content into existing page,
    # or create page otherwise
    page_name = page_data['page']
    path = os.path.join('sources', page_name)
    if os.path.exists(path):
        template = open(path).read()
        assert '{{autogenerated}}' in template, ('Template found for ' + path +
                                                 ' but missing {{autogenerated}} tag.')
        mkdown = template.replace('{{autogenerated}}', mkdown)
        print('...inserting autogenerated content into template:', path)
    else:
        print('...creating new page with autogenerated content:', path)
    subdir = os.path.dirname(path)
    if not os.path.exists(subdir):
        os.makedirs(subdir)
    open(path, 'w').write(mkdown)

shutil.copyfile('../CONTRIBUTING.md', 'sources/contributing.md')
<commit_msg>Update docs autogen script.<commit_after># -*- coding: utf-8 -*-
'''
General documentation architecture:

Home
Index

- Getting started
    Getting started with the sequential model
    Getting started with the functional api
    FAQ

- Models
    About Keras models
        explain when one should use Sequential or functional API
        explain compilation step
        explain weight saving, weight loading
        explain serialization, deserialization
    Sequential
    Model (functional API)

- Layers
    About Keras layers
        explain common layer functions: get_weights, set_weights, get_config
        explain input_shape
        explain usage on non-Keras tensors
    Core Layers
    Convolutional Layers
    Pooling Layers
    Locally-connected Layers
    Recurrent Layers
    Embedding Layers
    Merge Layers
    Advanced Activations Layers
    Normalization Layers
    Noise Layers
    Layer Wrappers
    Writing your own Keras layers

- Preprocessing
    Sequence Preprocessing
    Text Preprocessing
    Image Preprocessing

Losses
Metrics
Optimizers
Activations
Callbacks
Datasets
Applications
Backend
Initializers
Regularizers
Constraints
Visualization
Scikit-learn API
Utils
Contributing

'''
from __future__ import print_function
from __future__ import unicode_literals

import re
import inspect
import os
import shutil
import sys
if sys.version[0] == '2':
    reload(sys)
    sys.setdefaultencoding('utf8')

import keras
from keras import utils
from keras import layers
from keras import initializers
from keras.layers import pooling
from keras.layers import local
from keras.layers import recurrent
from keras.layers import core
from keras.layers import noise
from keras.layers import normalization
from keras.layers import advanced_activations
from keras.layers import embeddings
from keras.layers import wrappers
from keras import optimizers
from keras import callbacks
from keras import models
from keras.engine import topology
from keras import losses
from keras import metrics
from keras import backend
from keras import constraints
from keras import activations
from keras import regularizers
from keras.utils import data_utils
from keras.utils import io_utils
from keras.utils import layer_utils
from keras.utils import np_utils
from keras.utils import generic_utils

EXCLUDE = {
    'Optimizer',
    'Wrapper',
    'get_session',
    'set_session',
    'CallbackList',
    'serialize',
    'deserialize',
    'get',
}

PAGES = [
    {
        'page': 'models/sequential.md',
        'functions': [
            models.Sequential.compile,
            models.Sequential.fit,
            models.Sequential.evaluate,
            models.Sequential.predict,
            models.Sequential.train_on_batch,
            models.Sequential.test_on_batch,
            models.Sequential.predict_on_batch,
            models.Sequential.fit_generator,
            models.Sequential.evaluate_generator,
            models.Sequential.predict_generator,
            models.Sequential.get_layer,
        ],
    },
    {
        'page': 'models/model.md',
        'functions': [
            models.Model.compile,
            models.Model.fit,
            models.Model.evaluate,
            models.Model.predict,
            models.Model.train_on_batch,
            models.Model.test_on_batch,
            models.Model.predict_on_batch,
            models.Model.fit_generator,
            models.Model.evaluate_generator,
            models.Model.predict_generator,
            models.Model.get_layer,
        ]
    },
    {
        'page': 'layers/core.md',
        'classes': [
            layers.Dense,
            layers.Activation,
            layers.Dropout,
            layers.Flatten,
            layers.Reshape,
            layers.Permute,
            layers.RepeatVector,
            layers.Lambda,
            layers.ActivityRegularization,
            layers.Masking,
        ],
    },
    {
        'page': 'layers/convolutional.md',
        'classes': [
            layers.Conv1D,
            layers.Conv2D,
            layers.SeparableConv2D,
            layers.Conv2DTranspose,
            layers.Conv3D,
            layers.Cropping1D,
            layers.Cropping2D,
            layers.Cropping3D,
            layers.UpSampling1D,
            layers.UpSampling2D,
            layers.UpSampling3D,
            layers.ZeroPadding1D,
            layers.ZeroPadding2D,
            layers.ZeroPadding3D,
        ],
    },
    {
        'page': 'layers/pooling.md',
        'classes': [
            pooling.MaxPooling1D,
            pooling.MaxPooling2D,
            pooling.MaxPooling3D,
            pooling.AveragePooling1D,
            pooling.AveragePooling2D,
            pooling.AveragePooling3D,
            pooling.GlobalMaxPooling1D,
            pooling.GlobalAveragePooling1D,
            pooling.GlobalMaxPooling2D,
            pooling.GlobalAveragePooling2D,
        ],
    },
    {
        'page': 'layers/local.md',
        'classes': [
            local.LocallyConnected1D,
            local.LocallyConnected2D,
        ],
    },
    {
        'page': 'layers/recurrent.md',
        'classes': [
            recurrent.Recurrent,
            recurrent.SimpleRNN,
            recurrent.GRU,
            recurrent.LSTM,
        ],
    },
    {
        'page': 'layers/embeddings.md',
        'classes': [
            embeddings.Embedding,
        ],
    },
    {
        'page': 'layers/normalization.md',
        'classes': [
            normalization.BatchNormalization,
        ],
    },
    {
        'page': 'layers/advanced-activations.md',
        'all_module_classes': [advanced_activations],
    },
    {
        'page': 'layers/noise.md',
        'all_module_classes': [noise],
    },
    {
        'page': 'layers/merge.md',
        'classes': [
            layers.Add,
            layers.Multiply,
            layers.Average,
            layers.Maximum,
            layers.Concatenate,
            layers.Dot,
        ],
        'functions': [
            layers.add,
            layers.multiply,
            layers.average,
            layers.maximum,
            layers.concatenate,
            layers.dot,
        ]
    },
    {
        'page': 'layers/wrappers.md',
        'all_module_classes': [wrappers],
    },
    {
        'page': 'metrics.md',
        'all_module_functions': [metrics],
    },
    {
        'page': 'losses.md',
        'all_module_functions': [losses],
    },
    {
        'page': 'initializers.md',
        'all_module_functions': [initializers],
        'all_module_classes': [initializers],
    },
    {
        'page': 'optimizers.md',
        'all_module_classes': [optimizers],
    },
    {
        'page': 'callbacks.md',
        'all_module_classes': [callbacks],
    },
    {
        'page': 'activations.md',
        'all_module_functions': [activations],
    },
    {
        'page': 'backend.md',
        'all_module_functions': [backend],
    },
    {
        'page': 'utils.md',
        'all_module_functions': [utils],
        'classes': [utils.CustomObjectScope,
                    utils.HDF5Matrix]
    },
]

ROOT = 'http://keras.io/'

def get_earliest_class_that_defined_member(member, cls):
    ancestors = get_classes_ancestors([cls])
    result = None
    for ancestor in ancestors:
        if member in dir(ancestor):
            result = ancestor
    if not result:
        return cls
    return result

def get_classes_ancestors(classes):
    ancestors = []
    for cls in classes:
        ancestors += cls.__bases__
    filtered_ancestors = []
    for ancestor in ancestors:
        if ancestor.__name__ in ['object']:
            continue
        filtered_ancestors.append(ancestor)
    if filtered_ancestors:
        return filtered_ancestors + get_classes_ancestors(filtered_ancestors)
    else:
        return filtered_ancestors

def get_function_signature(function, method=True):
    signature = getattr(function, '_legacy_support_signature', None)
    if signature is None:
        signature = inspect.getargspec(function)
    defaults = signature.defaults
    if method:
        args = signature.args[1:]
    else:
        args = signature.args
    if defaults:
        kwargs = zip(args[-len(defaults):], defaults)
        args = args[:-len(defaults)]
    else:
        kwargs = []
    st = '%s.%s(' % (function.__module__, function.__name__)
    for a in args:
        st += str(a) + ', '
    for a, v in kwargs:
        if isinstance(v, str):
            v = '\'' + v + '\''
        st += str(a) + '=' + str(v) + ', '
    if kwargs or args:
        return st[:-2] + ')'
    else:
        return st + ')'

def get_class_signature(cls):
    try:
        class_signature = get_function_signature(cls.__init__)
        class_signature = class_signature.replace('__init__', cls.__name__)
    except:
        # in case the class inherits from object and does not
        # define __init__
        class_signature = cls.__module__ + '.' + cls.__name__ + '()'
    return class_signature

def class_to_docs_link(cls):
    module_name = cls.__module__
    assert module_name[:6] == 'keras.'
    module_name = module_name[6:]
    link = ROOT + module_name.replace('.', '/') + '#' + cls.__name__.lower()
    return link

def class_to_source_link(cls):
    module_name = cls.__module__
    assert module_name[:6] == 'keras.'
    path = module_name.replace('.', '/')
    path += '.py'
    line = inspect.getsourcelines(cls)[-1]
    link = 'https://github.com/fchollet/keras/blob/master/' + path + '#L' + str(line)
    return '[[source]](' + link + ')'

def code_snippet(snippet):
    result = '```python\n'
    result += snippet + '\n'
    result += '```\n'
    return result

def process_class_docstring(docstring):
    docstring = re.sub(r'\n    # (.*)\n',
                       r'\n    __\1__\n\n',
                       docstring)

    docstring = re.sub(r'    ([^\s\\]+):(.*)\n',
                       r'    - __\1__:\2\n',
                       docstring)

    docstring = docstring.replace('    ' * 5, '\t\t')
    docstring = docstring.replace('    ' * 3, '\t')
    docstring = docstring.replace('    ', '')
    return docstring

def process_function_docstring(docstring):
    docstring = re.sub(r'\n    # (.*)\n',
                       r'\n    __\1__\n\n',
                       docstring)
    docstring = re.sub(r'\n        # (.*)\n',
                       r'\n        __\1__\n\n',
                       docstring)

    docstring = re.sub(r'    ([^\s\\]+):(.*)\n',
                       r'    - __\1__:\2\n',
                       docstring)

    docstring = docstring.replace('    ' * 6, '\t\t')
    docstring = docstring.replace('    ' * 4, '\t')
    docstring = docstring.replace('    ', '')
    return docstring

print('Cleaning up existing sources directory.')
if os.path.exists('sources'):
    shutil.rmtree('sources')

print('Populating sources directory with templates.')
for subdir, dirs, fnames in os.walk('templates'):
    for fname in fnames:
        new_subdir = subdir.replace('templates', 'sources')
        if not os.path.exists(new_subdir):
            os.makedirs(new_subdir)
        if fname[-3:] == '.md':
            fpath = os.path.join(subdir, fname)
            new_fpath = fpath.replace('templates', 'sources')
            shutil.copy(fpath, new_fpath)

# Take care of index page.
readme = open('../README.md').read()
index = open('templates/index.md').read()
index = index.replace('{{autogenerated}}', readme[readme.find('##'):])
f = open('sources/index.md', 'w')
f.write(index)
f.close()

print('Starting autogeneration.')
for page_data in PAGES:
    blocks = []
    classes = page_data.get('classes', [])
    for module in page_data.get('all_module_classes', []):
        module_classes = []
        for name in dir(module):
            if name[0] == '_' or name in EXCLUDE:
                continue
            module_member = getattr(module, name)
            if inspect.isclass(module_member):
                cls = module_member
                if cls.__module__ == module.__name__:
                    if cls not in module_classes:
                        module_classes.append(cls)
        module_classes.sort(key=lambda x: id(x))
        classes += module_classes

    for cls in classes:
        subblocks = []
        signature = get_class_signature(cls)
        subblocks.append('' + class_to_source_link(cls) + '')
        subblocks.append('### ' + cls.__name__ + '\n')
        subblocks.append(code_snippet(signature))
        docstring = cls.__doc__
        if docstring:
            subblocks.append(process_class_docstring(docstring))
        blocks.append('\n'.join(subblocks))

    functions = page_data.get('functions', [])
    for module in page_data.get('all_module_functions', []):
        module_functions = []
        for name in dir(module):
            if name[0] == '_' or name in EXCLUDE:
                continue
            module_member = getattr(module, name)
            if inspect.isfunction(module_member):
                function = module_member
                if module.__name__ in function.__module__:
                    if function not in module_functions:
                        module_functions.append(function)
        module_functions.sort(key=lambda x: id(x))
        functions += module_functions

    for function in functions:
        subblocks = []
        signature = get_function_signature(function, method=False)
        signature = signature.replace(function.__module__ + '.', '')
        subblocks.append('### ' + function.__name__ + '\n')
        subblocks.append(code_snippet(signature))
        docstring = function.__doc__
        if docstring:
            subblocks.append(process_function_docstring(docstring))
        blocks.append('\n\n'.join(subblocks))

    if not blocks:
        raise RuntimeError('Found no content for page ' +
                           page_data['page'])

    mkdown = '\n----\n\n'.join(blocks)
    # save module page.
    # Either insert content into existing page,
    # or create page otherwise
    page_name = page_data['page']
    path = os.path.join('sources', page_name)
    if os.path.exists(path):
        template = open(path).read()
        assert '{{autogenerated}}' in template, ('Template found for ' + path +
                                                 ' but missing {{autogenerated}} tag.')
        mkdown = template.replace('{{autogenerated}}', mkdown)
        print('...inserting autogenerated content into template:', path)
    else:
        print('...creating new page with autogenerated content:', path)
    subdir = os.path.dirname(path)
    if not os.path.exists(subdir):
        os.makedirs(subdir)
    open(path, 'w').write(mkdown)

shutil.copyfile('../CONTRIBUTING.md', 'sources/contributing.md')
<|endoftext|>


<commit_before># Licensed GNU LGPL v3 or later: http://www.gnu.org/licenses/lgpl.html
import sys, re

rapicorn_debug_items, rapicorn_debug_items_set = '', set()
rapicorn_debug_keys, rapicorn_debug_keys_set = '', set()
rapicorn_flippers, rapicorn_flippers_set = '', set()
rapicorn_todo_lists = {}
rapicorn_bug_lists = {}

def process_start ():
  pass

def process_comment (txt, lines):
  if not txt.startswith (('///', '/**', '/*!')):
    return      # filter non-doxygen comments
  # @TODO @TODOS
  global rapicorn_todo_lists
  pattern = r'[@\\] TODO[Ss]? \s* ( : )'
  match = re.search (pattern, txt, re.MULTILINE | re.VERBOSE)
  if match:
    text = txt[match.end (1):].strip()                  # take todo text, whitespace-stripped
    text = text[:-2] if text.endswith ('*/') else text  # strip comment-closing
    text = re.sub (r'( ^ | \n) \s* \*+',                # match comment prefix at line start
                   r'\1', text, 0, re.X | re.M)         # strip comment prefix from all lines
    pattern = r'\s* ( [*+-] | [0-9]+ \. )'              # pattern for list bullet
    if not re.match (pattern, text, re.X):              # not a list
      l = lines + txt[0:match.start (1)].count ('\n')
      text = ' - %d: @b TODO: ' % l + text              # insert list bullet
    else:
      text = '%d: @b TODOS:\n' % lines + text
    blurb = rapicorn_todo_lists.get (filename, '')
    blurb += text.rstrip() + '\n'
    rapicorn_todo_lists[filename] = blurb
  # @BUG @BUGS
  global rapicorn_bug_lists
  pattern = r'[@\\] BUG[Ss]? \s* ( : )'
  match = re.search (pattern, txt, re.MULTILINE | re.VERBOSE)
  if match:
    text = txt[match.end (1):].strip()                  # take bug text, whitespace-stripped
    text = text[:-2] if text.endswith ('*/') else text  # strip comment-closing
    text = re.sub (r'( ^ | \n) \s* \*+',                # match comment prefix at line start
                   r'\1', text, 0, re.X | re.M)         # strip comment prefix from all lines
    pattern = r'\s* ( [*+-] | [0-9]+ \. )'              # pattern for list bullet
    if not re.match (pattern, text, re.X):              # not a list
      l = lines + txt[0:match.start (1)].count ('\n')
      text = ' - %d: @b BUG: ' % l + text               # insert list bullet
    else:
      text = '%d: @b BUGS:\n' % lines + text
    blurb = rapicorn_bug_lists.get (filename, '')
    blurb += text.rstrip() + '\n'
    rapicorn_bug_lists[filename] = blurb

def process_code (txt, lines):
  cstring = r' " ( (?: [^\\"] | \\ .) * ) " '
  # RAPICORN_DEBUG_OPTION (option, blurb)
  global rapicorn_debug_items, rapicorn_debug_items_set
  pattern = r'\b RAPICORN_DEBUG_OPTION \s* \( \s* ' + cstring + r' \s* , \s* ' + cstring + ' \s* \) \s* ;'
  matches = re.findall (pattern, txt, re.MULTILINE | re.VERBOSE)
  for m in matches:
    if not m[0] in rapicorn_debug_items_set:
      rapicorn_debug_items += '  * - @c %s - %s\n' % (m[0], m[1])
      rapicorn_debug_items_set.add (m[0])
  # RAPICORN_KEY_DEBUG (keys, ...)
  global rapicorn_debug_keys, rapicorn_debug_keys_set
  pattern = r'\b RAPICORN_KEY_DEBUG \s* \( \s* ' + cstring + r' \s* , \s* [^;] *? \)'
  matches = re.findall (pattern, txt, re.MULTILINE | re.VERBOSE)
  for m in matches:
    if not m in rapicorn_debug_keys_set:
      rapicorn_debug_keys += '  * - @c %s - Debugging message key, =1 enable, =0 disable.\n' % m
      rapicorn_debug_keys_set.add (m)
  # RAPICORN_FLIPPER (option, blurb)
  global rapicorn_flippers, rapicorn_flippers_set
  pattern = r'\b RAPICORN_FLIPPER \s* \( \s* ' + cstring + r' \s* , \s* ' + cstring + ' \s* \) \s* ;'
  matches = re.findall (pattern, txt, re.MULTILINE | re.VERBOSE)
  for m in matches:
    if not m[0] in rapicorn_flippers_set:
      rapicorn_flippers += '  * - @c %s - %s\n' % (m[0], m[1])
      rapicorn_flippers_set.add (m[0])

def sanitize_ident (txt):
  return re.sub (r'[^0-9_a-zA-Z]+', '_', txt).strip ('_')

def process_end ():
  if rapicorn_debug_items:
    print '/** @var $RAPICORN_DEBUG'
    print rapicorn_debug_items, '*/'
  if rapicorn_debug_keys:
    print '/** @var $RAPICORN_DEBUG'
    print rapicorn_debug_keys, '*/'
  if rapicorn_flippers:
    print '/** @var $RAPICORN_FLIPPER'
    print rapicorn_flippers, '*/'
  if rapicorn_todo_lists:
    # ('/** @page todo_lists Todo Lists', ' * @section %s %s' % (sanitize_ident (filename), filename), '*/')
    for filename, blurb in rapicorn_todo_lists.items():
      ident = sanitize_ident (filename)
      filename = filename[2:] if filename.startswith ('./') else filename
      print '/** @file %s' % filename
      print '@xrefitem todo "Todos" "Source Code TODO List"'
      print blurb.rstrip()
      print '*/'
  if rapicorn_bug_lists:
    # ('/** @page bug_lists Bug Lists', ' * @section %s %s' % (sanitize_ident (filename), filename), '*/')
    for filename, blurb in rapicorn_bug_lists.items():
      ident = sanitize_ident (filename)
      filename = filename[2:] if filename.startswith ('./') else filename
      print '/** @file %s' % filename
      print '@xrefitem bug "Bugs" "Source Code BUG List"'
      print blurb.rstrip()
      print '*/'

def process_specific (filename, text):
  def is_comment (t):
    return t.startswith ('//') or t.startswith ('/*')
  cxx_splitter = r"""(
    "   (?: [^\\"] | \\  .   ) * "        # double quoted string
  | '   (?: [^\\'] | \\  .   ) * '        # single quoted string
  | /\* (?: [^*]   | \* [^/] ) * \*/      # C comment
  | //  (?: [^\n]            ) * \n       # C++ comment
  )"""
  parts = re.split (cxx_splitter, text, 0, re.MULTILINE | re.VERBOSE)
  # parts = filter (len, parts) # strip empty parts
  # concatenate code vs. comment bits
  i, l = 0, 1
  while i < len (parts):
    s = parts[i]
    isc = is_comment (s)
    i += 1
    while i < len (parts) and isc == is_comment (parts[i]):
      s += parts[i]
      i += 1
    if isc:
      process_comment (s, l)
    else:
      process_code (s, l)
    l += s.count ('\n')

process_start()
# print '/// @file'
for iline in sys.stdin.readlines():
  filename = iline.rstrip() # removes trailing '\n'
  process_specific (filename, open (filename, 'r').read())
process_end()
<commit_msg>DOCS: allow 'todo' and 'bug' subpage commands<commit_after># Licensed GNU LGPL v3 or later: http://www.gnu.org/licenses/lgpl.html
import sys, re

rapicorn_debug_items, rapicorn_debug_items_set = '', set()
rapicorn_debug_keys, rapicorn_debug_keys_set = '', set()
rapicorn_flippers, rapicorn_flippers_set = '', set()
rapicorn_todo_lists = {}
rapicorn_bug_lists = {}

def process_start ():
  pass

def process_comment (txt, lines):
  if not txt.startswith (('///', '/**', '/*!')):
    return      # filter non-doxygen comments
  # @TODO @TODOS
  global rapicorn_todo_lists
  pattern = r'[@\\] TODO[Ss]? \s* ( : )'
  match = re.search (pattern, txt, re.MULTILINE | re.VERBOSE)
  if match:
    text = txt[match.end (1):].strip()                  # take todo text, whitespace-stripped
    text = text[:-2] if text.endswith ('*/') else text  # strip comment-closing
    text = re.sub (r'( ^ | \n) \s* \*+',                # match comment prefix at line start
                   r'\1', text, 0, re.X | re.M)         # strip comment prefix from all lines
    pattern = r'\s* ( [*+-] | [0-9]+ \. )'              # pattern for list bullet
    if not re.match (pattern, text, re.X):              # not a list
      l = lines + txt[0:match.start (1)].count ('\n')
      text = ' - %d: @b TODO: ' % l + text              # insert list bullet
    else:
      text = '%d: @b TODOS:\n' % lines + text
    blurb = rapicorn_todo_lists.get (filename, '')
    blurb += text.rstrip() + '\n'
    rapicorn_todo_lists[filename] = blurb
  # @BUG @BUGS
  global rapicorn_bug_lists
  pattern = r'[@\\] BUG[Ss]? \s* ( : )'
  match = re.search (pattern, txt, re.MULTILINE | re.VERBOSE)
  if match:
    text = txt[match.end (1):].strip()                  # take bug text, whitespace-stripped
    text = text[:-2] if text.endswith ('*/') else text  # strip comment-closing
    text = re.sub (r'( ^ | \n) \s* \*+',                # match comment prefix at line start
                   r'\1', text, 0, re.X | re.M)         # strip comment prefix from all lines
    pattern = r'\s* ( [*+-] | [0-9]+ \. )'              # pattern for list bullet
    if not re.match (pattern, text, re.X):              # not a list
      l = lines + txt[0:match.start (1)].count ('\n')
      text = ' - %d: @b BUG: ' % l + text               # insert list bullet
    else:
      text = '%d: @b BUGS:\n' % lines + text
    blurb = rapicorn_bug_lists.get (filename, '')
    blurb += text.rstrip() + '\n'
    rapicorn_bug_lists[filename] = blurb

def process_code (txt, lines):
  cstring = r' " ( (?: [^\\"] | \\ .) * ) " '
  # RAPICORN_DEBUG_OPTION (option, blurb)
  global rapicorn_debug_items, rapicorn_debug_items_set
  pattern = r'\b RAPICORN_DEBUG_OPTION \s* \( \s* ' + cstring + r' \s* , \s* ' + cstring + ' \s* \) \s* ;'
  matches = re.findall (pattern, txt, re.MULTILINE | re.VERBOSE)
  for m in matches:
    if not m[0] in rapicorn_debug_items_set:
      rapicorn_debug_items += '  * - @c %s - %s\n' % (m[0], m[1])
      rapicorn_debug_items_set.add (m[0])
  # RAPICORN_KEY_DEBUG (keys, ...)
  global rapicorn_debug_keys, rapicorn_debug_keys_set
  pattern = r'\b RAPICORN_KEY_DEBUG \s* \( \s* ' + cstring + r' \s* , \s* [^;] *? \)'
  matches = re.findall (pattern, txt, re.MULTILINE | re.VERBOSE)
  for m in matches:
    if not m in rapicorn_debug_keys_set:
      rapicorn_debug_keys += '  * - @c %s - Debugging message key, =1 enable, =0 disable.\n' % m
      rapicorn_debug_keys_set.add (m)
  # RAPICORN_FLIPPER (option, blurb)
  global rapicorn_flippers, rapicorn_flippers_set
  pattern = r'\b RAPICORN_FLIPPER \s* \( \s* ' + cstring + r' \s* , \s* ' + cstring + ' \s* \) \s* ;'
  matches = re.findall (pattern, txt, re.MULTILINE | re.VERBOSE)
  for m in matches:
    if not m[0] in rapicorn_flippers_set:
      rapicorn_flippers += '  * - @c %s - %s\n' % (m[0], m[1])
      rapicorn_flippers_set.add (m[0])

def sanitize_ident (txt):
  return re.sub (r'[^0-9_a-zA-Z]+', '_', txt).strip ('_')

def process_end ():
  if rapicorn_debug_items:
    print '/** @var $RAPICORN_DEBUG'
    print rapicorn_debug_items, '*/'
  if rapicorn_debug_keys:
    print '/** @var $RAPICORN_DEBUG'
    print rapicorn_debug_keys, '*/'
  if rapicorn_flippers:
    print '/** @var $RAPICORN_FLIPPER'
    print rapicorn_flippers, '*/'
  if rapicorn_todo_lists:
    # ('/** @page todo_lists Todo Lists', ' * @section %s %s' % (sanitize_ident (filename), filename), '*/')
    for filename, blurb in rapicorn_todo_lists.items():
      ident = sanitize_ident (filename)
      filename = filename[2:] if filename.startswith ('./') else filename
      print '/** @file %s' % filename
      print '@xrefitem todo "Todos" "Source Code TODO List"'
      print blurb.rstrip()
      print '*/'
  print '/**\n@page todo Source Code TODO List\n*/' # needed to refer to 'todo' page with @subpage
  if rapicorn_bug_lists:
    # ('/** @page bug_lists Bug Lists', ' * @section %s %s' % (sanitize_ident (filename), filename), '*/')
    for filename, blurb in rapicorn_bug_lists.items():
      ident = sanitize_ident (filename)
      filename = filename[2:] if filename.startswith ('./') else filename
      print '/** @file %s' % filename
      print '@xrefitem bug "Bugs" "Source Code BUG List"'
      print blurb.rstrip()
      print '*/'
    print '/**\n@page bug Source Code BUG List\n*/' # needed to refer to 'bug' page with @subpage

def process_specific (filename, text):
  def is_comment (t):
    return t.startswith ('//') or t.startswith ('/*')
  cxx_splitter = r"""(
    "   (?: [^\\"] | \\  .   ) * "        # double quoted string
  | '   (?: [^\\'] | \\  .   ) * '        # single quoted string
  | /\* (?: [^*]   | \* [^/] ) * \*/      # C comment
  | //  (?: [^\n]            ) * \n       # C++ comment
  )"""
  parts = re.split (cxx_splitter, text, 0, re.MULTILINE | re.VERBOSE)
  # parts = filter (len, parts) # strip empty parts
  # concatenate code vs. comment bits
  i, l = 0, 1
  while i < len (parts):
    s = parts[i]
    isc = is_comment (s)
    i += 1
    while i < len (parts) and isc == is_comment (parts[i]):
      s += parts[i]
      i += 1
    if isc:
      process_comment (s, l)
    else:
      process_code (s, l)
    l += s.count ('\n')

process_start()
# print '/// @file'
for iline in sys.stdin.readlines():
  filename = iline.rstrip() # removes trailing '\n'
  process_specific (filename, open (filename, 'r').read())
process_end()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Group modules and treat them as a single one.

This can be useful for example when adding modules to a group and you wish two
modules to be shown at the same time.

By adding the `{button}` placeholder in the format you can enable a toggle
button to hide or show the content.

Configuration parameters:
    button_toggle: Button used to toggle if one in format.
        Setting to None disables (default 1)
    format: Display format to use (default '{output}')
    format_button_closed: Format for the button when frame open (default '+')
    format_button_open: Format for the button when frame closed (default '-')
    format_separator: Specify separator between contents.
        If this is None then the default i3bar separator will be used
        (default None)
    open: If button then the frame can be set to be open or close
        (default True)

Format placeholders:
    {button} If used a button will be used that can be clicked to hide/show
        the contents of the frame.
    {output} The output of the modules in the frame

Examples:
```
# A frame showing times in different cities.
# We also have a button to hide/show the content
frame time {
    format = '{output}{button}'
    format_separator = ' '  # have space instead of usual i3bar separator

    tztime la {
        format = "LA %H:%M"
        timezone = "America/Los_Angeles"
    }
    tztime ny {
        format = "NY %H:%M"
        timezone = "America/New_York"
    }
    tztime du {
        format = "DU %H:%M"
        timezone = "Asia/Dubai"
    }
}

# Define a group which shows volume and battery info or the current time.
# The frame, volume_status and battery_level modules are named to prevent
# them clashing with any other defined modules of the same type.
group {
    frame {
        volume_status {}
        battery_level {}
    }
    time {}
}
```

@author tobes

SAMPLE OUTPUT
[
    {'full_text': u'module 1', 'separator': True},
    {'full_text': u'module 2', 'separator': True},
    {'full_text': u'module 3', 'separator': True},
    {'full_text': u'-', 'separator': True}
]

closed
{'full_text': u'+'}

"""

class Py3status:
    """
    """

    # available configuration parameters
    button_toggle = 1
    format = "{output}"
    format_button_closed = u"+"
    format_button_open = u"-"
    format_separator = None
    open = True

    class Meta:
        container = True

    def post_config_hook(self):
        self.urgent = False
        if not self.py3.format_contains(self.format, "button"):
            self.open = True
        self.py3.register_function("urgent_function", self._urgent_function)

    def _urgent_function(self, module_list):
        self.urgent = True

    def frame(self):

        if not self.items:
            return {"full_text": "", "cached_until": self.py3.CACHE_FOREVER}

        # get the child modules output.
        output = []
        if self.open:
            for item in self.items:
                out = self.py3.get_output(item)[:]
                if self.format_separator is None:
                    if out and "separator" not in out[-1]:
                        # we copy the item as we do not want to change the
                        # original.
                        last_item = out[-1].copy()
                        last_item["separator"] = True
                        out[-1] = last_item
                else:
                    if self.format_separator:
                        out += [{"full_text": self.format_separator}]
                output += out
            urgent = False

            # Remove last separator
            if self.format_separator:
                output = output[:-1]
        else:
            urgent = self.urgent

        if self.py3.format_contains(self.format, "button"):
            if self.open:
                format_control = self.format_button_open
            else:
                format_control = self.format_button_closed

            button = {"full_text": format_control, "index": "button"}
        else:
            button = None

        composites = {
            "output": self.py3.composite_create(output),
            "button": self.py3.composite_create(button),
        }
        output = self.py3.safe_format(self.format, composites)
        response = {"cached_until": self.py3.CACHE_FOREVER, "full_text": output}

        if urgent:
            response["urgent"] = urgent

        return response

    def on_click(self, event):
        """
        Switch the displayed module or pass the event on to the active module
        """
        if event["button"] == self.button_toggle:
            # we only toggle if button was used
            if event.get("index") == "button" and self.py3.is_my_event(event):
                self.urgent = False
                self.open = not self.open

if __name__ == "__main__":
    """
    Run module in test mode.
    """
    from py3status.module_test import module_test

    module_test(Py3status)
<commit_msg>frame: add safeformat for the separator (#1746)<commit_after># -*- coding: utf-8 -*-
"""
Group modules and treat them as a single one.

This can be useful for example when adding modules to a group and you wish two
modules to be shown at the same time.

By adding the `{button}` placeholder in the format you can enable a toggle
button to hide or show the content.

Configuration parameters:
    button_toggle: Button used to toggle if one in format.
        Setting to None disables (default 1)
    format: Display format to use (default '{output}')
    format_button_closed: Format for the button when frame open (default '+')
    format_button_open: Format for the button when frame closed (default '-')
    format_separator: Specify separator between contents.
        If this is None then the default i3bar separator will be used
        (default None)
    open: If button then the frame can be set to be open or close
        (default True)

Format placeholders:
    {button} If used a button will be used that can be clicked to hide/show
        the contents of the frame.
    {output} The output of the modules in the frame

Examples:
```
# A frame showing times in different cities.
# We also have a button to hide/show the content
frame time {
    format = '{output}{button}'
    format_separator = ' '  # have space instead of usual i3bar separator

    tztime la {
        format = "LA %H:%M"
        timezone = "America/Los_Angeles"
    }
    tztime ny {
        format = "NY %H:%M"
        timezone = "America/New_York"
    }
    tztime du {
        format = "DU %H:%M"
        timezone = "Asia/Dubai"
    }
}

# Define a group which shows volume and battery info or the current time.
# The frame, volume_status and battery_level modules are named to prevent
# them clashing with any other defined modules of the same type.
group {
    frame {
        volume_status {}
        battery_level {}
    }
    time {}
}
```

@author tobes

SAMPLE OUTPUT
[
    {'full_text': u'module 1', 'separator': True},
    {'full_text': u'module 2', 'separator': True},
    {'full_text': u'module 3', 'separator': True},
    {'full_text': u'-', 'separator': True}
]

closed
{'full_text': u'+'}

"""

class Py3status:
    """
    """

    # available configuration parameters
    button_toggle = 1
    format = "{output}"
    format_button_closed = u"+"
    format_button_open = u"-"
    format_separator = None
    open = True

    class Meta:
        container = True

    def post_config_hook(self):
        self.urgent = False
        if not self.py3.format_contains(self.format, "button"):
            self.open = True
        self.py3.register_function("urgent_function", self._urgent_function)

    def _urgent_function(self, module_list):
        self.urgent = True

    def frame(self):

        if not self.items:
            return {"full_text": "", "cached_until": self.py3.CACHE_FOREVER}

        # get the child modules output.
        output = []
        if self.open:
            if self.format_separator:
                format_separator = self.py3.safe_format(self.format_separator)
            for item in self.items:
                out = self.py3.get_output(item)[:]
                if self.format_separator is None:
                    if out and "separator" not in out[-1]:
                        # we copy the item as we do not want to change the
                        # original.
                        last_item = out[-1].copy()
                        last_item["separator"] = True
                        out[-1] = last_item
                elif self.format_separator:
                    out += format_separator
                output += out
            urgent = False

            # Remove last separator
            if self.format_separator:
                output = output[:-1]
        else:
            urgent = self.urgent

        if self.py3.format_contains(self.format, "button"):
            if self.open:
                format_control = self.format_button_open
            else:
                format_control = self.format_button_closed

            button = {"full_text": format_control, "index": "button"}
        else:
            button = None

        composites = {
            "output": self.py3.composite_create(output),
            "button": self.py3.composite_create(button),
        }
        output = self.py3.safe_format(self.format, composites)
        response = {"cached_until": self.py3.CACHE_FOREVER, "full_text": output}

        if urgent:
            response["urgent"] = urgent

        return response

    def on_click(self, event):
        """
        Switch the displayed module or pass the event on to the active module
        """
        if event["button"] == self.button_toggle:
            # we only toggle if button was used
            if event.get("index") == "button" and self.py3.is_my_event(event):
                self.urgent = False
                self.open = not self.open

if __name__ == "__main__":
    """
    Run module in test mode.
    """
    from py3status.module_test import module_test

    module_test(Py3status)
<|endoftext|>


<commit_before>"""
Task decorator (wrapper for `fabric3` @task decorator)
"""

from __future__ import with_statement
import timeit
import fabric.api
from py_mina.echo import echo_task, print_task_stats

def task(wrapped_function):
	"""
	Task function decorator
	"""

	wrapped_function_name = wrapped_function.__name__

	def task_wrapper(*args):
		"""
		Runs task and prints stats at the end
		"""

		echo_task('Running "%s" task' % wrapped_function_name)

		start_time = timeit.default_timer()

		with fabric.api.settings(colorize_errors=True):
			try:
				wrapped_function(*args)
			except Exception as e:
				print_task_stats(wrapped_function_name, start_time, e)

				raise e # escalate exception
			else:
				print_task_stats(wrapped_function_name, start_time)

	# Copy __name__ and __doc__ from decorated function to wrapper function
	task_wrapper.__name__ = wrapped_function_name or 'task'
	if wrapped_function.__doc__: task_wrapper.__doc__ = wrapped_function.__doc__

	# Decorate with `fabric3` task decorator
	return fabric.api.task(task_wrapper)
<commit_msg>style(cli): New line when echoing task name<commit_after>"""
Task decorator (wrapper for `fabric3` @task decorator)
"""

from __future__ import with_statement
import timeit
import fabric.api
from py_mina.echo import echo_task, print_task_stats

def task(wrapped_function):
	"""
	Task function decorator
	"""

	wrapped_function_name = wrapped_function.__name__

	def task_wrapper(*args):
		"""
		Runs task and prints stats at the end
		"""

		echo_task('Running "%s" task\n' % wrapped_function_name)

		start_time = timeit.default_timer()

		with fabric.api.settings(colorize_errors=True):
			try:
				wrapped_function(*args)
			except Exception as e:
				print_task_stats(wrapped_function_name, start_time, e)

				raise e # escalate exception
			else:
				print_task_stats(wrapped_function_name, start_time)

	# Copy __name__ and __doc__ from decorated function to wrapper function
	task_wrapper.__name__ = wrapped_function_name or 'task'
	if wrapped_function.__doc__: task_wrapper.__doc__ = wrapped_function.__doc__

	# Decorate with `fabric3` task decorator
	return fabric.api.task(task_wrapper)
<|endoftext|>