<commit_before>#!/usr/bin/python

# Contains class information for sensor nodes.
# Each plant is treated as a base, and each plant contains multiple sensors.
# Basic usage:
#   Create a plant record using:
#      plant1 = Plant(temp_addr, humidity_addr, lux_addr, adc_addr)
#   Updating individual sensor values can be done with
# Note that SMBus must be imported and initiated
#   in order to use these classes.
import smbus
import app.models as models
from control import ControlCluster
from i2c_utility import TCA_select, get_ADC_value, import_i2c_addr
from i2c_utility import IO_expander_output, get_IO_reg
from time import sleep, time  # needed to force a delay in humidity module
from math import e

class IterList(type):
    """ Metaclass for iterating over sensor objects in a _list
    """
    def __iter__(cls):
        return iter(cls._list)

class SensorCluster(object):
    'Base class for each individual plant containing sensor info'
    __metaclass__ = IterList
    _list = []
    analog_power_pin = 0
    power_bank = 0  # bank and pin used to toggle analog sensor power
    temp_addr = 0x48
    temp_chan = 3
    humidity_addr = 0x27
    humidity_chan = 1
    lux_addr = 0x39
    lux_chan = 0
    adc_addr = 0x68
    adc_chan = 2
    moisture_chan = 1
    tank_adc_adr = 0x6c
    tank_adc_chan = 0
    bus = None

    def __init__(self, ID, mux_addr=None):

        # Initializes cluster, enumeration, and sets up address info
        sensor_addr = import_i2c_addr(SensorCluster.bus)
        if (ID < 1 or ID > len(sensor_addr)):
            raise I2CBusError("Plant ID out of range.")
        self.mux_addr = mux_addr or (sensor_addr[ID-1])
        self.ID = ID  # Plant number specified by caller
        self.temp = 0
        self.humidity = 0
        self.lux = 0
        self.soil_moisture = 0
        self.acidity = 0
        self.timestamp = time()  # record time at instantiation
        self._list.append(self)
        self.update_count = 0

    def update_lux(self, extend=0):
        """ Communicates with the TSL2550D light sensor and returns a
            lux value.

        Note that this method contains approximately 1 second of total delay.
            This delay is necessary in order to obtain full resolution
            compensated lux values.

        Alternatively, the device could be put in extended mode,
            which drops some resolution in favor of shorter delays.

        """
        DEVICE_REG_OUT = 0x1d
        LUX_PWR_ON = 0x03
        if extend == 1:
            LUX_MODE = 0x1d
            delay = .08
            scale = 5
        else:
            LUX_MODE = 0x18
            delay = .4
            scale = 1
        LUX_READ_CH0 = 0x43
        LUX_READ_CH1 = 0x83
        # Select correct I2C mux channel on TCA module

        TCA_select(SensorCluster.bus, self.mux_addr, SensorCluster.lux_chan)
        # Make sure lux sensor is powered up.
        SensorCluster.bus.write_byte(SensorCluster.lux_addr, LUX_PWR_ON)
        lux_on = SensorCluster.bus.read_byte_data(SensorCluster.lux_addr, LUX_PWR_ON)

        # Check for successful powerup
        if (lux_on == LUX_PWR_ON):
            # Send command to initiate ADC on each channel
            # Read each channel after the new data is ready
            SensorCluster.bus.write_byte(SensorCluster.lux_addr, LUX_MODE)
            SensorCluster.bus.write_byte(SensorCluster.lux_addr, LUX_READ_CH0)
            sleep(delay)
            adc_ch0 = SensorCluster.bus.read_byte(SensorCluster.lux_addr)
            count0 = get_lux_count(adc_ch0) * scale  # 5x for extended mode
            SensorCluster.bus.write_byte(SensorCluster.lux_addr, LUX_READ_CH1)
            sleep(delay)
            adc_ch1 = SensorCluster.bus.read_byte(SensorCluster.lux_addr)
            count1 = get_lux_count(adc_ch1) * scale  # 5x for extended mode
            ratio = count1 / (count0 - count1)
            lux = (count0 - count1) * .39 * e**(-.181 * (ratio**2))
            self.lux = round(lux, 3)
            return TCA_select(SensorCluster.bus, self.mux_addr, "off")
        else:
            raise SensorError("The lux sensor is powered down.")

    def update_humidity_temp(self):
        """ This method utilizes the HIH7xxx sensor to read
            humidity and temperature in one call.
        """
        # Create mask for STATUS (first two bits of 64 bit wide result)
        STATUS = 0b11 << 6

        TCA_select(SensorCluster.bus, self.mux_addr, SensorCluster.humidity_chan)
        SensorCluster.bus.write_quick(SensorCluster.humidity_addr)  # Begin conversion
        sleep(.25)
        # wait 100ms to make sure the conversion takes place.
        data = SensorCluster.bus.read_i2c_block_data(SensorCluster.humidity_addr, 0, 4)
        status = (data[0] & STATUS) >> 6

        if status == 0 or status == 1:  # will always pass for now.
            humidity = round((((data[0] & 0x3f) << 8) |
                              data[1]) * 100.0 / (2**14 - 2), 3)
            self.humidity = humidity
            self.temp = (round((((data[2] << 6) + ((data[3] & 0xfc) >> 2))
                               * 165.0 / 16382.0 - 40.0), 3) * 9/5) + 32
            return TCA_select(SensorCluster.bus, self.mux_addr, "off")
        else:
            raise I2CBusError("Unable to retrieve humidity")

    def update_soil_moisture(self):
        """ Method will select the ADC module,
                turn on the analog sensor, wait for voltage settle,
                and then digitize the sensor voltage.
            Voltage division/signal loss is accounted for by
                scaling up the sensor output.
                This may need to be adjusted if a different sensor is used
        """
        SensorCluster.analog_sensor_power(SensorCluster.bus, "on")  # turn on sensor
        sleep(.2)
        TCA_select(SensorCluster.bus, self.mux_addr, SensorCluster.adc_chan)
        moisture = get_ADC_value(
            SensorCluster.bus, SensorCluster.adc_addr, SensorCluster.moisture_chan)
        status = TCA_select(SensorCluster.bus, self.mux_addr, "off")  # Turn off mux.
        SensorCluster.analog_sensor_power(SensorCluster.bus, "off")  # turn off sensor
        if (moisture >= 0):
            soil_moisture = moisture/2.048 # Scale to a percentage value
            self.soil_moisture = round(soil_moisture,3)
        else:
            raise SensorError(
                "The soil moisture meter is not configured correctly.")
        return status

    def update_instance_sensors(self, opt=None):

        """ Method runs through all sensor modules and updates
            to the latest sensor values.
        After running through each sensor module,
        The sensor head (the I2C multiplexer), is disabled
        in order to avoid address conflicts.
        Usage:
            plant_sensor_object.updateAllSensors(bus_object)
        """
        self.update_count += 1
        self.update_lux()
        self.update_humidity_temp()
        if opt == "all":
            try:
                self.update_soil_moisture()
            except SensorError:
                # This could be handled with a repeat request later.
                pass
        self.timestamp = time()
        # disable sensor module

        tca_status = TCA_select(SensorCluster.bus, self.mux_addr, "off")
        if tca_status != 0:
            raise I2CBusError(
                "Bus multiplexer was unable to switch off to prevent conflicts")

    def sensor_values(self):
        """
        Returns the values of all sensors for this cluster
        """
        self.update_instance_sensors(opt="all")
        return {
            "light": self.lux,
            "water": self.soil_moisture,
            "humidity": self.humidity,
            "temperature": self.temp
        }

    @classmethod
    def update_all_sensors(cls, opt=None):
        """ Method iterates over all SensorCluster objects and updates
            each sensor value and saves the values to the plant record.
                - Note that it must receive an open bus object.
            Usage:
            Update all sensors exluding analog sensors that need power.
            - update_all_sensors()

            Update all sensors including soil moisture.
            - update_all_sensors("all")

        """
        for sensorobj in cls:
            sensorobj.update_instance_sensors(opt)

    @classmethod
    def analog_sensor_power(cls, bus, operation):
        """ Method that turns on all of the analog sensor modules
            Includes all attached soil moisture sensors
            Note that all of the SensorCluster object should be attached
                in parallel and only 1 GPIO pin is available
                to toggle analog sensor power.
            The sensor power should be left on for at least 100ms
                in order to allow the sensors to stabilize before reading.
                Usage:  SensorCluster.analog_sensor_power(bus,"high")
                OR      SensorCluster.analog_sensor_power(bus,"low")
            This method should be removed if an off-board GPIO extender is used.
        """
        # Set appropriate analog sensor power bit in GPIO mask
        # using the ControlCluster bank_mask to avoid overwriting any data
        reg_data = get_IO_reg(bus, 0x20, cls.power_bank)

        if operation == "on":
            reg_data = reg_data | 1 << cls.analog_power_pin
        elif operation == "off":
            reg_data = reg_data & (0b11111111 ^ (1 << cls.analog_power_pin))
        else:
            raise SensorError(
                "Invalid command used while enabling analog sensors")
        # Send updated IO mask to output
        IO_expander_output(bus, 0x20, cls.power_bank, reg_data)

    @classmethod
    def get_water_level(cls):
        """ This method uses the ADC on the control module to measure
            the current water tank level and returns the water volume
            remaining in the tank.

            For this method, it is assumed that a simple voltage divider
            is used to interface the sensor to the ADC module.

            Testing shows that the sensor response is not completely linear,
                though it is quite close. To make the results more accurate,
                a mapping method approximated by a linear fit to data is used.
        """
        # ----------
        # These values should be updated based on the real system parameters
        vref = 4.95
        tank_height = 17.5 # in centimeters (height of container)
        rref = 2668  # Reference resistor
        # ----------
        val = 0
        for i in range(5):
            # Take five readings and do an average
            # Fetch value from ADC (0x69 - ch1)
            val = get_ADC_value(cls.bus, 0x6c, 1) + val
        avg = val / 5
        water_sensor_res = rref * avg/(vref - avg)
        depth_cm = water_sensor_res * \
                    (-.0163) + 28.127 # measured transfer adjusted offset
        if depth_cm < 1.0: # Below 1cm, the values should not be trusted.
            depth_cm = 0
        cls.water_remaining = depth_cm / tank_height
        # Return the current depth in case the user is interested in
        #   that parameter alone. (IE for automatic shut-off)
        return depth_cm/tank_height

def get_lux_count(lux_byte):
    """ Method to convert data from the TSL2550D lux sensor
    into more easily usable ADC count values.

    """
    LUX_VALID_MASK = 0b10000000
    LUX_CHORD_MASK = 0b01110000
    LUX_STEP_MASK = 0b00001111
    valid = lux_byte & LUX_VALID_MASK
    if valid != 0:
        step_num = (lux_byte & LUX_STEP_MASK)
        # Shift to normalize value
        chord_num = (lux_byte & LUX_CHORD_MASK) >> 4
        step_val = 2**chord_num
        chord_val = int(16.5 * (step_val - 1))
        count = chord_val + step_val * step_num
        return count
    else:
        raise SensorError("Invalid lux sensor data.")

class SensorError(Exception):
    """ Non-fatal
        Implies that a sensor is either turned off
            or unplugged from its slot.
        All I2C objects within the sensor cluster should
            be turned off before doing anything else.
    """
    pass

class I2CBusError(Exception):
    """ Typically fatal
        - Something on the bus has become unresponsive.
        - Should occur if the I2C multiplexer is not disabled
            after successive updates.
    """
    pass
<commit_msg>Models should stop being there now.<commit_after>#!/usr/bin/python

# Contains class information for sensor nodes.
# Each plant is treated as a base, and each plant contains multiple sensors.
# Basic usage:
#   Create a plant record using:
#      plant1 = Plant(temp_addr, humidity_addr, lux_addr, adc_addr)
#   Updating individual sensor values can be done with
# Note that SMBus must be imported and initiated
#   in order to use these classes.
import smbus
from control import ControlCluster
from i2c_utility import TCA_select, get_ADC_value, import_i2c_addr
from i2c_utility import IO_expander_output, get_IO_reg
from time import sleep, time  # needed to force a delay in humidity module
from math import e

class IterList(type):
    """ Metaclass for iterating over sensor objects in a _list
    """
    def __iter__(cls):
        return iter(cls._list)

class SensorCluster(object):
    'Base class for each individual plant containing sensor info'
    __metaclass__ = IterList
    _list = []
    analog_power_pin = 0
    power_bank = 0  # bank and pin used to toggle analog sensor power
    temp_addr = 0x48
    temp_chan = 3
    humidity_addr = 0x27
    humidity_chan = 1
    lux_addr = 0x39
    lux_chan = 0
    adc_addr = 0x68
    adc_chan = 2
    moisture_chan = 1
    tank_adc_adr = 0x6c
    tank_adc_chan = 0
    bus = None

    def __init__(self, ID, mux_addr=None):

        # Initializes cluster, enumeration, and sets up address info
        sensor_addr = import_i2c_addr(SensorCluster.bus)
        if (ID < 1 or ID > len(sensor_addr)):
            raise I2CBusError("Plant ID out of range.")
        self.mux_addr = mux_addr or (sensor_addr[ID-1])
        self.ID = ID  # Plant number specified by caller
        self.temp = 0
        self.humidity = 0
        self.lux = 0
        self.soil_moisture = 0
        self.acidity = 0
        self.timestamp = time()  # record time at instantiation
        self._list.append(self)
        self.update_count = 0

    def update_lux(self, extend=0):
        """ Communicates with the TSL2550D light sensor and returns a
            lux value.

        Note that this method contains approximately 1 second of total delay.
            This delay is necessary in order to obtain full resolution
            compensated lux values.

        Alternatively, the device could be put in extended mode,
            which drops some resolution in favor of shorter delays.

        """
        DEVICE_REG_OUT = 0x1d
        LUX_PWR_ON = 0x03
        if extend == 1:
            LUX_MODE = 0x1d
            delay = .08
            scale = 5
        else:
            LUX_MODE = 0x18
            delay = .4
            scale = 1
        LUX_READ_CH0 = 0x43
        LUX_READ_CH1 = 0x83
        # Select correct I2C mux channel on TCA module

        TCA_select(SensorCluster.bus, self.mux_addr, SensorCluster.lux_chan)
        # Make sure lux sensor is powered up.
        SensorCluster.bus.write_byte(SensorCluster.lux_addr, LUX_PWR_ON)
        lux_on = SensorCluster.bus.read_byte_data(SensorCluster.lux_addr, LUX_PWR_ON)

        # Check for successful powerup
        if (lux_on == LUX_PWR_ON):
            # Send command to initiate ADC on each channel
            # Read each channel after the new data is ready
            SensorCluster.bus.write_byte(SensorCluster.lux_addr, LUX_MODE)
            SensorCluster.bus.write_byte(SensorCluster.lux_addr, LUX_READ_CH0)
            sleep(delay)
            adc_ch0 = SensorCluster.bus.read_byte(SensorCluster.lux_addr)
            count0 = get_lux_count(adc_ch0) * scale  # 5x for extended mode
            SensorCluster.bus.write_byte(SensorCluster.lux_addr, LUX_READ_CH1)
            sleep(delay)
            adc_ch1 = SensorCluster.bus.read_byte(SensorCluster.lux_addr)
            count1 = get_lux_count(adc_ch1) * scale  # 5x for extended mode
            ratio = count1 / (count0 - count1)
            lux = (count0 - count1) * .39 * e**(-.181 * (ratio**2))
            self.lux = round(lux, 3)
            return TCA_select(SensorCluster.bus, self.mux_addr, "off")
        else:
            raise SensorError("The lux sensor is powered down.")

    def update_humidity_temp(self):
        """ This method utilizes the HIH7xxx sensor to read
            humidity and temperature in one call.
        """
        # Create mask for STATUS (first two bits of 64 bit wide result)
        STATUS = 0b11 << 6

        TCA_select(SensorCluster.bus, self.mux_addr, SensorCluster.humidity_chan)
        SensorCluster.bus.write_quick(SensorCluster.humidity_addr)  # Begin conversion
        sleep(.25)
        # wait 100ms to make sure the conversion takes place.
        data = SensorCluster.bus.read_i2c_block_data(SensorCluster.humidity_addr, 0, 4)
        status = (data[0] & STATUS) >> 6

        if status == 0 or status == 1:  # will always pass for now.
            humidity = round((((data[0] & 0x3f) << 8) |
                              data[1]) * 100.0 / (2**14 - 2), 3)
            self.humidity = humidity
            self.temp = (round((((data[2] << 6) + ((data[3] & 0xfc) >> 2))
                               * 165.0 / 16382.0 - 40.0), 3) * 9/5) + 32
            return TCA_select(SensorCluster.bus, self.mux_addr, "off")
        else:
            raise I2CBusError("Unable to retrieve humidity")

    def update_soil_moisture(self):
        """ Method will select the ADC module,
                turn on the analog sensor, wait for voltage settle,
                and then digitize the sensor voltage.
            Voltage division/signal loss is accounted for by
                scaling up the sensor output.
                This may need to be adjusted if a different sensor is used
        """
        SensorCluster.analog_sensor_power(SensorCluster.bus, "on")  # turn on sensor
        sleep(.2)
        TCA_select(SensorCluster.bus, self.mux_addr, SensorCluster.adc_chan)
        moisture = get_ADC_value(
            SensorCluster.bus, SensorCluster.adc_addr, SensorCluster.moisture_chan)
        status = TCA_select(SensorCluster.bus, self.mux_addr, "off")  # Turn off mux.
        SensorCluster.analog_sensor_power(SensorCluster.bus, "off")  # turn off sensor
        if (moisture >= 0):
            soil_moisture = moisture/2.048 # Scale to a percentage value
            self.soil_moisture = round(soil_moisture,3)
        else:
            raise SensorError(
                "The soil moisture meter is not configured correctly.")
        return status

    def update_instance_sensors(self, opt=None):

        """ Method runs through all sensor modules and updates
            to the latest sensor values.
        After running through each sensor module,
        The sensor head (the I2C multiplexer), is disabled
        in order to avoid address conflicts.
        Usage:
            plant_sensor_object.updateAllSensors(bus_object)
        """
        self.update_count += 1
        self.update_lux()
        self.update_humidity_temp()
        if opt == "all":
            try:
                self.update_soil_moisture()
            except SensorError:
                # This could be handled with a repeat request later.
                pass
        self.timestamp = time()
        # disable sensor module

        tca_status = TCA_select(SensorCluster.bus, self.mux_addr, "off")
        if tca_status != 0:
            raise I2CBusError(
                "Bus multiplexer was unable to switch off to prevent conflicts")

    def sensor_values(self):
        """
        Returns the values of all sensors for this cluster
        """
        self.update_instance_sensors(opt="all")
        return {
            "light": self.lux,
            "water": self.soil_moisture,
            "humidity": self.humidity,
            "temperature": self.temp
        }

    @classmethod
    def update_all_sensors(cls, opt=None):
        """ Method iterates over all SensorCluster objects and updates
            each sensor value and saves the values to the plant record.
                - Note that it must receive an open bus object.
            Usage:
            Update all sensors exluding analog sensors that need power.
            - update_all_sensors()

            Update all sensors including soil moisture.
            - update_all_sensors("all")

        """
        for sensorobj in cls:
            sensorobj.update_instance_sensors(opt)

    @classmethod
    def analog_sensor_power(cls, bus, operation):
        """ Method that turns on all of the analog sensor modules
            Includes all attached soil moisture sensors
            Note that all of the SensorCluster object should be attached
                in parallel and only 1 GPIO pin is available
                to toggle analog sensor power.
            The sensor power should be left on for at least 100ms
                in order to allow the sensors to stabilize before reading.
                Usage:  SensorCluster.analog_sensor_power(bus,"high")
                OR      SensorCluster.analog_sensor_power(bus,"low")
            This method should be removed if an off-board GPIO extender is used.
        """
        # Set appropriate analog sensor power bit in GPIO mask
        # using the ControlCluster bank_mask to avoid overwriting any data
        reg_data = get_IO_reg(bus, 0x20, cls.power_bank)

        if operation == "on":
            reg_data = reg_data | 1 << cls.analog_power_pin
        elif operation == "off":
            reg_data = reg_data & (0b11111111 ^ (1 << cls.analog_power_pin))
        else:
            raise SensorError(
                "Invalid command used while enabling analog sensors")
        # Send updated IO mask to output
        IO_expander_output(bus, 0x20, cls.power_bank, reg_data)

    @classmethod
    def get_water_level(cls):
        """ This method uses the ADC on the control module to measure
            the current water tank level and returns the water volume
            remaining in the tank.

            For this method, it is assumed that a simple voltage divider
            is used to interface the sensor to the ADC module.

            Testing shows that the sensor response is not completely linear,
                though it is quite close. To make the results more accurate,
                a mapping method approximated by a linear fit to data is used.
        """
        # ----------
        # These values should be updated based on the real system parameters
        vref = 4.95
        tank_height = 17.5 # in centimeters (height of container)
        rref = 2668  # Reference resistor
        # ----------
        val = 0
        for i in range(5):
            # Take five readings and do an average
            # Fetch value from ADC (0x69 - ch1)
            val = get_ADC_value(cls.bus, 0x6c, 1) + val
        avg = val / 5
        water_sensor_res = rref * avg/(vref - avg)
        depth_cm = water_sensor_res * \
                    (-.0163) + 28.127 # measured transfer adjusted offset
        if depth_cm < 1.0: # Below 1cm, the values should not be trusted.
            depth_cm = 0
        cls.water_remaining = depth_cm / tank_height
        # Return the current depth in case the user is interested in
        #   that parameter alone. (IE for automatic shut-off)
        return depth_cm/tank_height

def get_lux_count(lux_byte):
    """ Method to convert data from the TSL2550D lux sensor
    into more easily usable ADC count values.

    """
    LUX_VALID_MASK = 0b10000000
    LUX_CHORD_MASK = 0b01110000
    LUX_STEP_MASK = 0b00001111
    valid = lux_byte & LUX_VALID_MASK
    if valid != 0:
        step_num = (lux_byte & LUX_STEP_MASK)
        # Shift to normalize value
        chord_num = (lux_byte & LUX_CHORD_MASK) >> 4
        step_val = 2**chord_num
        chord_val = int(16.5 * (step_val - 1))
        count = chord_val + step_val * step_num
        return count
    else:
        raise SensorError("Invalid lux sensor data.")

class SensorError(Exception):
    """ Non-fatal
        Implies that a sensor is either turned off
            or unplugged from its slot.
        All I2C objects within the sensor cluster should
            be turned off before doing anything else.
    """
    pass

class I2CBusError(Exception):
    """ Typically fatal
        - Something on the bus has become unresponsive.
        - Should occur if the I2C multiplexer is not disabled
            after successive updates.
    """
    pass
<|endoftext|>


<commit_before>import multiprocessing
import aggregate_results_to_10_km
import subprocess
import os
import glob
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

# tile_list = uu.tile_list(cn.net_flux_dir)
# tile_id_list = ['00N_100E', '00N_110E', '00N_120E'] # test tiles
tile_id_list = ['00N_110E'] # test tiles
# tile_id_list = ['00N_110E', '80N_020E', '30N_080W', '00N_020E'] # test tiles: no mangrove or planted forest, mangrove only, planted forest only, mangrove and planted forest
print tile_id_list
print "There are {} tiles to process".format(str(len(tile_id_list)))

# # For copying individual tiles to spot machine for testing
# for tile_id in tile_id_list:
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.gross_emis_all_drivers_dir, tile_id, cn.pattern_gross_emis_all_drivers), '.')
#     # uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.annual_gain_combo_dir, tile_id, cn.pattern_annual_gain_combo), '.')
#     # uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.cumul_gain_combo_dir, tile_id, cn.pattern_cumul_gain_combo), '.')
#     # uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.net_flux_dir, tile_id, cn.pattern_net_flux), '.')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.pixel_area_dir, cn.pattern_pixel_area, tile_id), '.')

tile_list = uu.tile_list_spot_machine(".", ".tif")
# from https://stackoverflow.com/questions/12666897/removing-an-item-from-list-matching-a-substring
tile_list = [i for i in tile_list if not ('hanson_2013' in i)]
tile_list = [i for i in tile_list if not ('per_pixel' in i)]
tile_list = [i for i in tile_list if not ('average' in i)]
print "Tiles to process:", tile_list

pixel_count_dict = dict.fromkeys(tile_list, 0)
print pixel_count_dict

for tile in tile_list:
    aggregate_results_to_10_km.rewindow(tile)

# tile_list = glob.glob("*retile.tif")
print tile_list
# For single processor use
for tile in tile_list:
    aggregate_results_to_10_km.convert_to_per_pixel(tile, pixel_count_dict)

# # tile_list = glob.glob("*per_pixel.tif")
# print tile_list
# # For single processor use
# for tile in tile_list:
#     aggregate_results_to_10_km.average_10km(tile)

# out_vrt = "value_per_pixel.vrt"
# os.system('gdalbuildvrt {} *per_pixel.tif'.format(out_vrt))
#
# avg_10km = "average_10_km.tif"
# cmd = ['gdalwarp', '-co', 'COMPRESS=LZW', '-tr', '0.096342599', '0.096342599', '-overwrite', '-r', 'average',
#        '-tap', out_vrt, avg_10km]
#
# subprocess.check_call(cmd)
print "Tiles processed. Uploading to s3 now..."

# Uploads all output tiles to s3
uu.upload_final_set(cn.gross_emis_all_drivers_aggreg_dir, cn.pattern_gross_emis_all_drivers_aggreg)

# # Pixel area tiles-- necessary for calculating sum of pixels for any set of tiles
# uu.s3_folder_download(cn.pixel_area_dir, '.')
#
# # For downloading all tiles in selected folders
# download_list = [cn.gross_emis_all_drivers_dir, cn.cumul_gain_combo_dir, cn.net_flux_dir, cn.pixel_area_dir]
#
# # Iterates through each set of tiles and gets statistics of it
# for input in download_list:
#
#     # uu.s3_folder_download(input, '.')
#
#     # List of all the tiles on the spot machine to be summarized (excludes pixel area tiles and tiles created by gdal_calc
#     # (in case this script was already run on this spot machine and created output from gdal_calc)
#     tile_list = uu.tile_list_spot_machine(".", ".tif")
#     # from https://stackoverflow.com/questions/12666897/removing-an-item-from-list-matching-a-substring
#     tile_list = [i for i in tile_list if not ('value_per_pixel' in i)]
#     # tile_list = ['00N_000E_biomass.tif']
#     # tile_list = download_tile_list
#     print tile_list
#
#     tile_pattern = tile_list[0][9:-4]
#
#     # # For multiprocessor use
#     # count = multiprocessing.cpu_count()
#     # pool = multiprocessing.Pool(count/4)
#     # pool.map(aggregate_results_to_10_km.aggregate_results, tile_list)
#     # # Added these in response to error12: Cannot allocate memory error.
#     # # This fix was mentioned here: of https://stackoverflow.com/questions/26717120/python-cannot-allocate-memory-using-multiprocessing-pool
#     # # Could also try this: https://stackoverflow.com/questions/42584525/python-multiprocessing-debugging-oserror-errno-12-cannot-allocate-memory
#     # pool.close()
#     # pool.join()
#
#     # For single processor use
#     for tile in tile_list:
#         aggregate_results_to_10_km.aggregate_results(tile)
#
#     print "Tiles processed. Uploading to s3 now..."
#
#     # Uploads all output tiles to s3
#     uu.upload_final_set(cn.gross_emis_all_drivers_aggreg_dir, cn.pattern_gross_emis_all_drivers_aggreg)<commit_msg>Working on aggregating output tiles to 10 km.<commit_after>import multiprocessing
import aggregate_results_to_10_km
import subprocess
import os
import glob
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

# tile_list = uu.tile_list(cn.net_flux_dir)
# tile_id_list = ['00N_100E', '00N_110E', '00N_120E'] # test tiles
tile_id_list = ['00N_110E'] # test tiles
# tile_id_list = ['00N_110E', '80N_020E', '30N_080W', '00N_020E'] # test tiles: no mangrove or planted forest, mangrove only, planted forest only, mangrove and planted forest
print tile_id_list
print "There are {} tiles to process".format(str(len(tile_id_list)))

# # For copying individual tiles to spot machine for testing
# for tile_id in tile_id_list:
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.gross_emis_all_drivers_dir, tile_id, cn.pattern_gross_emis_all_drivers), '.')
#     # uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.annual_gain_combo_dir, tile_id, cn.pattern_annual_gain_combo), '.')
#     # uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.cumul_gain_combo_dir, tile_id, cn.pattern_cumul_gain_combo), '.')
#     # uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.net_flux_dir, tile_id, cn.pattern_net_flux), '.')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.pixel_area_dir, cn.pattern_pixel_area, tile_id), '.')

tile_list = uu.tile_list_spot_machine(".", ".tif")
# from https://stackoverflow.com/questions/12666897/removing-an-item-from-list-matching-a-substring
tile_list = [i for i in tile_list if not ('hanson_2013' in i)]
tile_list = [i for i in tile_list if not ('per_pixel' in i)]
tile_list = [i for i in tile_list if not ('average' in i)]
print "Tiles to process:", tile_list

pixel_count_dict = dict.fromkeys(tile_list, 0)
print pixel_count_dict

# for tile in tile_list:
#     aggregate_results_to_10_km.rewindow(tile)

# tile_list = glob.glob("*retile.tif")
print tile_list
# For single processor use
for tile in tile_list:
    aggregate_results_to_10_km.convert_to_per_pixel(tile, pixel_count_dict)

# # tile_list = glob.glob("*per_pixel.tif")
# print tile_list
# # For single processor use
# for tile in tile_list:
#     aggregate_results_to_10_km.average_10km(tile)

# out_vrt = "value_per_pixel.vrt"
# os.system('gdalbuildvrt {} *per_pixel.tif'.format(out_vrt))
#
# avg_10km = "average_10_km.tif"
# cmd = ['gdalwarp', '-co', 'COMPRESS=LZW', '-tr', '0.096342599', '0.096342599', '-overwrite', '-r', 'average',
#        '-tap', out_vrt, avg_10km]
#
# subprocess.check_call(cmd)
print "Tiles processed. Uploading to s3 now..."

# Uploads all output tiles to s3
uu.upload_final_set(cn.gross_emis_all_drivers_aggreg_dir, cn.pattern_gross_emis_all_drivers_aggreg)

# # Pixel area tiles-- necessary for calculating sum of pixels for any set of tiles
# uu.s3_folder_download(cn.pixel_area_dir, '.')
#
# # For downloading all tiles in selected folders
# download_list = [cn.gross_emis_all_drivers_dir, cn.cumul_gain_combo_dir, cn.net_flux_dir, cn.pixel_area_dir]
#
# # Iterates through each set of tiles and gets statistics of it
# for input in download_list:
#
#     # uu.s3_folder_download(input, '.')
#
#     # List of all the tiles on the spot machine to be summarized (excludes pixel area tiles and tiles created by gdal_calc
#     # (in case this script was already run on this spot machine and created output from gdal_calc)
#     tile_list = uu.tile_list_spot_machine(".", ".tif")
#     # from https://stackoverflow.com/questions/12666897/removing-an-item-from-list-matching-a-substring
#     tile_list = [i for i in tile_list if not ('value_per_pixel' in i)]
#     # tile_list = ['00N_000E_biomass.tif']
#     # tile_list = download_tile_list
#     print tile_list
#
#     tile_pattern = tile_list[0][9:-4]
#
#     # # For multiprocessor use
#     # count = multiprocessing.cpu_count()
#     # pool = multiprocessing.Pool(count/4)
#     # pool.map(aggregate_results_to_10_km.aggregate_results, tile_list)
#     # # Added these in response to error12: Cannot allocate memory error.
#     # # This fix was mentioned here: of https://stackoverflow.com/questions/26717120/python-cannot-allocate-memory-using-multiprocessing-pool
#     # # Could also try this: https://stackoverflow.com/questions/42584525/python-multiprocessing-debugging-oserror-errno-12-cannot-allocate-memory
#     # pool.close()
#     # pool.join()
#
#     # For single processor use
#     for tile in tile_list:
#         aggregate_results_to_10_km.aggregate_results(tile)
#
#     print "Tiles processed. Uploading to s3 now..."
#
#     # Uploads all output tiles to s3
#     uu.upload_final_set(cn.gross_emis_all_drivers_aggreg_dir, cn.pattern_gross_emis_all_drivers_aggreg)<|endoftext|>


<commit_before>"""
Created on 14/giu/2013

@author: Marco Pompili
"""

from django.conf import settings
from django.utils.safestring import mark_safe
from django import template

from twython import Twython
from twython import exceptions

register = template.Library()

class TwitterTimelineNode(template.Node):
    def __init__(self, args):
        self.username = args[1]
        self.count = args[2]

        self.twitter = Twython(
            settings.TWITTER_CONSUMER_KEY,
            settings.TWITTER_CONSUMER_SECRET,
            settings.TWITTER_ACCESS_TOKEN,
            settings.TWITTER_ACCESS_TOKEN_SECRET
        )

    def render(self, context):
        try:
            timeline = self.twitter.get_user_timeline(
                screen_name=self.username,
                count=self.count
            )

            context['timeline'] = timeline

            return ''
        except exceptions.TwythonError as e:
            print("Django Twython Error: %s" % e)

@register.tag
def twitter_user_timeline_data(parser, token):
    """
        User Time-line tag.

        Usage::
            {% twitter_user_timeline <twitter-username> <number-of-twits> %}
    """
    return TwitterTimelineNode(token.split_contents())

@register.inclusion_tag('django_twython/timeline.html')
def twitter_user_timeline(user, *args, **kwargs):
    try:
        api = Twython(
            settings.TWITTER_CONSUMER_KEY,
            settings.TWITTER_CONSUMER_SECRET,
            settings.TWITTER_ACCESS_TOKEN,
            settings.TWITTER_ACCESS_TOKEN_SECRET
        )

        timeline = api.get_user_timeline(screen_name=user, count=kwargs.get('count', '5'))

        return {
            'timeline': timeline
        }

    except exceptions.TwythonError as e:
        print("Django Twython Error: %s" % e)

@register.filter(name='html_for_tweet')
def twython_html_for_twit(value):
    """
        Transforms the text of the twit into an html version.

        Usage::
            {{ twit|html_for_tweet }}
    :param value: The twit
    :return:
    """
    return mark_safe(Twython.html_for_tweet(value));<commit_msg>Added helper method to get twython API.<commit_after>"""
Created on 14/giu/2013

@author: Marco Pompili
"""

from django.conf import settings
from django.utils.safestring import mark_safe
from django import template

from twython import Twython
from twython import exceptions

register = template.Library()

def twython_get_api():
    return Twython(
        settings.TWITTER_CONSUMER_KEY,
        settings.TWITTER_CONSUMER_SECRET,
        settings.TWITTER_ACCESS_TOKEN,
        settings.TWITTER_ACCESS_TOKEN_SECRET
    )

class TwitterTimelineNode(template.Node):
    def __init__(self, args):
        self.username = args[1]
        self.count = args[2]

        self.twitter = twython_get_api()

    def render(self, context):
        try:
            timeline = self.twitter.get_user_timeline(
                screen_name=self.username,
                count=self.count
            )

            context['timeline'] = timeline

            return ''
        except exceptions.TwythonError as e:
            print("Django Twython Error: %s" % e)

@register.tag
def twitter_user_timeline_data(parser, token):
    """
        User Time-line tag.

        Usage::
            {% twitter_user_timeline <twitter-username> <number-of-twits> %}
    """
    return TwitterTimelineNode(token.split_contents())

@register.inclusion_tag('django_twython/timeline.html')
def twitter_user_timeline(user, *args, **kwargs):
    try:
        api = twython_get_api()

        timeline = api.get_user_timeline(
            screen_name=user,
            count=kwargs.get('count', '5')
        )

        return {
            'timeline': timeline
        }

    except exceptions.TwythonError as e:
        print("Django Twython Error: %s" % e)

@register.filter(name='html_for_tweet')
def twython_html_for_twit(value):
    """
        Transforms the text of the twit into an html version.

        Usage::
            {{ twit|html_for_tweet }}
    :param value: The twit
    :return:
    """
    return mark_safe(Twython.html_for_tweet(value));<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 02-08-2017 16:40
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This module contains models for dealing with PoGo stuff
"""

class PogoRunResult:
    def __init__(self):
        self.__ncbi_taxonomy_id = None
        self.__pogo_source_file_path = None
        self.__protein_sequence_file_path = None
        self.__gtf_file_path = None
        self.__pogo_result_file_paths = set()

    def __init__(self,
                 ncbi_taxonomy_id,
                 pogo_source_file_path,
                 protein_sequence_file_path,
                 gtf_file_path):
        """
        Syntactic sugar constructor
        :param ncbi_taxonomy_id: ncbi taxonomy id for this PoGo run results
        :param pogo_source_file_path: path of the source file used to run PoGo
        """
        self.__init__()
        self.set_ncbi_taxonomy_id(ncbi_taxonomy_id)
        self.set_pogo_source_file_path(pogo_source_file_path)
        self.set_protein_sequence_file_path(protein_sequence_file_path)
        self.set_gtf_file_path(gtf_file_path)

    def set_ncbi_taxonomy_id(self, taxonomy_id):
        self.__ncbi_taxonomy_id = taxonomy_id

    def set_pogo_source_file_path(self, pogo_source_file_path):
        if not self.__pogo_source_file_path or (self.__pogo_source_file_path != pogo_source_file_path):
            self.__pogo_source_file_path = pogo_source_file_path
            # TODO - Regenerate pogo result file paths

    def set_protein_sequence_file_path(self, protein_sequence_file_path):
        self.__protein_sequence_file_path = protein_sequence_file_path

    def set_gtf_file_path(self, gtf_file_path):
        self.__gtf_file_path = gtf_file_path

    def add_pogo_result_file_path(self, pogo_result_file_path):
        self.__pogo_result_file_paths.add(pogo_result_file_path)
<commit_msg>__pogo_result_file_paths refactor<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 02-08-2017 16:40
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This module contains models for dealing with PoGo stuff
"""

class PogoRunResult:
    def __init__(self):
        self.__ncbi_taxonomy_id = None
        self.__pogo_source_file_path = None
        self.__protein_sequence_file_path = None
        self.__gtf_file_path = None
        # Map<pogo_result_file_extension, pogo_result_file_path>
        self.__pogo_result_file_paths = {}

    def __init__(self,
                 ncbi_taxonomy_id,
                 pogo_source_file_path,
                 protein_sequence_file_path,
                 gtf_file_path):
        """
        Syntactic sugar constructor
        :param ncbi_taxonomy_id: ncbi taxonomy id for this PoGo run results
        :param pogo_source_file_path: path of the source file used to run PoGo
        """
        self.__init__()
        self.set_ncbi_taxonomy_id(ncbi_taxonomy_id)
        self.set_pogo_source_file_path(pogo_source_file_path)
        self.set_protein_sequence_file_path(protein_sequence_file_path)
        self.set_gtf_file_path(gtf_file_path)

    def set_ncbi_taxonomy_id(self, taxonomy_id):
        self.__ncbi_taxonomy_id = taxonomy_id

    def set_pogo_source_file_path(self, pogo_source_file_path):
        if not self.__pogo_source_file_path or (self.__pogo_source_file_path != pogo_source_file_path):
            self.__pogo_source_file_path = pogo_source_file_path
            # TODO - Regenerate pogo result file paths

    def set_protein_sequence_file_path(self, protein_sequence_file_path):
        self.__protein_sequence_file_path = protein_sequence_file_path

    def set_gtf_file_path(self, gtf_file_path):
        self.__gtf_file_path = gtf_file_path

    def add_pogo_result_file_path(self, pogo_result_file_path):
        self.__pogo_result_file_paths.add(pogo_result_file_path)
<|endoftext|>


<commit_before>from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response
from django.template import Context, Template
from django.template import RequestContext

from alerts.alert import Alerts, alert_key
from alerts.models import Alert
from alerts.highlighter import Highlighter

class AlertMiddleware(object):
    """
    A Middleware used to add notifications and alerts to the current user
    """

    def process_request(self, request):
        """
        Adds the alert interface into the request
        """
        request.alert = Alerts(
            request.session,
            alert_key(request.session)
        )

    def process_response(self, request, response):
        """
        Delays any messages if this view redirects you
        """
        if isinstance(response, HttpResponseRedirect):
            # Force delay any current messages
            request.alert.delay()
        return response

class FieldHighlightMiddleware(object):
    """
    A Middleware used to highlight fields based on input validation
    """

    def process_request(self, request):
        """
        Adds the highlight interface into the request
        """
        request.highlight = Highlighter()

<commit_msg>Removed unused imports.<commit_after>from django.http import HttpResponseRedirect

from alerts.alert import Alerts, alert_key
from alerts.highlighter import Highlighter

class AlertMiddleware(object):
    """
    A Middleware used to add notifications and alerts to the current user
    """

    def process_request(self, request):
        """
        Adds the alert interface into the request
        """
        request.alert = Alerts(
            request.session,
            alert_key(request.session)
        )

    def process_response(self, request, response):
        """
        Delays any messages if this view redirects you
        """
        if isinstance(response, HttpResponseRedirect):
            # Force delay any current messages
            request.alert.delay()
        return response

class FieldHighlightMiddleware(object):
    """
    A Middleware used to highlight fields based on input validation
    """

    def process_request(self, request):
        """
        Adds the highlight interface into the request
        """
        request.highlight = Highlighter()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Graph axis methods

Class for traversing and querying (sub)graph hierarchy with respect to a root
node.
"""

from lie_graph.graph import Graph
from lie_graph.graph_helpers import GraphException
from lie_graph.graph_axis.graph_axis_methods import (node_ancestors, node_children, node_descendants, node_neighbors,
    node_parent, node_all_parents, node_siblings, node_leaves)

class GraphAxis(Graph):
    """
    Hierarchical graph query and traversal with respect to a root node.

    The GraphAxis class extends the Graph base class with methods that enable
    query and traversal of graphs that have a hierachy with respect to a
    certain root node.
    A root node is chosen by node ID (nid) and is the one with the lowest ID
    (number) by default or any other one chosen at a later time.
    Hierarchy is queried with respect to the root node by means of axis that
    include: children, descendants, leaves, neighours, siblings, parents and
    ancestors.

    These methods return selections, or sub-graphs, that represent the
    respective axis. By default these selections maintain full connectivity to
    the full graphs and thus represent a selective 'view' on the full graph.
    This behaviour can be changed by enabling a 'mask' on the selection or by
    copying the selection into a new independent graph. The behaviour of both
    options are the same, connectivity is maintained with the selection but
    outside of it. The benefit of a mask, enabled by settings 'is_masked' to
    True, is that it is non-destructive to the data and can be easily lifted
    again.

    The behaviour of axis methods is furthermore affected by the directivity
    of the graph. The default non-directive graphs can be fully traversed
    forwards and backwards. Directed graphs or mixed graphs block traversal
    to parts of the graph by lack of an edge to traverse over.

    All of the axis methods are wrappers around their respective function in
    lie_graph.graph_axis.graph_axis_methods. These functions return node ID's
    that are used by the wrapper method to return the respective sub-graph or
    node ID's with 'return_nids' equals True.
    """

    def _resolve_nid(self):
        """
        Return the node ID (nid) of the current node

        When using single node graph objects this method will return the nid of
        the given node, in multi-node graphs it will return the first nid in
        the keys list and in empty graphs it will return None.
        """

        if self.root is None:
            raise GraphException('Graph node descendant requires a root node')

        try:
            return self.nid
        except AttributeError:
            return self.root

    def ancestors(self, node=None, include_self=False, return_nids=False):
        """
        Return the ancestors of the source node

        Traversal path is determined as the shortest path between the root node
        and the target node (Dijkstra shortest path).

        Directed graphs and/or is_masked behaviour: if the target node is not
        reachable using the Dijkstra shortest path method, the selection will
        remain empty.

        :param node:         source node to start search from
        :type node:          mixed
        :param include_self: include source nid in results
        :type include_self:  bool
        :param return_nids:  return a list of node ID's (nid) instead of a new
                             graph object representing the selection
        :type return_nids:   bool
        """

        nid = node or self._resolve_nid()
        anc = node_ancestors(self, nid, self.root, include_self=include_self)

        # mask node check
        if self.is_masked:
            anc = [n for n in anc if n in self.nodes]

        if return_nids:
            return anc
        return self.getnodes(anc)

    def children(self, node=None, include_self=False, return_nids=False):
        """
        Return the children of the source node.

        Traversal path is determined from the parent node with respect to the
        root via the source node to the children.

        Directed graphs and/or is_masked behaviour: masked child nodes or child
        nodes with directed connections from child to parent but not vice-versa
        will not be returned.

        :param node:         source node to start search from
        :type node:          mixed
        :param include_self: include source nid in results
        :type include_self:  bool
        :param return_nids:  return a list of node ID's (nid) instead of a new
                             graph object representing the selection
        :type return_nids:   bool

        :rtype:              Graph object or :py:list
        """

        nid = node or self._resolve_nid()
        nch = node_children(self, nid, self.root, include_self=include_self)

        if return_nids:
            return nch
        return self.getnodes(nch)

    def descendants(self, node=None, include_self=False, return_nids=False):
        """
        Return all descendants nodes to the source node

        Traversal path is determined from the parent with respect to the root
        node to the source node and then all descendants of the source.

        Directed graphs and/or is_masked behaviour: masked descendant linage's
        or linage's unreachable by directed edges are not returned.

        :param node:         source node to start search from
        :type node:          mixed
        :param include_self: include source nid in results
        :type include_self:  bool
        :param return_nids:  return a list of node ID's (nid) instead of a new
                             graph object representing the selection
        :type return_nids:   bool
        """

        nid = node or self._resolve_nid()
        nds = node_descendants(self, nid, self.root, include_self=include_self)

        if return_nids:
            return nds
        return self.getnodes(nds)

    def leaves(self, include_root=False, return_nids=False, include_isolated=False):
        """
        Return all leaf nodes in the (sub)graph

        Directed graphs and/or is_masked behaviour: leaf nodes are identified
        as those nodes having one edge only. This equals one adjacency node in
        an undirectional graph and no adjacency nodes in a directed graph.

        Graph root nodes do not affect the character of a node being a leave,
        neither will unidirectional versus directional edges.

        :param include_root:     include the root node if it is a leaf
        :type include_root:      bool
        :param return_nids:      return a list of node ID's (nid) instead of a new
                                 graph object representing the selection
        :type return_nids:       bool
        :param include_isolated: Include isolated nodes in the result
        :type include_isolated:  :py:bool

        :return:                 leaf node nids
        """

        leaves = node_leaves(self, include_isolated=include_isolated)
        if not include_root and self.root in leaves:
            leaves.remove(self.root)

        if return_nids:
            return sorted(leaves)
        return self.getnodes(leaves)

    def neighbors(self, node=None, return_nids=False):
        """
        Return de neighbor nodes of the source node.

        This method is not hierarchical and thus the root node has no effect.

        Directed graphs and/or is_masked behaviour: masked nodes or directed
        nodes not having an edge from source to node will not be returned.

        :param node:         node to return neighbors for
        :type node:          mixed
        :param return_nids:  return a list of node ID's (nid) instead of a new
                             graph object representing the selection
        :type return_nids:   bool
        """

        nid = node or self._resolve_nid()
        nng = node_neighbors(self, nid)

        if return_nids:
            return sorted(nng)
        return self.getnodes(nng)

    def parent(self, node=None, return_nids=False):
        """
        Get the parent node of the source node relative to the graph root
        when following the shortest path (Dijkstra shortest path).

        :param node:         node to define parent of
        :type node:          mixed
        :param return_nids:  return parent nid instead of a new graph object
                             representing the selection
        :type return_nids:   bool

        :return:             parent node
        :rtype:              Graph object
        """

        nid = node or self._resolve_nid()
        np = None
        if self.root != nid:
            np = node_parent(self, nid, self.root)

            # For masked graphs, parent might not be in nodes
            if self.is_masked and not np in self.nodes:
                np = None

        if return_nids:
            return np
        return self.getnodes(np)

    def all_parents(self, node=None, return_nids=False):
        """
        Get all parent nodes to the source node relative to the graph root

        Directed graphs and/or is_masked behaviour: masked nodes or directed
        nodes not having an edge from source to node will not be returned.

        :param node:         node to define parents of
        :type node:          mixed
        :param return_nids:  return parent nid instead of a new graph object
                             representing the selection
        :type return_nids:   bool

        :return:             parent nodes
        :rtype:              Graph object or list
        """

        nid = node or self._resolve_nid()
        anp = node_all_parents(self, nid, self.root)

        if return_nids:
            return anp
        return self.getnodes(anp)

    def siblings(self, node=None, return_nids=False):
        """
        Get the siblings of the source node

        Directed graphs and/or is_masked behaviour: masked nodes or directed
        nodes not having an edge from source to node will not be returned.

        :param node:         source node to start search from
        :type node:          mixed
        :param return_nids:  return a list of node ID's (nid) instead of a new
                             graph object representing the selection
        :type return_nids:   bool

        :return:             sibling nodes
        :rtype:              Graph object or list
        """

        nid = node or self._resolve_nid()
        nsb = node_siblings(self, nid, self.root)

        if return_nids:
            return sorted(nsb)
        return self.getnodes(nsb)
<commit_msg>Add support for nested graphs in GraphAxis items and values methods<commit_after># -*- coding: utf-8 -*-

"""
Graph axis methods

Class for traversing and querying (sub)graph hierarchy with respect to a root
node.
"""

from lie_graph.graph import Graph
from lie_graph.graph_helpers import GraphException
from lie_graph.graph_axis.graph_axis_methods import (node_ancestors, node_children, node_descendants, node_neighbors,
    node_parent, node_all_parents, node_siblings, node_leaves)

class GraphAxis(Graph):
    """
    Hierarchical graph query and traversal with respect to a root node.

    The GraphAxis class extends the Graph base class with methods that enable
    query and traversal of graphs that have a hierachy with respect to a
    certain root node.
    A root node is chosen by node ID (nid) and is the one with the lowest ID
    (number) by default or any other one chosen at a later time.
    Hierarchy is queried with respect to the root node by means of axis that
    include: children, descendants, leaves, neighours, siblings, parents and
    ancestors.

    These methods return selections, or sub-graphs, that represent the
    respective axis. By default these selections maintain full connectivity to
    the full graphs and thus represent a selective 'view' on the full graph.
    This behaviour can be changed by enabling a 'mask' on the selection or by
    copying the selection into a new independent graph. The behaviour of both
    options are the same, connectivity is maintained with the selection but
    outside of it. The benefit of a mask, enabled by settings 'is_masked' to
    True, is that it is non-destructive to the data and can be easily lifted
    again.

    The behaviour of axis methods is furthermore affected by the directivity
    of the graph. The default non-directive graphs can be fully traversed
    forwards and backwards. Directed graphs or mixed graphs block traversal
    to parts of the graph by lack of an edge to traverse over.

    All of the axis methods are wrappers around their respective function in
    lie_graph.graph_axis.graph_axis_methods. These functions return node ID's
    that are used by the wrapper method to return the respective sub-graph or
    node ID's with 'return_nids' equals True.
    """

    def _resolve_nid(self):
        """
        Return the node ID (nid) of the current node

        When using single node graph objects this method will return the nid of
        the given node, in multi-node graphs it will return the first nid in
        the keys list and in empty graphs it will return None.
        """

        if self.root is None:
            raise GraphException('Graph node descendant requires a root node')

        try:
            return self.nid
        except AttributeError:
            return self.root

    def ancestors(self, node=None, include_self=False, return_nids=False):
        """
        Return the ancestors of the source node

        Traversal path is determined as the shortest path between the root node
        and the target node (Dijkstra shortest path).

        Directed graphs and/or is_masked behaviour: if the target node is not
        reachable using the Dijkstra shortest path method, the selection will
        remain empty.

        :param node:         source node to start search from
        :type node:          mixed
        :param include_self: include source nid in results
        :type include_self:  bool
        :param return_nids:  return a list of node ID's (nid) instead of a new
                             graph object representing the selection
        :type return_nids:   bool
        """

        nid = node or self._resolve_nid()
        anc = node_ancestors(self, nid, self.root, include_self=include_self)

        # mask node check
        if self.is_masked:
            anc = [n for n in anc if n in self.nodes]

        if return_nids:
            return anc
        return self.getnodes(anc)

    def children(self, node=None, include_self=False, return_nids=False):
        """
        Return the children of the source node.

        Traversal path is determined from the parent node with respect to the
        root via the source node to the children.

        Directed graphs and/or is_masked behaviour: masked child nodes or child
        nodes with directed connections from child to parent but not vice-versa
        will not be returned.

        :param node:         source node to start search from
        :type node:          mixed
        :param include_self: include source nid in results
        :type include_self:  bool
        :param return_nids:  return a list of node ID's (nid) instead of a new
                             graph object representing the selection
        :type return_nids:   bool

        :rtype:              Graph object or :py:list
        """

        nid = node or self._resolve_nid()
        nch = node_children(self, nid, self.root, include_self=include_self)

        if return_nids:
            return nch
        return self.getnodes(nch)

    def descendants(self, node=None, include_self=False, return_nids=False):
        """
        Return all descendants nodes to the source node

        Traversal path is determined from the parent with respect to the root
        node to the source node and then all descendants of the source.

        Directed graphs and/or is_masked behaviour: masked descendant linage's
        or linage's unreachable by directed edges are not returned.

        :param node:         source node to start search from
        :type node:          mixed
        :param include_self: include source nid in results
        :type include_self:  bool
        :param return_nids:  return a list of node ID's (nid) instead of a new
                             graph object representing the selection
        :type return_nids:   bool
        """

        nid = node or self._resolve_nid()
        nds = node_descendants(self, nid, self.root, include_self=include_self)

        if return_nids:
            return nds
        return self.getnodes(nds)

    def leaves(self, include_root=False, return_nids=False, include_isolated=False):
        """
        Return all leaf nodes in the (sub)graph

        Directed graphs and/or is_masked behaviour: leaf nodes are identified
        as those nodes having one edge only. This equals one adjacency node in
        an undirectional graph and no adjacency nodes in a directed graph.

        Graph root nodes do not affect the character of a node being a leave,
        neither will unidirectional versus directional edges.

        :param include_root:     include the root node if it is a leaf
        :type include_root:      bool
        :param return_nids:      return a list of node ID's (nid) instead of a new
                                 graph object representing the selection
        :type return_nids:       bool
        :param include_isolated: Include isolated nodes in the result
        :type include_isolated:  :py:bool

        :return:                 leaf node nids
        """

        leaves = node_leaves(self, include_isolated=include_isolated)
        if not include_root and self.root in leaves:
            leaves.remove(self.root)

        if return_nids:
            return sorted(leaves)
        return self.getnodes(leaves)

    def neighbors(self, node=None, return_nids=False):
        """
        Return de neighbor nodes of the source node.

        This method is not hierarchical and thus the root node has no effect.

        Directed graphs and/or is_masked behaviour: masked nodes or directed
        nodes not having an edge from source to node will not be returned.

        :param node:         node to return neighbors for
        :type node:          mixed
        :param return_nids:  return a list of node ID's (nid) instead of a new
                             graph object representing the selection
        :type return_nids:   bool
        """

        nid = node or self._resolve_nid()
        nng = node_neighbors(self, nid)

        if return_nids:
            return sorted(nng)
        return self.getnodes(nng)

    def parent(self, node=None, return_nids=False):
        """
        Get the parent node of the source node relative to the graph root
        when following the shortest path (Dijkstra shortest path).

        :param node:         node to define parent of
        :type node:          mixed
        :param return_nids:  return parent nid instead of a new graph object
                             representing the selection
        :type return_nids:   bool

        :return:             parent node
        :rtype:              Graph object
        """

        nid = node or self._resolve_nid()
        np = None
        if self.root != nid:
            np = node_parent(self, nid, self.root)

            # For masked graphs, parent might not be in nodes
            if self.is_masked and not np in self.nodes:
                np = None

        if return_nids:
            return np
        return self.getnodes(np)

    def all_parents(self, node=None, return_nids=False):
        """
        Get all parent nodes to the source node relative to the graph root

        Directed graphs and/or is_masked behaviour: masked nodes or directed
        nodes not having an edge from source to node will not be returned.

        :param node:         node to define parents of
        :type node:          mixed
        :param return_nids:  return parent nid instead of a new graph object
                             representing the selection
        :type return_nids:   bool

        :return:             parent nodes
        :rtype:              Graph object or list
        """

        nid = node or self._resolve_nid()
        anp = node_all_parents(self, nid, self.root)

        if return_nids:
            return anp
        return self.getnodes(anp)

    def siblings(self, node=None, return_nids=False):
        """
        Get the siblings of the source node

        Directed graphs and/or is_masked behaviour: masked nodes or directed
        nodes not having an edge from source to node will not be returned.

        :param node:         source node to start search from
        :type node:          mixed
        :param return_nids:  return a list of node ID's (nid) instead of a new
                             graph object representing the selection
        :type return_nids:   bool

        :return:             sibling nodes
        :rtype:              Graph object or list
        """

        nid = node or self._resolve_nid()
        nsb = node_siblings(self, nid, self.root)

        if return_nids:
            return sorted(nsb)
        return self.getnodes(nsb)

    # DICTIONARY LIKE NODE ACCESS
    def items(self, keystring=None, valuestring=None, desc=True):
        """
        Python dict-like function to return node items in the (sub)graph.

        Keystring defines the value lookup key in the node data dict.
        This defaults to the graph node_key_tag.
        Valuestring defines the value lookup key in the node data dict.

        :param keystring:   Data key to use for dictionary keys.
        :type keystring:    :py:str
        :param valuestring: Data key to use for dictionary values.
        :type valuestring:  :py:str
        :param desc:        return nested values as new GraphAxis objects
        :type desc:         :py:bool

        :return:            List of keys, value pairs
        :rtype:             :py:list
        """

        keystring = keystring or self.node_key_tag
        valuestring = valuestring or self.node_value_tag

        if desc:
            return [(n.get(keystring), n.get(valuestring)) if n.isleaf else (n.get(keystring), n)
                    for n in self.iternodes()]
        return [(n.get(keystring), n.get(valuestring)) for n in self.iternodes()]

    def values(self, valuestring=None, desc=True):
        """
        Python dict-like function to return node values in the (sub)graph.

        Valuestring defines the value lookup key in the node data dict.

        :param valuestring: Data key to use for dictionary values.
        :type valuestring:  :py:str
        :param desc:        return nested values as new GraphAxis objects
        :type desc:         :py:bool

        :return:            List of values
        :rtype:             :py:list
        """

        valuestring = valuestring or self.node_value_tag

        if desc:
            return [n.get(valuestring) if n.isleaf else n for n in self.iternodes()]
        return [n.get(valuestring) for n in self.iternodes()]

<|endoftext|>


<commit_before>import numpy as np

import tokenizer
import file_handling as fh
from feature_extractor import FeatureExtractorCounts

class FeatureExtractorCountsNgrams(FeatureExtractorCounts):

    n = None

    def __init__(self, basedir, n=1, min_doc_threshold=1, binarize=True):
        name = 'ngrams'
        self.n = int(n)
        prefix = '_n' + str(n) + '_'
        FeatureExtractorCounts.__init__(self, basedir, name, prefix, min_doc_threshold=min_doc_threshold,
                                        binarize=binarize)
        self.extend_dirname()

    def extend_dirname(self):
        self.dirname += ',n=' + str(self.n)

    def get_n(self):
        return self.n

    def extract_tokens_from_text(self, data):
        token_dict = {}
        for key, text in data.items():
            text = text.lower()
            text = text.lstrip()
            text = text.rstrip()
            tokens = []

            sentences = tokenizer.split_sentences(text)
            for s in sentences:
                sent_tokens = tokenizer.make_ngrams(s, n)
                tokens = tokens + sent_tokens

            tokens = [self.get_prefix() + t for t in tokens]
            token_dict[key] = tokens
        return token_dict

    def do_transformations(self):
        FeatureExtractorCounts.do_transformations(self)

<commit_msg>corrected one minor type<commit_after>import numpy as np

import tokenizer
import file_handling as fh
from feature_extractor import FeatureExtractorCounts

class FeatureExtractorCountsNgrams(FeatureExtractorCounts):

    n = None

    def __init__(self, basedir, n=1, min_doc_threshold=1, binarize=True):
        name = 'ngrams'
        self.n = int(n)
        prefix = '_n' + str(n) + '_'
        FeatureExtractorCounts.__init__(self, basedir, name, prefix, min_doc_threshold=min_doc_threshold,
                                        binarize=binarize)
        self.extend_dirname()

    def extend_dirname(self):
        self.dirname += ',n=' + str(self.n)

    def get_n(self):
        return self.n

    def extract_tokens_from_text(self, data):
        token_dict = {}
        for key, text in data.items():
            text = text.lower()
            text = text.lstrip()
            text = text.rstrip()
            tokens = []

            sentences = tokenizer.split_sentences(text)
            for s in sentences:
                sent_tokens = tokenizer.make_ngrams(s, self.n)
                tokens = tokens + sent_tokens

            tokens = [self.get_prefix() + t for t in tokens]
            token_dict[key] = tokens
        return token_dict

    def do_transformations(self):
        FeatureExtractorCounts.do_transformations(self)

<|endoftext|>


<commit_before># Load libraries
import numpy as np
from scipy import optimize, sparse
from scipy.sparse import sparsetools

# Define functions
def csr_scale_rows(A, x):
    sparsetools.csr_scale_rows(A.shape[0], A.shape[1],
                               A.indptr, A.indices, A.data,
                               x)
def csr_scale_columns(A, x):
    sparsetools.csr_scale_columns(A.shape[0], A.shape[1],
                                  A.indptr, A.indices, A.data,
                                  x)

def l2_error(x1, x2):
    return np.sqrt( np.mean( (x1 - x2)**2 ) )

def l1_error(x1, x2):
    return np.mean(np.abs(x1-x2))

def loglik(theta, y, region_types, X, Xt, subset, theta0,
           mu, sigmasq, omega=1.0, log=False):
    b = theta0.copy()
    b[subset] = theta
    logb = b
    if log: b = np.exp(logb)
    else: logb = np.log(b)

    lam = X * (omega*b)

    u = logb - mu[region_types]

    val = np.sum(lam) - np.sum( y * np.log(lam) )
    val += np.sum(u*u/sigmasq[region_types])/2.0
    val += np.log(sigmasq[region_types]).sum()/2.0
    return val

def loglik_convolve(theta, y, region_types, template, subset, theta0,
                    mu, sigmasq, omega=1.0, log=False):
    b = theta0.copy()
    b[subset] = theta
    logb = b
    if log: b = np.exp(logb)
    else: logb = np.log(b)

    lam = omega * np.convolve(b, template, mode='same')

    u = logb - mu[region_types]

    val = np.sum(lam) - np.sum( y * np.log(lam) )
    val += np.sum(u*u/sigmasq[region_types])/2.0
    val += np.log(sigmasq[region_types]).sum()/2.0

    return val

def dloglik(theta, y, region_types, X, Xt, subset, theta0,
            mu, sigmasq, omega=1.0, log=False):
    b = theta0.copy()
    b[subset] = theta
    logb = b
    if log: b = np.exp(logb)
    else: logb = np.log(b)
    #
    lam = X * (omega*b)
    #
    u = logb - mu[region_types]
    #
    grad = Xt * ( omega*(lam-y)/lam )
    grad += u/sigmasq[region_types]/b
    if log: grad *= b
    #
    return grad[subset]

def dloglik_convolve(theta, y, region_types, template, subset, theta0,
                     mu, sigmasq, omega=1.0, log=False):
    b = theta0.copy()
    b[subset] = theta
    logb = b
    if log: b = np.exp(logb)
    else: logb = np.log(b)

    lam = omega * np.convolve(b, template, mode='same')

    u = logb - mu[region_types]

    grad = omega * np.convolve((lam-y)/lam, template, mode='same')
    grad += u/sigmasq[region_types]/b
    if log: grad *= b
    #
    return grad[subset]

def ddloglik(theta, y, region_types, X, Xt, subset, theta0,
             mu, sigmasq, omega=1.0, log=True):
    b = theta0.copy()
    b[subset] = theta
    logb = b
    if log: b = np.exp(logb)
    else: logb = np.log(b)
    #
    lam = X * (omega*b)
    w = np.sqrt(y) / lam
    #
    Z = omega * X
    Zt = omega * Xt
    csr_scale_rows(Z, w)
    csr_scale_columns(Zt, w)
    if log:
        csr_scale_columns(Z, b)
        csr_scale_rows(Zt, b)
    H = Zt * Z
    if log: H = H + sparse.spdiags( Xt * ( omega*(lam-y)/lam )*b,
                                    0, X.shape[1], X.shape[1], 'csr' )
    #
    Sigma_inv = sparse.spdiags( np.ones(X.shape[1])/sigmasq[region_types], 0,
                               X.shape[1], X.shape[1],
                               'csr' )
    if not log:
        w = 1.0/b
        csr_scale_rows(Sigma_inv, w)
        csr_scale_columns(Sigma_inv, w)
        #
        u = logb - mu[region_types]
        #
        grad = u/sigmasq[region_types]/b
        #
        Sigma_inv = Sigma_inv - sparse.dia_matrix( (grad, 0),
                                                 (X.shape[1], X.shape[1]) )
    #
    H = H + Sigma_inv
    return H

def ddloglik_p(theta, p, y, region_types, X, Xt, subset, theta0,
               mu, sigmasq, omega=1.0, log=False):
    b = theta0.copy()
    b[subset] = theta
    logb = b
    if log: b = np.exp(logb)
    else: logb = np.log(b)
    #
    lam = X * (omega*b)
    w = np.sqrt(y) / lam
    #
    Z = omega * X
    Zt = omega * Xt
    csr_scale_rows(Z, w)
    csr_scale_columns(Zt, w)
    if log:
        csr_scale_columns(Z, b)
        csr_scale_rows(Zt, b)
    H = Zt * Z
    if log: H = H + sparse.spdiags( Xt * ( omega*(lam-y)/lam )*b,
                                    0, X.shape[1], X.shape[1], 'csr' )
    #
    Sigma_inv = sparse.spdiags( np.ones(X.shape[1])/sigmasq[region_types], 0,
                               X.shape[1], X.shape[1],
                               'csr' )
    if not log:
        w = 1.0/b
        csr_scale_rows(Sigma_inv, w)
        csr_scale_columns(Sigma_inv, w)
        #
        u = logb - mu[region_types]
        #
        grad = u/sigmasq[region_types]/b
        #
        Sigma_inv = Sigma_inv - sparse.dia_matrix( (grad, 0),
                                                 (X.shape[1], X.shape[1]) )
    #
    H = H + Sigma_inv
    H = H.tocsr()
    H = H[subset]
    H = H.tocsc()
    H = H[:,subset]
    H = H.tocsr()
    #
    return H * p

def ddloglik_diag(theta, y, region_types, X, Xt, subset, theta0,
                  mu, sigmasq, omega=1.0, log=True):
    b = theta0.copy()
    b[subset] = theta
    logb = b
    if log: b = np.exp(logb)
    else: logb = np.log(b)
    #
    lam = X * (omega*b)
    w = np.sqrt(y) / lam
    #
    Z = omega * X
    csr_scale_rows(Z, w)
    if log:
        csr_scale_columns(Z, b)
    Z.data = Z.data**2
    Hdiag = Z.sum(0)
    Hdiag = np.asarray(Hdiag).flatten()
    if log: Hdiag = Hdiag + (Xt * ( omega*(lam-y)/lam )*b)
    #
    Sigma_inv_diag = np.ones(X.shape[1])/sigmasq[region_types]
    if not log:
        Sigma_inv_diag /= b**2
        #
        u = logb - mu[region_types]
        #
        grad = u/sigmasq[region_types]/b
        #
        Sigma_inv_diag = Sigma_inv_diag - grad
    #
    Hdiag = Hdiag + Sigma_inv_diag
    return Hdiag

# Identify active components of basis
def find_active(y, w=38):
    N = y.shape[0]
    active = np.zeros(N, np.bool)
    for n in range(N):
        active[n] = (y[ np.maximum(0,n-w+1):np.minimum(n+w,N) ].max() > 0)
    return active

def deconvolve(loglik, dloglik, y, region_types, template,
               mu, sigmasq,
               subset=slice(None), theta0=None, omega=1.0, log=False,
               lower_bound = np.sqrt( np.finfo(float).eps ),
               **kwargs):
    m = subset.stop - subset.start

    if theta0 is None:
        theta0 = y + 1
        if log: theta0 = np.log(theta0)
    if log:
        lower_bound = np.log(lower_bound)

    result = optimize.fmin_tnc( loglik, theta0[subset], dloglik,
                                args=(y, region_types, template, subset, theta0,
                                      mu, sigmasq, omega, log),
                                bounds = zip( np.ones(m)*lower_bound,
                                              np.ones(m)*np.Inf ),
                                **kwargs )
    return result

<commit_msg>Minor tweaks to Hessian calculation for clarity.<commit_after># Load libraries
import numpy as np
from scipy import optimize, sparse
from scipy.sparse import sparsetools

# Define functions
def csr_scale_rows(A, x):
    sparsetools.csr_scale_rows(A.shape[0], A.shape[1],
                               A.indptr, A.indices, A.data,
                               x)
def csr_scale_columns(A, x):
    sparsetools.csr_scale_columns(A.shape[0], A.shape[1],
                                  A.indptr, A.indices, A.data,
                                  x)

def l2_error(x1, x2):
    return np.sqrt( np.mean( (x1 - x2)**2 ) )

def l1_error(x1, x2):
    return np.mean(np.abs(x1-x2))

def loglik(theta, y, region_types, X, Xt, subset, theta0,
           mu, sigmasq, omega=1.0, log=False):
    b = theta0.copy()
    b[subset] = theta
    logb = b
    if log: b = np.exp(logb)
    else: logb = np.log(b)

    lam = X * (omega*b)

    u = logb - mu[region_types]

    val = np.sum(lam) - np.sum( y * np.log(lam) )
    val += np.sum(u*u/sigmasq[region_types])/2.0
    val += np.log(sigmasq[region_types]).sum()/2.0
    return val

def loglik_convolve(theta, y, region_types, template, subset, theta0,
                    mu, sigmasq, omega=1.0, log=False):
    b = theta0.copy()
    b[subset] = theta
    logb = b
    if log: b = np.exp(logb)
    else: logb = np.log(b)

    lam = omega * np.convolve(b, template, mode='same')

    u = logb - mu[region_types]

    val = np.sum(lam) - np.sum( y * np.log(lam) )
    val += np.sum(u*u/sigmasq[region_types])/2.0
    val += np.log(sigmasq[region_types]).sum()/2.0

    return val

def dloglik(theta, y, region_types, X, Xt, subset, theta0,
            mu, sigmasq, omega=1.0, log=False):
    b = theta0.copy()
    b[subset] = theta
    logb = b
    if log: b = np.exp(logb)
    else: logb = np.log(b)
    #
    lam = X * (omega*b)
    #
    u = logb - mu[region_types]
    #
    grad = Xt * ( omega*(lam-y)/lam )
    grad += u/sigmasq[region_types]/b
    if log: grad *= b
    #
    return grad[subset]

def dloglik_convolve(theta, y, region_types, template, subset, theta0,
                     mu, sigmasq, omega=1.0, log=False):
    b = theta0.copy()
    b[subset] = theta
    logb = b
    if log: b = np.exp(logb)
    else: logb = np.log(b)

    lam = omega * np.convolve(b, template, mode='same')

    u = logb - mu[region_types]

    grad = omega * np.convolve((lam-y)/lam, template, mode='same')
    grad += u/sigmasq[region_types]/b
    if log: grad *= b
    #
    return grad[subset]

def ddloglik(theta, y, region_types, X, Xt, subset, theta0,
             mu, sigmasq, omega=1.0, log=True):
    b = theta0.copy()
    b[subset] = theta
    logb = b
    if log: b = np.exp(logb)
    else: logb = np.log(b)
    #
    lam = X * (omega*b)
    w = np.sqrt(y) / lam
    #
    Z = omega * X
    Zt = omega * Xt
    csr_scale_rows(Z, w)
    csr_scale_columns(Zt, w)
    if log:
        csr_scale_columns(Z, b)
        csr_scale_rows(Zt, b)
    H = Zt * Z
    if log:
        grad = Xt * ( omega*(lam-y)/lam )
        H = H + sparse.spdiags(grad*b, 0, X.shape[1], X.shape[1], 'csr' )
    #
    Sigma_inv = sparse.spdiags(np.ones(X.shape[1])/sigmasq[region_types], 0,
                               X.shape[1], X.shape[1],
                               'csr' )
    if not log:
        w = 1.0/b
        csr_scale_rows(Sigma_inv, w)
        csr_scale_columns(Sigma_inv, w)
        #
        u = logb - mu[region_types]
        #
        grad = u/sigmasq[region_types]/b
        #
        Sigma_inv = Sigma_inv - sparse.dia_matrix( (grad, 0),
                                                 (X.shape[1], X.shape[1]) )
    #
    H = H + Sigma_inv
    return H

def ddloglik_p(theta, p, y, region_types, X, Xt, subset, theta0,
               mu, sigmasq, omega=1.0, log=False):
    b = theta0.copy()
    b[subset] = theta
    logb = b
    if log: b = np.exp(logb)
    else: logb = np.log(b)
    #
    lam = X * (omega*b)
    w = np.sqrt(y) / lam
    #
    Z = omega * X
    Zt = omega * Xt
    csr_scale_rows(Z, w)
    csr_scale_columns(Zt, w)
    if log:
        csr_scale_columns(Z, b)
        csr_scale_rows(Zt, b)
    H = Zt * Z
    if log: H = H + sparse.spdiags( Xt * ( omega*(lam-y)/lam )*b,
                                    0, X.shape[1], X.shape[1], 'csr' )
    #
    Sigma_inv = sparse.spdiags( np.ones(X.shape[1])/sigmasq[region_types], 0,
                               X.shape[1], X.shape[1],
                               'csr' )
    if not log:
        w = 1.0/b
        csr_scale_rows(Sigma_inv, w)
        csr_scale_columns(Sigma_inv, w)
        #
        u = logb - mu[region_types]
        #
        grad = u/sigmasq[region_types]/b
        #
        Sigma_inv = Sigma_inv - sparse.dia_matrix( (grad, 0),
                                                 (X.shape[1], X.shape[1]) )
    #
    H = H + Sigma_inv
    H = H.tocsr()
    H = H[subset]
    H = H.tocsc()
    H = H[:,subset]
    H = H.tocsr()
    #
    return H * p

def ddloglik_diag(theta, y, region_types, X, Xt, subset, theta0,
                  mu, sigmasq, omega=1.0, log=True):
    b = theta0.copy()
    b[subset] = theta
    logb = b
    if log: b = np.exp(logb)
    else: logb = np.log(b)
    #
    lam = X * (omega*b)
    w = np.sqrt(y) / lam
    #
    Z = omega * X
    csr_scale_rows(Z, w)
    if log:
        csr_scale_columns(Z, b)
    Z.data = Z.data**2
    Hdiag = Z.sum(0)
    Hdiag = np.asarray(Hdiag).flatten()
    if log: Hdiag = Hdiag + (Xt * ( omega*(lam-y)/lam )*b)
    #
    Sigma_inv_diag = np.ones(X.shape[1])/sigmasq[region_types]
    if not log:
        Sigma_inv_diag /= b**2
        #
        u = logb - mu[region_types]
        #
        grad = u/sigmasq[region_types]/b
        #
        Sigma_inv_diag = Sigma_inv_diag - grad
    #
    Hdiag = Hdiag + Sigma_inv_diag
    return Hdiag

# Identify active components of basis
def find_active(y, w=38):
    N = y.shape[0]
    active = np.zeros(N, np.bool)
    for n in range(N):
        active[n] = (y[ np.maximum(0,n-w+1):np.minimum(n+w,N) ].max() > 0)
    return active

def deconvolve(loglik, dloglik, y, region_types, template,
               mu, sigmasq,
               subset=slice(None), theta0=None, omega=1.0, log=False,
               lower_bound = np.sqrt( np.finfo(float).eps ),
               **kwargs):
    m = subset.stop - subset.start

    if theta0 is None:
        theta0 = y + 1
        if log: theta0 = np.log(theta0)
    if log:
        lower_bound = np.log(lower_bound)

    result = optimize.fmin_tnc( loglik, theta0[subset], dloglik,
                                args=(y, region_types, template, subset, theta0,
                                      mu, sigmasq, omega, log),
                                bounds = zip( np.ones(m)*lower_bound,
                                              np.ones(m)*np.Inf ),
                                **kwargs )
    return result

<|endoftext|>