<commit_before>import numpy

import cupy
from cupy import core

def place(arr, mask, vals):
    """Change elements of an array based on conditional and input values.

    This function uses the first N elements of `vals`, where N is the number
    of true values in `mask`.

    Args:
        arr (cupy.ndarray): Array to put data into.
        mask (array-like): Boolean mask array. Must have the same size as `a`.
        vals (array-like): Values to put into `a`. Only the first
            N elements are used, where N is the number of True values in
            `mask`. If `vals` is smaller than N, it will be repeated, and if
            elements of `a` are to be masked, this sequence must be non-empty.

    Examples
    --------
    >>> arr = np.arange(6).reshape(2, 3)
    >>> np.place(arr, arr>2, [44, 55])
    >>> arr
    array([[ 0,  1,  2],
           [44, 55, 44]])

    .. warning::

        This function may synchronize the device.

    .. seealso:: :func:`numpy.place`
    """
    # TODO(niboshi): Avoid nonzero which may synchronize the device.
    mask = cupy.asarray(mask)
    if arr.size != mask.size:
        raise ValueError('Mask and data must be the same size.')
    vals = cupy.asarray(vals)

    mask_indices = mask.ravel().nonzero()[0]  # may synchronize
    if mask_indices.size == 0:
        return
    if vals.size == 0:
        raise ValueError('Cannot insert from an empty array.')
    arr.put(mask_indices, vals, mode='wrap')

def put(a, ind, v, mode='wrap'):
    """Replaces specified elements of an array with given values.

    Args:
        a (cupy.ndarray): Target array.
        ind (array-like): Target indices, interpreted as integers.
        v (array-like): Values to place in `a` at target indices.
            If `v` is shorter than `ind` it will be repeated as necessary.
        mode (str): How out-of-bounds indices will behave. Its value must be
            either `'raise'`, `'wrap'` or `'clip'`. Otherwise,
            :class:`TypeError` is raised.

    .. note::
        Default `mode` is set to `'wrap'` to avoid unintended performance drop.
        If you need NumPy's behavior, please pass `mode='raise'` manually.

    .. seealso:: :func:`numpy.put`
    """
    a.put(ind, v, mode=mode)

_putmask_kernel = core.ElementwiseKernel(
    'Q mask, raw S values, uint64 len_vals', 'T out',
    '''
    if (mask) out = (T) values[i % len_vals];
    ''',
    'putmask_kernel'
)

def putmask(a, mask, values):
    """
    Changes elements of an array inplace, based on a conditional mask and
    input values.

    Sets ``a.flat[n] = values[n]`` for each n where ``mask.flat[n]==True``.
    If `values` is not the same size as `a` and `mask` then it will repeat.

    Args:
        a (cupy.ndarray): Target array.
        mask (cupy.ndarray): Boolean mask array. It has to be
            the same shape as `a`.
        values (cupy.ndarray or scalar): Values to put into `a` where `mask`
            is True. If `values` is smaller than `a`, then it will be
            repeated.

    Examples
    --------
    >>> x = cupy.arange(6).reshape(2, 3)
    >>> cupy.putmask(x, x>2, x**2)
    >>> x
    array([[ 0,  1,  2],
           [ 9, 16, 25]])

    If `values` is smaller than `a` it is repeated:

    >>> x = cupy.arange(6)
    >>> cupy.putmask(x, x>2, cupy.array([-33, -44]))
    >>> x
    array([  0,   1,   2, -44, -33, -44])

    .. seealso:: :func:`numpy.putmask`

    """

    if not isinstance(a, cupy.ndarray):
        raise TypeError('`a` should be of type cupy.ndarray')
    if not isinstance(mask, cupy.ndarray):
        raise TypeError('`mask` should be of type cupy.ndarray')
    if not (cupy.isscalar(values) or isinstance(values, cupy.ndarray)):
        raise TypeError('`values` should be of type cupy.ndarray')

    if not a.shape == mask.shape:
        raise ValueError('mask and data must be the same size')

    mask = mask.astype(numpy.bool_)

    if cupy.isscalar(values):
        a[mask] = values

    elif not numpy.can_cast(values.dtype, a.dtype):
        raise TypeError('Cannot cast array data from'
                        ' {} to {} according to the rule \'safe\''
                        .format(values.dtype, a.dtype))

    elif a.shape == values.shape:
        a[mask] = values[mask]

    else:
        values = values.ravel()
        _putmask_kernel(mask, values, len(values), a)

def fill_diagonal(a, val, wrap=False):
    """Fills the main diagonal of the given array of any dimensionality.

    For an array `a` with ``a.ndim > 2``, the diagonal is the list of
    locations with indices ``a[i, i, ..., i]`` all identical. This function
    modifies the input array in-place, it does not return a value.

    Args:
        a (cupy.ndarray): The array, at least 2-D.
        val (scalar): The value to be written on the diagonal.
            Its type must be compatible with that of the array a.
        wrap (bool): If specified, the diagonal is "wrapped" after N columns.
            This affects only tall matrices.

    Examples
    --------
    >>> a = cupy.zeros((3, 3), int)
    >>> cupy.fill_diagonal(a, 5)
    >>> a
    array([[5, 0, 0],
           [0, 5, 0],
           [0, 0, 5]])

    .. seealso:: :func:`numpy.fill_diagonal`
    """
    # The followings are imported from the original numpy
    if a.ndim < 2:
        raise ValueError('array must be at least 2-d')
    end = None
    if a.ndim == 2:
        step = a.shape[1] + 1
        if not wrap:
            end = a.shape[1] * a.shape[1]
    else:
        if not numpy.alltrue(numpy.diff(a.shape) == 0):
            raise ValueError('All dimensions of input must be of equal length')
        step = 1 + numpy.cumprod(a.shape[:-1]).sum()

    a.flat[:end:step] = val

def diag_indices(n, ndim=2):
    """Return the indices to access the main diagonal of an array.

    Returns a tuple of indices that can be used to access the main
    diagonal of an array with ``ndim >= 2`` dimensions and shape
    (n, n, ..., n).

    Args:
        n (int): The size, along each dimension of the arrays for which
            the indices are to be returned.
        ndim (int): The number of dimensions. default `2`.

    Examples
    --------
    Create a set of indices to access the diagonal of a (4, 4) array:

    >>> dig = cupy.diag_indices(4)
    >>> dig
    (array([0, 1, 2, 3]), array([0, 1, 2, 3]))
    >>> a = cupy.arange(16).reshape(4, 4)
    >>> a
    array([[ 0,  1,  2,  3],
           [ 4,  5,  6,  7],
           [ 8,  9, 10, 11],
           [12, 13, 14, 15]])
    >>> a[di] = 100
    >>> a
    array([[100,   1,   2,   3],
           [  4, 100,   6,   7],
           [  8,   9, 100,  11],
           [ 12,  13,  14, 100]])

    Create indices to manipulate a 3-D array:

    >>> d3 = cupy.diag_indices(2, 3)
    >>> d3
    (array([0, 1]), array([0, 1]), array([0, 1]))

    And use it to set the diagonal of an array of zeros to 1:

    >>> a = cupy.zeros((2, 2, 2), dtype=int)
    >>> a[d3] = 1
    >>> a
    array([[[1, 0],
            [0, 0]],
           [[0, 0],
            [0, 1]]])

    .. seealso:: :func:`numpy.diag_indices`

    """
    idx = cupy.arange(n)
    return (idx,) * ndim

def diag_indices_from(arr):
    """
    Return the indices to access the main diagonal of an n-dimensional array.
    See `diag_indices` for full details.

    Args:
        arr (cupy.ndarray): At least 2-D.

    .. seealso:: :func:`numpy.diag_indices_from`

    """
    if not isinstance(arr, cupy.ndarray):
        raise TypeError("Argument must be cupy.ndarray")

    if not arr.ndim >= 2:
        raise ValueError("input array must be at least 2-d")
    # For more than d=2, the strided formula is only valid for arrays with
    # all dimensions equal, so we check first.
    if not cupy.all(cupy.diff(arr.shape) == 0):
        raise ValueError("All dimensions of input must be of equal length")

    return diag_indices(arr.shape[0], arr.ndim)
<commit_msg>Fix doctest<commit_after>import numpy

import cupy
from cupy import core

def place(arr, mask, vals):
    """Change elements of an array based on conditional and input values.

    This function uses the first N elements of `vals`, where N is the number
    of true values in `mask`.

    Args:
        arr (cupy.ndarray): Array to put data into.
        mask (array-like): Boolean mask array. Must have the same size as `a`.
        vals (array-like): Values to put into `a`. Only the first
            N elements are used, where N is the number of True values in
            `mask`. If `vals` is smaller than N, it will be repeated, and if
            elements of `a` are to be masked, this sequence must be non-empty.

    Examples
    --------
    >>> arr = np.arange(6).reshape(2, 3)
    >>> np.place(arr, arr>2, [44, 55])
    >>> arr
    array([[ 0,  1,  2],
           [44, 55, 44]])

    .. warning::

        This function may synchronize the device.

    .. seealso:: :func:`numpy.place`
    """
    # TODO(niboshi): Avoid nonzero which may synchronize the device.
    mask = cupy.asarray(mask)
    if arr.size != mask.size:
        raise ValueError('Mask and data must be the same size.')
    vals = cupy.asarray(vals)

    mask_indices = mask.ravel().nonzero()[0]  # may synchronize
    if mask_indices.size == 0:
        return
    if vals.size == 0:
        raise ValueError('Cannot insert from an empty array.')
    arr.put(mask_indices, vals, mode='wrap')

def put(a, ind, v, mode='wrap'):
    """Replaces specified elements of an array with given values.

    Args:
        a (cupy.ndarray): Target array.
        ind (array-like): Target indices, interpreted as integers.
        v (array-like): Values to place in `a` at target indices.
            If `v` is shorter than `ind` it will be repeated as necessary.
        mode (str): How out-of-bounds indices will behave. Its value must be
            either `'raise'`, `'wrap'` or `'clip'`. Otherwise,
            :class:`TypeError` is raised.

    .. note::
        Default `mode` is set to `'wrap'` to avoid unintended performance drop.
        If you need NumPy's behavior, please pass `mode='raise'` manually.

    .. seealso:: :func:`numpy.put`
    """
    a.put(ind, v, mode=mode)

_putmask_kernel = core.ElementwiseKernel(
    'Q mask, raw S values, uint64 len_vals', 'T out',
    '''
    if (mask) out = (T) values[i % len_vals];
    ''',
    'putmask_kernel'
)

def putmask(a, mask, values):
    """
    Changes elements of an array inplace, based on a conditional mask and
    input values.

    Sets ``a.flat[n] = values[n]`` for each n where ``mask.flat[n]==True``.
    If `values` is not the same size as `a` and `mask` then it will repeat.

    Args:
        a (cupy.ndarray): Target array.
        mask (cupy.ndarray): Boolean mask array. It has to be
            the same shape as `a`.
        values (cupy.ndarray or scalar): Values to put into `a` where `mask`
            is True. If `values` is smaller than `a`, then it will be
            repeated.

    Examples
    --------
    >>> x = cupy.arange(6).reshape(2, 3)
    >>> cupy.putmask(x, x>2, x**2)
    >>> x
    array([[ 0,  1,  2],
           [ 9, 16, 25]])

    If `values` is smaller than `a` it is repeated:

    >>> x = cupy.arange(6)
    >>> cupy.putmask(x, x>2, cupy.array([-33, -44]))
    >>> x
    array([  0,   1,   2, -44, -33, -44])

    .. seealso:: :func:`numpy.putmask`

    """

    if not isinstance(a, cupy.ndarray):
        raise TypeError('`a` should be of type cupy.ndarray')
    if not isinstance(mask, cupy.ndarray):
        raise TypeError('`mask` should be of type cupy.ndarray')
    if not (cupy.isscalar(values) or isinstance(values, cupy.ndarray)):
        raise TypeError('`values` should be of type cupy.ndarray')

    if not a.shape == mask.shape:
        raise ValueError('mask and data must be the same size')

    mask = mask.astype(numpy.bool_)

    if cupy.isscalar(values):
        a[mask] = values

    elif not numpy.can_cast(values.dtype, a.dtype):
        raise TypeError('Cannot cast array data from'
                        ' {} to {} according to the rule \'safe\''
                        .format(values.dtype, a.dtype))

    elif a.shape == values.shape:
        a[mask] = values[mask]

    else:
        values = values.ravel()
        _putmask_kernel(mask, values, len(values), a)

def fill_diagonal(a, val, wrap=False):
    """Fills the main diagonal of the given array of any dimensionality.

    For an array `a` with ``a.ndim > 2``, the diagonal is the list of
    locations with indices ``a[i, i, ..., i]`` all identical. This function
    modifies the input array in-place, it does not return a value.

    Args:
        a (cupy.ndarray): The array, at least 2-D.
        val (scalar): The value to be written on the diagonal.
            Its type must be compatible with that of the array a.
        wrap (bool): If specified, the diagonal is "wrapped" after N columns.
            This affects only tall matrices.

    Examples
    --------
    >>> a = cupy.zeros((3, 3), int)
    >>> cupy.fill_diagonal(a, 5)
    >>> a
    array([[5, 0, 0],
           [0, 5, 0],
           [0, 0, 5]])

    .. seealso:: :func:`numpy.fill_diagonal`
    """
    # The followings are imported from the original numpy
    if a.ndim < 2:
        raise ValueError('array must be at least 2-d')
    end = None
    if a.ndim == 2:
        step = a.shape[1] + 1
        if not wrap:
            end = a.shape[1] * a.shape[1]
    else:
        if not numpy.alltrue(numpy.diff(a.shape) == 0):
            raise ValueError('All dimensions of input must be of equal length')
        step = 1 + numpy.cumprod(a.shape[:-1]).sum()

    a.flat[:end:step] = val

def diag_indices(n, ndim=2):
    """Return the indices to access the main diagonal of an array.

    Returns a tuple of indices that can be used to access the main
    diagonal of an array with ``ndim >= 2`` dimensions and shape
    (n, n, ..., n).

    Args:
        n (int): The size, along each dimension of the arrays for which
            the indices are to be returned.
        ndim (int): The number of dimensions. default `2`.

    Examples
    --------
    Create a set of indices to access the diagonal of a (4, 4) array:

    >>> di = cupy.diag_indices(4)
    >>> di
    (array([0, 1, 2, 3]), array([0, 1, 2, 3]))
    >>> a = cupy.arange(16).reshape(4, 4)
    >>> a
    array([[ 0,  1,  2,  3],
           [ 4,  5,  6,  7],
           [ 8,  9, 10, 11],
           [12, 13, 14, 15]])
    >>> a[di] = 100
    >>> a
    array([[100,   1,   2,   3],
           [  4, 100,   6,   7],
           [  8,   9, 100,  11],
           [ 12,  13,  14, 100]])

    Create indices to manipulate a 3-D array:

    >>> d3 = cupy.diag_indices(2, 3)
    >>> d3
    (array([0, 1]), array([0, 1]), array([0, 1]))

    And use it to set the diagonal of an array of zeros to 1:

    >>> a = cupy.zeros((2, 2, 2), dtype=int)
    >>> a[d3] = 1
    >>> a
    array([[[1, 0],
            [0, 0]],
    <BLANKLINE>
           [[0, 0],
            [0, 1]]])

    .. seealso:: :func:`numpy.diag_indices`

    """
    idx = cupy.arange(n)
    return (idx,) * ndim

def diag_indices_from(arr):
    """
    Return the indices to access the main diagonal of an n-dimensional array.
    See `diag_indices` for full details.

    Args:
        arr (cupy.ndarray): At least 2-D.

    .. seealso:: :func:`numpy.diag_indices_from`

    """
    if not isinstance(arr, cupy.ndarray):
        raise TypeError("Argument must be cupy.ndarray")

    if not arr.ndim >= 2:
        raise ValueError("input array must be at least 2-d")
    # For more than d=2, the strided formula is only valid for arrays with
    # all dimensions equal, so we check first.
    if not cupy.all(cupy.diff(arr.shape) == 0):
        raise ValueError("All dimensions of input must be of equal length")

    return diag_indices(arr.shape[0], arr.ndim)
<|endoftext|>


<commit_before>"""Allows to run HIRE locally outside the SMIF framework

    Tools
    # -------
    Profiling:  https://jiffyclub.github.io/snakeviz/
    python -m cProfile -o program.prof main.py
    snakeviz program.prof
"""
import os
import sys
import time
import logging
import numpy as np
from energy_demand import model
from energy_demand.basic import testing_functions
from energy_demand.basic import lookup_tables
from energy_demand.assumptions import non_param_assumptions
from energy_demand.assumptions import param_assumptions
from energy_demand.read_write import data_loader
from energy_demand.basic import logger_setup
from energy_demand.read_write import write_data
from energy_demand.read_write import read_data
from energy_demand.basic import basic_functions
from energy_demand.scripts import s_disaggregation

import fiona

def energy_demand_model(regions, data, assumptions):
    """Main function of energy demand model to calculate yearly demand

    Arguments
    ----------
    data : dict
        Data container

    Returns
    -------
    result_dict : dict
        A nested dictionary containing all data for energy supply model with
        timesteps for every hour in a year.
        [fueltype : region : timestep]
    modelrun_obj : dict
        Object of a yearly model run

    Note
    ----
    This function is executed in the wrapper
    """
    modelrun = model.EnergyDemandModel(
        regions=regions, #data['regions'],
        data=data,
        assumptions=assumptions)

    # Calculate base year demand
    fuels_in = testing_functions.test_function_fuel_sum(
        data,
        data['fuel_disagg'],
        data['criterias']['mode_constrained'],
        assumptions.enduse_space_heating)

    # Log model results
    write_data.logg_info(modelrun, fuels_in, data)
    logging.info("...finished running energy demand model simulation")
    return modelrun

def get_region_names(path):
    ''' Returns names of shapes within a shapefile
    '''
    with fiona.open(path, 'r') as source:
        return [elem['properties']['name'] for elem in source]

if __name__ == "__main__":
    """
    """
    data = {}

    # Paths
    if len(sys.argv) != 2:
        print("Please provide a local data path:")
        #local_data_path = os.path.abspath(
        #    os.path.join(
        #        os.path.dirname(__file__), "..", "..", "data"))

        local_data_path = os.path.abspath('data')
    else:
        local_data_path = sys.argv[1]

    path_main = os.path.abspath(
        os.path.join(
            os.path.dirname(__file__), '..', "energy_demand/config_data"))

    # Initialise logger
    #logger_setup.set_up_logger(os.path.join(local_data_path, "..", "logging_local_run.log"))
    # logger_setup.set_up_logger(os.path.join(local_data_path, "logging_local_run.log"))

    # Load data
    data['criterias'] = {}
    data['criterias']['mode_constrained'] = True                    # True: Technologies are defined in ED model and fuel is provided, False: Heat is delievered not per technologies
    data['criterias']['virtual_building_stock_criteria'] = True     # True: Run virtual building stock model

    fast_model_run = False
    if fast_model_run == True:
        data['criterias']['write_to_txt'] = False
        data['criterias']['beyond_supply_outputs'] = False
        data['criterias']['validation_criteria'] = False    # For validation, the mode_constrained must be True
        data['criterias']['plot_tech_lp'] = False
        data['criterias']['plot_crit'] = False
        data['criterias']['crit_plot_enduse_lp'] = False
        data['criterias']['plot_HDD_chart'] = False
        data['criterias']['writeYAML'] = False
    else:
        data['criterias']['write_to_txt'] = True
        data['criterias']['beyond_supply_outputs'] = True
        data['criterias']['validation_criteria'] = True
        data['criterias']['plot_tech_lp'] = False
        data['criterias']['plot_crit'] = False
        data['criterias']['crit_plot_enduse_lp'] = True
        data['criterias']['plot_HDD_chart'] = False
        data['criterias']['writeYAML'] = True #set to false

    # ----------------------------
    # Model running configurations
    # ----------------------------
    simulated_yrs = [2015]
    name_scenario_run = "_result_data_{}".format(str(time.ctime()).replace(":", "_").replace(" ", "_"))

    # Paths
    data['paths'] = data_loader.load_paths(path_main)
    data['local_paths'] = data_loader.get_local_paths(local_data_path)
    data['result_paths'] = data_loader.get_result_paths(
        os.path.join(os.path.join(local_data_path, "..", "results"), name_scenario_run))

    data['lookups'] = lookup_tables.basic_lookups()
    data['enduses'], data['sectors'], data['fuels'] = data_loader.load_fuels(data['paths'], data['lookups'])

    # data['regions'] = data_loader.load_regions_localmodelrun(
    #     os.path.join(local_data_path, 'region_definitions', 'regions_local_modelrun.csv'))

    data['regions'] = get_region_names(os.path.join(local_data_path, 'region_definitions', 'same_as_scenario_data', 'lad_2016_uk_simplified.shp'))
    print(data['regions'])
    raise Exception

    data['reg_nrs'] = len(data['regions'])

    data['population'] = data_loader.read_scenario_data(
        os.path.join(local_data_path, 'scenarios', 'uk_pop_high_migration_2015_2050.csv'))

    data['gva'] = data_loader.read_scenario_data(
        os.path.join(local_data_path, 'scenarios', 'gva_sven.csv'))

    data['industry_gva'] = "TST"

    #Dummy data
    pop_density = {}
    data['reg_coord'] = {}
    for reg in data['regions']:
        data['reg_coord'][reg] = {'longitude': 52.58, 'latitude': -1.091}
        pop_density[reg] = 1
    data['pop_density'] = pop_density

    # ------------------------------
    # Assumptions
    # ------------------------------
    # Parameters not defined within smif
    data['assumptions'] = non_param_assumptions.Assumptions(
        base_yr=2015,
        curr_yr=2015,
        simulated_yrs=simulated_yrs,
        paths=data['paths'],
        enduses=data['enduses'],
        sectors=data['sectors'],
        fueltypes=data['lookups']['fueltypes'],
        fueltypes_nr=data['lookups']['fueltypes_nr'])

    # Parameters defined within smif
    strategy_variables = param_assumptions.load_param_assump(
        data['paths'], data['local_paths'], data['assumptions'])
    data['assumptions'].update('strategy_variables', strategy_variables)

    data['tech_lp'] = data_loader.load_data_profiles(
        data['paths'], data['local_paths'],
        data['assumptions'].model_yeardays,
        data['assumptions'].model_yeardays_daytype,
        data['criterias']['plot_tech_lp'])

    technologies = non_param_assumptions.update_technology_assumption(
        data['assumptions'].technologies,
        data['assumptions'].strategy_variables['f_eff_achieved']['scenario_value'],
        data['assumptions'].strategy_variables['gshp_fraction_ey']['scenario_value'])
    data['technologies'] = technologies

    data['weather_stations'], data['temp_data'] = data_loader.load_temp_data(data['local_paths'])

    # ------------------------------

    if data['criterias']['virtual_building_stock_criteria']:
        rs_floorarea, ss_floorarea, data['service_building_count'], rs_regions_without_floorarea, ss_regions_without_floorarea = data_loader.floor_area_virtual_dw(
            data['regions'],
            data['sectors']['all_sectors'],
            data['local_paths'],
            data['population'][data['assumptions'].base_yr],
            data['assumptions'].base_yr)

        # Add all areas with no floor area data
        data['assumptions'].update("rs_regions_without_floorarea", rs_regions_without_floorarea)
        data['assumptions'].update("ss_regions_without_floorarea", ss_regions_without_floorarea)
    # Lookup table to import industry sectoral gva
    lookup_tables.industrydemand_name_sic2007()

    #Scenario data
    data['scenario_data'] = {
        'gva': data['gva'],
        'population': data['population'],
        'industry_gva': data['industry_gva'],
        'floor_area': {
            'rs_floorarea': rs_floorarea,
            'ss_floorarea': ss_floorarea}}

    print("Start Energy Demand Model with python version: " + str(sys.version))
    print("Info model run")
    print("Nr of Regions " + str(data['reg_nrs']))

    # --------------------
    # Disaggregate fuel
    # --------------------
    data['fuel_disagg'] = s_disaggregation.disaggregate_demand(data)

    # In order to load these data, the initialisation scripts need to be run
    print("... Load data from script calculations")
    data = read_data.load_script_data(data)

    #-------------------
    # Folder cleaning
    #--------------------
    print("... delete previous model run results")
    basic_functions.del_previous_setup(data['result_paths']['data_results'])
    basic_functions.create_folder(data['result_paths']['data_results'])
    basic_functions.create_folder(data['result_paths']['data_results_PDF'])
    basic_functions.create_folder(data['result_paths']['data_results_model_run_pop'])

    # Create .ini file with simulation information
    write_data.write_simulation_inifile(data['result_paths']['data_results'], data)

    for sim_yr in data['assumptions'].simulated_yrs:
        setattr(data['assumptions'], 'curr_yr', sim_yr)

        print("Simulation for year --------------:  " + str(sim_yr))
        fuel_in, fuel_in_biomass, fuel_in_elec, fuel_in_gas, fuel_in_heat, fuel_in_hydro, fuel_in_solid_fuel, fuel_in_oil, tot_heating = testing_functions.test_function_fuel_sum(
            data,
            data['fuel_disagg'],
            data['criterias']['mode_constrained'],
            data['assumptions'].enduse_space_heating)

        # Main model run function
        modelrun_obj = energy_demand_model(data, data['assumptions'])

        # --------------------
        # Result unconstrained
        #
        # Sum according to first element in array (sectors)
        # which aggregtes over the sectors
        # ---
        supply_results_unconstrained = sum(modelrun_obj.results_unconstrained[:, ])

        # Write out all calculations which are not used for SMIF
        if data['criterias']['beyond_supply_outputs']:

            ed_fueltype_regs_yh = modelrun_obj.ed_fueltype_regs_yh
            out_enduse_specific = modelrun_obj.tot_fuel_y_enduse_specific_yh
            tot_fuel_y_max_enduses = modelrun_obj.tot_fuel_y_max_enduses
            ed_fueltype_national_yh = modelrun_obj.ed_fueltype_national_yh

            reg_load_factor_y = modelrun_obj.reg_load_factor_y
            reg_load_factor_yd = modelrun_obj.reg_load_factor_yd
            reg_load_factor_winter = modelrun_obj.reg_seasons_lf['winter']
            reg_load_factor_spring = modelrun_obj.reg_seasons_lf['spring']
            reg_load_factor_summer = modelrun_obj.reg_seasons_lf['summer']
            reg_load_factor_autumn = modelrun_obj.reg_seasons_lf['autumn']

            # -------------------------------------------
            # Write annual results to txt files
            # -------------------------------------------
            print("... Start writing results to file")
            path_runs = data['result_paths']['data_results_model_runs']

            write_data.write_supply_results(
                sim_yr,
                "result_tot_yh",
                path_runs,
                modelrun_obj.ed_fueltype_regs_yh,
                "result_tot_submodels_fueltypes")
            write_data.write_enduse_specific(
                sim_yr,
                path_runs,
                out_enduse_specific,
                "out_enduse_specific")
            write_data.write_lf(
                path_runs,
                "result_reg_load_factor_y",
                [sim_yr],
                reg_load_factor_y,
                'reg_load_factor_y')
            write_data.write_lf(
                path_runs,
                "result_reg_load_factor_yd",
                [sim_yr],
                reg_load_factor_yd,
                'reg_load_factor_yd')
            write_data.write_lf(
                path_runs,
                "result_reg_load_factor_winter",
                [sim_yr],
                reg_load_factor_winter,
                'reg_load_factor_winter')
            write_data.write_lf(
                path_runs,
                "result_reg_load_factor_spring",
                [sim_yr],
                reg_load_factor_spring,
                'reg_load_factor_spring')
            write_data.write_lf(
                path_runs,
                "result_reg_load_factor_summer",
                [sim_yr],
                reg_load_factor_summer,
                'reg_load_factor_summer')
            write_data.write_lf(
                path_runs,
                "result_reg_load_factor_autumn",
                [sim_yr],
                reg_load_factor_autumn,
                'reg_load_factor_autumn')

            # -------------------------------------------
            # Write population files of simulation year
            # -------------------------------------------
            pop_array_reg = np.zeros((len(data['regions'])), dtype="float")
            for reg_array_nr, reg in enumerate(data['regions']):
                pop_array_reg[reg_array_nr] = data['scenario_data']['population'][sim_yr][reg]

            write_data.write_scenaric_population_data(
                sim_yr,
                data['result_paths']['model_run_pop'],
                pop_array_reg)
            print("... Finished writing results to file")

    print("-------------------------")
    print("... Finished running HIRE")
    print("-------------------------")<commit_msg>Implement method to get centroids from shapefile<commit_after>"""Allows to run HIRE locally outside the SMIF framework

    Tools
    # -------
    Profiling:  https://jiffyclub.github.io/snakeviz/
    python -m cProfile -o program.prof main.py
    snakeviz program.prof
"""
import os
import sys
import time
import logging
import numpy as np
from energy_demand import model
from energy_demand.basic import testing_functions
from energy_demand.basic import lookup_tables
from energy_demand.assumptions import non_param_assumptions
from energy_demand.assumptions import param_assumptions
from energy_demand.read_write import data_loader
from energy_demand.basic import logger_setup
from energy_demand.read_write import write_data
from energy_demand.read_write import read_data
from energy_demand.basic import basic_functions
from energy_demand.scripts import s_disaggregation

import fiona
from pprint import pprint
from shapely.geometry import shape, mapping

def energy_demand_model(regions, data, assumptions):
    """Main function of energy demand model to calculate yearly demand

    Arguments
    ----------
    data : dict
        Data container

    Returns
    -------
    result_dict : dict
        A nested dictionary containing all data for energy supply model with
        timesteps for every hour in a year.
        [fueltype : region : timestep]
    modelrun_obj : dict
        Object of a yearly model run

    Note
    ----
    This function is executed in the wrapper
    """
    modelrun = model.EnergyDemandModel(
        regions=regions, #data['regions'],
        data=data,
        assumptions=assumptions)

    # Calculate base year demand
    fuels_in = testing_functions.test_function_fuel_sum(
        data,
        data['fuel_disagg'],
        data['criterias']['mode_constrained'],
        assumptions.enduse_space_heating)

    # Log model results
    write_data.logg_info(modelrun, fuels_in, data)
    logging.info("...finished running energy demand model simulation")
    return modelrun

def get_region_names(path):
    ''' Returns names of shapes within a shapefile
    '''
    with fiona.open(path, 'r') as source:
        return [elem['properties']['name'] for elem in source]

def get_region_centroids(path):
    ''' Returns centroids of shapes within a shapefile
    '''
    with fiona.open(path, 'r') as source:
        geoms = [elem for elem in source]

    for geom in geoms:
        my_shape = shape(geom['geometry'])
        geom['geometry'] = mapping(my_shape.centroid)

    return geoms

if __name__ == "__main__":
    """
    """
    data = {}

    # Paths
    if len(sys.argv) != 2:
        print("Please provide a local data path:")
        #local_data_path = os.path.abspath(
        #    os.path.join(
        #        os.path.dirname(__file__), "..", "..", "data"))

        local_data_path = os.path.abspath('data')
    else:
        local_data_path = sys.argv[1]

    path_main = os.path.abspath(
        os.path.join(
            os.path.dirname(__file__), '..', "energy_demand/config_data"))

    # Initialise logger
    #logger_setup.set_up_logger(os.path.join(local_data_path, "..", "logging_local_run.log"))
    # logger_setup.set_up_logger(os.path.join(local_data_path, "logging_local_run.log"))

    # Load data
    data['criterias'] = {}
    data['criterias']['mode_constrained'] = True                    # True: Technologies are defined in ED model and fuel is provided, False: Heat is delievered not per technologies
    data['criterias']['virtual_building_stock_criteria'] = True     # True: Run virtual building stock model

    fast_model_run = False
    if fast_model_run == True:
        data['criterias']['write_to_txt'] = False
        data['criterias']['beyond_supply_outputs'] = False
        data['criterias']['validation_criteria'] = False    # For validation, the mode_constrained must be True
        data['criterias']['plot_tech_lp'] = False
        data['criterias']['plot_crit'] = False
        data['criterias']['crit_plot_enduse_lp'] = False
        data['criterias']['plot_HDD_chart'] = False
        data['criterias']['writeYAML'] = False
    else:
        data['criterias']['write_to_txt'] = True
        data['criterias']['beyond_supply_outputs'] = True
        data['criterias']['validation_criteria'] = True
        data['criterias']['plot_tech_lp'] = False
        data['criterias']['plot_crit'] = False
        data['criterias']['crit_plot_enduse_lp'] = True
        data['criterias']['plot_HDD_chart'] = False
        data['criterias']['writeYAML'] = True #set to false

    # ----------------------------
    # Model running configurations
    # ----------------------------
    simulated_yrs = [2015]
    name_scenario_run = "_result_data_{}".format(str(time.ctime()).replace(":", "_").replace(" ", "_"))

    # Paths
    data['paths'] = data_loader.load_paths(path_main)
    data['local_paths'] = data_loader.get_local_paths(local_data_path)
    data['result_paths'] = data_loader.get_result_paths(
        os.path.join(os.path.join(local_data_path, "..", "results"), name_scenario_run))

    data['lookups'] = lookup_tables.basic_lookups()
    data['enduses'], data['sectors'], data['fuels'] = data_loader.load_fuels(data['paths'], data['lookups'])

    # data['regions'] = data_loader.load_regions_localmodelrun(
    #     os.path.join(local_data_path, 'region_definitions', 'regions_local_modelrun.csv'))

    data['regions'] = get_region_names(os.path.join(local_data_path, 'region_definitions', 'same_as_scenario_data', 'lad_2016_uk_simplified.shp'))
    reg_centroids = get_region_centroids(os.path.join(local_data_path, 'region_definitions', 'same_as_scenario_data', 'lad_2016_uk_simplified.shp'))

    pprint(reg_centroids[0])
    pprint(reg_centroids[10])

    raise Exception

    data['reg_nrs'] = len(data['regions'])

    data['population'] = data_loader.read_scenario_data(
        os.path.join(local_data_path, 'scenarios', 'uk_pop_high_migration_2015_2050.csv'))

    data['gva'] = data_loader.read_scenario_data(
        os.path.join(local_data_path, 'scenarios', 'gva_sven.csv'))

    data['industry_gva'] = "TST"

    #Dummy data
    pop_density = {}
    data['reg_coord'] = {}
    for reg in data['regions']:
        data['reg_coord'][reg] = {'longitude': 52.58, 'latitude': -1.091}
        pop_density[reg] = 1
    data['pop_density'] = pop_density

    # ------------------------------
    # Assumptions
    # ------------------------------
    # Parameters not defined within smif
    data['assumptions'] = non_param_assumptions.Assumptions(
        base_yr=2015,
        curr_yr=2015,
        simulated_yrs=simulated_yrs,
        paths=data['paths'],
        enduses=data['enduses'],
        sectors=data['sectors'],
        fueltypes=data['lookups']['fueltypes'],
        fueltypes_nr=data['lookups']['fueltypes_nr'])

    # Parameters defined within smif
    strategy_variables = param_assumptions.load_param_assump(
        data['paths'], data['local_paths'], data['assumptions'])
    data['assumptions'].update('strategy_variables', strategy_variables)

    data['tech_lp'] = data_loader.load_data_profiles(
        data['paths'], data['local_paths'],
        data['assumptions'].model_yeardays,
        data['assumptions'].model_yeardays_daytype,
        data['criterias']['plot_tech_lp'])

    technologies = non_param_assumptions.update_technology_assumption(
        data['assumptions'].technologies,
        data['assumptions'].strategy_variables['f_eff_achieved']['scenario_value'],
        data['assumptions'].strategy_variables['gshp_fraction_ey']['scenario_value'])
    data['technologies'] = technologies

    data['weather_stations'], data['temp_data'] = data_loader.load_temp_data(data['local_paths'])

    # ------------------------------

    if data['criterias']['virtual_building_stock_criteria']:
        rs_floorarea, ss_floorarea, data['service_building_count'], rs_regions_without_floorarea, ss_regions_without_floorarea = data_loader.floor_area_virtual_dw(
            data['regions'],
            data['sectors']['all_sectors'],
            data['local_paths'],
            data['population'][data['assumptions'].base_yr],
            data['assumptions'].base_yr)

        # Add all areas with no floor area data
        data['assumptions'].update("rs_regions_without_floorarea", rs_regions_without_floorarea)
        data['assumptions'].update("ss_regions_without_floorarea", ss_regions_without_floorarea)
    # Lookup table to import industry sectoral gva
    lookup_tables.industrydemand_name_sic2007()

    #Scenario data
    data['scenario_data'] = {
        'gva': data['gva'],
        'population': data['population'],
        'industry_gva': data['industry_gva'],
        'floor_area': {
            'rs_floorarea': rs_floorarea,
            'ss_floorarea': ss_floorarea}}

    print("Start Energy Demand Model with python version: " + str(sys.version))
    print("Info model run")
    print("Nr of Regions " + str(data['reg_nrs']))

    # --------------------
    # Disaggregate fuel
    # --------------------
    data['fuel_disagg'] = s_disaggregation.disaggregate_demand(data)

    # In order to load these data, the initialisation scripts need to be run
    print("... Load data from script calculations")
    data = read_data.load_script_data(data)

    #-------------------
    # Folder cleaning
    #--------------------
    print("... delete previous model run results")
    basic_functions.del_previous_setup(data['result_paths']['data_results'])
    basic_functions.create_folder(data['result_paths']['data_results'])
    basic_functions.create_folder(data['result_paths']['data_results_PDF'])
    basic_functions.create_folder(data['result_paths']['data_results_model_run_pop'])

    # Create .ini file with simulation information
    write_data.write_simulation_inifile(data['result_paths']['data_results'], data)

    for sim_yr in data['assumptions'].simulated_yrs:
        setattr(data['assumptions'], 'curr_yr', sim_yr)

        print("Simulation for year --------------:  " + str(sim_yr))
        fuel_in, fuel_in_biomass, fuel_in_elec, fuel_in_gas, fuel_in_heat, fuel_in_hydro, fuel_in_solid_fuel, fuel_in_oil, tot_heating = testing_functions.test_function_fuel_sum(
            data,
            data['fuel_disagg'],
            data['criterias']['mode_constrained'],
            data['assumptions'].enduse_space_heating)

        # Main model run function
        modelrun_obj = energy_demand_model(data, data['assumptions'])

        # --------------------
        # Result unconstrained
        #
        # Sum according to first element in array (sectors)
        # which aggregtes over the sectors
        # ---
        supply_results_unconstrained = sum(modelrun_obj.results_unconstrained[:, ])

        # Write out all calculations which are not used for SMIF
        if data['criterias']['beyond_supply_outputs']:

            ed_fueltype_regs_yh = modelrun_obj.ed_fueltype_regs_yh
            out_enduse_specific = modelrun_obj.tot_fuel_y_enduse_specific_yh
            tot_fuel_y_max_enduses = modelrun_obj.tot_fuel_y_max_enduses
            ed_fueltype_national_yh = modelrun_obj.ed_fueltype_national_yh

            reg_load_factor_y = modelrun_obj.reg_load_factor_y
            reg_load_factor_yd = modelrun_obj.reg_load_factor_yd
            reg_load_factor_winter = modelrun_obj.reg_seasons_lf['winter']
            reg_load_factor_spring = modelrun_obj.reg_seasons_lf['spring']
            reg_load_factor_summer = modelrun_obj.reg_seasons_lf['summer']
            reg_load_factor_autumn = modelrun_obj.reg_seasons_lf['autumn']

            # -------------------------------------------
            # Write annual results to txt files
            # -------------------------------------------
            print("... Start writing results to file")
            path_runs = data['result_paths']['data_results_model_runs']

            write_data.write_supply_results(
                sim_yr,
                "result_tot_yh",
                path_runs,
                modelrun_obj.ed_fueltype_regs_yh,
                "result_tot_submodels_fueltypes")
            write_data.write_enduse_specific(
                sim_yr,
                path_runs,
                out_enduse_specific,
                "out_enduse_specific")
            write_data.write_lf(
                path_runs,
                "result_reg_load_factor_y",
                [sim_yr],
                reg_load_factor_y,
                'reg_load_factor_y')
            write_data.write_lf(
                path_runs,
                "result_reg_load_factor_yd",
                [sim_yr],
                reg_load_factor_yd,
                'reg_load_factor_yd')
            write_data.write_lf(
                path_runs,
                "result_reg_load_factor_winter",
                [sim_yr],
                reg_load_factor_winter,
                'reg_load_factor_winter')
            write_data.write_lf(
                path_runs,
                "result_reg_load_factor_spring",
                [sim_yr],
                reg_load_factor_spring,
                'reg_load_factor_spring')
            write_data.write_lf(
                path_runs,
                "result_reg_load_factor_summer",
                [sim_yr],
                reg_load_factor_summer,
                'reg_load_factor_summer')
            write_data.write_lf(
                path_runs,
                "result_reg_load_factor_autumn",
                [sim_yr],
                reg_load_factor_autumn,
                'reg_load_factor_autumn')

            # -------------------------------------------
            # Write population files of simulation year
            # -------------------------------------------
            pop_array_reg = np.zeros((len(data['regions'])), dtype="float")
            for reg_array_nr, reg in enumerate(data['regions']):
                pop_array_reg[reg_array_nr] = data['scenario_data']['population'][sim_yr][reg]

            write_data.write_scenaric_population_data(
                sim_yr,
                data['result_paths']['model_run_pop'],
                pop_array_reg)
            print("... Finished writing results to file")

    print("-------------------------")
    print("... Finished running HIRE")
    print("-------------------------")<|endoftext|>


<commit_before>class RaftLog(object):
    def __init__(self, log):
        if not log:
            le = logentry(0, '', {})
            le['index'] = 0
            le['committed'] = True
            log = {0: le}
        self.log_by_index = log
        self.log_by_msgid = {}
        for ent in self.log_by_index.values():
            msgid = ent['msgid']
            self.log_by_msgid[msgid] = ent

    def dump(self):
        return self.log_by_index

    def get_max_index_term(self):
        maxindex = self.maxindex()
        maxterm = self.log_by_index.get(maxindex, {}).get('term', None)
        return maxindex, maxterm

    def has_uuid(self, uuid):
        return uuid in self.log_by_msgid

    def maxindex(self):
        return max(self.log_by_index)

    def get(self, idx):
        return self.log_by_index.get(idx, None)

    def get_by_uuid(self, uuid):
        return self.log_by_msgid.get(uuid, None)

    def get_by_index(self, index):
        return self.get(index)

    def get_term_of(self, idx):
        le = self.get(idx)
        return le['term']

    def remove(self, idx):
        ent = self.log_by_index[idx]
        del self.log_by_index[idx]
        msgid = ent['msgid']
        del self.log_by_msgid[msgid]

    def add(self, logentry):
        if not 'index' in logentry:
            # this is being appended to a leader's log; reject if msgid is
            # known and allocate a new index for it
            if logentry['msgid'] in self.log_by_msgid:
                return
            index = self.maxindex() + 1
            logentry['index'] = index
        else:
            # this is a follower being told to put logentry in a specific spot
            index = logentry['index']
            mi = self.maxindex()
            if mi + 1 != index:
                # remove everything in the log after and including the current
                # index
                remove = [x for x in self.log_by_index if x >= index]
                for rem in remove:
                    self.remove(rem)
        self.log_by_index[index] = logentry
        msgid = logentry['msgid']
        self.log_by_msgid[msgid] = logentry
        return index

    def add_ack(self, index, term, uuid):
        ent = self.log_by_index[index]
        if uuid in ent['acked']:
            return
        ent['acked'].append(uuid)

    def num_acked(self, index):
        ent = self.log_by_index[index]
        return len(ent['acked'])

    def commit(self, index, term):
        ent = self.log_by_index[index]
        assert ent['term'] == term
        ent['committed'] = True

    def force_commit(self, index):
        # this is more dangerous; only call it from followers on orders
        # from the leader
        ent = self.log_by_index.get(index, None)
        if ent is None:
            return
        ent['committed'] = True

    def is_committed(self, index, term):
        ent = self.log_by_index[index]
        if ent['term'] != term:
            return False
        index = ent['index']
        return index <= self.get_commit_index()

    def is_committed_by_uuid(self, uuid):
        ent = self.log_by_msgid.get(uuid, None)
        if ent is None:
            return False
        index = ent['index']
        return index <= self.get_commit_index()

    def logs_after_index(self, index):
        last = self.maxindex()
        logs = {}
        for x in range(index, min(last, index + 50)):
            logs[x+1] = self.log_by_index[x+1]
        return logs

    def committed_logs_after_index(self, index):
        last = self.get_commit_index()
        logs = {}
        for x in range(index, last):
            logs[x+1] = self.log_by_index[x+1]
        return logs

    def get_commit_index(self):
        for k in reversed(sorted(self.log_by_index)):
            v = self.log_by_index[k]
            if v['committed']:
                return v['index']
        return 0

    def exists(self, index, term):
        if self.log_by_index.get(index, {}).get('term', None) == term:
            return True
        return False

    def __le__(self, other):
        mi, mt = self.get_max_index_term()
        oi, ot = other.get_max_index_term()
        if ot > mt:
            return True
        if ot == mt and oi >= mi:
            return True
        return False

    def __gt__(self, other):
        return not self <= other

def logentry(term, uuid, msg):
    rpc = {
        'term': term,
        'msgid': uuid,
        'committed': False,
        'acked': [],
        'msg': msg,
    }
    return rpc
<commit_msg>golf it<commit_after>class RaftLog(object):
    def __init__(self, log):
        if not log:
            le = logentry(0, '', {})
            le['index'] = 0
            le['committed'] = True
            log = {0: le}
        self.log_by_index = log
        self.log_by_msgid = {}
        for ent in self.log_by_index.values():
            msgid = ent['msgid']
            self.log_by_msgid[msgid] = ent

    def dump(self):
        return self.log_by_index

    def get_max_index_term(self):
        maxindex = self.maxindex()
        maxterm = self.log_by_index.get(maxindex, {}).get('term', None)
        return maxindex, maxterm

    def has_uuid(self, uuid):
        return uuid in self.log_by_msgid

    def maxindex(self):
        return max(self.log_by_index)

    def get(self, idx):
        return self.log_by_index.get(idx, None)

    def get_by_uuid(self, uuid):
        return self.log_by_msgid.get(uuid, None)

    def get_by_index(self, index):
        return self.get(index)

    def get_term_of(self, idx):
        le = self.get(idx)
        return le['term']

    def remove(self, idx):
        ent = self.log_by_index[idx]
        del self.log_by_index[idx]
        msgid = ent['msgid']
        del self.log_by_msgid[msgid]

    def add(self, logentry):
        if not 'index' in logentry:
            # this is being appended to a leader's log; reject if msgid is
            # known and allocate a new index for it
            if logentry['msgid'] in self.log_by_msgid:
                return
            index = self.maxindex() + 1
            logentry['index'] = index
        else:
            # this is a follower being told to put logentry in a specific spot
            index = logentry['index']
            mi = self.maxindex()
            if mi + 1 != index:
                # remove everything in the log after and including the current
                # index
                remove = [x for x in self.log_by_index if x >= index]
                for rem in remove:
                    self.remove(rem)
        self.log_by_index[index] = logentry
        msgid = logentry['msgid']
        self.log_by_msgid[msgid] = logentry
        return index

    def add_ack(self, index, term, uuid):
        ent = self.log_by_index[index]
        if uuid in ent['acked']:
            return
        ent['acked'].append(uuid)

    def num_acked(self, index):
        ent = self.log_by_index[index]
        return len(ent['acked'])

    def commit(self, index, term):
        ent = self.log_by_index[index]
        assert ent['term'] == term
        ent['committed'] = True

    def force_commit(self, index):
        # this is more dangerous; only call it from followers on orders
        # from the leader
        ent = self.log_by_index.get(index, None)
        if ent is None:
            return
        ent['committed'] = True

    def is_committed(self, index, term):
        ent = self.log_by_index[index]
        if ent['term'] != term:
            return False
        index = ent['index']
        return index <= self.get_commit_index()

    def is_committed_by_uuid(self, uuid):
        ent = self.log_by_msgid.get(uuid, None)
        if ent is None:
            return False
        index = ent['index']
        return index <= self.get_commit_index()

    def logs_after_index(self, index):
        last = self.maxindex()
        logs = {}
        for x in range(index, min(last, index + 50)):
            logs[x+1] = self.log_by_index[x+1]
        return logs

    def committed_logs_after_index(self, index):
        last = self.get_commit_index()
        logs = {}
        for x in range(index, last):
            logs[x+1] = self.log_by_index[x+1]
        return logs

    def get_commit_index(self):
        for k in reversed(sorted(self.log_by_index)):
            v = self.log_by_index[k]
            if v['committed']:
                return v['index']
        return 0

    def exists(self, index, term):
        return self.log_by_index.get(index, {}).get('term', None) == term

    def __le__(self, other):
        mi, mt = self.get_max_index_term()
        oi, ot = other.get_max_index_term()
        if ot > mt:
            return True
        if ot == mt and oi >= mi:
            return True
        return False

    def __gt__(self, other):
        return not self <= other

def logentry(term, uuid, msg):
    rpc = {
        'term': term,
        'msgid': uuid,
        'committed': False,
        'acked': [],
        'msg': msg,
    }
    return rpc
<|endoftext|>


<commit_before>"""Reads the fasttext word embeddings

Visit: https://fasttext.cc/docs/en/english-vectors.html#format"""

import os
import numpy as np
import dataset.utils as utils
from collections import defaultdict
from . import fetcher

NAME = "wiki-news-300d-1M.vec"

def fetch_and_parse(data_dir):
    """
    Fetches and parses the fasttext word embeddings dataset. The dataset is
    also cached as a pickle for further calls.

    Args:
        data_dir (str): absolute path to the dir where datasets are stored

    Returns:
        voc (dict): bimap of word <-> id
        emb (numpy array): array of embeddings for each word in `voc`
    """
    fasttext_dir = fetcher.fetch(data_dir)

    return parse(fasttext_dir)

@utils.cache(2)
def parse(fasttext_dir):
    """
    Parses the fasttext word embeddings.

    Args:
        fastext_dir (str): absolute path to the extracted word embeddings

    Returns:
        voc (dict): bimap of word <-> id
        emb (numpy.array): array of embeddings for each word in `voc`
    """
    # The first embedding is reserved for the UNK token
    emb = [[]]
    voc = defaultdict(int)
    voc[0] = "UNK"
    voc["UNK"] = 0

    first = True
    fasttext_path = os.path.join(fasttext_dir, NAME)
    with open(fasttext_path) as f:
        for line in f:
            # First line contains # words and embedding sizes, skip
            if first:
                first = False
                continue

            i = len(emb)
            parts = line.split(" ")
            voc[parts[0]] = i
            voc[i] = parts[0]
            emb.append([float(n) for n in parts[1:-1]])

    # We copy the last embedding for the UNK token
    emb[0] = emb[-1]
    return voc, np.array(emb)
<commit_msg>embeddings: add special tokens in the vocabulary<commit_after>"""Reads the fasttext word embeddings

Visit: https://fasttext.cc/docs/en/english-vectors.html#format"""

import os
import numpy as np
import dataset.utils as utils
from collections import defaultdict
from . import fetcher

NAME = "wiki-news-300d-1M.vec"

def fetch_and_parse(data_dir):
    """
    Fetches and parses the fasttext word embeddings dataset. The dataset is
    also cached as a pickle for further calls.

    Args:
        data_dir (str): absolute path to the dir where datasets are stored

    Returns:
        voc (dict): bimap of word <-> id
        emb (numpy array): array of embeddings for each word in `voc`
    """
    fasttext_dir = fetcher.fetch(data_dir)

    return parse(fasttext_dir)

@utils.cache(2)
def parse(fasttext_dir):
    """
    Parses the fasttext word embeddings.

    Args:
        fastext_dir (str): absolute path to the extracted word embeddings

    Returns:
        voc (dict): bimap of word <-> id
        emb (numpy.array): array of embeddings for each word in `voc`
    """
    # Reserve 0 for special padding token, 1 for unknown, 2 for end of stream,
    # and default any token not in the vocabulary to unknown
    voc = defaultdict(lambda: 1)
    voc[0] = "PAD"
    voc["_PAD_"] = 0
    voc[1] = "UNK"
    voc["_UNK_"] = 1
    voc[2] = "END_STREAM"
    voc["_END_STREAM_"] = 1

    # Reserve first embeddings for special tokens
    emb = [[], [], []]

    first = True
    fasttext_path = os.path.join(fasttext_dir, NAME)
    with open(fasttext_path) as f:
        for line in f:
            # First line contains # words and embedding sizes, skip
            if first:
                first = False
                continue

            i = len(emb)
            parts = line.split(" ")
            if parts[0] not in voc:
                voc[parts[0]] = i
                voc[i] = parts[0]
                emb.append([float(n) for n in parts[1:-1]])

    # We copy the last embedding for the special token
    emb[0] = emb[-1]
    emb[1] = emb[-1]
    emb[2] = emb[-1]
    return voc, np.array(emb)
<|endoftext|>


<commit_before>"""
Usage:
    rapy -r ROUTER -p PROFILE -i INPUT_FILE [-o OUTPUT_DIR] [-x PARAMS] [-v | --verbose]
    rapy -h | --help
    rapy --version

Fetch optimal routes between every two locations in the INPUT_FILE csv file
with an online routing service of ROUTER, write the results in OUTPUT_DIR.
The extra parameters for different routing services can be provided in the
JSON file specified by the PARAMS argument.  The test area information are
configured in appconf.json file.

Options:
    -r ROUTER      Set routing service provider (required)
    -p PROFILE     Set the preferred routing profile indicating the
                   transportation mode to use for routing (required)
                   [default: walking]
    -i INPUT_FILE  Set the input csv file containing all the points (required)
    -o OUTPUT_DIR  Set the directory for saving routing results, create a new
                   directory if not exists (optional) [default: ./output]
    -x PARAMS      Extra parameters for the router, a plain text file in JSON
                   format (optional)
    -v --verbose   Show running log in detail
    -h --help      Show this help
    --version      Show version number

Arguments:
    ROUTER         Routing service API provider name. The valid values are
                   configured in appconf.json file
    PROFILE        Routing profile name indicating what kind of transportation
                   mode should be use, default to walking
    INPUT_FILE     Points information file in csv format. Must have `x`, `y`,
                   and `id` fields at least to record the longtitude and
                   latitude coordinates and id
    OUTPUT_DIR     Directory for saving routing results, default to ./output
    PARAMS         JSON file containing extra parameters for the router

Examples:
    rapy -r mapbox -p walking -i ./input/munich.csv
    rapy -r graphhopper -p driving -i ./input/heidelberg.csv -o ./gh_results -v
"""
import json
import os
import csv
from docopt import docopt
try:
    from schema import Schema, And, Or, Use, SchemaError
except ImportError:
    exit('This example requires that `schema` data-validation library'
         ' is installed: \n    pip install schema\n'
         'https://github.com/halst/schema')
from rap import __version__, RoutingServiceFactory

def validate_arguments(raw_args, conf):
    sch = Schema({
        'ROUTER': And(str,
                      Use(str.lower),
                      lambda r: r in conf['routers'],
                      error='ROUTER should be one of {0}'.format(', '.join(
                          conf['routers']))),
        'PROFILE': And(str,
                       Use(str.lower),
                       lambda p: p in conf['profiles'],
                       error='PROFILE should be one of {0}'.format(', '.join(
                           conf['profiles']))),
        'INPUT_FILE': Use(os.path.isfile,
                          error='Input data file {0} does not exist'.format(
                              raw_args['INPUT_FILE'])),
        'OUTPUT_DIR': Use(os.path.isdir,
                          error='{0} is not a valid directory'.format(raw_args[
                              'OUTPUT_DIR'])),
        'PARAMS': Or(None,
                     Use(os.path.isfile,
                         error='Can not find extra parameters file {0}'.format(
                             raw_args['PARAMS'])))
    })
    try:
        args = sch.validate(raw_args)
    except docopt.DocoptExit as e:
        print(e.message)
    except SchemaError as e:
        print(e.message)
        exit(e)
    return args

def save_route_to(route, filepath):
    with open(filepath, 'w') as f:
        json.dump(route, f)

def try_touching(router, source, target, output_dir, params=None):
    res = router.find_path(source['x'], source['y'], target['x'], target['y'],
                           params)
    if res is None:
        return 0
    # The found routes will be stored in the form that looks like
    # /OUTPUT_DIR_ROOT/ROUTER/PROFILE/SOURCE_TARGET.json
    os.makedirs(output_dir, exist_ok=True)
    save_route_to(res.geojson,
                  os.path.join(output_dir, '{0}_{1}.json'.format(
                      source['id'], target['id'])))
    return 1

def cal_accessibility(router, target, all_pts, output_dir, params=None):
    i = all_pts.index(target)
    # Put all the points except the target into other_pts list
    other_pts = all_pts[:i] + all_pts[(i + 1):]
    # Calculate how many points can reach the target, and use this value as the
    # target's accessibility index
    return reduce(
        lambda x, y: x + y,
        map(lambda p: try_touching(router, p, target, output_dir, params),
            other_pts), 0)

def main():
    """Entrypoint of command line interface.
    """
    args = docopt(__doc__, version=__version__)
    with open('appconf.json', 'r') as f:
        appconf = json.load(f)
    args = validate_arguments(args, appconf)
    print(args)
    router = RoutingServiceFactory(args['ROUTER'], args['PROFILE'])
    points = []
    with open(args['INPUT_FILE'], 'r') as f:
        pf = csv.DictReader(f)
        for r in pf:
            points.append({
                'x': float(r['x']),
                'y': float(r['y']),
                'id': int(r['id'])
            })

    if args['PARAMS'] is None:
        params = None
    else:
        with open(args['PARAMS']) as f:
            params = json.load(f)

    points_with_accessibility = list(
        map(lambda p: cal_accessibility(
                router, p, points, args['PROFILE'],
                os.path.join(
                    args['OUTPUT_DIR'], args['ROUTER'], args['PROFILE']),
                params),
            points))

    with open(
            os.path.join(args['OUTPUT_DIR'], '{0}.csv'.format(args['ROUTER'])),
            'w') as f:
        fieldnames = points_with_accessibility[0].keys()
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        writer.writeheader()
        writer.writerows(points_with_accessibility)
<commit_msg>bug fix, add logging. finally works<commit_after>"""
Usage:
    rapy -r ROUTER -p PROFILE -i INPUT_FILE [-o OUTPUT_DIR] [-x PARAMS] [-v | --verbose]
    rapy -h | --help
    rapy --version

Fetch optimal routes between every two locations in the INPUT_FILE csv file
with an online routing service of ROUTER, write the results in OUTPUT_DIR.
The extra parameters for different routing services can be provided in the
JSON file specified by the PARAMS argument.  The test area information are
configured in appconf.json file.

Options:
    -r ROUTER      Set routing service provider (required)
    -p PROFILE     Set the preferred routing profile indicating the
                   transportation mode to use for routing (required)
                   [default: walking]
    -i INPUT_FILE  Set the input csv file containing all the points (required)
    -o OUTPUT_DIR  Set the directory for saving routing results, create a new
                   directory if not exists (optional) [default: ./output]
    -x PARAMS      Extra parameters for the router, a plain text file in JSON
                   format (optional)
    -v --verbose   Show running log in detail
    -h --help      Show this help
    --version      Show version number

Arguments:
    ROUTER         Routing service API provider name. The valid values are
                   configured in appconf.json file
    PROFILE        Routing profile name indicating what kind of transportation
                   mode should be use, default to walking
    INPUT_FILE     Points information file in csv format. Must have `x`, `y`,
                   and `id` fields at least to record the longtitude and
                   latitude coordinates and id
    OUTPUT_DIR     Directory for saving routing results, default to ./output
    PARAMS         JSON file containing extra parameters for the router

Examples:
    rapy -r mapbox -p walking -i ./input/munich.csv
    rapy -r graphhopper -p driving -i ./input/heidelberg.csv -o ./gh_results -v
"""
import json
import os
import csv
from functools import reduce
import logging.config
import logging
from docopt import docopt, DocoptExit
try:
    from schema import Schema, And, Or, Optional, Use, SchemaError
except ImportError:
    exit('This example requires that `schema` data-validation library'
         ' is installed: \n    pip install schema\n'
         'https://github.com/halst/schema')
from rap import __version__, RoutingServiceFactory

LOGGING_CONF_FILE = 'logging.json'
DEFAULT_LOGGING_LVL = logging.WARNING
path = LOGGING_CONF_FILE
value = os.getenv('RAPY_LOG_CFG', None)
if value:
    path = value
if os.path.exists(path):
    with open(path, 'rt') as f:
        config = json.load(f)
    logging.config.dictConfig(config)
else:
    logging.basicConfig(
        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
        level=logging.INFO)
logger = logging.getLogger(__name__)

def validate_arguments(raw_args, conf):
    logger.info("Validating input arguments")
    sch = Schema({
        '-r': And(Use(str.lower),
                  lambda r: str.lower(r) in conf['routers'],
                  error="ROUTER should be one of {0}".format(', '.join(conf[
                      'routers']))),
        '-p': And(Use(str.lower),
                  lambda p: str.lower(p) in conf['profiles'],
                  error="PROFILE should be one of {0}".format(', '.join(conf[
                      'profiles']))),
        '-i': And(lambda i: os.path.isfile(i),
                  error="File {0} does not exist".format(raw_args['-i'])),
        Optional(
            '-o', default='./output'):
        And(lambda o: os.path.isdir(o),
            error="{0} is not a valid directory".format(raw_args['-o'])),
        Optional('-x'): Or(
            None,
            lambda x: os.path.isfile(x),
            error="Parameters file {0} does not exist".format(raw_args['-x'])),
        Optional('--help'): Or(True, False),
        Optional('--version'): Or(True, False),
        Optional('--verbose'): Or(True, False)
    })
    try:
        args = sch.validate(raw_args)
    except DocoptExit as e:
        logger.error("Parse command-line arguments failed!")
        print(e.message)
    except SchemaError as e:
        logger.error("Validate command-line arguments failed!")
        print(e.autos)
        exit(e)
    return args

def save_route_to(route, filepath):
    logger.debug("Save the found route information to {0} ".format(filepath))
    with open(filepath, 'w') as f:
        json.dump(route, f)

def try_touching(router, source, target, output_dir, params=None):
    logger.debug("Try searching for a path from {0} to {1}".format(
        str(source), str(target)))
    res = router.find_path(source['x'], source['y'], target['x'], target['y'],
                           params)
    if res is None:
        return 0
    # The found routes will be stored in a directory like
    # /OUTPUT_DIR_ROOT/ROUTER/PROFILE/SOURCE_TARGET.json
    os.makedirs(output_dir, exist_ok=True)
    save_route_to(res,
                  os.path.join(output_dir, '{0}_{1}.json'.format(
                      source['id'], target['id'])))
    return 1

def cal_accessibility(router, target, all_pts, output_dir, params=None):
    logger.debug("Calculating the accessibilities of point {0}...".format(
        str(target)))
    i = all_pts.index(target)
    # Put all the points except the target into other_pts list
    other_pts = all_pts[:i] + all_pts[(i + 1):]
    # Calculate how many points can reach the target, and use this value as the
    # target's accessibility index
    p_accessibility = {k: v for k, v in target.items()}
    p_accessibility.update({
        'accessibility': reduce(
            lambda x, y: x + y,
            map(lambda p: try_touching(router, p, target, output_dir, params),
                other_pts), 0)
    })
    return p_accessibility

def main():
    """Entrypoint of command line interface.
    """
    args = docopt(__doc__, version=__version__)
    if args['--verbose']:
        logging.getLogger().setLevel(logging.DEBUG)
        logger.info(
            "==== Routers As Probes pYthon command-line tool (rapy) ====")
        logger.info("== Running in verbose mode with DEBUG info ==")
        logger.info("Project page: http://github.com/tumluliu/rap")
        logger.info("Contact: Lu Liu via nudtlliu@gmail.com")
        logger.info("Start working...")
        logger.debug("Arguments for rapy: {0}".format(str(args)))
    else:
        logging.getLogger().setLevel(logging.WARNING)
    with open('appconf.json', 'r') as f:
        appconf = json.load(f)
    args = validate_arguments(args, appconf)
    logger.debug("Arguments after validation: {0}".format(args))
    router = RoutingServiceFactory(args['-r'], args['-p'])
    logger.debug("Router {0} instance has been created".format(
        router.__class__.__name__))
    points = []
    logger.info("Open input data file with stub points")
    with open(args['-i'], 'r') as f:
        logger.debug("Data file {0} has been opened for reading".format(args[
            '-i']))
        pf = csv.DictReader(f)
        for r in pf:
            logger.debug("Current row in the points info file: {0}".format(
                str(r)))
            points.append({
                'x': float(r['x']),
                'y': float(r['y']),
                'id': int(r['id'])
            })

    logger.info("Load extra parameter file for the current router")
    if args['-x'] is None:
        params = None
    else:
        with open(args['-x']) as f:
            params = json.load(f)

    logger.info("Calculate accessibilities for all the stub points")
    points_with_accessibility = list(
        map(lambda p: cal_accessibility(
                router, p, points,
                os.path.join(
                    args['-o'], args['-r'], args['-p']),
                params),
            points))
    logger.debug("And we get the points with accessibilities: {0}".format(
        str(points_with_accessibility)))

    with open(os.path.join(args['-o'], '{0}.csv'.format(args['-r'])),
              'w') as f:
        fieldnames = points_with_accessibility[0].keys()
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        writer.writeheader()
        writer.writerows(points_with_accessibility)

    logger.info("All done!")
<|endoftext|>


<commit_before>import os

from django.utils.crypto import get_random_string

import herokuify

from antxetamedia.settings import * #noqa

DEBUG = False

DATABASES = herokuify.get_db_config()

INSTALLED_APPS = [
    'collectfast',
] + INSTALLED_APPS + [
    'djangosecure',
    'antxetamedia.heroku',
]

###########
# Storage #
###########

AWS_STORAGE_BUCKET_NAME = 'amv2'
AWS_S3_CUSTOM_DOMAIN = '{0}.s3.amazonaws.com'.format(AWS_STORAGE_BUCKET_NAME)
AWS_QUERYSTRING_AUTH = False
AWS_PRELOAD_METADATA = True

DEFAULT_FILE_STORAGE = 'herokuify.storage.S3MediaStorage'
STATICFILES_STORAGE = 'herokuify.storage.CachedS3StaticStorage'
COMPRESS_STORAGE = 'herokuify.storage.CachedS3StaticStorage'
MEDIA_URL = "https://{0}.s3.amazonaws.com/media/".format(AWS_STORAGE_BUCKET_NAME)
STATIC_URL = "https://{0}.s3.amazonaws.com/static/".format(AWS_STORAGE_BUCKET_NAME)
CKEDITOR_JQUERY_URL = os.path.join(STATIC_URL, 'bower_components/jquery/dist/jquery.min.js')

#####################
# Security settings #
#####################

SECRET_KEY = os.environ.get('SECRET_KEY', get_random_string(
    50, 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'))

MIDDLEWARE_CLASSES += [
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'djangosecure.middleware.SecurityMiddleware',
]
X_FRAME_OPTIONS = 'DENY'
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
SECURE_SSL_REDIRECT = True
SECURE_HSTS_SECONDS = 60*60*24
SECURE_HSTS_INCLUDE_SUBDOMAINS = True
SECURE_CONTENT_TYPE_NOSNIFF = True
SECURE_BROWSER_XSS_FILTER = True
SESSION_COOKIE_SECURE = True
SESSION_COOKIE_HTTPONLY = True
CSRF_COOKIE_SECURE = True

ALLOWED_HOSTS = ['localhost',
                 'antxetamedia.herokuapp.com',
                 'antxetamedia.info',
                 'www.antxetamedia.info',
                 'beta.antxetamedia.info']

#########
# Cache #
#########

CACHES = herokuify.get_cache_config()
SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db'

##########
# Celery #
##########

BROKER_URL = os.environ.get('REDISCLOUD_URL')
CELERY_ALWAYS_EAGER = False
CELERYD_TASK_SOFT_TIME_LIMIT = 500

########
# SMTP #
########

ADMINS = [
    ('Unai Zalakain', 'unai@gisa-elkartea.org'),
]

from herokuify.mail.sendgrid import EMAIL_HOST, EMAIL_HOST_USER, EMAIL_HOST_PASSWORD, EMAIL_PORT, EMAIL_USE_TLS  # noqa
SERVER_EMAIL = 'system@antxetamedia.info'
EMAIL_SUBJECT_PREFIX = '[antxetamedia] '

###########
# Logging #
###########

LOGGING = {
    "version": 1,
    "disable_existing_loggers": False,
    "handlers": {
        "console": {
            "level": "INFO",
            "class": "logging.StreamHandler",
        },
    },
    "loggers": {
        "django": {
            "handlers": ["console"],
        }
    }
}

MIDDLEWARE_CLASSES += [
    'raygun_dot_io.middleware.RaygunDotIOMiddleware',
]

RAYGUN_API_KEY = os.getenv('RAYGUN_APIKEY')
RAYGUN_API_ENABLED = True
<commit_msg>Do not use pickle to encode celery messages<commit_after>import os

from django.utils.crypto import get_random_string

import herokuify

from antxetamedia.settings import * #noqa

DEBUG = False

DATABASES = herokuify.get_db_config()

INSTALLED_APPS = [
    'collectfast',
] + INSTALLED_APPS + [
    'djangosecure',
    'antxetamedia.heroku',
]

###########
# Storage #
###########

AWS_STORAGE_BUCKET_NAME = 'amv2'
AWS_S3_CUSTOM_DOMAIN = '{0}.s3.amazonaws.com'.format(AWS_STORAGE_BUCKET_NAME)
AWS_QUERYSTRING_AUTH = False
AWS_PRELOAD_METADATA = True

DEFAULT_FILE_STORAGE = 'herokuify.storage.S3MediaStorage'
STATICFILES_STORAGE = 'herokuify.storage.CachedS3StaticStorage'
COMPRESS_STORAGE = 'herokuify.storage.CachedS3StaticStorage'
MEDIA_URL = "https://{0}.s3.amazonaws.com/media/".format(AWS_STORAGE_BUCKET_NAME)
STATIC_URL = "https://{0}.s3.amazonaws.com/static/".format(AWS_STORAGE_BUCKET_NAME)
CKEDITOR_JQUERY_URL = os.path.join(STATIC_URL, 'bower_components/jquery/dist/jquery.min.js')

#####################
# Security settings #
#####################

SECRET_KEY = os.environ.get('SECRET_KEY', get_random_string(
    50, 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'))

MIDDLEWARE_CLASSES += [
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'djangosecure.middleware.SecurityMiddleware',
]
X_FRAME_OPTIONS = 'DENY'
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
SECURE_SSL_REDIRECT = True
SECURE_HSTS_SECONDS = 60*60*24
SECURE_HSTS_INCLUDE_SUBDOMAINS = True
SECURE_CONTENT_TYPE_NOSNIFF = True
SECURE_BROWSER_XSS_FILTER = True
SESSION_COOKIE_SECURE = True
SESSION_COOKIE_HTTPONLY = True
CSRF_COOKIE_SECURE = True

ALLOWED_HOSTS = ['localhost',
                 'antxetamedia.herokuapp.com',
                 'antxetamedia.info',
                 'www.antxetamedia.info',
                 'beta.antxetamedia.info']

#########
# Cache #
#########

CACHES = herokuify.get_cache_config()
SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db'

##########
# Celery #
##########

BROKER_URL = os.environ.get('REDISCLOUD_URL')
CELERY_ALWAYS_EAGER = False
CELERYD_TASK_SOFT_TIME_LIMIT = 500
CELERY_ACCEPT_CONTENT = ['json']

########
# SMTP #
########

ADMINS = [
    ('Unai Zalakain', 'unai@gisa-elkartea.org'),
]

from herokuify.mail.sendgrid import EMAIL_HOST, EMAIL_HOST_USER, EMAIL_HOST_PASSWORD, EMAIL_PORT, EMAIL_USE_TLS  # noqa
SERVER_EMAIL = 'system@antxetamedia.info'
EMAIL_SUBJECT_PREFIX = '[antxetamedia] '

###########
# Logging #
###########

LOGGING = {
    "version": 1,
    "disable_existing_loggers": False,
    "handlers": {
        "console": {
            "level": "INFO",
            "class": "logging.StreamHandler",
        },
    },
    "loggers": {
        "django": {
            "handlers": ["console"],
        }
    }
}

MIDDLEWARE_CLASSES += [
    'raygun_dot_io.middleware.RaygunDotIOMiddleware',
]

RAYGUN_API_KEY = os.getenv('RAYGUN_APIKEY')
RAYGUN_API_ENABLED = True
<|endoftext|>


<commit_before>import socket
from pysimplesoap.client import SoapClient

def discover(service, timeout=2, retries=1):
    group = ("239.255.255.250", 1900)
    message = "\r\n".join([
        'M-SEARCH * HTTP/1.1',
        'HOST: {0}:{1}'.format(*group),
        'MAN: "ssdp:discover"',
        'ST: {st}', 'MX: 1', '', ''])
    socket.setdefaulttimeout(timeout)
    responses = {}
    for _ in range(retries):
        sock = socket.socket(socket.AF_INET,
                             socket.SOCK_DGRAM,
                             socket.IPPROTO_UDP)
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 2)
        sock.sendto(message.format(st=service), group)
        while True:
            try:
                response, addr = sock.recvfrom(1024)
                responses[addr[0]] = response
                print response
            except socket.timeout:
                break
    return responses

# devices = discover('ssdp:urn:schemas-upnp-org:device:MediaRenderer:1')
# for device in devices:
#     print device

rendering_control = SoapClient(
    location="http://192.168.178.37:42342/RenderingService/Control",
    action="urn:upnp-org:serviceId:RenderingControl#",
    namespace="http://schemas.xmlsoap.org/soap/envelope/",
    soap_ns='soap', ns='s', exceptions=True)

av_transport = SoapClient(
    location="http://192.168.178.37:42342/TransportService/Control",
    action="urn:schemas-upnp-org:service:AVTransport:1#",
    namespace="http://schemas.xmlsoap.org/soap/envelope/",
    soap_ns='soap', ns='s', exceptions=True)

# response = av_transport.Play(InstanceID=1, Speed=2)
response = av_transport.Pause(InstanceID=1)
# response = av_transport.Play(InstanceID=1)
# response = rendering_control.SetVolume(InstanceID=1, DesiredVolume=0)
# response = rendering_control.SetMute(InstanceID=1, DesiredMute=0)

# response = rendering_control.GetVolume(InstanceID=1)
# print response.CurrentVolume
<commit_msg>created the raumfeld class<commit_after>import socket
from pysimplesoap.client import SoapClient

def discover(service='ssdp:all', timeout=2, retries=1):
    group = ("239.255.255.250", 1900)
    message = "\r\n".join([
        'M-SEARCH * HTTP/1.1',
        'HOST: {0}:{1}'.format(*group),
        'MAN: "ssdp:discover"',
        'ST: {st}', 'MX: 1', '', ''])
    socket.setdefaulttimeout(timeout)
    responses = {}
    for _ in range(retries):
        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM,
                             socket.IPPROTO_UDP)
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 2)
        sock.sendto(message.format(st=service), group)
        while True:
            try:
                response, addr = sock.recvfrom(1024)
                responses[addr[0]] = response
                print response
            except socket.timeout:
                break
    return responses

class RaumfeldDevice(object):

    def __init__(self, ip, port):
        self.address = 'http://%s:%s' % (ip, port)
        self.rendering_control = SoapClient(
            location='%s/RenderingService/Control' % self.address,
            action='urn:upnp-org:serviceId:RenderingControl#',
            namespace='http://schemas.xmlsoap.org/soap/envelope/',
            soap_ns='soap', ns='s', exceptions=True)

        self.av_transport = SoapClient(
            location='%s/TransportService/Control' % self.address,
            action='urn:schemas-upnp-org:service:AVTransport:1#',
            namespace='http://schemas.xmlsoap.org/soap/envelope/',
            soap_ns='soap', ns='s', exceptions=True)

    def play(self):
        self.av_transport.Play(InstanceID=1, Speed=2)

    def pause(self):
        self.av_transport.Pause(InstanceID=1)

    def set_volume(self, volume):
        self.rendering_control.SetVolume(InstanceID=1, DesiredVolume=volume)

    def get_volume(self):
        response = self.rendering_control.GetVolume(InstanceID=1)
        return response.CurrentVolume

    def set_mute(self, mute):
        self.rendering_control.SetMute(InstanceID=1, DesiredMute=mute)

if __name__ == '__main__':
    devices = discover('ssdp:urn:schemas-upnp-org:device:MediaRenderer:1')
    print devices
    box = RaumfeldDevice(ip='192.168.178.37', port=42342)
    box.pause()
<|endoftext|>


<commit_before>#------------------------------------------------------------------------------
#
#  Copyright (c) 2007 by Enthought, Inc.
#  All rights reserved.
#
#------------------------------------------------------------------------------

from __future__ import absolute_import

try:
    __import__('pkg_resources').declare_namespace(__name__)
except:
    pass

# For py2app / py2exe support
try:
    import modulefinder
    for p in __path__:
        modulefinder.AddPackagePath(__name__, p)
except:
    pass
<commit_msg>remove enthought namespace<commit_after><|endoftext|>


<commit_before>import tdl

import instances
from entities import entity

class Animation(entity.Entity):
    def __init__(self):
        super().__init__(' ')

    def on_done(self):
        pass

class FlashBackground(Animation):
    def __init__(self, bg=(255, 255, 255), interval=0.25, repeat=2):
        super().__init__()

        self.time = 0
        self.interval = interval
        self.times_flashed = 0
        self.times_to_flash = repeat * 2
        self.bg = bg
        self.current_bg = bg

    def update(self, time):
        self.time += time
        if self.time > self.interval:
            self.time = 0
            self.times_flashed += 1

            if self.current_bg == self.bg:
                self.current_bg = self.parent.bg
            else:
                self.current_bg = self.bg

        if self.times_flashed > self.times_to_flash:
            self.parent.children.remove(self)
            self.on_done()

    def draw(self, console):
        if not self.parent.visible:
            return

        pos = self.parent.position
        char = self.parent.char
        fg = self.parent.fg

        console.draw_char(*pos, char, fg, self.current_bg)

class Flash(Animation):
    def __init__(self, char, fg=(255, 255, 255), bg=(0, 0, 0), interval=0.25, repeat=2):
        super().__init__()

        self.time = 0
        self.interval = interval
        self.times_flashed = 0
        self.times_to_flash = repeat * 2
        self.char = char
        self.fg = fg
        self.bg = bg
        self.current_bg = bg
        self.hidden = False

    def update(self, time):
        self.time += time
        if self.time > self.interval:
            self.time = 0
            self.times_flashed += 1

            self.hidden = not self.hidden

        if self.times_flashed > self.times_to_flash:
            self.parent.children.remove(self)
            self.on_done()

    def draw(self, console):
        if not self.visible or not self.parent.visible:
            return

        pos = self.parent.position
        console.draw_char(*pos, self.char, self.fg, self.bg)

class ThrowMotion(Animation):
    def __init__(self, source, dest, time):
        super().__init__()

        self.points = tdl.map.bresenham(*source, *dest)
        self.dest = dest
        self.time = time
        self.time_to_next = 0
        self.frame_time = time / len(self.points)
        self.current_point = self.points.pop(0)

    def update(self, time):
        self.parent.hidden = True
        self.time_to_next += time
        if self.time_to_next >= self.frame_time:
            self.time_to_next = 0

            if self.points:
                self.current_point = self.points.pop(0)

            else:
                self.parent.hidden = False
                self.parent.remove(self)
                self.on_done()

    def draw(self, console):
        if not instances.scene_root.check_visibility(*self.current_point):
            return

        p = self.parent
        console.draw_char(*self.current_point, p.char, p.fg, p.bg)
<commit_msg>Flash animation should use offset property<commit_after>import tdl

import instances
from entities import entity

class Animation(entity.Entity):
    def __init__(self):
        super().__init__(' ')

    def on_done(self):
        pass

class FlashBackground(Animation):
    def __init__(self, bg=(255, 255, 255), interval=0.25, repeat=2):
        super().__init__()

        self.time = 0
        self.interval = interval
        self.times_flashed = 0
        self.times_to_flash = repeat * 2
        self.bg = bg
        self.current_bg = bg

    def update(self, time):
        self.time += time
        if self.time > self.interval:
            self.time = 0
            self.times_flashed += 1

            if self.current_bg == self.bg:
                self.current_bg = self.parent.bg
            else:
                self.current_bg = self.bg

        if self.times_flashed > self.times_to_flash:
            self.parent.children.remove(self)
            self.on_done()

    def draw(self, console):
        if not self.parent.visible:
            return

        pos = self.parent.position
        char = self.parent.char
        fg = self.parent.fg

        console.draw_char(*pos, char, fg, self.current_bg)

class Flash(Animation):
    def __init__(self, char, fg=(255, 255, 255), bg=(0, 0, 0), interval=0.25, repeat=2):
        super().__init__()

        self.time = 0
        self.interval = interval
        self.times_flashed = 0
        self.times_to_flash = repeat * 2
        self.char = char
        self.fg = fg
        self.bg = bg
        self.current_bg = bg
        self.hidden = False

    def update(self, time):
        self.time += time
        if self.time > self.interval:
            self.time = 0
            self.times_flashed += 1

            self.hidden = not self.hidden

        if self.times_flashed > self.times_to_flash:
            self.parent.children.remove(self)
            self.on_done()

    def draw(self, console):
        if not self.visible or not self.parent.visible:
            return

        console.draw_char(*self.offset, self.char, self.fg, self.bg)

class ThrowMotion(Animation):
    def __init__(self, source, dest, time):
        super().__init__()

        self.points = tdl.map.bresenham(*source, *dest)
        self.dest = dest
        self.time = time
        self.time_to_next = 0
        self.frame_time = time / len(self.points)
        self.current_point = self.points.pop(0)

    def update(self, time):
        self.parent.hidden = True
        self.time_to_next += time
        if self.time_to_next >= self.frame_time:
            self.time_to_next = 0

            if self.points:
                self.current_point = self.points.pop(0)

            else:
                self.parent.hidden = False
                self.parent.remove(self)
                self.on_done()

    def draw(self, console):
        if not instances.scene_root.check_visibility(*self.current_point):
            return

        p = self.parent
        console.draw_char(*self.current_point, p.char, p.fg, p.bg)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
This is a script to run the C++ implementation of the Full TSQR algorithm
with direct computation of the matrix Q.

See options:
     python run_full_tsqr_cxx.py --help

Example usage:
     python run_full_tsqr_cxx.py --input=A_800M_10.bseq \
            --ncols=10 --schedule=100,100,100 \
            --local_output=tsqr-tmp --output=FULL_TESTING

This script is designed to run on ICME's MapReduce cluster, icme-hadoop1.

Austin R. Benson     arbenson@stanford.edu
David F. Gleich
Copyright (c) 2012
"""

import os
import shutil
import subprocess
import sys
import time
from optparse import OptionParser
lib_path = os.path.abspath('../dumbo')
sys.path.append(lib_path)
import util

# Parse command-line options
#
# TODO(arbenson): use argparse instead of optparse when icme-hadoop1 defaults
# to python 2.7
parser = OptionParser()
parser.add_option('-i', '--input', dest='input', default='',
                  help='input matrix')
parser.add_option('-o', '--output', dest='out', default='',
                  help='base string for output of Hadoop jobs')
parser.add_option('-l', '--local_output', dest='local_out', default='full_out_tmp',
                  help='Base directory for placing local files')
parser.add_option('-t', '--times_output', dest='times_out', default='times',
                  help='Base directory for placing local files')
parser.add_option('-n', '--ncols', type='int', dest='ncols', default=0,
                  help='number of columns in the matrix')
parser.add_option('-s', '--schedule', dest='sched', default='100,100,100',
                  help='comma separated list of number of map tasks to use for'
                       + ' the three jobs')

# TODO(arbenson): add option that computes singular vectors but not the Q in
# QR.  This will be the go-to option for computing the SVD of a
# tall-and-skinny matrix.
parser.add_option('-x', '--svd', type='int', dest='svd', default=0,
                  help="""0: no SVD computed ;
1: compute the singular values (R = USV^t) ;
2: compute the singular vectors as well as QR
"""
)
parser.add_option('-q', '--quiet', action='store_false', dest='verbose',
                  default=True, help='turn off some statement printing')

(options, args) = parser.parse_args()
cm = util.CommandManager(verbose=options.verbose)

STREAMING_JAR='/usr/lib/hadoop/contrib/streaming/hadoop-streaming-0.20.2-cdh3u4.jar'

# Store options in the appropriate variables
in1 = options.input
if in1 == '':
  cm.error('no input matrix provided, use --input')

out = options.out
if out == '':
  # TODO(arbenson): make sure in1 is clean
  out = in1 + '_FULL'

local_out = options.local_out
out_file = lambda f: local_out + '/' + f
if os.path.exists(local_out):
  shutil.rmtree(local_out)
os.mkdir(local_out)

times_out = options.times_out

ncols = options.ncols
if ncols == 0:
  cm.error('number of columns not provided, use --ncols')

svd_opt = options.svd

sched = options.sched
try:
  sched = [int(s) for s in sched.split(',')]
  sched[2]
except:
  cm.error('invalid schedule provided')

def form_cmd(hadoop_opts):
  cmd = 'hadoop jar %s -libjars feathers.jar ' % STREAMING_JAR
  for opt_type in hadoop_opts:
    for opt in hadoop_opts[opt_type]:
      cmd += '-%s %s ' % (opt_type, opt)
  return cmd

def run_step(hadoop_opts):
  cm.exec_cmd('hadoop fs -rmr ' + hadoop_opts['output'][0])
  cm.exec_cmd(form_cmd(hadoop_opts))

hadoop_opts = {'jobconf': ['mapreduce.job.name=tsqr_cxx',
                           'stream.map.input=typedbytes',
                           'stream.reduce.input=typedbytes',
                           'stream.map.output=typedbytes',
                           'stream.reduce.output=typedbytes',],
               'inputformat': ['org.apache.hadoop.streaming.AutoInputFormat'],
               'outputformat': ['fm.last.feathers.output.MultipleSequenceFiles'],
               'file': ['tsqr', 'tsqr_wrapper.sh', 'libblas.so.3'],
               'input': [in1],
               'mapper': ["'./tsqr_wrapper.sh map 1'"],
               'reducer': ['org.apache.hadoop.mapred.lib.IdentityReducer'],
               'numReduceTasks': ['0'],
               }

# Now run the MapReduce jobs
out1 = out + '_1'
hadoop_opts['output'] = [out1]
run_step(hadoop_opts)

out2 = out + '_2'
hadoop_opts['input'] = [out1 + '/R_*']
hadoop_opts['output'] = [out2]
hadoop_opts['mapper'] =  ['org.apache.hadoop.mapred.lib.IdentityMapper']
hadoop_opts['reducer'] =  ["'./tsqr_wrapper.sh map 2 %d'" % ncols]
hadoop_opts['numReduceTasks'] = ['1']
hadoop_opts['jobconf'] += ['mapred.map.tasks=%d' % sched[0]]
run_step(hadoop_opts)

# Q2 file needs parsing before being distributed to phase 3
Q2_file = out_file('Q2.txt')

if os.path.exists(Q2_file):
  os.remove(Q2_file)

if os.path.exists(Q2_file + '.out'):
  os.remove(Q2_file + '.out')

cm.copy_from_hdfs(out2 + '/Q2', Q2_file)
cm.parse_seq_file(Q2_file)

out3 = out + '_3'
hadoop_opts['input'] = [out1 + '/Q_*']
hadoop_opts['output'] = [out3]
hadoop_opts['file'] += [Q2_file + '.out']
hadoop_opts['mapper'] =  ["'./tsqr_wrapper.sh map 3 %d'" % ncols]
hadoop_opts['reducer'] = ['org.apache.hadoop.mapred.lib.IdentityReducer']
hadoop_opts['outputformat'] = ['org.apache.hadoop.mapred.SequenceFileOutputFormat']
hadoop_opts['numReduceTasks'] = ['0']
hadoop_opts['jobconf'] += ['mapred.map.tasks=%d' % sched[1]]
run_step(hadoop_opts)

try:
  f = open(times_out, 'a')
  f.write('times: ' + str(cm.times) + '\n')
  f.close
except:
  pass
<commit_msg>honor scheduling options properly<commit_after>#!/usr/bin/env python

"""
This is a script to run the C++ implementation of the Full TSQR algorithm
with direct computation of the matrix Q.

See options:
     python run_full_tsqr_cxx.py --help

Example usage:
     python run_full_tsqr_cxx.py --input=A_800M_10.bseq \
            --ncols=10 --schedule=100,100,100 \
            --local_output=tsqr-tmp --output=FULL_TESTING

This script is designed to run on ICME's MapReduce cluster, icme-hadoop1.

Austin R. Benson     arbenson@stanford.edu
David F. Gleich
Copyright (c) 2012
"""

import os
import shutil
import subprocess
import sys
import time
from optparse import OptionParser
lib_path = os.path.abspath('../dumbo')
sys.path.append(lib_path)
import util

# Parse command-line options
#
# TODO(arbenson): use argparse instead of optparse when icme-hadoop1 defaults
# to python 2.7
parser = OptionParser()
parser.add_option('-i', '--input', dest='input', default='',
                  help='input matrix')
parser.add_option('-o', '--output', dest='out', default='',
                  help='base string for output of Hadoop jobs')
parser.add_option('-l', '--local_output', dest='local_out', default='full_out_tmp',
                  help='Base directory for placing local files')
parser.add_option('-t', '--times_output', dest='times_out', default='times',
                  help='Base directory for placing local files')
parser.add_option('-n', '--ncols', type='int', dest='ncols', default=0,
                  help='number of columns in the matrix')
parser.add_option('-s', '--schedule', dest='sched', default='100,100,100',
                  help='comma separated list of number of map tasks to use for'
                       + ' the three jobs')

# TODO(arbenson): add option that computes singular vectors but not the Q in
# QR.  This will be the go-to option for computing the SVD of a
# tall-and-skinny matrix.
parser.add_option('-x', '--svd', type='int', dest='svd', default=0,
                  help="""0: no SVD computed ;
1: compute the singular values (R = USV^t) ;
2: compute the singular vectors as well as QR
"""
)
parser.add_option('-q', '--quiet', action='store_false', dest='verbose',
                  default=True, help='turn off some statement printing')

(options, args) = parser.parse_args()
cm = util.CommandManager(verbose=options.verbose)

STREAMING_JAR='/usr/lib/hadoop/contrib/streaming/hadoop-streaming-0.20.2-cdh3u4.jar'

# Store options in the appropriate variables
in1 = options.input
if in1 == '':
  cm.error('no input matrix provided, use --input')

out = options.out
if out == '':
  # TODO(arbenson): make sure in1 is clean
  out = in1 + '_FULL'

local_out = options.local_out
out_file = lambda f: local_out + '/' + f
if os.path.exists(local_out):
  shutil.rmtree(local_out)
os.mkdir(local_out)

times_out = options.times_out

ncols = options.ncols
if ncols == 0:
  cm.error('number of columns not provided, use --ncols')

svd_opt = options.svd

sched = options.sched
try:
  sched = [int(s) for s in sched.split(',')]
  sched[2]
except:
  cm.error('invalid schedule provided')

def form_cmd(hadoop_opts):
  cmd = 'hadoop jar %s -libjars feathers.jar ' % STREAMING_JAR
  for opt_type in hadoop_opts:
    for opt in hadoop_opts[opt_type]:
      cmd += '-%s %s ' % (opt_type, opt)
  return cmd

def run_step(hadoop_opts):
  cm.exec_cmd('hadoop fs -rmr ' + hadoop_opts['output'][0])
  cm.exec_cmd(form_cmd(hadoop_opts))

hadoop_opts = {'jobconf': ['mapreduce.job.name=tsqr_cxx',
                           'stream.map.input=typedbytes',
                           'stream.reduce.input=typedbytes',
                           'stream.map.output=typedbytes',
                           'stream.reduce.output=typedbytes',],
               'inputformat': ['org.apache.hadoop.streaming.AutoInputFormat'],
               'outputformat': ['fm.last.feathers.output.MultipleSequenceFiles'],
               'file': ['tsqr', 'tsqr_wrapper.sh'],
               'input': [in1],
               'mapper': ["'./tsqr_wrapper.sh map 1'"],
               'reducer': ['org.apache.hadoop.mapred.lib.IdentityReducer'],
               'numReduceTasks': ['0'],
               }

# Now run the MapReduce jobs
out1 = out + '_1'
hadoop_opts['output'] = [out1]
jobconf = [x for x in hadoop_opts['jobconf']]
hadoop_opts['jobconf'] = jobconf + ['mapred.map.tasks=%d' % sched[0]]
run_step(hadoop_opts)

out2 = out + '_2'
hadoop_opts['input'] = [out1 + '/R_*']
hadoop_opts['output'] = [out2]
hadoop_opts['mapper'] =  ['org.apache.hadoop.mapred.lib.IdentityMapper']
hadoop_opts['reducer'] =  ["'./tsqr_wrapper.sh map 2 %d'" % ncols]
hadoop_opts['numReduceTasks'] = ['1']
hadoop_opts['jobconf'] = jobconf + ['mapred.map.tasks=%d' % sched[1]]
run_step(hadoop_opts)

# Q2 file needs parsing before being distributed to phase 3
Q2_file = out_file('Q2.txt')

if os.path.exists(Q2_file):
  os.remove(Q2_file)

if os.path.exists(Q2_file + '.out'):
  os.remove(Q2_file + '.out')

cm.copy_from_hdfs(out2 + '/Q2', Q2_file)
cm.parse_seq_file(Q2_file)

out3 = out + '_3'
hadoop_opts['input'] = [out1 + '/Q_*']
hadoop_opts['output'] = [out3]
hadoop_opts['file'] += [Q2_file + '.out']
hadoop_opts['mapper'] =  ["'./tsqr_wrapper.sh map 3 %d'" % ncols]
hadoop_opts['reducer'] = ['org.apache.hadoop.mapred.lib.IdentityReducer']
hadoop_opts['outputformat'] = ['org.apache.hadoop.mapred.SequenceFileOutputFormat']
hadoop_opts['numReduceTasks'] = ['0']
hadoop_opts['jobconf'] = jobconf + ['mapred.map.tasks=%d' % sched[2]]
run_step(hadoop_opts)

try:
  f = open(times_out, 'a')
  f.write('times: ' + str(cm.times) + '\n')
  f.close
except:
  pass
<|endoftext|>