<commit_before>"""The sector model wrapper for smif to run the energy demand model
"""
import logging
from datetime import date
from smif.model.sector_model import SectorModel
from energy_demand.scripts.init_scripts import scenario_initalisation
from energy_demand.cli import run_model
from energy_demand.dwelling_stock import dw_stock
from energy_demand.read_write import read_data
from energy_demand.main import energy_demand_model
from energy_demand.read_write import data_loader
from energy_demand.assumptions import assumptions
from energy_demand.basic import date_handling
from pkg_resources import Requirement, resource_filename

class EDWrapper(SectorModel):
    """Energy Demand Wrapper
    """

    def before_model_run(self):
        """Runs prior to any ``simulate()`` step

        Writes scenario data out into the scenario files

        Saves useful data into the ``self.user_data`` dictionary for access
        in the ``simulate()`` method

        Data is accessed using the `get_scenario_data()` method is provided
        as a numpy array with the dimensions timesteps-by-regions-by-intervals.

        Info
        -----
        `self.user_data` allows to pass data from before_model_run to main model
        """
        self.user_data['data_path'] = '/vagrant/data_energy_demand'
        self.processed_path = '/vagrant/data_energy_demand/_processed_data'
        self.result_path = '/vagrant/data_energy_demand/_result_data'

        # Obtain scenario data
        ed_data = {}
        ed_data['print_criteria'] = True #Print criteria
        ed_data['population'] = self.get_scenario_data('population')
        ed_data['GVA'] = self.get_scenario_data('gva')
        ed_data['rs_floorarea'] = self.get_scenario_data('floor_area')
        ed_data['ss_floorarea'] = self.get_scenario_data('floor_area')
        ed_data['reg_floorarea_resid'] = self.get_scenario_data('floor_area')
        ed_data['lu_reg'] = self.get_region_names('lad')
        #ed_data['reg_coord'] = regions.get_region_centroids('lad') #TO BE IMPLEMENTED BY THE SMIF GUYS

        # Load data (is to be replaced partly by scenario data (e.g. scenario assumptions))
        path_main = resource_filename(Requirement.parse("energy_demand"), "")
        ed_data['paths'] = data_loader.load_paths(path_main)
        ed_data['local_paths'] = data_loader.load_local_paths(self.user_data['data_path'])
        ed_data['lookups'] = data_loader.load_basic_lookups()
        ed_data['reg_coord'], ed_data['lu_reg'] = data_loader.get_dummy_coordinates_and_regions(ed_data['local_paths']) #REMOVE IF CORRECT DATA IN
        ed_data['weather_stations'], ed_data['temp_data'] = data_loader.load_temp_data(ed_data['local_paths'])
        ed_data['enduses'], ed_data['sectors'], ed_data['fuels'] = data_loader.load_fuels(ed_data['paths'], ed_data['lookups'])
        ed_data['tech_load_profiles'] = data_loader.load_data_profiles(ed_data['paths'], ed_data['local_paths'])
        ed_data['sim_param'], ed_data['assumptions'] = assumptions.load_assumptions(ed_data, write_sim_param=True)

        #========SCRAP (POP.....) THIS OVERRITES SMIF INPUT REMOVE
        ed_data = data_loader.dummy_data_generation(ed_data)
        #========SCRAP (POP.....) THIS OVERRITES SMIF INPUT REMOVE

        # Initialise scenario
        self.user_data['temp_data'], self.user_data['fts_cont'], self.user_data['sgs_cont'], self.user_data['sd_cont'] = scenario_initalisation(
            self.user_data['data_path'],
            ed_data)

        # Generate dwelling stocks over whole simulation period
        self.user_data['rs_dw_stock'] = dw_stock.rs_dw_stock(ed_data['lu_reg'], ed_data)
        self.user_data['ss_dw_stock'] = dw_stock.ss_dw_stock(ed_data['lu_reg'], ed_data)

    def initialise(self, initial_conditions):
        """
        """
        pass

    def simulate(self, timestep, data=None):
        """Runs the Energy Demand model for one `timestep`

        Arguments
        ---------
        timestep : int
            The name of the current timestep
        data : dict
            A dictionary containing all parameters and model inputs defined in
            the smif configuration by name

        Notes
        -----
        1. Get scenario data

        Population data is required as a nested dict::

            data[year][region_geocode]

        GVA is the same::

            data[year][region_geocode]

        Floor area::

            data[year][region_geoode][sector]

        2. Run initialise scenarios
        3. For each timestep, run the model

        Data is provided to these methods in the format::

            {'parameter_name': value_array}

        where ``value_array`` is a regions-by-intervals numpy array.

        """
        self.user_data['data_path'] = '/vagrant/data_energy_demand'

        # ---------
        # Scenario data
        # ---------
        ed_data = {}
        ed_data['print_criteria'] = False # No plt.show() functions are exectued if False
        ed_data['population'] = data['population']
        ed_data['GVA'] = data['gva']
        ed_data['rs_floorarea'] = data['floor_area']
        ed_data['ss_floorarea'] = data['floor_area']
        ed_data['reg_floorarea_resid'] = data['floor_area']

        # ---------
        # Replace data in ed_data with data provided from wrapper or before_model_run
        # Write data from smif to data container from energy demand model
        # ---------
        path_main = resource_filename(Requirement.parse("energy_demand"), "")
        ed_data['lu_reg'] = self.get_region_names('lad')

        #ed_data['reg_coord'] = regions.get_region_centroids('lad') #TO BE IMPLEMENTED BY THE SMIF GUYS
        ed_data['paths'] = data_loader.load_paths(path_main)
        ed_data['local_paths'] = data_loader.load_local_paths(self.user_data['data_path'])
        ed_data['lookups'] = data_loader.load_basic_lookups()
        ed_data['enduses'], ed_data['sectors'], ed_data['fuels'] = data_loader.load_fuels(ed_data['paths'], ed_data['lookups'])
        ed_data['tech_load_profiles'] = data_loader.load_data_profiles(ed_data['paths'], ed_data['local_paths'])
        ed_data['sim_param'], ed_data['assumptions'] = assumptions.load_assumptions(ed_data, write_sim_param=True)
        ed_data['weather_stations'], _ = data_loader.load_temp_data(ed_data['local_paths'])
        ed_data['reg_coord'], ed_data['lu_reg'] = data_loader.get_dummy_coordinates_and_regions(ed_data['local_paths']) #REPLACE BY SMIF INPUT
        ed_data['sim_param']['current_year'] = timestep
        ed_data['sim_param']['end_year'] = 2020
        ed_data['sim_param']['sim_years_intervall'] = 1

        ed_data['assumptions']['assump_diff_floorarea_pp'] = data['assump_diff_floorarea_pp']
        ed_data['assumptions']['climate_change_temp_diff_month'] = data['climate_change_temp_diff_month']
        ed_data['assumptions']['rs_t_base_heating']['end_yr'] = data['rs_t_base_heating_ey']
        ed_data['assumptions']['efficiency_achieving_factor'] = data['efficiency_achieving_factor']

        # Update: Necessary updates after external data definition
        ed_data['sim_param']['sim_period'] = range(ed_data['sim_param']['base_yr'], ed_data['sim_param']['end_yr'] + 1, ed_data['sim_param']['sim_years_intervall'])
        ed_data['sim_param']['sim_period_yrs'] = int(ed_data['sim_param']['end_yr'] + 1 - ed_data['sim_param']['base_yr'])
        ed_data['sim_param']['list_dates'] = date_handling.fullyear_dates(start=date(ed_data['sim_param']['base_yr'], 1, 1), end=date(ed_data['sim_param']['base_yr'], 12, 31))
        ed_data['assumptions'] = assumptions.update_assumptions(ed_data['assumptions']) #Maybe write s_script

        ed_data['rs_dw_stock'] = self.user_data['rs_dw_stock']
        ed_data['ss_dw_stock'] = self.user_data['ss_dw_stock']

        #========SCRAP (POP.....) THIS OVERRITES SMIF INPUT REMOVE
        ed_data = data_loader.dummy_data_generation(ed_data)
        #========SCRAP (POP.....) THIS OVERRITES SMIF INPUT REMOVE

        # -----------------------
        # Load data from scripts (replacing #ed_data = read_data.load_script_data(ed_data))
        # -----------------------
        # fts_cont['ss_service_tech_by_p'], fts_cont['ss_service_fueltype_tech_by_p'], fts_cont['ss_service_fueltype_by_p']
        # Insert from script calculations which are stored in memory
        ed_data['temp_data'] = self.user_data['temp_data']
        ed_data['assumptions']['rs_service_tech_by_p'] = self.user_data['fts_cont']['rs_service_tech_by_p']
        ed_data['assumptions']['ss_service_tech_by_p'] = self.user_data['fts_cont']['ss_service_tech_by_p']
        ed_data['assumptions']['is_service_tech_by_p'] = self.user_data['fts_cont']['is_service_tech_by_p']
        ed_data['assumptions']['rs_service_fueltype_by_p'] = self.user_data['fts_cont']['rs_service_fueltype_by_p']
        ed_data['assumptions']['ss_service_fueltype_by_p'] = self.user_data['fts_cont']['ss_service_fueltype_by_p']
        ed_data['assumptions']['is_service_fueltype_by_p'] = self.user_data['fts_cont']['is_service_fueltype_by_p']
        ed_data['assumptions']['rs_service_fueltype_tech_by_p'] = self.user_data['fts_cont']['rs_service_fueltype_tech_by_p']
        ed_data['assumptions']['ss_service_fueltype_tech_by_p'] = self.user_data['fts_cont']['ss_service_fueltype_tech_by_p']
        ed_data['assumptions']['is_service_fueltype_tech_by_p'] = self.user_data['fts_cont']['is_service_fueltype_tech_by_p']
        ed_data['assumptions']['rs_tech_increased_service'] = self.user_data['sgs_cont']['rs_tech_increased_service']
        ed_data['assumptions']['ss_tech_increased_service'] = self.user_data['sgs_cont']['ss_tech_increased_service']
        ed_data['assumptions']['is_tech_increased_service'] = self.user_data['sgs_cont']['is_tech_increased_service']
        ed_data['assumptions']['rs_tech_decreased_share'] = self.user_data['sgs_cont']['rs_tech_decreased_share']
        ed_data['assumptions']['ss_tech_decreased_share'] = self.user_data['sgs_cont']['ss_tech_decreased_share']
        ed_data['assumptions']['is_tech_decreased_share'] = self.user_data['sgs_cont']['is_tech_decreased_share']
        ed_data['assumptions']['rs_tech_constant_share'] = self.user_data['sgs_cont']['rs_tech_constant_share']
        ed_data['assumptions']['ss_tech_constant_share'] = self.user_data['sgs_cont']['ss_tech_constant_share']
        ed_data['assumptions']['is_tech_constant_share'] = self.user_data['sgs_cont']['is_tech_constant_share']
        ed_data['assumptions']['rs_sig_param_tech'] = self.user_data['sgs_cont']['rs_sig_param_tech']
        ed_data['assumptions']['ss_sig_param_tech'] = self.user_data['sgs_cont']['ss_sig_param_tech']
        ed_data['assumptions']['is_sig_param_tech'] = self.user_data['sgs_cont']['is_sig_param_tech']
        ed_data['assumptions']['rs_installed_tech'] = self.user_data['sgs_cont']['rs_installed_tech']
        ed_data['assumptions']['ss_installed_tech'] = self.user_data['sgs_cont']['ss_installed_tech']
        ed_data['assumptions']['is_installed_tech'] = self.user_data['sgs_cont']['is_installed_tech']
        ed_data['rs_fuel_disagg'] = self.user_data['sd_cont']['rs_fuel_disagg']
        ed_data['ss_fuel_disagg'] = self.user_data['sd_cont']['ss_fuel_disagg']
        ed_data['is_fuel_disagg'] = self.user_data['sd_cont']['is_fuel_disagg']
        ed_data['ts_fuel_disagg'] = self.user_data['sd_cont']['ts_fuel_disagg']

        # ---------
        # Run model
        # ---------
        _, results = energy_demand_model(ed_data)

        # ---------
        # Process results
        # ---------
        out_to_supply = results.fuel_indiv_regions_yh

        #logging.debug("FINISHED WRAPPER CALCULATIONS")
        return results

    def extract_obj(self, results):
        """Implement this method to return a scalar value objective function

        This method should take the results from the output of the `simulate`
        method, process the results, and return a scalar value which can be
        used as the objective function

        Arguments
        =========
        results : :class:`dict`
            The results from the `simulate` method

        Returns
        =======
        float
            A scalar component generated from the simulation model results
        """
        pass

'''
if __name__ == '__main__':

    data = {'population': {},
            'gva': {},
            'floor_area': {}}

    data['assump_diff_floorarea_pp'] = 1
    data['climate_change_temp_diff_month'] = 1
    data['rs_t_base_heating_ey'] = 1
    data['efficiency_achieving_factor'] = 1

    ed = EDWrapper('ed')
    from smif.convert.area import get_register as get_region_register
    from smif.convert.area import RegionSet
    from smif.convert.interval import get_register as get_interval_register
    from smif.convert.interval import IntervalSet
    from unittest.mock import Mock
    regions = get_region_register()
    regions.register(RegionSet('lad', []))

    ed.before_model_run()

    ed.simulate(2010, data)
'''
<commit_msg>Implementing array_to_dict<commit_after>"""The sector model wrapper for smif to run the energy demand model
"""
import logging
from datetime import date
from smif.model.sector_model import SectorModel
from energy_demand.scripts.init_scripts import scenario_initalisation
from energy_demand.cli import run_model
from energy_demand.dwelling_stock import dw_stock
from energy_demand.read_write import read_data
from energy_demand.main import energy_demand_model
from energy_demand.read_write import data_loader
from energy_demand.assumptions import assumptions
from energy_demand.basic import date_handling
from pkg_resources import Requirement, resource_filename

from collections import defaultdict

class EDWrapper(SectorModel):
    """Energy Demand Wrapper
    """

    def array_to_dict(self, input_array):
        """

        Arguments
        ---------
        input_array : numpy.ndarray
            timesteps, regions, interval

        Returns
        -------
        {}

        """
        data = defaultdict(dict)
        for t_idx, timestep in enumerate(self.timesteps):
            for r_idx, region in enumerate(self.get_region_names('lad')):
                data[timestep][region] = input_array[t_idx, r_idx, 0]
        return data

    def before_model_run(self):
        """Runs prior to any ``simulate()`` step

        Writes scenario data out into the scenario files

        Saves useful data into the ``self.user_data`` dictionary for access
        in the ``simulate()`` method

        Data is accessed using the `get_scenario_data()` method is provided
        as a numpy array with the dimensions timesteps-by-regions-by-intervals.

        Info
        -----
        `self.user_data` allows to pass data from before_model_run to main model
        """
        self.user_data['data_path'] = '/vagrant/data_energy_demand'
        self.processed_path = '/vagrant/data_energy_demand/_processed_data'
        self.result_path = '/vagrant/data_energy_demand/_result_data'

        # Obtain scenario data
        ed_data = {}
        ed_data['print_criteria'] = True #Print criteria
        pop_array = self.get_scenario_data('population')
        self.user_data['population'] = self.array_to_dict(pop_array)
        gva_array = self.get_scenario_data('gva')
        self.user_data['GVA'] = self.array_to_dict(gva_array)
        ed_data['rs_floorarea'] = self.get_scenario_data('floor_area')
        floor_array = self.get_scenario_data('floor_area')
        floor_dict = self.array_to_dict(floor_array)
        self.user_data['ss_floorarea'] = floor_dict
        self.user_data['reg_floorarea_resid'] = floor_dict
        ed_data['lu_reg'] = self.get_region_names('lad')
        #ed_data['reg_coord'] = regions.get_region_centroids('lad') #TO BE IMPLEMENTED BY THE SMIF GUYS

        # Load data (is to be replaced partly by scenario data (e.g. scenario assumptions))
        path_main = resource_filename(Requirement.parse("energy_demand"), "")
        ed_data['paths'] = data_loader.load_paths(path_main)
        ed_data['local_paths'] = data_loader.load_local_paths(self.user_data['data_path'])
        ed_data['lookups'] = data_loader.load_basic_lookups()
        ed_data['reg_coord'], ed_data['lu_reg'] = data_loader.get_dummy_coordinates_and_regions(ed_data['local_paths']) #REMOVE IF CORRECT DATA IN
        ed_data['weather_stations'], ed_data['temp_data'] = data_loader.load_temp_data(ed_data['local_paths'])
        ed_data['enduses'], ed_data['sectors'], ed_data['fuels'] = data_loader.load_fuels(ed_data['paths'], ed_data['lookups'])
        ed_data['tech_load_profiles'] = data_loader.load_data_profiles(ed_data['paths'], ed_data['local_paths'])
        ed_data['sim_param'], ed_data['assumptions'] = assumptions.load_assumptions(ed_data, write_sim_param=True)

        self.user_data['population_by'] = data['population'][2015]

        #========SCRAP (POP.....) THIS OVERRITES SMIF INPUT REMOVE
        ed_data = data_loader.dummy_data_generation(ed_data)
        #========SCRAP (POP.....) THIS OVERRITES SMIF INPUT REMOVE

        # Initialise scenario
        self.user_data['temp_data'], self.user_data['fts_cont'], self.user_data['sgs_cont'], self.user_data['sd_cont'] = scenario_initalisation(
            self.user_data['data_path'],
            ed_data)

        # Generate dwelling stocks over whole simulation period
        self.user_data['rs_dw_stock'] = dw_stock.rs_dw_stock(ed_data['lu_reg'], ed_data)
        self.user_data['ss_dw_stock'] = dw_stock.ss_dw_stock(ed_data['lu_reg'], ed_data)

    def initialise(self, initial_conditions):
        """
        """
        pass

    def simulate(self, timestep, data=None):
        """Runs the Energy Demand model for one `timestep`

        Arguments
        ---------
        timestep : int
            The name of the current timestep
        data : dict
            A dictionary containing all parameters and model inputs defined in
            the smif configuration by name

        Notes
        -----
        1. Get scenario data

        Population data is required as a nested dict::

            data[year][region_geocode]

        GVA is the same::

            data[year][region_geocode]

        Floor area::

            data[year][region_geoode][sector]

        2. Run initialise scenarios
        3. For each timestep, run the model

        Data is provided to these methods in the format::

            {'parameter_name': value_array}

        where ``value_array`` is a regions-by-intervals numpy array.

        """
        self.user_data['data_path'] = '/vagrant/data_energy_demand'

        # ---------
        # Scenario data
        # ---------
        ed_data = {}
        ed_data['print_criteria'] = False # No plt.show() functions are exectued if False
        ed_data['population'] = {}
        ed_data['population'][self.timesteps[0]] = self.user_data['population_by']
        ed_data['population'] = self.user_data['population']
        ed_data['GVA'] = data['gva']
        ed_data['rs_floorarea'] = data['floor_area']
        ed_data['ss_floorarea'] = data['floor_area']
        ed_data['reg_floorarea_resid'] = data['floor_area']

        # ---------
        # Replace data in ed_data with data provided from wrapper or before_model_run
        # Write data from smif to data container from energy demand model
        # ---------
        path_main = resource_filename(Requirement.parse("energy_demand"), "")
        ed_data['lu_reg'] = self.get_region_names('lad')

        #ed_data['reg_coord'] = regions.get_region_centroids('lad') #TO BE IMPLEMENTED BY THE SMIF GUYS
        ed_data['paths'] = data_loader.load_paths(path_main)
        ed_data['local_paths'] = data_loader.load_local_paths(self.user_data['data_path'])
        ed_data['lookups'] = data_loader.load_basic_lookups()
        ed_data['enduses'], ed_data['sectors'], ed_data['fuels'] = data_loader.load_fuels(ed_data['paths'], ed_data['lookups'])
        ed_data['tech_load_profiles'] = data_loader.load_data_profiles(ed_data['paths'], ed_data['local_paths'])
        ed_data['sim_param'], ed_data['assumptions'] = assumptions.load_assumptions(ed_data, write_sim_param=True)
        ed_data['weather_stations'], _ = data_loader.load_temp_data(ed_data['local_paths'])
        ed_data['reg_coord'], ed_data['lu_reg'] = data_loader.get_dummy_coordinates_and_regions(ed_data['local_paths']) #REPLACE BY SMIF INPUT
        ed_data['sim_param']['current_year'] = timestep
        ed_data['sim_param']['end_year'] = 2020
        ed_data['sim_param']['sim_years_intervall'] = 1

        ed_data['assumptions']['assump_diff_floorarea_pp'] = data['assump_diff_floorarea_pp']
        ed_data['assumptions']['climate_change_temp_diff_month'] = data['climate_change_temp_diff_month']
        ed_data['assumptions']['rs_t_base_heating']['end_yr'] = data['rs_t_base_heating_ey']
        ed_data['assumptions']['efficiency_achieving_factor'] = data['efficiency_achieving_factor']

        # Update: Necessary updates after external data definition
        ed_data['sim_param']['sim_period'] = range(ed_data['sim_param']['base_yr'], ed_data['sim_param']['end_yr'] + 1, ed_data['sim_param']['sim_years_intervall'])
        ed_data['sim_param']['sim_period_yrs'] = int(ed_data['sim_param']['end_yr'] + 1 - ed_data['sim_param']['base_yr'])
        ed_data['sim_param']['list_dates'] = date_handling.fullyear_dates(start=date(ed_data['sim_param']['base_yr'], 1, 1), end=date(ed_data['sim_param']['base_yr'], 12, 31))
        ed_data['assumptions'] = assumptions.update_assumptions(ed_data['assumptions']) #Maybe write s_script

        ed_data['rs_dw_stock'] = self.user_data['rs_dw_stock']
        ed_data['ss_dw_stock'] = self.user_data['ss_dw_stock']

        #========SCRAP (POP.....) THIS OVERRITES SMIF INPUT REMOVE
        ed_data = data_loader.dummy_data_generation(ed_data)
        #========SCRAP (POP.....) THIS OVERRITES SMIF INPUT REMOVE

        # -----------------------
        # Load data from scripts (replacing #ed_data = read_data.load_script_data(ed_data))
        # -----------------------
        # fts_cont['ss_service_tech_by_p'], fts_cont['ss_service_fueltype_tech_by_p'], fts_cont['ss_service_fueltype_by_p']
        # Insert from script calculations which are stored in memory
        ed_data['temp_data'] = self.user_data['temp_data']
        ed_data['assumptions']['rs_service_tech_by_p'] = self.user_data['fts_cont']['rs_service_tech_by_p']
        ed_data['assumptions']['ss_service_tech_by_p'] = self.user_data['fts_cont']['ss_service_tech_by_p']
        ed_data['assumptions']['is_service_tech_by_p'] = self.user_data['fts_cont']['is_service_tech_by_p']
        ed_data['assumptions']['rs_service_fueltype_by_p'] = self.user_data['fts_cont']['rs_service_fueltype_by_p']
        ed_data['assumptions']['ss_service_fueltype_by_p'] = self.user_data['fts_cont']['ss_service_fueltype_by_p']
        ed_data['assumptions']['is_service_fueltype_by_p'] = self.user_data['fts_cont']['is_service_fueltype_by_p']
        ed_data['assumptions']['rs_service_fueltype_tech_by_p'] = self.user_data['fts_cont']['rs_service_fueltype_tech_by_p']
        ed_data['assumptions']['ss_service_fueltype_tech_by_p'] = self.user_data['fts_cont']['ss_service_fueltype_tech_by_p']
        ed_data['assumptions']['is_service_fueltype_tech_by_p'] = self.user_data['fts_cont']['is_service_fueltype_tech_by_p']
        ed_data['assumptions']['rs_tech_increased_service'] = self.user_data['sgs_cont']['rs_tech_increased_service']
        ed_data['assumptions']['ss_tech_increased_service'] = self.user_data['sgs_cont']['ss_tech_increased_service']
        ed_data['assumptions']['is_tech_increased_service'] = self.user_data['sgs_cont']['is_tech_increased_service']
        ed_data['assumptions']['rs_tech_decreased_share'] = self.user_data['sgs_cont']['rs_tech_decreased_share']
        ed_data['assumptions']['ss_tech_decreased_share'] = self.user_data['sgs_cont']['ss_tech_decreased_share']
        ed_data['assumptions']['is_tech_decreased_share'] = self.user_data['sgs_cont']['is_tech_decreased_share']
        ed_data['assumptions']['rs_tech_constant_share'] = self.user_data['sgs_cont']['rs_tech_constant_share']
        ed_data['assumptions']['ss_tech_constant_share'] = self.user_data['sgs_cont']['ss_tech_constant_share']
        ed_data['assumptions']['is_tech_constant_share'] = self.user_data['sgs_cont']['is_tech_constant_share']
        ed_data['assumptions']['rs_sig_param_tech'] = self.user_data['sgs_cont']['rs_sig_param_tech']
        ed_data['assumptions']['ss_sig_param_tech'] = self.user_data['sgs_cont']['ss_sig_param_tech']
        ed_data['assumptions']['is_sig_param_tech'] = self.user_data['sgs_cont']['is_sig_param_tech']
        ed_data['assumptions']['rs_installed_tech'] = self.user_data['sgs_cont']['rs_installed_tech']
        ed_data['assumptions']['ss_installed_tech'] = self.user_data['sgs_cont']['ss_installed_tech']
        ed_data['assumptions']['is_installed_tech'] = self.user_data['sgs_cont']['is_installed_tech']
        ed_data['rs_fuel_disagg'] = self.user_data['sd_cont']['rs_fuel_disagg']
        ed_data['ss_fuel_disagg'] = self.user_data['sd_cont']['ss_fuel_disagg']
        ed_data['is_fuel_disagg'] = self.user_data['sd_cont']['is_fuel_disagg']
        ed_data['ts_fuel_disagg'] = self.user_data['sd_cont']['ts_fuel_disagg']

        # ---------
        # Run model
        # ---------
        _, results = energy_demand_model(ed_data)

        # ---------
        # Process results
        # ---------
        out_to_supply = results.fuel_indiv_regions_yh

        #logging.debug("FINISHED WRAPPER CALCULATIONS")
        return results

    def extract_obj(self, results):
        """Implement this method to return a scalar value objective function

        This method should take the results from the output of the `simulate`
        method, process the results, and return a scalar value which can be
        used as the objective function

        Arguments
        =========
        results : :class:`dict`
            The results from the `simulate` method

        Returns
        =======
        float
            A scalar component generated from the simulation model results
        """
        pass

'''
if __name__ == '__main__':

    data = {'population': {},
            'gva': {},
            'floor_area': {}}

    data['assump_diff_floorarea_pp'] = 1
    data['climate_change_temp_diff_month'] = 1
    data['rs_t_base_heating_ey'] = 1
    data['efficiency_achieving_factor'] = 1

    ed = EDWrapper('ed')
    from smif.convert.area import get_register as get_region_register
    from smif.convert.area import RegionSet
    from smif.convert.interval import get_register as get_interval_register
    from smif.convert.interval import IntervalSet
    from unittest.mock import Mock
    regions = get_region_register()
    regions.register(RegionSet('lad', []))

    ed.before_model_run()

    ed.simulate(2010, data)
'''
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Kurisu by 916253 & ihaveamac
# license: Apache License 2.0
# https://github.com/916253/Kurisu

description = """
Robocop, the moderation bot of ReSwitched.
"""

# import dependencies
import os
from discord.ext import commands
import discord
import datetime
import json, asyncio
import copy
import configparser
import traceback
import sys
import os
import re

# sets working directory to bot's folder
dir_path = os.path.dirname(os.path.realpath(__file__))
os.chdir(dir_path)

# read config for token
config = configparser.ConfigParser()
config.read("config.ini")

os.makedirs("data", exist_ok=True)
# create warnsv2.json if it doesn't exist, and convert warns.json if needed
if not os.path.isfile("data/warnsv2.json"):
    if os.path.isfile("data/warns.json"):
        print("Converting warns.json to warnsv2 format")
        with open("data/warns.json", "r") as f:
            warns = json.load(f)
        warnsv2 = {}
        for user_id, info in warns.items():
            warnsv2[user_id] = {"name": info["name"], "warns": []}
            for w_idx in range(len(info["warns"])):
                warnsv2[user_id]["warns"].append(info["warns"][str(w_idx + 1)])
        with open("data/warnsv2.json", "w") as f:
            json.dump(warnsv2, f)
    else:
        with open("data/warnsv2.json", "w") as f:
            f.write("{}")

# create restrictions.json if it doesn't exist
if not os.path.isfile("data/restrictions.json"):
    with open("data/restrictions.json", "w") as f:
        f.write("{}")

# create helpers.json if it doesn't exist
if not os.path.isfile("data/helpers.json"):
    with open("data/helpers.json", "w") as f:
        f.write("{}")

# create timebans.json if it doesn't exist
if not os.path.isfile("data/timebans.json"):
    with open("data/timebans.json", "w") as f:
        f.write("{}")

prefix = ['!', '.']
bot = commands.Bot(command_prefix=prefix, description=description, pm_help=None)

bot.actions = []  # changes messages in mod-/server-logs

# http://stackoverflow.com/questions/3411771/multiple-character-replace-with-python
chars = "\\`*_<>#@:~"
def escape_name(name):
    name = str(name)
    for c in chars:
        if c in name:
            name = name.replace(c, "\\" + c)
    return name.replace("@", "@\u200b")  # prevent mentions
bot.escape_name = escape_name

bot.pruning = False  # used to disable leave logs if pruning, maybe.

# mostly taken from https://github.com/Rapptz/discord.py/blob/async/discord/ext/commands/bot.py
@bot.event
async def on_command_error(error, ctx):
    if isinstance(error, discord.ext.commands.errors.CommandNotFound):
        pass  # ...don't need to know if commands don't exist
    if isinstance(error, discord.ext.commands.errors.CheckFailure):
        await bot.send_message(ctx.message.channel, "{} You don't have permission to use this command.".format(ctx.message.author.mention))
    elif isinstance(error, discord.ext.commands.errors.MissingRequiredArgument):
        formatter = commands.formatter.HelpFormatter()
        await bot.send_message(ctx.message.channel, "{} You are missing required arguments.\n{}".format(ctx.message.author.mention, formatter.format_help_for(ctx, ctx.command)[0]))
    else:
        if ctx.command:
            await bot.send_message(ctx.message.channel, "An error occured while processing the `{}` command.".format(ctx.command.name))
        print('Ignoring exception in command {}'.format(ctx.command), file=sys.stderr)
        traceback.print_exception(type(error), error, error.__traceback__, file=sys.stderr)

bot.all_ready = False
bot._is_all_ready = asyncio.Event(loop=bot.loop)
async def wait_until_all_ready():
    """Wait until the entire bot is ready."""
    await bot._is_all_ready.wait()
bot.wait_until_all_ready = wait_until_all_ready

@bot.event
async def on_ready():
    if bot.all_ready:
        return
    # this bot should only ever be in one server anyway
    for server in bot.servers:
        print("{} has started! {} has {:,} members!".format(bot.user.name, server.name, server.member_count))
        bot.server = server

        bot.config = config

        # channels
        bot.welcome_channel = discord.utils.get(server.channels, name="newcomers")
        bot.announcements_channel = discord.utils.get(server.channels, name="news")
        bot.helpers_channel = discord.utils.get(server.channels, name="general")
        bot.mods_channel = discord.utils.get(server.channels, name="server-logs")
        bot.modlogs_channel = discord.utils.get(server.channels, name="server-logs")
        bot.serverlogs_channel = discord.utils.get(server.channels, name="server-logs")
        bot.messagelogs_channel = discord.utils.get(server.channels, name="server-logs")

        # this is here until the new robocop is finished with new discord.py which supports categories.
        bot.community_channels = (
            discord.utils.get(server.channels, name="switch-hacking-general"),
            discord.utils.get(server.channels, name="homebrew-development"),
            discord.utils.get(server.channels, name="off-topic"),
            discord.utils.get(server.channels, name="temp-atmosphere-logo"),
        )

        # TODO: remove some of these roles that are useless on Reswitched. need to find their use around the bot first.
        # roles
        bot.staff_role = discord.utils.get(server.roles, name="mod")
        bot.halfop_role = discord.utils.get(server.roles, name="hop")
        bot.op_role = discord.utils.get(server.roles, name="OP")
        bot.superop_role = discord.utils.get(server.roles, name="moderator")
        bot.owner_role = discord.utils.get(server.roles, name="wizards")
        bot.helpers_role = discord.utils.get(server.roles, name="Helpers")
        bot.verified_role = discord.utils.get(server.roles, name="Verified")
        bot.trusted_role = discord.utils.get(server.roles, name="Trusted")
        bot.probation_role = discord.utils.get(server.roles, name="Probation")
        bot.muted_role = discord.utils.get(server.roles, name="Muted")
        bot.nomemes_role = discord.utils.get(server.roles, name="No-Memes")
        bot.nohelp_role = discord.utils.get(server.roles, name="hackers")
        bot.noembed_role = discord.utils.get(server.roles, name="No-Embed")
        bot.elsewhere_role = discord.utils.get(server.roles, name="#elsewhere")

        bot.private_role = discord.utils.get(server.roles, name="private")
        bot.hacker_role = discord.utils.get(server.roles, name="hacker")
        bot.community_role = discord.utils.get(server.roles, name="community")
        bot.everyone_role = server.default_role

        bot.staff_ranks = {
            "HalfOP": bot.halfop_role,
            "OP": bot.op_role,
            "moderator": bot.superop_role,
            "Wizards": bot.owner_role,
        }

        # load timebans
        with open("data/timebans.json", "r") as f:
            timebans = json.load(f)
        bot.timebans = {}
        timebans_i = copy.copy(timebans)
        for user_id, timestamp in timebans_i.items():
            found = False
            for user in await bot.get_bans(server):
                if user.id == user_id:
                    bot.timebans[user_id] = [user, datetime.datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S"), False]  # last variable is "notified", for <=30 minute notifications
                    found = True
                    break
            if not found:
                timebans.pop(user_id) # somehow not in the banned list anymore so let's just remove it
        with open("data/timebans.json", "w") as f:
            json.dump(timebans, f)

        bot.all_ready = True
        bot._is_all_ready.set()

        msg = "{} has started! {} has {:,} members!".format(bot.user.name, server.name, server.member_count)
        if len(failed_addons) != 0:
            msg += "\n\nSome addons failed to load:\n"
            for f in failed_addons:
                msg += "\n{}: `{}: {}`".format(*f)
        await bot.send_message(bot.serverlogs_channel, msg)

        break

# loads extensions
addons = [
    'addons.blah',
    'addons.events',
    'addons.extras',
    'addons.kickban',
    'addons.load',
    'addons.lockdown',
    'addons.logs',
    'addons.loop',
    'addons.mod',
    'addons.links',
    'addons.err',
    'addons.auto_probation',
    'addons.nxerr',
    'addons.things',
    'addons.mod_warn',
]

failed_addons = []

for extension in addons:
    try:
        bot.load_extension(extension)
    except Exception as e:
        print('{} failed to load.\n{}: {}'.format(extension, type(e).__name__, e))
        failed_addons.append([extension, type(e).__name__, e])

# Execute
print('Bot directory: ', dir_path)
bot.run(config['Main']['token'])
<commit_msg>Revert "add temp channel to community channel list"<commit_after>#!/usr/bin/env python3

# Kurisu by 916253 & ihaveamac
# license: Apache License 2.0
# https://github.com/916253/Kurisu

description = """
Robocop, the moderation bot of ReSwitched.
"""

# import dependencies
import os
from discord.ext import commands
import discord
import datetime
import json, asyncio
import copy
import configparser
import traceback
import sys
import os
import re

# sets working directory to bot's folder
dir_path = os.path.dirname(os.path.realpath(__file__))
os.chdir(dir_path)

# read config for token
config = configparser.ConfigParser()
config.read("config.ini")

os.makedirs("data", exist_ok=True)
# create warnsv2.json if it doesn't exist, and convert warns.json if needed
if not os.path.isfile("data/warnsv2.json"):
    if os.path.isfile("data/warns.json"):
        print("Converting warns.json to warnsv2 format")
        with open("data/warns.json", "r") as f:
            warns = json.load(f)
        warnsv2 = {}
        for user_id, info in warns.items():
            warnsv2[user_id] = {"name": info["name"], "warns": []}
            for w_idx in range(len(info["warns"])):
                warnsv2[user_id]["warns"].append(info["warns"][str(w_idx + 1)])
        with open("data/warnsv2.json", "w") as f:
            json.dump(warnsv2, f)
    else:
        with open("data/warnsv2.json", "w") as f:
            f.write("{}")

# create restrictions.json if it doesn't exist
if not os.path.isfile("data/restrictions.json"):
    with open("data/restrictions.json", "w") as f:
        f.write("{}")

# create helpers.json if it doesn't exist
if not os.path.isfile("data/helpers.json"):
    with open("data/helpers.json", "w") as f:
        f.write("{}")

# create timebans.json if it doesn't exist
if not os.path.isfile("data/timebans.json"):
    with open("data/timebans.json", "w") as f:
        f.write("{}")

prefix = ['!', '.']
bot = commands.Bot(command_prefix=prefix, description=description, pm_help=None)

bot.actions = []  # changes messages in mod-/server-logs

# http://stackoverflow.com/questions/3411771/multiple-character-replace-with-python
chars = "\\`*_<>#@:~"
def escape_name(name):
    name = str(name)
    for c in chars:
        if c in name:
            name = name.replace(c, "\\" + c)
    return name.replace("@", "@\u200b")  # prevent mentions
bot.escape_name = escape_name

bot.pruning = False  # used to disable leave logs if pruning, maybe.

# mostly taken from https://github.com/Rapptz/discord.py/blob/async/discord/ext/commands/bot.py
@bot.event
async def on_command_error(error, ctx):
    if isinstance(error, discord.ext.commands.errors.CommandNotFound):
        pass  # ...don't need to know if commands don't exist
    if isinstance(error, discord.ext.commands.errors.CheckFailure):
        await bot.send_message(ctx.message.channel, "{} You don't have permission to use this command.".format(ctx.message.author.mention))
    elif isinstance(error, discord.ext.commands.errors.MissingRequiredArgument):
        formatter = commands.formatter.HelpFormatter()
        await bot.send_message(ctx.message.channel, "{} You are missing required arguments.\n{}".format(ctx.message.author.mention, formatter.format_help_for(ctx, ctx.command)[0]))
    else:
        if ctx.command:
            await bot.send_message(ctx.message.channel, "An error occured while processing the `{}` command.".format(ctx.command.name))
        print('Ignoring exception in command {}'.format(ctx.command), file=sys.stderr)
        traceback.print_exception(type(error), error, error.__traceback__, file=sys.stderr)

bot.all_ready = False
bot._is_all_ready = asyncio.Event(loop=bot.loop)
async def wait_until_all_ready():
    """Wait until the entire bot is ready."""
    await bot._is_all_ready.wait()
bot.wait_until_all_ready = wait_until_all_ready

@bot.event
async def on_ready():
    if bot.all_ready:
        return
    # this bot should only ever be in one server anyway
    for server in bot.servers:
        print("{} has started! {} has {:,} members!".format(bot.user.name, server.name, server.member_count))
        bot.server = server

        bot.config = config

        # channels
        bot.welcome_channel = discord.utils.get(server.channels, name="newcomers")
        bot.announcements_channel = discord.utils.get(server.channels, name="news")
        bot.helpers_channel = discord.utils.get(server.channels, name="general")
        bot.mods_channel = discord.utils.get(server.channels, name="server-logs")
        bot.modlogs_channel = discord.utils.get(server.channels, name="server-logs")
        bot.serverlogs_channel = discord.utils.get(server.channels, name="server-logs")
        bot.messagelogs_channel = discord.utils.get(server.channels, name="server-logs")

        bot.community_channels = (
            discord.utils.get(server.channels, name="switch-hacking-general"),
            discord.utils.get(server.channels, name="homebrew-development"),
            discord.utils.get(server.channels, name="off-topic"),
        )

        # TODO: remove some of these roles that are useless on Reswitched. need to find their use around the bot first.
        # roles
        bot.staff_role = discord.utils.get(server.roles, name="mod")
        bot.halfop_role = discord.utils.get(server.roles, name="hop")
        bot.op_role = discord.utils.get(server.roles, name="OP")
        bot.superop_role = discord.utils.get(server.roles, name="moderator")
        bot.owner_role = discord.utils.get(server.roles, name="wizards")
        bot.helpers_role = discord.utils.get(server.roles, name="Helpers")
        bot.verified_role = discord.utils.get(server.roles, name="Verified")
        bot.trusted_role = discord.utils.get(server.roles, name="Trusted")
        bot.probation_role = discord.utils.get(server.roles, name="Probation")
        bot.muted_role = discord.utils.get(server.roles, name="Muted")
        bot.nomemes_role = discord.utils.get(server.roles, name="No-Memes")
        bot.nohelp_role = discord.utils.get(server.roles, name="hackers")
        bot.noembed_role = discord.utils.get(server.roles, name="No-Embed")
        bot.elsewhere_role = discord.utils.get(server.roles, name="#elsewhere")

        bot.private_role = discord.utils.get(server.roles, name="private")
        bot.hacker_role = discord.utils.get(server.roles, name="hacker")
        bot.community_role = discord.utils.get(server.roles, name="community")
        bot.everyone_role = server.default_role

        bot.staff_ranks = {
            "HalfOP": bot.halfop_role,
            "OP": bot.op_role,
            "moderator": bot.superop_role,
            "Wizards": bot.owner_role,
        }

        # load timebans
        with open("data/timebans.json", "r") as f:
            timebans = json.load(f)
        bot.timebans = {}
        timebans_i = copy.copy(timebans)
        for user_id, timestamp in timebans_i.items():
            found = False
            for user in await bot.get_bans(server):
                if user.id == user_id:
                    bot.timebans[user_id] = [user, datetime.datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S"), False]  # last variable is "notified", for <=30 minute notifications
                    found = True
                    break
            if not found:
                timebans.pop(user_id) # somehow not in the banned list anymore so let's just remove it
        with open("data/timebans.json", "w") as f:
            json.dump(timebans, f)

        bot.all_ready = True
        bot._is_all_ready.set()

        msg = "{} has started! {} has {:,} members!".format(bot.user.name, server.name, server.member_count)
        if len(failed_addons) != 0:
            msg += "\n\nSome addons failed to load:\n"
            for f in failed_addons:
                msg += "\n{}: `{}: {}`".format(*f)
        await bot.send_message(bot.serverlogs_channel, msg)

        break

# loads extensions
addons = [
    'addons.blah',
    'addons.events',
    'addons.extras',
    'addons.kickban',
    'addons.load',
    'addons.lockdown',
    'addons.logs',
    'addons.loop',
    'addons.mod',
    'addons.links',
    'addons.err',
    'addons.auto_probation',
    'addons.nxerr',
    'addons.things',
    'addons.mod_warn',
]

failed_addons = []

for extension in addons:
    try:
        bot.load_extension(extension)
    except Exception as e:
        print('{} failed to load.\n{}: {}'.format(extension, type(e).__name__, e))
        failed_addons.append([extension, type(e).__name__, e])

# Execute
print('Bot directory: ', dir_path)
bot.run(config['Main']['token'])
<|endoftext|>


<commit_before>from datetime import datetime
from enum import IntEnum, Enum
import random
import time
import math
import curses
import os
import subprocess

class Block(object):
    def __init__(self, x, y, sym, color, on_hit=lambda x: None, size=1):
        self.x, self.y = x, y
        self.sym = sym
        self.color = color
        self.on_hit = on_hit
        self.size = size

    def hit(self):
        self.on_hit((self.x, self.y))

    def render(self, renderer):
        renderer.render(self.x, self.y, self.sym * self.size, self.color)

class Vec2(object):
    def __init__(self, x, y):
        self.x = x
        self.y = y

    def translate(self, target):
        return Vec2(self.x + target.x, self.y + target.y)

    def scale(self, scalar):
        return Vec2(self.x * scalar, self.y * scalar)

class Plane(Enum):
    X = 0
    Y = 1
    BOTH = 2

class Ball:
    def __init__(self, width, miss_callback):
        self.pos = Vec2(width // 2, 1)
        self.angle = math.pi/2 + random.uniform(-0.5, 0.5)
        self.direction_vec = Vec2(math.cos(self.angle),
                                  math.sin(self.angle))
        self.speed = 12 # move 12 blocks per second
        self.miss_callback = miss_callback

    def update(self, paddle, blocks, dt):
        def get_new_position():
            scaled_dir = self.direction_vec.scale(dt * self.speed)
            return self.pos.translate(scaled_dir)

        # todo: technically we could cross more than one cell...
        def get_crossed_cell(old_pos, new_pos):
            new_cell_x = math.floor(new_pos.x)
            new_cell_y = math.floor(new_pos.y)
            crossed_x = new_cell_x != math.floor(old_pos.x)
            crossed_y = new_cell_y != math.floor(old_pos.y)
            if crossed_x and crossed_y:
                return (new_cell_x, new_cell_y, Plane.BOTH)
            if crossed_x:
                return (new_cell_x, new_cell_y, Plane.Y)
            if crossed_y:
                return (new_cell_x, new_cell_y, Plane.X)

        def get_hit_block(x, y):
            block = blocks.get((x, y), None)
            if block:
                return block
            if paddle.occupies_cell(x, y):
                return paddle

        # todo
        def get_time_to_hit(x, y, plane):
            return 0

        new_pos = get_new_position()
        # todo: i don't like hard-coding this here
        if new_pos.y < 0:
            self.miss_callback()
        else:
            crossed_cell = get_crossed_cell(self.pos, new_pos)
            if not crossed_cell:
                self.pos = new_pos
            else:
                crossed_x, crossed_y, crossed_plane = crossed_cell
                hit_block = get_hit_block(crossed_x, crossed_y)
                if not hit_block:
                    self.pos = new_pos
                else:
                    time_to_hit = get_time_to_hit(crossed_x, crossed_y, crossed_plane)
                    time_remaining = dt - time_to_hit
                    self.bounce(crossed_plane)
                    hit_block.hit()
                    self.update(paddle, blocks, time_remaining)

    def bounce(self, plane):
        if plane == Plane.X or plane == Plane.BOTH:
            self.angle = -self.angle
        if plane == Plane.Y or plane == Plane.BOTH:
            if self.angle >= 0:
                self.angle = math.pi - self.angle
            else:
                self.angle = -math.pi - self.angle
        self.direction_vec = Vec2(math.cos(self.angle), math.sin(self.angle))

    def render(self, renderer):
        renderer.render(self.pos.x, self.pos.y, 'o', Color.YELLOW)

# todo: this probably isn't a very pythonic way to do this
class Observable:
    def __init__(self):
        self.observers = {}

    def observe(self, event, observer):
        existing = self.observers.get(event, None)
        if existing:
            existing.append(observer)
        else:
            self.observers[event] = [observer]

    def emit(self, event):
        if event in self.observers:
            for observer in self.observers[event]:
                observer()

class Paddle(Observable):
    def __init__(self, max_x):
        super().__init__()
        self.width = max_x // 5
        self.max_x = max_x
        self.x = max_x // 2 - self.width // 2
        self.y = 0

    def occupies_cell(self, x, y):
        return self.y == y and x <= self.x + self.width and x >= self.x

    def hit(self):
        self.emit('hit')

    def move_left(self):
        if self.x > 0:
            self.x -= 1

    def move_right(self):
        if self.x + self.width < self.max_x:
            self.x += 1

    def render(self, renderer):
        paddle_icon = '=' * self.width
        renderer.render(self.x, self.y, paddle_icon, Color.YELLOW)

class Renderer:
    def __init__(self, screen, max_x, max_y):
        self.screen = screen
        self.max_x = max_x
        self.max_y = max_y

    def clear(self):
        for row in range(0, self.max_y):
            self.render(0, row, ' ' * self.max_x)

    def render(self, x, y, string, color = 1):
        x = int(x)
        y = int(y)
        string_len = len(string)
        last_x = x + string_len
        if last_x > self.max_x:
            max_len = string_len - (last_x - self.max_x)
            string = string[:max_len]

        if y > self.max_y:
            y = self.max_y

        curses_color = curses.color_pair(color)
        flip_y = self.max_y - 1 - y

        if y == 0 and x + len(string) == self.max_x:
            # curses cannot directly output to the last cell
            all_but_first = string[1:]
            self.screen.addstr(flip_y, x, all_but_first, curses_color)
            self.screen.insstr(flip_y, x, string[0], curses_color)
        else:
            self.screen.addstr(flip_y, x, string, curses_color)

    def render_points(self, points):
        self.render(1, self.max_y - 1, str(points), Color.RED)

    def render_message(self, message):
        # todo: handle small screen widths
        starting_x = self.max_x // 2 - (len(message) // 2)
        self.render(starting_x, self.max_y // 2, message, Color.YELLOW)

class Color(IntEnum):
    # todo: where would be a more natural place for this logic?
    @classmethod
    def init(cls):
        curses.init_pair(1, curses.COLOR_RED, curses.COLOR_BLACK)
        curses.init_pair(2, curses.COLOR_GREEN, curses.COLOR_BLACK)
        curses.init_pair(3, curses.COLOR_YELLOW, curses.COLOR_BLACK)
        curses.init_pair(4, curses.COLOR_BLUE, curses.COLOR_BLACK)

    RED = 1
    GREEN = 2
    YELLOW = 3
    BLUE = 4

class Game(object):
    def __init__(self, width, height, renderer):
        self.size = (width, height)
        self.in_play = False
        self.finished = False
        self.renderer = renderer # todo: remove
        self.paddle = Paddle(width)
        self.paddle.observe('hit', self.beep)
        self.ball = Ball(width, self.handle_miss)
        self.blocks = {}
        self.add_blocks(width, height)
        self.points = 0

    def beep(self):
        print('\a')

    def handle_miss(self):
        self.finished = True
        self.in_play = False

    def handle_block_hit(self, coords):
        self.blocks.pop(coords)
        x, y = coords
        self.blocks.pop((x + 1, y))
        self.points += 50

    def add_blocks(self, width, height):
        self.add_border(width, height)
        for y in range(height // 2, height - 2):
            for x in range(2, width - 3, 2):
                block = Block(x, y, '%', Color.YELLOW, self.handle_block_hit, 2)
                self.blocks[(x, y)] = block
                self.blocks[(x + 1, y)] = block

    def add_border(self, width, height):
        noop = lambda x: None
        green = Color.GREEN

        self.blocks[(0, height - 1)] = Block(0, height - 1, '╔', green)
        self.blocks[(width - 1, height - 1)] = Block(width - 1, height - 1, '╗', green)

        for x in range(1, width - 1):
            self.blocks[(x, height - 1)] = Block(x, height - 1, '═', green)

        for y in range (1, height - 1):
            self.blocks[(0, y)] = Block(0, y, '║', green)
            self.blocks[(width - 1, y)] = Block(width - 1, y, '║', green)

    def handle_input(self, user_input):
        if user_input == ord(' ') and not self.finished:
            self.in_play = not self.in_play
        if self.in_play:
            if user_input == ord('h'):
                self.paddle.move_left()
            if user_input == ord('l'):
                self.paddle.move_right()

    def update(self, dt):
        # todo: handle resetting if we touch the bottom
        if self.in_play:
            self.ball.update(self.paddle, self.blocks, dt)

    def render(self, screen):
        self.renderer.clear()
        if self.finished:
            self.renderer.render_message('GAME OVER')
        else:
            self.ball.render(self.renderer)
            self.paddle.render(self.renderer)
            for pos, block in self.blocks.items():
                block.render(self.renderer)
        self.renderer.render_points(self.points)
        screen.refresh()

def main(screen):
    screen.nodelay(1) # user input is non-blocking
    curses.curs_set(0) # don't display cursor
    Color.init()

    max_y, max_x = screen.getmaxyx()
    renderer = Renderer(screen, max_x, max_y)
    game = Game(max_x, max_y, renderer)

    last_frame_time = datetime.now()
    while True:
        now = datetime.now()
        dt = (now - last_frame_time).microseconds / float(1000000)
        last_frame_time = now

        user_input = screen.getch()
        no_user_input = user_input == -1
        if not no_user_input:
            game.handle_input(user_input)

        game.update(dt)
        game.render(screen)
        # todo: put fps limit in?
        time.sleep(0.01)

if __name__ == '__main__':
    curses.wrapper(main)

<commit_msg>use dict comprehension for blocks<commit_after>from datetime import datetime
from enum import IntEnum, Enum
import random
import time
import math
import curses
import os
import subprocess

class Block(object):
    def __init__(self, x, y, sym, color, on_hit=lambda x: None, size=1):
        self.x, self.y = x, y
        self.sym = sym
        self.color = color
        self.on_hit = on_hit
        self.size = size

    def hit(self):
        self.on_hit((self.x, self.y))

    def render(self, renderer):
        renderer.render(self.x, self.y, self.sym * self.size, self.color)

class Vec2(object):
    def __init__(self, x, y):
        self.x = x
        self.y = y

    def translate(self, target):
        return Vec2(self.x + target.x, self.y + target.y)

    def scale(self, scalar):
        return Vec2(self.x * scalar, self.y * scalar)

class Plane(Enum):
    X = 0
    Y = 1
    BOTH = 2

class Ball:
    def __init__(self, width, miss_callback):
        self.pos = Vec2(width // 2, 1)
        self.angle = math.pi/2 + random.uniform(-0.5, 0.5)
        self.direction_vec = Vec2(math.cos(self.angle),
                                  math.sin(self.angle))
        self.speed = 12 # move 12 blocks per second
        self.miss_callback = miss_callback

    def update(self, paddle, blocks, dt):
        def get_new_position():
            scaled_dir = self.direction_vec.scale(dt * self.speed)
            return self.pos.translate(scaled_dir)

        # todo: technically we could cross more than one cell...
        def get_crossed_cell(old_pos, new_pos):
            new_cell_x = math.floor(new_pos.x)
            new_cell_y = math.floor(new_pos.y)
            crossed_x = new_cell_x != math.floor(old_pos.x)
            crossed_y = new_cell_y != math.floor(old_pos.y)
            if crossed_x and crossed_y:
                return (new_cell_x, new_cell_y, Plane.BOTH)
            if crossed_x:
                return (new_cell_x, new_cell_y, Plane.Y)
            if crossed_y:
                return (new_cell_x, new_cell_y, Plane.X)

        def get_hit_block(x, y):
            block = blocks.get((x, y), None) or blocks.get((x + 1, y))
            if block:
                return block
            if paddle.occupies_cell(x, y):
                return paddle

        # todo
        def get_time_to_hit(x, y, plane):
            return 0

        new_pos = get_new_position()
        # todo: i don't like hard-coding this here
        if new_pos.y < 0:
            self.miss_callback()
        else:
            crossed_cell = get_crossed_cell(self.pos, new_pos)
            if not crossed_cell:
                self.pos = new_pos
            else:
                crossed_x, crossed_y, crossed_plane = crossed_cell
                hit_block = get_hit_block(crossed_x, crossed_y)
                if not hit_block:
                    self.pos = new_pos
                else:
                    time_to_hit = get_time_to_hit(crossed_x, crossed_y, crossed_plane)
                    time_remaining = dt - time_to_hit
                    self.bounce(crossed_plane)
                    hit_block.hit()
                    self.update(paddle, blocks, time_remaining)

    def bounce(self, plane):
        if plane == Plane.X or plane == Plane.BOTH:
            self.angle = -self.angle
        if plane == Plane.Y or plane == Plane.BOTH:
            if self.angle >= 0:
                self.angle = math.pi - self.angle
            else:
                self.angle = -math.pi - self.angle
        self.direction_vec = Vec2(math.cos(self.angle), math.sin(self.angle))

    def render(self, renderer):
        renderer.render(self.pos.x, self.pos.y, 'o', Color.YELLOW)

# todo: this probably isn't a very pythonic way to do this
class Observable:
    def __init__(self):
        self.observers = {}

    def observe(self, event, observer):
        existing = self.observers.get(event, None)
        if existing:
            existing.append(observer)
        else:
            self.observers[event] = [observer]

    def emit(self, event):
        if event in self.observers:
            for observer in self.observers[event]:
                observer()

class Paddle(Observable):
    def __init__(self, max_x):
        super().__init__()
        self.width = max_x // 5
        self.max_x = max_x
        self.x = max_x // 2 - self.width // 2
        self.y = 0

    def occupies_cell(self, x, y):
        return self.y == y and x <= self.x + self.width and x >= self.x

    def hit(self):
        self.emit('hit')

    def move_left(self):
        if self.x > 0:
            self.x -= 1

    def move_right(self):
        if self.x + self.width < self.max_x:
            self.x += 1

    def render(self, renderer):
        paddle_icon = '=' * self.width
        renderer.render(self.x, self.y, paddle_icon, Color.YELLOW)

class Renderer:
    def __init__(self, screen, max_x, max_y):
        self.screen = screen
        self.max_x = max_x
        self.max_y = max_y

    def clear(self):
        for row in range(0, self.max_y):
            self.render(0, row, ' ' * self.max_x)

    def render(self, x, y, string, color = 1):
        x = int(x)
        y = int(y)
        string_len = len(string)
        last_x = x + string_len
        if last_x > self.max_x:
            max_len = string_len - (last_x - self.max_x)
            string = string[:max_len]

        if y > self.max_y:
            y = self.max_y

        curses_color = curses.color_pair(color)
        flip_y = self.max_y - 1 - y

        if y == 0 and x + len(string) == self.max_x:
            # curses cannot directly output to the last cell
            all_but_first = string[1:]
            self.screen.addstr(flip_y, x, all_but_first, curses_color)
            self.screen.insstr(flip_y, x, string[0], curses_color)
        else:
            self.screen.addstr(flip_y, x, string, curses_color)

    def render_points(self, points):
        self.render(1, self.max_y - 1, str(points), Color.RED)

    def render_message(self, message):
        # todo: handle small screen widths
        starting_x = self.max_x // 2 - (len(message) // 2)
        self.render(starting_x, self.max_y // 2, message, Color.YELLOW)

class Color(IntEnum):
    # todo: where would be a more natural place for this logic?
    @classmethod
    def init(cls):
        curses.init_pair(1, curses.COLOR_RED, curses.COLOR_BLACK)
        curses.init_pair(2, curses.COLOR_GREEN, curses.COLOR_BLACK)
        curses.init_pair(3, curses.COLOR_YELLOW, curses.COLOR_BLACK)
        curses.init_pair(4, curses.COLOR_BLUE, curses.COLOR_BLACK)

    RED = 1
    GREEN = 2
    YELLOW = 3
    BLUE = 4

class Game(object):
    def __init__(self, width, height, renderer):
        self.size = (width, height)
        self.in_play = False
        self.finished = False
        self.renderer = renderer # todo: remove
        self.paddle = Paddle(width)
        self.paddle.observe('hit', self.beep)
        self.ball = Ball(width, self.handle_miss)
        self.blocks = {(x, y):Block(x, y, '%', Color.YELLOW, self.block_hit, 2)
                       for x in range(2, width - 3, 2)
                       for y in range(height // 2, height - 2)}
        self.add_border(width, height)
        self.points = 0

    def beep(self):
        print('\a')

    def handle_miss(self):
        self.finished = True
        self.in_play = False

    def block_hit(self, coords):
        x, y = coords
        if not self.blocks.pop((x, y), None):
            self.blocks.pop((x + 1, y), None)
        self.points += 50

    def add_border(self, width, height):
        green = Color.GREEN

        self.blocks[(0, height - 1)] = Block(0, height - 1, '╔', green)
        self.blocks[(width - 1, height - 1)] = Block(width - 1, height - 1, '╗', green)

        for x in range(1, width - 1):
            self.blocks[(x, height - 1)] = Block(x, height - 1, '═', green)

        for y in range (1, height - 1):
            self.blocks[(0, y)] = Block(0, y, '║', green)
            self.blocks[(width - 1, y)] = Block(width - 1, y, '║', green)

    def handle_input(self, user_input):
        if user_input == ord(' ') and not self.finished:
            self.in_play = not self.in_play
        if self.in_play:
            if user_input == ord('h'):
                self.paddle.move_left()
            if user_input == ord('l'):
                self.paddle.move_right()

    def update(self, dt):
        # todo: handle resetting if we touch the bottom
        if self.in_play:
            self.ball.update(self.paddle, self.blocks, dt)

    def render(self, screen):
        self.renderer.clear()
        if self.finished:
            self.renderer.render_message('GAME OVER')
        else:
            self.ball.render(self.renderer)
            self.paddle.render(self.renderer)
            for pos, block in self.blocks.items():
                block.render(self.renderer)
        self.renderer.render_points(self.points)
        screen.refresh()

def main(screen):
    screen.nodelay(1) # user input is non-blocking
    curses.curs_set(0) # don't display cursor
    Color.init()

    max_y, max_x = screen.getmaxyx()
    renderer = Renderer(screen, max_x, max_y)
    game = Game(max_x, max_y, renderer)

    last_frame_time = datetime.now()
    while True:
        now = datetime.now()
        dt = (now - last_frame_time).microseconds / float(1000000)
        last_frame_time = now

        user_input = screen.getch()
        no_user_input = user_input == -1
        if not no_user_input:
            game.handle_input(user_input)

        game.update(dt)
        game.render(screen)
        # todo: put fps limit in?
        time.sleep(0.01)

if __name__ == '__main__':
    curses.wrapper(main)

<|endoftext|>


<commit_before>import numpy as np
import os
import speakerInfo as sinfo
from unpackMFC import run as unmfc
from pyAudioAnalysis import audioBasicIO, audioFeatureExtraction
from datetime import datetime
import sklearn
from sklearn import neighbors, svm, cluster

# primary inputs
inputPath = "/home/jkih/Music/sukwoo/"
outputPath = inputPath + str(datetime.now().time()) + '/'
num_sets = 20

# pAA settings
# https://github.com/tyiannak/pyAudioAnalysis/wiki/3.-Feature-Extraction
# -1 for all
paaFunction = -1
# in ms
windowSize = 50
timeStep = 25

# don't change unless necessary
zeroThresh = 1e-10
featureVectorSize = 13

# no touch
featureVectors = dict()
groundTruths = dict()
lastSpeaker = -1

def clearVariables():
	global featureVectors
	global groundTruths
	global lastSpeaker

	featureVectors = dict()
	groundTruths = dict()
	lastSpeaker = -1

def forgivingFloatEquivalence(value, standard):
	return value < -1 * standard - zeroThresh or value > standard + zeroThresh

def pairwiseComparison(a, b):
	retval = []
	for i, j in zip(a, b):
		if i == j:
			retval.append(True)
		else:
			retval.append(False)
	return retval

def recallCalc(test, truth):
	correct = 0
	dividend = 0
	for tst, trt in zip(test, truth):
		if trt:
			dividend += 1
			if tst:
				correct +=1
	return float(correct) / dividend

def validateNormalization(featureVector):
	for mean in featureVector.mean(axis=0):
		if forgivingFloatEquivalence(mean, 0):
			print "WARN: validationNormalization failed with mean " + str(mean)
			return False
	for stdv in featureVector.std(axis=0):
		if forgivingFloatEquivalence(stdv, 1):
			print "WARN: validationNormalization failed with stddev " + str(stdv)
			return False
	return True

def loadFeatureVector(inputPath, featureType):
	if featureType == 'mfcc':
		loadMFCCFiles(inputPath)
	elif featureType == 'paa':
		loadWAVwithPAA(inputPath)
	else:
		print "ERR: unknown feature type", featureType
		assert False

def storeFeature(sid, data, filePath):
	global featureVectors
	global groundTruths

	if sid in featureVectors:
		featureVectors[sid].append(data)
		groundTruths[sid].append(np.full(len(data), sinfo.getTruthValue(filePath), dtype='int8'))
	else:
		if type(data) is np.ndarray:
			data = data.tolist()
		featureVectors[sid] = [data]
		groundTruths[sid] = [np.full(len(data), sinfo.getTruthValue(filePath), dtype='int8').tolist()]

def loadMFCCFiles(inputPath):
	filePaths = [inputPath+f for f in os.listdir(inputPath) if os.path.isfile(inputPath+f) and f.endswith('.mfc')]
	for filePath in filePaths:
		sid = sinfo.getSpeakerID(filePath)
		data = unmfc(filePath, featureVectorSize)
		storeFeature(sid, data, filePath)

def loadWAVwithPAA(inputPath):
	filePaths = [inputPath+f for f in os.listdir(inputPath) if os.path.isfile(inputPath+f) and f.endswith('.wav')]
	for filePath in filePaths:
		sid = sinfo.getSpeakerID(filePath)
		[Fs, x] = audioBasicIO.readAudioFile(filePath)
		assert paaFunction > 0 and paaFunction < 35
		data = audioFeatureExtraction.stFeatureExtraction(x, Fs, 0.001 * windowSize * Fs, 0.001 * timeStep * Fs)[paaFunction,:]
		# using 1D feature vector breaks my code, sklearn code, and probably the law
		if len(np.array(data).shape) < 2:
			data = [[datum] for datum in data]
		storeFeature(sid, data, filePath)

# returns: feature vector array (2D), ground truth array (1D)
def collateData(speakerList):
	x = []
	y = []

	for speaker in speakerList:
		if speaker in featureVectors:
			data = featureVectors[speaker]
		else:
			print "ERR: unknown speaker", str(speaker)
			print featureVectors.keys()
			print groundTruths.keys()
			assert False
		for i in range(len(data)):
			x.extend(data[i])
			y.extend(groundTruths[speaker][i])

	sklSS = sklearn.preprocessing.StandardScaler()
	x = sklSS.fit_transform(x)
	if not validateNormalization(x):
		print "ERR: data not normalized for speakers " + str(speakerList)
		print "Check if bounds are too close"
		assert False
	return x, y

def getSubset():
	global lastSpeaker

	testSpeaker = lastSpeaker + 1
	if testSpeaker >= len(featureVectors.keys()):
		testSpeaker = 0
	speakers = featureVectors.keys()
	testFeatureVector, testTruthVector = collateData([speakers[testSpeaker]])
	trainFeatureVector, trainTruthVector = collateData([speaker for speaker in speakers if speaker != speakers[testSpeaker]])

	lastSpeaker = testSpeaker
	print "Testing with speaker #" + str(testSpeaker) + ", label: " + str(speakers[testSpeaker])
	return trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector

def model_KNN():
	print 'Running KNN'
	return sklearn.neighbors.KNeighborsClassifier(n_neighbors=sinfo.getNbClasses())

# data set too small
def model_RNC():
	print 'Running RNC'
	return sklearn.neighbors.RadiusNeighborsClassifier()

def model_SVM_linear():
	print 'Running SVM Linear'
	return sklearn.svm.SVC(kernel='linear')

def model_SVM_poly():
	print 'Running SVM Poly'
	return sklearn.svm.SVC(kernel='poly')

# Radial Basis Function
def model_SVM_rbf():
	print 'Running SVM RBF'
	return sklearn.svm.SVC(kernel='rbf')

# not enough memory
def model_Spectral():
	print 'Running Spectral Clustering'
	return sklearn.cluster.SpectralClustering(n_clusters=sinfo.getNbClasses())

# not enough memory
def model_MiniK():
	print 'Running Mini K'
	return sklearn.cluster.MiniBatchKMeans(n_clusters=sinfo.getNbClasses())

def model_ACWard():
	print 'Running Ward'
	return sklearn.cluster.AgglomerativeClustering(n_clusters=sinfo.getNbClasses(), linkage='ward')

def model_ACComplete():
	print 'Running Agglomerative'
	return sklearn.cluster.AgglomerativeClustering(n_clusters=sinfo.getNbClasses(), linkage='complete')

def model_ACAverage():
	print 'Running Agglomerative'
	return sklearn.cluster.AgglomerativeClustering(n_clusters=sinfo.getNbClasses(), linkage='average')

# clustering class incompatible with classifiers
# clustering is deterministic, non-ML? Doesn't seem to use training at all.
# http://scikit-learn.org/stable/auto_examples/cluster/plot_cluster_comparison.html
def model_Birch():
	print 'Running Birch'
	return sklearn.cluster.Birch(n_clusters=sinfo.getNbClasses())

def runModel(model, tag, trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector):
	model.fit(trainFeatureVector, trainTruthVector)
	accuracy = -1
	f1 = -1
	try:
		predicted_labels = model.predict(testFeatureVector)
		accuracy = model.score(testFeatureVector, testTruthVector)
		f1 = sklearn.metrics.f1_score(testTruthVector, predicted_labels)
	except: AttributeError:
		# some models only have online modes
		predicted_labels = model.fit_predict(testFeatureVector)
		accuracy = float(pairwiseComparison(predicted_labels, testTruthVector).count(True)) / len(testTruthVector)
		recall = recallCalc(predicted_labels, testTruthVector)
		f1 = float(2) * accuracy * recall / (accuracy + recall)

	f = open(outputPath + tag + '.log', 'w')
	f.write('accuracy: ' + str(accuracy) + '\tf1: ' + str(f1))
	f.write('\n')
	f.write('predicted labels followed by truth values')
	f.write('\n')
	f.write(str(predicted_labels.tolist()))
	f.write('\n')
	f.write(str(testTruthVector))
	f.close()

	return accuracy, f1

def runAllModels(i, trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector):
	runModel(model_KNN(), 'PAA_' + str(paaFunction) + '_KNN_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	# runModel(model_RNC(), 'PAA_' + str(paaFunction) + '_RNC_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	runModel(model_SVM_linear(), 'PAA_' + str(paaFunction) + '_SVM_Linear_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	runModel(model_SVM_poly(), 'PAA_' + str(paaFunction) + '_SVM_Poly_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	runModel(model_SVM_rbf(), 'PAA_' + str(paaFunction) + '_SVM_RBF_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	# runModel(model_Spectral(), 'PAA_' + str(paaFunction) + '_SpectralClustering_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	# runModel(model_MiniK(), 'PAA_' + str(paaFunction) + '_MiniK_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	runModel(model_ACWard(), 'PAA_' + str(paaFunction) + '_ACWard_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	runModel(model_ACAverage(), 'PAA_' + str(paaFunction) + '_ACAvg_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	runModel(model_ACComplete(), 'PAA_' + str(paaFunction) + '_ACComplete_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	runModel(model_Birch(), 'PAA_' + str(paaFunction) + '_Birch_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)

def script():
	global paaFunction

	if not os.path.exists(outputPath):
		os.mkdir(outputPath)
	if paaFunction < 0:
		for paaFunction in range(1, 35):
			print "Running feature extraction #" + str(paaFunction)
			clearVariables()
			loadFeatureVector(inputPath, 'paa')
			for i in range(num_sets * len(featureVectors.keys())):
				trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector = getSubset()
				runAllModels(i, trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	else:
		clearVariables()
		loadFeatureVector(inputPath, 'paa')
		for i in range(num_sets * len(featureVectors.keys())):
			trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector = getSubset()
			runAllModels(i, trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)

script()<commit_msg>bug fixes<commit_after>import numpy as np
import os
import speakerInfo as sinfo
from unpackMFC import run as unmfc
from pyAudioAnalysis import audioBasicIO, audioFeatureExtraction
from datetime import datetime
import sklearn
from sklearn import neighbors, svm, cluster

# primary inputs
inputPath = "/home/jkih/Music/sukwoo/"
outputPath = inputPath + str(datetime.now().time()) + '/'
num_sets = 20

# pAA settings
# https://github.com/tyiannak/pyAudioAnalysis/wiki/3.-Feature-Extraction
# -1 for all
paaFunction = -1
# in ms
windowSize = 50
timeStep = 25

# don't change unless necessary
zeroThresh = 1e-10
featureVectorSize = 13

# no touch
featureVectors = dict()
groundTruths = dict()
lastSpeaker = -1

def clearVariables():
	global featureVectors
	global groundTruths
	global lastSpeaker

	featureVectors = dict()
	groundTruths = dict()
	lastSpeaker = -1

def forgivingFloatEquivalence(value, standard):
	return value < -1 * standard - zeroThresh or value > standard + zeroThresh

def pairwiseComparison(a, b):
	retval = []
	for i, j in zip(a, b):
		if i == j:
			retval.append(True)
		else:
			retval.append(False)
	return retval

def recallCalc(test, truth):
	correct = 0
	dividend = 0
	for tst, trt in zip(test, truth):
		if trt:
			dividend += 1
			if tst:
				correct +=1
	return float(correct) / dividend

def validateNormalization(featureVector):
	for mean in featureVector.mean(axis=0):
		if forgivingFloatEquivalence(mean, 0):
			print "WARN: validationNormalization failed with mean " + str(mean)
			return False
	for stdv in featureVector.std(axis=0):
		if forgivingFloatEquivalence(stdv, 1):
			print "WARN: validationNormalization failed with stddev " + str(stdv)
			return False
	return True

def loadFeatureVector(inputPath, featureType):
	if featureType == 'mfcc':
		loadMFCCFiles(inputPath)
	elif featureType == 'paa':
		loadWAVwithPAA(inputPath)
	else:
		print "ERR: unknown feature type", featureType
		assert False

def storeFeature(sid, data, filePath):
	global featureVectors
	global groundTruths

	if sid in featureVectors:
		featureVectors[sid].append(data)
		groundTruths[sid].append(np.full(len(data), sinfo.getTruthValue(filePath), dtype='int8'))
	else:
		if type(data) is np.ndarray:
			data = data.tolist()
		featureVectors[sid] = [data]
		groundTruths[sid] = [np.full(len(data), sinfo.getTruthValue(filePath), dtype='int8').tolist()]

def loadMFCCFiles(inputPath):
	filePaths = [inputPath+f for f in os.listdir(inputPath) if os.path.isfile(inputPath+f) and f.endswith('.mfc')]
	for filePath in filePaths:
		sid = sinfo.getSpeakerID(filePath)
		data = unmfc(filePath, featureVectorSize)
		storeFeature(sid, data, filePath)

def loadWAVwithPAA(inputPath):
	filePaths = [inputPath+f for f in os.listdir(inputPath) if os.path.isfile(inputPath+f) and f.endswith('.wav')]
	for filePath in filePaths:
		sid = sinfo.getSpeakerID(filePath)
		[Fs, x] = audioBasicIO.readAudioFile(filePath)
		assert paaFunction > 0 and paaFunction < 35
		data = audioFeatureExtraction.stFeatureExtraction(x, Fs, 0.001 * windowSize * Fs, 0.001 * timeStep * Fs)[paaFunction,:]
		# using 1D feature vector breaks my code, sklearn code, and probably the law
		if len(np.array(data).shape) < 2:
			data = [[datum] for datum in data]
		storeFeature(sid, data, filePath)

# returns: feature vector array (2D), ground truth array (1D)
def collateData(speakerList):
	x = []
	y = []

	for speaker in speakerList:
		if speaker in featureVectors:
			data = featureVectors[speaker]
		else:
			print "ERR: unknown speaker", str(speaker)
			print featureVectors.keys()
			print groundTruths.keys()
			assert False
		for i in range(len(data)):
			x.extend(data[i])
			y.extend(groundTruths[speaker][i])

	sklSS = sklearn.preprocessing.StandardScaler()
	x = sklSS.fit_transform(x)
	if not validateNormalization(x):
		print "ERR: data not normalized for speakers " + str(speakerList)
		print "Check if bounds are too close"
		assert False
	return x, y

def getSubset():
	global lastSpeaker

	testSpeaker = lastSpeaker + 1
	if testSpeaker >= len(featureVectors.keys()):
		testSpeaker = 0
	speakers = featureVectors.keys()
	testFeatureVector, testTruthVector = collateData([speakers[testSpeaker]])
	trainFeatureVector, trainTruthVector = collateData([speaker for speaker in speakers if speaker != speakers[testSpeaker]])

	lastSpeaker = testSpeaker
	print "Testing with speaker #" + str(testSpeaker) + ", label: " + str(speakers[testSpeaker])
	return trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector

def model_KNN():
	print 'Running KNN'
	return sklearn.neighbors.KNeighborsClassifier(n_neighbors=sinfo.getNbClasses())

# data set too small
def model_RNC():
	print 'Running RNC'
	return sklearn.neighbors.RadiusNeighborsClassifier()

def model_SVM_linear():
	print 'Running SVM Linear'
	return sklearn.svm.SVC(kernel='linear')

def model_SVM_poly():
	print 'Running SVM Poly'
	return sklearn.svm.SVC(kernel='poly')

# Radial Basis Function
def model_SVM_rbf():
	print 'Running SVM RBF'
	return sklearn.svm.SVC(kernel='rbf')

# not enough memory
def model_Spectral():
	print 'Running Spectral Clustering'
	return sklearn.cluster.SpectralClustering(n_clusters=sinfo.getNbClasses())

# not enough memory
def model_MiniK():
	print 'Running Mini K'
	return sklearn.cluster.MiniBatchKMeans(n_clusters=sinfo.getNbClasses())

def model_ACWard():
	print 'Running AC Ward'
	return sklearn.cluster.AgglomerativeClustering(n_clusters=sinfo.getNbClasses(), linkage='ward')

def model_ACComplete():
	print 'Running AC Complete'
	return sklearn.cluster.AgglomerativeClustering(n_clusters=sinfo.getNbClasses(), linkage='complete')

def model_ACAverage():
	print 'Running AC Average'
	return sklearn.cluster.AgglomerativeClustering(n_clusters=sinfo.getNbClasses(), linkage='average')

# clustering class incompatible with classifiers
# clustering is deterministic, non-ML? Doesn't seem to use training at all.
# http://scikit-learn.org/stable/auto_examples/cluster/plot_cluster_comparison.html
def model_Birch():
	print 'Running Birch'
	return sklearn.cluster.Birch(n_clusters=sinfo.getNbClasses())

def runModel(model, tag, trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector):
	model.fit(trainFeatureVector, trainTruthVector)
	accuracy = -1
	f1 = -1
	try:
		predicted_labels = model.predict(testFeatureVector)
		accuracy = model.score(testFeatureVector, testTruthVector)
		f1 = sklearn.metrics.f1_score(testTruthVector, predicted_labels)
	except AttributeError:
		# some models only have online modes
		predicted_labels = model.fit_predict(testFeatureVector)
		accuracy = float(pairwiseComparison(predicted_labels, testTruthVector).count(True)) / len(testTruthVector)
		recall = recallCalc(predicted_labels, testTruthVector)
		f1 = float(2) * accuracy * recall / (accuracy + recall)

	f = open(outputPath + tag + '.log', 'w')
	f.write('accuracy: ' + str(accuracy) + '\tf1: ' + str(f1))
	f.write('\n')
	f.write('predicted labels followed by truth values')
	f.write('\n')
	f.write(str(predicted_labels.tolist()))
	f.write('\n')
	f.write(str(testTruthVector))
	f.close()

	return accuracy, f1

def runAllModels(i, trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector):
	runModel(model_KNN(), 'PAA_' + str(paaFunction) + '_KNN_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	# runModel(model_RNC(), 'PAA_' + str(paaFunction) + '_RNC_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	runModel(model_SVM_linear(), 'PAA_' + str(paaFunction) + '_SVM_Linear_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	runModel(model_SVM_poly(), 'PAA_' + str(paaFunction) + '_SVM_Poly_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	runModel(model_SVM_rbf(), 'PAA_' + str(paaFunction) + '_SVM_RBF_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	# runModel(model_Spectral(), 'PAA_' + str(paaFunction) + '_SpectralClustering_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	# runModel(model_MiniK(), 'PAA_' + str(paaFunction) + '_MiniK_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	runModel(model_ACWard(), 'PAA_' + str(paaFunction) + '_ACWard_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	runModel(model_ACAverage(), 'PAA_' + str(paaFunction) + '_ACAvg_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	runModel(model_ACComplete(), 'PAA_' + str(paaFunction) + '_ACComplete_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	runModel(model_Birch(), 'PAA_' + str(paaFunction) + '_Birch_' + str(i) + '_' + featureVectors.keys()[lastSpeaker], trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)

def script():
	global paaFunction

	if not os.path.exists(outputPath):
		os.mkdir(outputPath)
	if paaFunction < 0:
		for paaFunction in range(1, 35):
			print "Running feature extraction #" + str(paaFunction)
			clearVariables()
			loadFeatureVector(inputPath, 'paa')
			for i in range(num_sets * len(featureVectors.keys())):
				trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector = getSubset()
				runAllModels(i, trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)
	else:
		clearVariables()
		loadFeatureVector(inputPath, 'paa')
		for i in range(num_sets * len(featureVectors.keys())):
			trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector = getSubset()
			runAllModels(i, trainFeatureVector, testFeatureVector, trainTruthVector, testTruthVector)

script()<|endoftext|>