<commit_before>"""Plots spatial distribution of false alarms."""

import argparse
import numpy
import pandas
import matplotlib
matplotlib.use('agg')
from matplotlib import pyplot
from gewittergefahr.gg_io import storm_tracking_io as tracking_io
from gewittergefahr.gg_utils import grids
from gewittergefahr.gg_utils import projections
from gewittergefahr.gg_utils import time_conversion
from gewittergefahr.gg_utils import storm_tracking_utils as tracking_utils
from gewittergefahr.gg_utils import echo_top_tracking
from gewittergefahr.gg_utils import file_system_utils
from gewittergefahr.gg_utils import error_checking
from gewittergefahr.deep_learning import prediction_io
from gewittergefahr.scripts import plot_spatially_subset_eval as plotter

SEPARATOR_STRING = '\n\n' + '*' * 50 + '\n\n'
DUMMY_TRACKING_SCALE_METRES2 = echo_top_tracking.DUMMY_TRACKING_SCALE_METRES2

MIN_LATITUDE_DEG = 25.
MAX_LATITUDE_DEG = 50.
MIN_LONGITUDE_DEG = 230.
MAX_LONGITUDE_DEG = 300.

COLOUR_MAP_OBJECT = pyplot.get_cmap('viridis')
MAX_COLOUR_PERCENTILE = 99.
FIGURE_RESOLUTION_DPI = 600

PREDICTION_FILE_ARG_NAME = 'input_prediction_file_name'
TRACKING_DIR_ARG_NAME = 'input_tracking_dir_name'
PROB_THRESHOLD_ARG_NAME = 'prob_threshold'
GRID_SPACING_ARG_NAME = 'grid_spacing_metres'
OUTPUT_FILE_ARG_NAME = 'output_file_name'

PREDICTION_FILE_HELP_STRING = (
    'Path to prediction file.  Will be read by '
    '`prediction_io.read_ungridded_predictions`.'
)

TRACKING_DIR_HELP_STRING = (
    'Name of top-level directory with tracking data.  Files therein will be '
    'found by `storm_tracking_io.find_file` and read by'
    ' `storm_tracking_io.read_file`.'
)

PROB_THRESHOLD_HELP_STRING = (
    'Probability threshold.  Any negative example with forecast >= `{0:s}` will'
    ' be considered a false alarm.'
).format(PROB_THRESHOLD_ARG_NAME)

GRID_SPACING_HELP_STRING = 'Grid spacing.'
OUTPUT_FILE_HELP_STRING = 'Path to output file (figure will be saved here).'

INPUT_ARG_PARSER = argparse.ArgumentParser()
INPUT_ARG_PARSER.add_argument(
    '--' + PREDICTION_FILE_ARG_NAME, type=str, required=True,
    help=PREDICTION_FILE_HELP_STRING
)
INPUT_ARG_PARSER.add_argument(
    '--' + TRACKING_DIR_ARG_NAME, type=str, required=True,
    help=TRACKING_DIR_HELP_STRING
)
INPUT_ARG_PARSER.add_argument(
    '--' + PROB_THRESHOLD_ARG_NAME, type=float, required=True,
    help=PROB_THRESHOLD_HELP_STRING
)
INPUT_ARG_PARSER.add_argument(
    '--' + GRID_SPACING_ARG_NAME, type=float, required=False, default=1e5,
    help=GRID_SPACING_HELP_STRING
)
INPUT_ARG_PARSER.add_argument(
    '--' + OUTPUT_FILE_ARG_NAME, type=str, required=True,
    help=OUTPUT_FILE_HELP_STRING
)

def _run(prediction_file_name, top_tracking_dir_name, prob_threshold,
         grid_spacing_metres, output_file_name):
    """Plots spatial distribution of false alarms.

    This is effectively the main method.

    :param prediction_file_name: See documentation at top of file.
    :param top_tracking_dir_name: Same.
    :param prob_threshold: Same.
    :param grid_spacing_metres: Same.
    :param output_file_name: Same.
    """

    # Process input args.
    file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
    error_checking.assert_is_greater(prob_threshold, 0.)
    error_checking.assert_is_less_than(prob_threshold, 1.)

    grid_metadata_dict = grids.create_equidistant_grid(
        min_latitude_deg=MIN_LATITUDE_DEG, max_latitude_deg=MAX_LATITUDE_DEG,
        min_longitude_deg=MIN_LONGITUDE_DEG,
        max_longitude_deg=MAX_LONGITUDE_DEG,
        x_spacing_metres=grid_spacing_metres,
        y_spacing_metres=grid_spacing_metres, azimuthal=False)

    # Read predictions and find false alarms.
    print('Reading predictions from: "{0:s}"...'.format(prediction_file_name))
    prediction_dict = prediction_io.read_ungridded_predictions(
        prediction_file_name
    )

    observed_labels = prediction_dict[prediction_io.OBSERVED_LABELS_KEY]
    forecast_labels = (
        prediction_dict[prediction_io.PROBABILITY_MATRIX_KEY][:, -1] >=
        prob_threshold
    ).astype(int)

    false_alarm_indices = numpy.where(numpy.logical_and(
        observed_labels == 0, forecast_labels == 1
    ))[0]

    num_examples = len(observed_labels)
    num_false_alarms = len(false_alarm_indices)
    print((
        'Number of false alarms (at probability threshold of {0:.3f}) = '
        '{1:d} of {2:d}'
    ).format(
        prob_threshold, num_false_alarms, num_examples
    ))

    # Find and read tracking files.
    false_alarm_id_strings = [
        prediction_dict[prediction_io.STORM_IDS_KEY][k]
        for k in false_alarm_indices
    ]
    false_alarm_times_unix_sec = (
        prediction_dict[prediction_io.STORM_TIMES_KEY][false_alarm_indices]
    )

    file_times_unix_sec = numpy.unique(false_alarm_times_unix_sec)
    num_files = len(file_times_unix_sec)
    storm_object_tables = [None] * num_files

    print(SEPARATOR_STRING)

    for i in range(num_files):
        this_tracking_file_name = tracking_io.find_file(
            top_tracking_dir_name=top_tracking_dir_name,
            tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2,
            source_name=tracking_utils.SEGMOTION_NAME,
            valid_time_unix_sec=file_times_unix_sec[i],
            spc_date_string=time_conversion.time_to_spc_date_string(
                file_times_unix_sec[i]
            ),
            raise_error_if_missing=True
        )

        print('Reading data from: "{0:s}"...'.format(this_tracking_file_name))
        this_table = tracking_io.read_file(this_tracking_file_name)
        storm_object_tables[i] = this_table.loc[
            this_table[tracking_utils.FULL_ID_COLUMN].isin(
                false_alarm_id_strings)
        ]

        if i == 0:
            continue

        storm_object_tables[i] = storm_object_tables[i].align(
            storm_object_tables[0], axis=1
        )[0]

    storm_object_table = pandas.concat(
        storm_object_tables, axis=0, ignore_index=True
    )
    print(SEPARATOR_STRING)

    # Find latitudes and longitudes of false alarms.
    all_id_strings = (
        storm_object_table[tracking_utils.FULL_ID_COLUMN].values.tolist()
    )
    all_times_unix_sec = (
        storm_object_table[tracking_utils.VALID_TIME_COLUMN].values
    )

    good_indices = tracking_utils.find_storm_objects(
        all_id_strings=all_id_strings, all_times_unix_sec=all_times_unix_sec,
        id_strings_to_keep=false_alarm_id_strings,
        times_to_keep_unix_sec=false_alarm_times_unix_sec,
        allow_missing=False)

    false_alarm_latitudes_deg = storm_object_table[
        tracking_utils.CENTROID_LATITUDE_COLUMN
    ].values[good_indices]

    false_alarm_longitudes_deg = storm_object_table[
        tracking_utils.CENTROID_LONGITUDE_COLUMN
    ].values[good_indices]

    false_alarm_x_coords_metres, false_alarm_y_coords_metres = (
        projections.project_latlng_to_xy(
            latitudes_deg=false_alarm_latitudes_deg,
            longitudes_deg=false_alarm_longitudes_deg,
            projection_object=grid_metadata_dict[grids.PROJECTION_KEY]
        )
    )

    num_false_alarms_matrix = grids.count_events_on_equidistant_grid(
        event_x_coords_metres=false_alarm_x_coords_metres,
        event_y_coords_metres=false_alarm_y_coords_metres,
        grid_point_x_coords_metres=grid_metadata_dict[grids.X_COORDS_KEY],
        grid_point_y_coords_metres=grid_metadata_dict[grids.Y_COORDS_KEY]
    )[0]
    print(SEPARATOR_STRING)

    num_false_alarms_matrix = num_false_alarms_matrix.astype(float)
    num_false_alarms_matrix[num_false_alarms_matrix == 0] = numpy.nan

    max_colour_value = numpy.percentile(
        num_false_alarms_matrix, MAX_COLOUR_PERCENTILE
    )
    if max_colour_value < 10:
        max_colour_value = numpy.max(num_false_alarms_matrix)

    figure_object = plotter._plot_one_value(
        data_matrix=num_false_alarms_matrix,
        grid_metadata_dict=grid_metadata_dict,
        colour_map_object=COLOUR_MAP_OBJECT, min_colour_value=0,
        max_colour_value=max_colour_value, plot_cbar_min_arrow=False,
        plot_cbar_max_arrow=True
    )[0]

    print('Saving figure to: "{0:s}"...'.format(output_file_name))
    figure_object.savefig(
        output_file_name, dpi=FIGURE_RESOLUTION_DPI, pad_inches=0,
        bbox_inches='tight'
    )
    pyplot.close(figure_object)

if __name__ == '__main__':
    INPUT_ARG_OBJECT = INPUT_ARG_PARSER.parse_args()

    _run(
        prediction_file_name=getattr(
            INPUT_ARG_OBJECT, PREDICTION_FILE_ARG_NAME
        ),
        top_tracking_dir_name=getattr(INPUT_ARG_OBJECT, TRACKING_DIR_ARG_NAME),
        prob_threshold=getattr(INPUT_ARG_OBJECT, PROB_THRESHOLD_ARG_NAME),
        grid_spacing_metres=getattr(INPUT_ARG_OBJECT, GRID_SPACING_ARG_NAME),
        output_file_name=getattr(INPUT_ARG_OBJECT, OUTPUT_FILE_ARG_NAME)
    )
<commit_msg>Fixed stupid bug in plot_false_alarms_spatially.py.<commit_after>"""Plots spatial distribution of false alarms."""

import argparse
import numpy
import pandas
import matplotlib
matplotlib.use('agg')
from matplotlib import pyplot
from gewittergefahr.gg_io import storm_tracking_io as tracking_io
from gewittergefahr.gg_utils import grids
from gewittergefahr.gg_utils import projections
from gewittergefahr.gg_utils import time_conversion
from gewittergefahr.gg_utils import storm_tracking_utils as tracking_utils
from gewittergefahr.gg_utils import echo_top_tracking
from gewittergefahr.gg_utils import file_system_utils
from gewittergefahr.gg_utils import error_checking
from gewittergefahr.deep_learning import prediction_io
from gewittergefahr.scripts import plot_spatially_subset_eval as plotter

SEPARATOR_STRING = '\n\n' + '*' * 50 + '\n\n'
DUMMY_TRACKING_SCALE_METRES2 = echo_top_tracking.DUMMY_TRACKING_SCALE_METRES2

MIN_LATITUDE_DEG = 25.
MAX_LATITUDE_DEG = 50.
MIN_LONGITUDE_DEG = 230.
MAX_LONGITUDE_DEG = 300.

COLOUR_MAP_OBJECT = pyplot.get_cmap('viridis')
MAX_COLOUR_PERCENTILE = 99.
FIGURE_RESOLUTION_DPI = 600

PREDICTION_FILE_ARG_NAME = 'input_prediction_file_name'
TRACKING_DIR_ARG_NAME = 'input_tracking_dir_name'
PROB_THRESHOLD_ARG_NAME = 'prob_threshold'
GRID_SPACING_ARG_NAME = 'grid_spacing_metres'
OUTPUT_FILE_ARG_NAME = 'output_file_name'

PREDICTION_FILE_HELP_STRING = (
    'Path to prediction file.  Will be read by '
    '`prediction_io.read_ungridded_predictions`.'
)

TRACKING_DIR_HELP_STRING = (
    'Name of top-level directory with tracking data.  Files therein will be '
    'found by `storm_tracking_io.find_file` and read by'
    ' `storm_tracking_io.read_file`.'
)

PROB_THRESHOLD_HELP_STRING = (
    'Probability threshold.  Any negative example with forecast >= `{0:s}` will'
    ' be considered a false alarm.'
).format(PROB_THRESHOLD_ARG_NAME)

GRID_SPACING_HELP_STRING = 'Grid spacing.'
OUTPUT_FILE_HELP_STRING = 'Path to output file (figure will be saved here).'

INPUT_ARG_PARSER = argparse.ArgumentParser()
INPUT_ARG_PARSER.add_argument(
    '--' + PREDICTION_FILE_ARG_NAME, type=str, required=True,
    help=PREDICTION_FILE_HELP_STRING
)
INPUT_ARG_PARSER.add_argument(
    '--' + TRACKING_DIR_ARG_NAME, type=str, required=True,
    help=TRACKING_DIR_HELP_STRING
)
INPUT_ARG_PARSER.add_argument(
    '--' + PROB_THRESHOLD_ARG_NAME, type=float, required=True,
    help=PROB_THRESHOLD_HELP_STRING
)
INPUT_ARG_PARSER.add_argument(
    '--' + GRID_SPACING_ARG_NAME, type=float, required=False, default=1e5,
    help=GRID_SPACING_HELP_STRING
)
INPUT_ARG_PARSER.add_argument(
    '--' + OUTPUT_FILE_ARG_NAME, type=str, required=True,
    help=OUTPUT_FILE_HELP_STRING
)

def _run(prediction_file_name, top_tracking_dir_name, prob_threshold,
         grid_spacing_metres, output_file_name):
    """Plots spatial distribution of false alarms.

    This is effectively the main method.

    :param prediction_file_name: See documentation at top of file.
    :param top_tracking_dir_name: Same.
    :param prob_threshold: Same.
    :param grid_spacing_metres: Same.
    :param output_file_name: Same.
    """

    # Process input args.
    file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
    error_checking.assert_is_greater(prob_threshold, 0.)
    error_checking.assert_is_less_than(prob_threshold, 1.)

    grid_metadata_dict = grids.create_equidistant_grid(
        min_latitude_deg=MIN_LATITUDE_DEG, max_latitude_deg=MAX_LATITUDE_DEG,
        min_longitude_deg=MIN_LONGITUDE_DEG,
        max_longitude_deg=MAX_LONGITUDE_DEG,
        x_spacing_metres=grid_spacing_metres,
        y_spacing_metres=grid_spacing_metres, azimuthal=False)

    # Read predictions and find false alarms.
    print('Reading predictions from: "{0:s}"...'.format(prediction_file_name))
    prediction_dict = prediction_io.read_ungridded_predictions(
        prediction_file_name
    )

    observed_labels = prediction_dict[prediction_io.OBSERVED_LABELS_KEY]
    forecast_labels = (
        prediction_dict[prediction_io.PROBABILITY_MATRIX_KEY][:, -1] >=
        prob_threshold
    ).astype(int)

    false_alarm_indices = numpy.where(numpy.logical_and(
        observed_labels == 0, forecast_labels == 1
    ))[0]

    num_examples = len(observed_labels)
    num_false_alarms = len(false_alarm_indices)
    print((
        'Number of false alarms (at probability threshold of {0:.3f}) = '
        '{1:d} of {2:d}'
    ).format(
        prob_threshold, num_false_alarms, num_examples
    ))

    # Find and read tracking files.
    false_alarm_id_strings = [
        prediction_dict[prediction_io.STORM_IDS_KEY][k]
        for k in false_alarm_indices
    ]
    false_alarm_times_unix_sec = (
        prediction_dict[prediction_io.STORM_TIMES_KEY][false_alarm_indices]
    )

    file_times_unix_sec = numpy.unique(false_alarm_times_unix_sec)
    num_files = len(file_times_unix_sec)
    storm_object_tables = [None] * num_files

    print(SEPARATOR_STRING)

    for i in range(num_files):
        this_tracking_file_name = tracking_io.find_file(
            top_tracking_dir_name=top_tracking_dir_name,
            tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2,
            source_name=tracking_utils.SEGMOTION_NAME,
            valid_time_unix_sec=file_times_unix_sec[i],
            spc_date_string=time_conversion.time_to_spc_date_string(
                file_times_unix_sec[i]
            ),
            raise_error_if_missing=True
        )

        print('Reading data from: "{0:s}"...'.format(this_tracking_file_name))
        this_table = tracking_io.read_file(this_tracking_file_name)
        storm_object_tables[i] = this_table.loc[
            this_table[tracking_utils.FULL_ID_COLUMN].isin(
                false_alarm_id_strings)
        ]

        if i == 0:
            continue

        storm_object_tables[i] = storm_object_tables[i].align(
            storm_object_tables[0], axis=1
        )[0]

    storm_object_table = pandas.concat(
        storm_object_tables, axis=0, ignore_index=True
    )
    print(SEPARATOR_STRING)

    # Find latitudes and longitudes of false alarms.
    all_id_strings = (
        storm_object_table[tracking_utils.FULL_ID_COLUMN].values.tolist()
    )
    all_times_unix_sec = (
        storm_object_table[tracking_utils.VALID_TIME_COLUMN].values
    )

    good_indices = tracking_utils.find_storm_objects(
        all_id_strings=all_id_strings, all_times_unix_sec=all_times_unix_sec,
        id_strings_to_keep=false_alarm_id_strings,
        times_to_keep_unix_sec=false_alarm_times_unix_sec,
        allow_missing=False)

    false_alarm_latitudes_deg = storm_object_table[
        tracking_utils.CENTROID_LATITUDE_COLUMN
    ].values[good_indices]

    false_alarm_longitudes_deg = storm_object_table[
        tracking_utils.CENTROID_LONGITUDE_COLUMN
    ].values[good_indices]

    false_alarm_x_coords_metres, false_alarm_y_coords_metres = (
        projections.project_latlng_to_xy(
            latitudes_deg=false_alarm_latitudes_deg,
            longitudes_deg=false_alarm_longitudes_deg,
            projection_object=grid_metadata_dict[grids.PROJECTION_KEY]
        )
    )

    num_false_alarms_matrix = grids.count_events_on_equidistant_grid(
        event_x_coords_metres=false_alarm_x_coords_metres,
        event_y_coords_metres=false_alarm_y_coords_metres,
        grid_point_x_coords_metres=grid_metadata_dict[grids.X_COORDS_KEY],
        grid_point_y_coords_metres=grid_metadata_dict[grids.Y_COORDS_KEY]
    )[0]
    print(SEPARATOR_STRING)

    num_false_alarms_matrix = num_false_alarms_matrix.astype(float)
    num_false_alarms_matrix[num_false_alarms_matrix == 0] = numpy.nan

    max_colour_value = numpy.nanpercentile(
        num_false_alarms_matrix, MAX_COLOUR_PERCENTILE
    )
    if max_colour_value < 10:
        max_colour_value = numpy.nanmax(num_false_alarms_matrix)

    figure_object = plotter._plot_one_value(
        data_matrix=num_false_alarms_matrix,
        grid_metadata_dict=grid_metadata_dict,
        colour_map_object=COLOUR_MAP_OBJECT, min_colour_value=0,
        max_colour_value=max_colour_value, plot_cbar_min_arrow=False,
        plot_cbar_max_arrow=True
    )[0]

    print('Saving figure to: "{0:s}"...'.format(output_file_name))
    figure_object.savefig(
        output_file_name, dpi=FIGURE_RESOLUTION_DPI, pad_inches=0,
        bbox_inches='tight'
    )
    pyplot.close(figure_object)

if __name__ == '__main__':
    INPUT_ARG_OBJECT = INPUT_ARG_PARSER.parse_args()

    _run(
        prediction_file_name=getattr(
            INPUT_ARG_OBJECT, PREDICTION_FILE_ARG_NAME
        ),
        top_tracking_dir_name=getattr(INPUT_ARG_OBJECT, TRACKING_DIR_ARG_NAME),
        prob_threshold=getattr(INPUT_ARG_OBJECT, PROB_THRESHOLD_ARG_NAME),
        grid_spacing_metres=getattr(INPUT_ARG_OBJECT, GRID_SPACING_ARG_NAME),
        output_file_name=getattr(INPUT_ARG_OBJECT, OUTPUT_FILE_ARG_NAME)
    )
<|endoftext|>


<commit_before>"""Module that defines the Atom Models that back the Search Views"""

import six
from collections import deque
from atom.api import Atom, Typed, List, Range, Dict, observe, Str
from databroker.api import BrokerStruct
from databroker.broker import simple_broker
from pprint import pprint

class GetLastModel(Atom):
    """Class that defines the model for the 'get last N datasets view'

    Attributes
    ----------
    num_to_retrieve : range, min=1
    begin_run_events : list
    selected : databroker.api.BrokerStruct
    """
    num_to_retrieve = Range(low=1)
    begin_run_events = List()
    selected = Typed(BrokerStruct)
    selected_as_dict = Dict()
    selected_keys = List()
    __begin_run_events_as_dict = Dict()
    __begin_run_events_keys = Dict()

    @observe('selected')
    def selected_changed(self, changed):
        # set the summary dictionary
        self.selected_as_dict = {}
        self.selected_as_dict = self.__begin_run_events_as_dict[self.selected]
        # set the keys dictionary
        print('selected_changed in GetLastModel. self.selected_keys: {}'.format(self.__begin_run_events_keys[self.selected]))
        self.selected_keys = []
        self.selected_keys = self.__begin_run_events_keys[self.selected]

    @observe('num_to_retrieve')
    def num_changed(self, changed):
        self.begin_run_events = simple_broker.get_last_headers(self.num_to_retrieve)
        begin_run_events_as_dict = {}
        begin_run_events_keys = {}
        for bre in self.begin_run_events:
            bre_vars = vars(bre)
            event_descriptors = bre_vars.pop('event_descriptors', [])
            sample = bre_vars.pop('sample', {})
            beamline_config = bre_vars.pop('beamline_config', {})
            dct = bre_vars
            begin_run_events_as_dict[bre] = dct
            data_keys = deque(['KEY NAME', 'PV NAME', 'DATA LOCATION'])
            for evd in event_descriptors:
                dk = evd.data_keys
                for data_key, data_key_dict in six.iteritems(dk):
                    while data_key in data_keys:
                        data_key += '_1'
                    print(data_key, data_key_dict)
                    data_keys.append(data_key)
                    try:
                        data_keys.append(data_key_dict['source'])
                        try:
                            data_keys.append(data_key_dict['EXTERNAL'])
                        except KeyError:
                            data_keys.append('metadatastore')
                    except (KeyError, TypeError):
                        data_keys.append(data_key_dict)
                        data_keys.append('metadatastore')

            begin_run_events_keys[bre] = list(data_keys)

        self.__begin_run_events_as_dict = begin_run_events_as_dict
        self.__begin_run_events_keys = begin_run_events_keys

<commit_msg>DOC: Improved docs and removed blank lines<commit_after>"""Module that defines the Atom Models that back the Search Views"""

import six
from collections import deque
from atom.api import Atom, Typed, List, Range, Dict, observe, Str
from databroker.api import BrokerStruct
from databroker.broker import simple_broker
from pprint import pprint

class GetLastModel(Atom):
    """Class that defines the model for the 'get last N datasets view'

    Attributes
    ----------
    num_to_retrieve : range, min=1
    begin_run_events : list
    selected : databroker.api.BrokerStruct
    """
    num_to_retrieve = Range(low=1)
    begin_run_events = List()
    selected = Typed(BrokerStruct)
    selected_as_dict = Dict()
    selected_keys = List()
    __begin_run_events_as_dict = Dict()
    __begin_run_events_keys = Dict()

    @observe('selected')
    def selected_changed(self, changed):
        # set the summary dictionary
        self.selected_as_dict = {}
        self.selected_as_dict = self.__begin_run_events_as_dict[self.selected]
        # set the keys dictionary
        print('selected_changed in GetLastModel. self.selected_keys: {}'.format(self.__begin_run_events_keys[self.selected]))
        self.selected_keys = []
        self.selected_keys = self.__begin_run_events_keys[self.selected]

    @observe('num_to_retrieve')
    def num_changed(self, changed):
        self.begin_run_events = simple_broker.get_last_headers(self.num_to_retrieve)
        begin_run_events_as_dict = {}
        begin_run_events_keys = {}
        for bre in self.begin_run_events:
            bre_vars = vars(bre)
            event_descriptors = bre_vars.pop('event_descriptors', [])
            sample = bre_vars.pop('sample', {})
            beamline_config = bre_vars.pop('beamline_config', {})
            dct = bre_vars
            begin_run_events_as_dict[bre] = dct
            # format the data keys into a single list that enaml will unpack
            # into a N rows by 3 columns grid
            data_keys = deque(['KEY NAME', 'PV NAME', 'DATA LOCATION'])
            for evd in event_descriptors:
                dk = evd.data_keys
                for data_key, data_key_dict in six.iteritems(dk):
                    while data_key in data_keys:
                        data_key += '_1'
                    print(data_key, data_key_dict)
                    data_keys.append(data_key)
                    try:
                        data_keys.append(data_key_dict['source'])
                        try:
                            data_keys.append(data_key_dict['EXTERNAL'])
                        except KeyError:
                            data_keys.append('metadatastore')
                    except (KeyError, TypeError):
                        data_keys.append(data_key_dict)
                        data_keys.append('metadatastore')
            begin_run_events_keys[bre] = list(data_keys)

        self.__begin_run_events_as_dict = begin_run_events_as_dict
        self.__begin_run_events_keys = begin_run_events_keys

<|endoftext|>


<commit_before>import taichi as ti

@ti.all_archs
def test_dynamic():
  x = ti.var(ti.i32)
  y = ti.var(ti.i32, shape=())
  # ti.cfg.print_ir = True
  # ti.cfg.print_kernel_llvm_ir = True

  n = 128

  @ti.layout
  def place():
    ti.root.dynamic(ti.i, n).place(x)

  @ti.kernel
  def count():
    for i in x:
      y[None] += 1

  x[n // 3] = 1

  count()

  assert y[None] == n // 3 + 1

<commit_msg>test_dense_dynamic<commit_after>import taichi as ti

@ti.all_archs
def test_dynamic():
  x = ti.var(ti.i32)
  y = ti.var(ti.i32, shape=())
  # ti.cfg.print_ir = True
  # ti.cfg.print_kernel_llvm_ir = True

  n = 128

  @ti.layout
  def place():
    ti.root.dynamic(ti.i, n).place(x)

  @ti.kernel
  def count():
    for i in x:
      y[None] += 1

  x[n // 3] = 1

  count()

  assert y[None] == n // 3 + 1

@ti.all_archs
def test_dense_dynamic():
  n = 128

  x = ti.var(ti.i32)
  y = ti.var(ti.i32, shape=(n))

  @ti.layout
  def place():
    ti.root.dense(ti.i, n).dynamic(ti.j, n * 2).place(x)

  @ti.kernel
  def append():
    for i in range(n):
      for j in range(i):
        ti.append(x, i, j * 2)

  append()

  @ti.kernel
  def count():
    for i, j in x:
      y[i] += x[i, j]

  count()

  for i in range(n):
    print(i)
    assert y[i] == i * (i - 1)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from flask import Blueprint, render_template, g
from dataviva.apps.general.views import get_locale

from dataviva.api.attrs.models import Cbo, Bra, Cnae
from dataviva.api.rais.models import Yo, Ybo, Yio, Ybio

from dataviva.api.rais.services import Occupation as RaisOccupationService

from dataviva import db
from sqlalchemy.sql.expression import func, desc, asc

mod = Blueprint('occupation', __name__,
                template_folder='templates',
                url_prefix='/<lang_code>/occupation',
                static_folder='static')

@mod.url_value_preprocessor
def pull_lang_code(endpoint, values):
    g.locale = values.pop('lang_code')

@mod.url_defaults
def add_language_code(endpoint, values):
    values.setdefault('lang_code', get_locale())

@mod.route('/<occupation_id>')
def index(occupation_id):

    #occupation_id = '2122'
    bra_id = None #'4mg'
    header = {}
    body = {}

    if bra_id:
        rais_occupation_service = RaisOccupationService(occupation_id = occupation_id, bra_id = bra_id)
        header['year'] = rais_occupation_service.year
        header.update(rais_occupation_service.get_ybo_header())
        body.update(rais_occupation_service.get_ybo_county_num_jobs_with_bra_id())
        body.update(rais_occupation_service.get_ybo_county_wage_avg_with_bra_id())
        body.update(rais_occupation_service.get_ybio_activity_num_jobs())
        body.update(rais_occupation_service.get_ybio_activity_wage_avg())

    else:
        rais_occupation_service = RaisOccupationService(occupation_id = occupation_id)
        header['year'] = rais_occupation_service.year
        header.update(rais_occupation_service.get_yo_header())
        body.update(rais_occupation_service.get_ybo_county_num_jobs())
        body.update(rais_occupation_service.get_ybo_county_wage_avg())
        body.update(rais_occupation_service.get_yio_activity_num_jobs())
        body.update(rais_occupation_service.get_yio_activity_wage_avg())

    context = {
        'portrait' : 'https://www.google.com/maps/embed?pb=!1m18!1m12!1m3!1d7748245.803118934!2d-49.94643868147362!3d-18.514293729997753!2m3!1f0!2f0!3f0!3m2!1i1024!2i768!4f13.1!3m3!1m2!1s0xa690a165324289%3A0x112170c9379de7b3!2sMinas+Gerais!5e0!3m2!1spt-BR!2sbr!4v1450524997110',
        #unidades
        'average_monthly_income_unity' : 'Milhares', #'unidade_renda_media_mensal'
        'salary_mass_unity' : 'mil',
        'total_employment_unity' : 'milhares',
        'total_establishments_unity' : 'milhares', #'unidade_total_estabelecimentos'
        'jobs_county_unity' : 'milhares de', #'unidade_empregos_principal_municipio'
        'activity_for_job_unity': unicode('bilhões','utf8'), #unidade_atividade_por_empregos
        'bigger_average_monsthly_income_unity': unicode('bilhões','utf8'),
        'activity_for_job_unity' : unicode('bilhoes','utf8'),
        #tab-salario-emprego
        'text_salario_e_emprego': unicode('Minas Gerais é uma das 27 unidades feder...','utf8'),
        #tab-oportunidades-economicas
        'text_oportunidades_economicas' : unicode('Minas Gerais é uma das 27 unidades federativas do Brasil, localizada na Região Sudeste ','utf8')

    }

    if len(occupation_id) == 4:
        context['family'] = True
    else:
         context['family'] = False
    #acessar o contex do diogo com context.id.oqeuquero

    return render_template('occupation/index.html', body_class='perfil-estado', context=context, header = header, body = body)
<commit_msg>Removendo imports desnecessarios.<commit_after># -*- coding: utf-8 -*-
from flask import Blueprint, render_template, g
from dataviva.apps.general.views import get_locale
from dataviva.api.rais.services import Occupation as RaisOccupationService

mod = Blueprint('occupation', __name__,
                template_folder='templates',
                url_prefix='/<lang_code>/occupation',
                static_folder='static')

@mod.url_value_preprocessor
def pull_lang_code(endpoint, values):
    g.locale = values.pop('lang_code')

@mod.url_defaults
def add_language_code(endpoint, values):
    values.setdefault('lang_code', get_locale())

@mod.route('/<occupation_id>')
def index(occupation_id):

    bra_id = None #'4mg'
    header = {}
    body = {}

    if bra_id:
        rais_occupation_service = RaisOccupationService(occupation_id = occupation_id, bra_id = bra_id)
        header['year'] = rais_occupation_service.year
        header.update(rais_occupation_service.get_ybo_header())
        body.update(rais_occupation_service.get_ybo_county_num_jobs_with_bra_id())
        body.update(rais_occupation_service.get_ybo_county_wage_avg_with_bra_id())
        body.update(rais_occupation_service.get_ybio_activity_num_jobs())
        body.update(rais_occupation_service.get_ybio_activity_wage_avg())

    else:
        rais_occupation_service = RaisOccupationService(occupation_id = occupation_id)
        header['year'] = rais_occupation_service.year
        header.update(rais_occupation_service.get_yo_header())
        body.update(rais_occupation_service.get_ybo_county_num_jobs())
        body.update(rais_occupation_service.get_ybo_county_wage_avg())
        body.update(rais_occupation_service.get_yio_activity_num_jobs())
        body.update(rais_occupation_service.get_yio_activity_wage_avg())

    context = {
        'portrait' : 'https://www.google.com/maps/embed?pb=!1m18!1m12!1m3!1d7748245.803118934!2d-49.94643868147362!3d-18.514293729997753!2m3!1f0!2f0!3f0!3m2!1i1024!2i768!4f13.1!3m3!1m2!1s0xa690a165324289%3A0x112170c9379de7b3!2sMinas+Gerais!5e0!3m2!1spt-BR!2sbr!4v1450524997110',
        #unidades
        'average_monthly_income_unity' : 'Milhares', #'unidade_renda_media_mensal'
        'salary_mass_unity' : 'mil',
        'total_employment_unity' : 'milhares',
        'total_establishments_unity' : 'milhares', #'unidade_total_estabelecimentos'
        'jobs_county_unity' : 'milhares de', #'unidade_empregos_principal_municipio'
        'activity_for_job_unity': unicode('bilhões','utf8'), #unidade_atividade_por_empregos
        'bigger_average_monsthly_income_unity': unicode('bilhões','utf8'),
        'activity_for_job_unity' : unicode('bilhoes','utf8'),
        #tab-salario-emprego
        'text_salario_e_emprego': unicode('Minas Gerais é uma das 27 unidades feder...','utf8'),
        #tab-oportunidades-economicas
        'text_oportunidades_economicas' : unicode('Minas Gerais é uma das 27 unidades federativas do Brasil, localizada na Região Sudeste ','utf8')

    }

    if len(occupation_id) == 4:
        context['family'] = True
    else:
         context['family'] = False

    return render_template('occupation/index.html', body_class='perfil-estado', context=context, header = header, body = body)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Unit and doctests for specific database backends.
import datetime
import models
import unittest
from django.db import backend, connection, DEFAULT_DB_ALIAS
from django.db.backends.signals import connection_created
from django.conf import settings
from django.test import TestCase

class Callproc(unittest.TestCase):

    def test_dbms_session(self):
        # If the backend is Oracle, test that we can call a standard
        # stored procedure through our cursor wrapper.
        if settings.DATABASES[DEFAULT_DB_ALIAS]['ENGINE'] == 'django.db.backends.oracle':
            convert_unicode = backend.convert_unicode
            cursor = connection.cursor()
            cursor.callproc(convert_unicode('DBMS_SESSION.SET_IDENTIFIER'),
                            [convert_unicode('_django_testing!'),])
            return True
        else:
            return True

class LongString(unittest.TestCase):

    def test_long_string(self):
        # If the backend is Oracle, test that we can save a text longer
        # than 4000 chars and read it properly
        if settings.DATABASES[DEFAULT_DB_ALIAS]['ENGINE'] == 'django.db.backends.oracle':
            c = connection.cursor()
            c.execute('CREATE TABLE ltext ("TEXT" NCLOB)')
            long_str = ''.join([unicode(x) for x in xrange(4000)])
            c.execute('INSERT INTO ltext VALUES (%s)',[long_str])
            c.execute('SELECT text FROM ltext')
            row = c.fetchone()
            c.execute('DROP TABLE ltext')
            self.assertEquals(long_str, row[0].read())

class DateQuotingTest(TestCase):

    def test_django_date_trunc(self):
        """
        Test the custom ``django_date_trunc method``, in particular against
        fields which clash with strings passed to it (e.g. 'year') - see
        #12818__.

        __: http://code.djangoproject.com/ticket/12818

        """
        updated = datetime.datetime(2010, 2, 20)
        models.SchoolClass.objects.create(year=2009, last_updated=updated)
        years = models.SchoolClass.objects.dates('last_updated', 'year')
        self.assertEqual(list(years), [datetime.datetime(2010, 1, 1, 0, 0)])

    def test_django_extract(self):
        """
        Test the custom ``django_extract method``, in particular against fields
        which clash with strings passed to it (e.g. 'day') - see #12818__.

        __: http://code.djangoproject.com/ticket/12818

        """
        updated = datetime.datetime(2010, 2, 20)
        models.SchoolClass.objects.create(year=2009, last_updated=updated)
        classes = models.SchoolClass.objects.filter(last_updated__day=20)
        self.assertEqual(len(classes), 1)

class ParameterHandlingTest(TestCase):
    def test_bad_parameter_count(self):
        "An executemany call with too many/not enough parameters will raise an exception (Refs #12612)"
        cursor = connection.cursor()
        query = ('INSERT INTO %s (%s, %s) VALUES (%%s, %%s)' % (
            connection.introspection.table_name_converter('backends_square'),
            connection.ops.quote_name('root'),
            connection.ops.quote_name('square')
        ))
        self.assertRaises(Exception, cursor.executemany, query, [(1,2,3),])
        self.assertRaises(Exception, cursor.executemany, query, [(1,),])

def connection_created_test(sender, **kwargs):
    print 'connection_created signal'

__test__ = {'API_TESTS': """
# Check Postgres version parsing
>>> from django.db.backends.postgresql import version as pg_version

>>> pg_version._parse_version("PostgreSQL 8.3.1 on i386-apple-darwin9.2.2, compiled by GCC i686-apple-darwin9-gcc-4.0.1 (GCC) 4.0.1 (Apple Inc. build 5478)")
(8, 3, 1)

>>> pg_version._parse_version("PostgreSQL 8.3.6")
(8, 3, 6)

>>> pg_version._parse_version("PostgreSQL 8.3")
(8, 3, None)

>>> pg_version._parse_version("EnterpriseDB 8.3")
(8, 3, None)

>>> pg_version._parse_version("PostgreSQL 8.3 beta4")
(8, 3, None)

>>> pg_version._parse_version("PostgreSQL 8.4beta1")
(8, 4, None)

"""}

# Unfortunately with sqlite3 the in-memory test database cannot be
# closed, and so it cannot be re-opened during testing, and so we
# sadly disable this test for now.
if settings.DATABASES[DEFAULT_DB_ALIAS]['ENGINE'] != 'django.db.backends.sqlite3':
    __test__['API_TESTS'] += """
>>> connection_created.connect(connection_created_test)
>>> connection.close() # Ensure the connection is closed
>>> cursor = connection.cursor()
connection_created signal
>>> connection_created.disconnect(connection_created_test)
>>> cursor = connection.cursor()
"""

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fixed #13082 -- Slight modification to an Oracle test to avoid a problem cause by teardown behavior. Thanks to stvsmth for the report and patch.<commit_after># -*- coding: utf-8 -*-
# Unit and doctests for specific database backends.
import datetime
import models
import unittest
from django.db import backend, connection, DEFAULT_DB_ALIAS
from django.db.backends.signals import connection_created
from django.conf import settings
from django.test import TestCase

class Callproc(unittest.TestCase):

    def test_dbms_session(self):
        # If the backend is Oracle, test that we can call a standard
        # stored procedure through our cursor wrapper.
        if settings.DATABASES[DEFAULT_DB_ALIAS]['ENGINE'] == 'django.db.backends.oracle':
            convert_unicode = backend.convert_unicode
            cursor = connection.cursor()
            cursor.callproc(convert_unicode('DBMS_SESSION.SET_IDENTIFIER'),
                            [convert_unicode('_django_testing!'),])
            return True
        else:
            return True

class LongString(unittest.TestCase):

    def test_long_string(self):
        # If the backend is Oracle, test that we can save a text longer
        # than 4000 chars and read it properly
        if settings.DATABASES[DEFAULT_DB_ALIAS]['ENGINE'] == 'django.db.backends.oracle':
            c = connection.cursor()
            c.execute('CREATE TABLE ltext ("TEXT" NCLOB)')
            long_str = ''.join([unicode(x) for x in xrange(4000)])
            c.execute('INSERT INTO ltext VALUES (%s)',[long_str])
            c.execute('SELECT text FROM ltext')
            row = c.fetchone()
            self.assertEquals(long_str, row[0].read())
            c.execute('DROP TABLE ltext')

class DateQuotingTest(TestCase):

    def test_django_date_trunc(self):
        """
        Test the custom ``django_date_trunc method``, in particular against
        fields which clash with strings passed to it (e.g. 'year') - see
        #12818__.

        __: http://code.djangoproject.com/ticket/12818

        """
        updated = datetime.datetime(2010, 2, 20)
        models.SchoolClass.objects.create(year=2009, last_updated=updated)
        years = models.SchoolClass.objects.dates('last_updated', 'year')
        self.assertEqual(list(years), [datetime.datetime(2010, 1, 1, 0, 0)])

    def test_django_extract(self):
        """
        Test the custom ``django_extract method``, in particular against fields
        which clash with strings passed to it (e.g. 'day') - see #12818__.

        __: http://code.djangoproject.com/ticket/12818

        """
        updated = datetime.datetime(2010, 2, 20)
        models.SchoolClass.objects.create(year=2009, last_updated=updated)
        classes = models.SchoolClass.objects.filter(last_updated__day=20)
        self.assertEqual(len(classes), 1)

class ParameterHandlingTest(TestCase):
    def test_bad_parameter_count(self):
        "An executemany call with too many/not enough parameters will raise an exception (Refs #12612)"
        cursor = connection.cursor()
        query = ('INSERT INTO %s (%s, %s) VALUES (%%s, %%s)' % (
            connection.introspection.table_name_converter('backends_square'),
            connection.ops.quote_name('root'),
            connection.ops.quote_name('square')
        ))
        self.assertRaises(Exception, cursor.executemany, query, [(1,2,3),])
        self.assertRaises(Exception, cursor.executemany, query, [(1,),])

def connection_created_test(sender, **kwargs):
    print 'connection_created signal'

__test__ = {'API_TESTS': """
# Check Postgres version parsing
>>> from django.db.backends.postgresql import version as pg_version

>>> pg_version._parse_version("PostgreSQL 8.3.1 on i386-apple-darwin9.2.2, compiled by GCC i686-apple-darwin9-gcc-4.0.1 (GCC) 4.0.1 (Apple Inc. build 5478)")
(8, 3, 1)

>>> pg_version._parse_version("PostgreSQL 8.3.6")
(8, 3, 6)

>>> pg_version._parse_version("PostgreSQL 8.3")
(8, 3, None)

>>> pg_version._parse_version("EnterpriseDB 8.3")
(8, 3, None)

>>> pg_version._parse_version("PostgreSQL 8.3 beta4")
(8, 3, None)

>>> pg_version._parse_version("PostgreSQL 8.4beta1")
(8, 4, None)

"""}

# Unfortunately with sqlite3 the in-memory test database cannot be
# closed, and so it cannot be re-opened during testing, and so we
# sadly disable this test for now.
if settings.DATABASES[DEFAULT_DB_ALIAS]['ENGINE'] != 'django.db.backends.sqlite3':
    __test__['API_TESTS'] += """
>>> connection_created.connect(connection_created_test)
>>> connection.close() # Ensure the connection is closed
>>> cursor = connection.cursor()
connection_created signal
>>> connection_created.disconnect(connection_created_test)
>>> cursor = connection.cursor()
"""

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>examples = [
    'dba1.mysql.*\.select group by 3',
    'plugin:network-lan:baggins',
    'thorin.iostat group by 3',
    'bandwidth.*(in|out)',
    'beansdb group by 4',
    'beansmq group by 3',
    'doubanmemcache.*11211 group by 3',
    'doubanmemcache.*11212 group by 3',
    'doubanmemcache.*11213 group by 3',
    'doubanmemcache.*11214 group by 3',
]
<commit_msg>use it<commit_after>examples = [
    'dba1.mysql.*\.select group by 3',
    'plugin:network.lan:baggins',
    'thorin.iostat group by 3',
    'bandwidth.*(in|out)',
    'beansdb group by 4',
    'beansmq group by 3',
    'doubanmemcache.*11211 group by 3',
    'doubanmemcache.*11212 group by 3',
    'doubanmemcache.*11213 group by 3',
    'doubanmemcache.*11214 group by 3',
]
<|endoftext|>


<commit_before># Copyright 2016 Sean McGinnis
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from datetime import datetime
import fileinput
import logging
import os
from threading import Thread
import time

import paramiko
from retrying import retry
from scp import SCPClient

from instance import Instance
from vmwinstance import VMWInstance

TIMEOUT_DURATION = (60 * 120)

def wait_for_completion(session, err_session, return_output=False):
    """Wait for the executed command completes.

    Call after calling exec_command to wait for the
    command to complete and get the results.
    """
    session.setblocking(0)
    output = ''
    start = datetime.now().replace(microsecond=0)
    timeout = time.time() + TIMEOUT_DURATION
    session_closed = False
    while True:
        if session.recv_ready():
            data = session.recv(4096)
            if not data:
                # Closed connection?
                logging.debug('Session closed!')
                session_closed = True
            if return_output:
                output += data

        if err_session.recv_ready():
            data = err_session.recv(4096)
            if not data:
                session_closed = True
            if return_output:
                output += data

        if (time.time() > timeout or session_closed or
                session.exit_status_ready()):
            exit_code = (
                session.recv_exit_status() if session.exit_status_ready()
                else -1)
            end = datetime.now().replace(microsecond=0)
            return exit_code, output, (end-start)
        time.sleep(0.05)

@retry(wait_exponential_multiplier=1000,
       wait_exponential_max=10000,
       stop_max_delay=30000)
def scp_put(scp, from_path, to_path):
    scp.put(from_path, to_path)

class Executor(Thread):
    """CI test execution handler.

    Performs the actual setup and run of the CI tempest tests.
    """

    def __init__(self, conf, event, dependencies):
        """Constructor.

        :param conf: The configuration settings.
        :param event: The gerrit event.
        :param dependencies: Any cross-repo dependencies.
        """
        Thread.__init__(self)
        self.patchset_ref = event['patchSet']['ref']
        self.name = '%s-%s-%s' % (
            conf['test-name'],
            event['change']['number'],
            event['patchSet']['number'])
        self.conf = conf
        self.results = None
        self.passed = False
        self.commit_id = None
        self.dependencies = dependencies
        self.concurrency = conf.get('test-concurrency', 1)
        logging.debug('Instantiated test execution %s', self.name)

    def get_name(self):
        return self.name

    def get_commit_id(self):
        return self.commit_id

    def get_change_id(self):
        return self.change_number

    def test_passed(self):
        return self.passed

    def _set_results(self, passing):
        status = 'FAILURE'
        if passing:
            status = 'SUCCESS'
        self.results = '* %s-dsvm-volume http://oslogs.' \
                       'compellent.com/%s/ : %s ' % \
                       (self.conf['test-name'],
                        self.name,
                        status)
        self.passed = passing

    def get_results(self):
        """Gets the test run results."""
        return self.results

    def run(self):
        """Thread run.

        Executes the tests.
        """
        name = self.name
        vmw_platform = self.conf.get('test-platform') == 'vmware'
        if vmw_platform:
            instance = VMWInstance(name, self.conf)
        else:
            instance = Instance(name, self.conf)
        instance.wait_for_ready()
        try:
            self.run_test(instance)
        except Exception:
            logging.exception('Test execution failed:')

        if instance:
            instance.delete_instance()

    def run_test(self, instance):
        """The test run."""
        name = self.name
        ip = instance.get_instance_ip()

        logging.info('Starting CI test execution...')
        ssh_client = paramiko.SSHClient()
        ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        connected = False
        for i in range(1, 10):
            try:
                ssh_client.connect(
                    ip,
                    username=self.conf.get('image-user') or 'ubuntu',
                    key_filename='./jenkins_key')
                logging.debug('Connected to %s', ip)
                connected = True
                break
            except:
                logging.debug('Waiting to connect to instance %s...', ip)
                time.sleep(15 * i)
        if connected:
            scp = SCPClient(ssh_client.get_transport())
        else:
            logging.warning('Timed out waiting for SSH connection '
                            'to instance.')
            instance.delete_instance()
            return

        # Upload necessary files to test host
        ipoct = ip.split('.')
        outfile = open('./scripts/%slocal.conf' % name, 'a')
        input_filename = './scripts/local.conf.template'
        custom_file = './scripts/%s-%slocal.conf.template' % (
            self.conf['test-name'],
            self.change_number)
        if os.path.exists(custom_file):
            input_filename = custom_file

        for line in fileinput.FileInput(input_filename):
            if 'CINDER_BRANCH' in line:
                line = 'CINDER_BRANCH=%s\n' % (self.patchset_ref)
                # Handle any cross-repo dependencies here
                for dep in self.dependencies:
                    project = dep.get('change', {}).get('project', '')
                    cross_rep = ("%s_BRANCH=%s" % (
                                 project.replace('openstack/', '').upper(),
                                 project.get('patchSet', {}).get('ref')))
                    if cross_rep.startswith("_BRANCH") or cross_rep.endswith(
                            '='):
                        continue
                    line = "%s%s\n" % (line, cross_rep)
            if 'TEMPEST_STORAGE_PROTOCOL' in line:
                line = 'TEMPEST_STORAGE_PROTOCOL=%s\n' % (
                    self.conf.get('test-type', 'iSCSI'))
            if 'FIXED_RANGE' in line:
                line = 'FIXED_RANGE=192.168.%s.0/24\n' % ipoct[3]
            outfile.write(line)
        outfile.write('[%s]\n' % self.conf['backend-name'])
        for line in self.conf['config-opts']:
            outfile.write('%s\n' % line)
        outfile.close()

        scp_put(scp, './scripts/%slocal.conf' % name, '~/devstack/local.conf')
        scp_put(scp, './scripts/gather_logs.sh', '~/')
        scp_put(scp, './scripts/subunit2html.py', '/opt/stack/tempest/')
        os.remove('./scripts/%slocal.conf' % name)

        # Disable selinux enforcement to make sure no conflicts
        stdin, stdout, stderr = ssh_client.exec_command('sudo setenforce 0')

        logging.debug('Stacking...')
        # Make sure no leftover config
        stdin, stdout, stderr = ssh_client.exec_command(
            'rm -f /etc/cinder/cinder.conf')

        stdin, stdout, stderr = ssh_client.exec_command(
                'cd ~/devstack && ./stack.sh > '
                '/tmp/stack.sh.log 2>&1')
        exit_code, output, timespan = wait_for_completion(stdout.channel,
                                                          stderr.channel)
        logging.debug('Stacking operation took %s', timespan)
        if exit_code != 0:
            logging.warning('Stacking returned %d, failing run.',
                            exit_code)
            try:
                scp.get('/tmp/stack.sh.log', './%sstack.log' % name)
            except:
                pass
            return

        # Get the commit ID for later
        stdin, stdout, stderr = ssh_client.exec_command(
            "cd /opt/stack/cinder && git log --abbrev-commit --pretty=oneline "
            "-n1 | cut -f1 -d' '")
        exit_code, output, timespan = wait_for_completion(stdout.channel,
                                                          stderr.channel,
                                                          True)

        if exit_code != 0:
            logging.warning('Unable to extract commit id!')
        else:
            self.commit_id = output.rstrip('\r\n').strip()

        # Seeing some dnsmasq weirdness, block it
        stdin, stdout, stderr = ssh_client.exec_command(
            'sudo ebtables -I INPUT -i eth0 '
            '--protocol ipv4 --ip-proto udp '
            '--ip-dport 67:68 -j DROP')

        # Perform the actual tests
        logging.debug('Running tempest...')
        stdin, stdout, stderr = ssh_client.exec_command(
            "cd /opt/stack/tempest && "
            "tox -e all -- "
            "'^(?=.*volume)(?!.*test_encrypted_cinder_volumes)"
            "/*' --concurrency=%d > "
            "console.log.out 2>&1" % self.concurrency)
        exit_code, output, timespan = wait_for_completion(stdout.channel,
                                                          stderr.channel)
        logging.info('Tempest test run took %s', timespan)
        if exit_code != 0:
            logging.warning('Tempest test returned %d. Bummer.',
                            exit_code)
            self._set_results(False)
        else:
            self._set_results(True)

        # Generate the testr report
        stdin, stdout, stderr = ssh_client.exec_command(
            'cd /opt/stack/tempest && testr last --subunit > testrepository.'
            'subunit && python subunit2html.py testrepository.subunit')
        exit_code, output, timespan = wait_for_completion(stdout.channel,
                                                          stderr.channel)

        logging.debug('Collecting logs...')
        stdin, stdout, stderr = ssh_client.exec_command(
            'bash ~/gather_logs.sh %s' % name)
        exit_code, output, timespan = wait_for_completion(stdout.channel,
                                                          stderr.channel)
        scp.get('~/%s.tar.gz' % name, './%s.tgz' % self.name)
<commit_msg>Capture change number for later use<commit_after># Copyright 2016 Sean McGinnis
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from datetime import datetime
import fileinput
import logging
import os
from threading import Thread
import time

import paramiko
from retrying import retry
from scp import SCPClient

from instance import Instance
from vmwinstance import VMWInstance

TIMEOUT_DURATION = (60 * 120)

def wait_for_completion(session, err_session, return_output=False):
    """Wait for the executed command completes.

    Call after calling exec_command to wait for the
    command to complete and get the results.
    """
    session.setblocking(0)
    output = ''
    start = datetime.now().replace(microsecond=0)
    timeout = time.time() + TIMEOUT_DURATION
    session_closed = False
    while True:
        if session.recv_ready():
            data = session.recv(4096)
            if not data:
                # Closed connection?
                logging.debug('Session closed!')
                session_closed = True
            if return_output:
                output += data

        if err_session.recv_ready():
            data = err_session.recv(4096)
            if not data:
                session_closed = True
            if return_output:
                output += data

        if (time.time() > timeout or session_closed or
                session.exit_status_ready()):
            exit_code = (
                session.recv_exit_status() if session.exit_status_ready()
                else -1)
            end = datetime.now().replace(microsecond=0)
            return exit_code, output, (end-start)
        time.sleep(0.05)

@retry(wait_exponential_multiplier=1000,
       wait_exponential_max=10000,
       stop_max_delay=30000)
def scp_put(scp, from_path, to_path):
    scp.put(from_path, to_path)

class Executor(Thread):
    """CI test execution handler.

    Performs the actual setup and run of the CI tempest tests.
    """

    def __init__(self, conf, event, dependencies):
        """Constructor.

        :param conf: The configuration settings.
        :param event: The gerrit event.
        :param dependencies: Any cross-repo dependencies.
        """
        Thread.__init__(self)
        self.patchset_ref = event['patchSet']['ref']
        self.change_number = event['change']['number']
        self.name = '%s-%s-%s' % (
            conf['test-name'],
            event['change']['number'],
            event['patchSet']['number'])
        self.conf = conf
        self.results = None
        self.passed = False
        self.commit_id = None
        self.dependencies = dependencies
        self.concurrency = conf.get('test-concurrency', 1)
        logging.debug('Instantiated test execution %s', self.name)

    def get_name(self):
        return self.name

    def get_commit_id(self):
        return self.commit_id

    def get_change_id(self):
        return self.change_number

    def test_passed(self):
        return self.passed

    def _set_results(self, passing):
        status = 'FAILURE'
        if passing:
            status = 'SUCCESS'
        self.results = '* %s-dsvm-volume http://oslogs.' \
                       'compellent.com/%s/ : %s ' % \
                       (self.conf['test-name'],
                        self.name,
                        status)
        self.passed = passing

    def get_results(self):
        """Gets the test run results."""
        return self.results

    def run(self):
        """Thread run.

        Executes the tests.
        """
        name = self.name
        vmw_platform = self.conf.get('test-platform') == 'vmware'
        if vmw_platform:
            instance = VMWInstance(name, self.conf)
        else:
            instance = Instance(name, self.conf)
        instance.wait_for_ready()
        try:
            self.run_test(instance)
        except Exception:
            logging.exception('Test execution failed:')

        if instance:
            instance.delete_instance()

    def run_test(self, instance):
        """The test run."""
        name = self.name
        ip = instance.get_instance_ip()

        logging.info('Starting CI test execution...')
        ssh_client = paramiko.SSHClient()
        ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        connected = False
        time.sleep(15)
        for i in range(1, 10):
            try:
                ssh_client.connect(
                    ip,
                    username=self.conf.get('image-user') or 'ubuntu',
                    key_filename='./jenkins_key')
                logging.debug('Connected to %s', ip)
                connected = True
                break
            except:
                logging.debug('Waiting to connect to instance %s...', ip)
                time.sleep(15 * i)
        if connected:
            scp = SCPClient(ssh_client.get_transport())
        else:
            logging.warning('Timed out waiting for SSH connection '
                            'to instance.')
            instance.delete_instance()
            return

        # Upload necessary files to test host
        ipoct = ip.split('.')
        outfile = open('./scripts/%slocal.conf' % name, 'a')
        input_filename = './scripts/local.conf.template'
        custom_file = './scripts/%s-%slocal.conf.template' % (
            self.conf['test-name'],
            self.change_number)
        if os.path.exists(custom_file):
            input_filename = custom_file

        for line in fileinput.FileInput(input_filename):
            if 'CINDER_BRANCH' in line:
                line = 'CINDER_BRANCH=%s\n' % (self.patchset_ref)
                # Handle any cross-repo dependencies here
                for dep in self.dependencies:
                    project = dep.get('change', {}).get('project', '')
                    cross_rep = ("%s_BRANCH=%s" % (
                                 project.replace('openstack/', '').upper(),
                                 project.get('patchSet', {}).get('ref')))
                    if cross_rep.startswith("_BRANCH") or cross_rep.endswith(
                            '='):
                        continue
                    line = "%s%s\n" % (line, cross_rep)
            if 'TEMPEST_STORAGE_PROTOCOL' in line:
                line = 'TEMPEST_STORAGE_PROTOCOL=%s\n' % (
                    self.conf.get('test-type', 'iSCSI'))
            if 'FIXED_RANGE' in line:
                line = 'FIXED_RANGE=192.168.%s.0/24\n' % ipoct[3]
            outfile.write(line)
        outfile.write('[%s]\n' % self.conf['backend-name'])
        for line in self.conf['config-opts']:
            outfile.write('%s\n' % line)
        outfile.close()

        scp_put(scp, './scripts/%slocal.conf' % name, '~/devstack/local.conf')
        scp_put(scp, './scripts/gather_logs.sh', '~/')
        scp_put(scp, './scripts/subunit2html.py', '/opt/stack/tempest/')
        os.remove('./scripts/%slocal.conf' % name)

        # Disable selinux enforcement to make sure no conflicts
        stdin, stdout, stderr = ssh_client.exec_command('sudo setenforce 0')

        logging.debug('Stacking...')
        # Make sure no leftover config
        stdin, stdout, stderr = ssh_client.exec_command(
            'rm -f /etc/cinder/cinder.conf')

        stdin, stdout, stderr = ssh_client.exec_command(
                'cd ~/devstack && ./stack.sh > '
                '/tmp/stack.sh.log 2>&1')
        exit_code, output, timespan = wait_for_completion(stdout.channel,
                                                          stderr.channel)
        logging.debug('Stacking operation took %s', timespan)
        if exit_code != 0:
            logging.warning('Stacking returned %d, failing run.',
                            exit_code)
            try:
                scp.get('/tmp/stack.sh.log', './%sstack.log' % name)
            except:
                pass
            return

        # Get the commit ID for later
        stdin, stdout, stderr = ssh_client.exec_command(
            "cd /opt/stack/cinder && git log --abbrev-commit --pretty=oneline "
            "-n1 | cut -f1 -d' '")
        exit_code, output, timespan = wait_for_completion(stdout.channel,
                                                          stderr.channel,
                                                          True)

        if exit_code != 0:
            logging.warning('Unable to extract commit id!')
        else:
            self.commit_id = output.rstrip('\r\n').strip()

        # Seeing some dnsmasq weirdness, block it
        stdin, stdout, stderr = ssh_client.exec_command(
            'sudo ebtables -I INPUT -i eth0 '
            '--protocol ipv4 --ip-proto udp '
            '--ip-dport 67:68 -j DROP')

        # Perform the actual tests
        logging.debug('Running tempest...')
        stdin, stdout, stderr = ssh_client.exec_command(
            "cd /opt/stack/tempest && "
            "tox -e all -- "
            "'^(?=.*volume)(?!.*test_encrypted_cinder_volumes)"
            "/*' --concurrency=%d > "
            "console.log.out 2>&1" % self.concurrency)
        exit_code, output, timespan = wait_for_completion(stdout.channel,
                                                          stderr.channel)
        logging.info('Tempest test run took %s', timespan)
        if exit_code != 0:
            logging.warning('Tempest test returned %d. Bummer.',
                            exit_code)
            self._set_results(False)
        else:
            self._set_results(True)

        # Generate the testr report
        stdin, stdout, stderr = ssh_client.exec_command(
            'cd /opt/stack/tempest && testr last --subunit > testrepository.'
            'subunit && python subunit2html.py testrepository.subunit')
        exit_code, output, timespan = wait_for_completion(stdout.channel,
                                                          stderr.channel)

        logging.debug('Collecting logs...')
        stdin, stdout, stderr = ssh_client.exec_command(
            'bash ~/gather_logs.sh %s' % name)
        exit_code, output, timespan = wait_for_completion(stdout.channel,
                                                          stderr.channel)
        scp.get('~/%s.tar.gz' % name, './%s.tgz' % self.name)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import rospy
from geometry_msgs.msg import PoseStamped

rospy.init_node('excesize')

goal_pub = rospy.Publisher('/move_base_simple/goal', PoseStamped, queue_size=1)

goal = PoseStamped()
goal.header.seq = 3
goal.header.frame_id = 'world'
goal.header.stamp = rospy.Time.now()
goal.pose.position.x = 3.5
goal.pose.position.y = 3.5
goal.pose.position.z = 0.0
goal.pose.orientation.w = 1.0
goal.pose.orientation.x = 0.0
goal.pose.orientation.y = 0.0
goal.pose.orientation.z = 0.0

goal_pub.publish(goal)

rate = rospy.Rate(10)
while not rospy.is_shutdown():
    rate.sleep()
<commit_msg>add latch option<commit_after>#!/usr/bin/env python

import rospy
from geometry_msgs.msg import PoseStamped

def current_goal_callback(msg):
    pass

rospy.init_node('excesize')

goal_pub = rospy.Publisher('/move_base_simple/goal', PoseStamped, queue_size=1, latch=True)

goal = PoseStamped()
goal.header.seq = 0
goal.header.frame_id = 'world'
goal.header.stamp = rospy.Time.now() #+ rospy.Duration(5, 0)
goal.pose.position.x = 3.5
goal.pose.position.y = 3.5
goal.pose.position.z = 0.0
goal.pose.orientation.x = 0.0
goal.pose.orientation.y = 0.0
goal.pose.orientation.z = 0.0
goal.pose.orientation.w = 1.0

goal_pub.publish(goal)

rate = rospy.Rate(10)
while not rospy.is_shutdown():
    rate.sleep()
<|endoftext|>


<commit_before># This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Custom kernel-ci JSON serializer/deserializer to be used with Celery.

The following module defines two custom functions to serialize and deserialize
JSON objects that use BSON notation. These functions are intended to be used
as the defual encoder/decoder functions that Celery uses to send messages.
"""

try:
    import simplejson as json
except ImportError:
    import json

import bson

def kernelci_json_encoder(obj):
    """Custom JSON serialization function.

    :param obj: The object to serialize.
    :type obj: dict
    :return A unicode string.
    """
    return json.dumps(
        obj,
        default=bson.json_util.default,
        ensure_ascii=False,
        separators=(",", ":")
    )

def kernelci_json_decoder(obj):
    """Custome JSON deserialization function.

    :param obj: The object to deserialize.
    :type obj: string or unicode
    :return A JSON object.
    """
    return json.loads(obj, object_hook=bson.json_util.object_hook)
<commit_msg>Fix import to make celery cli work<commit_after># This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Custom kernel-ci JSON serializer/deserializer to be used with Celery.

The following module defines two custom functions to serialize and deserialize
JSON objects that use BSON notation. These functions are intended to be used
as the defual encoder/decoder functions that Celery uses to send messages.
"""

try:
    import simplejson as json
except ImportError:
    import json

from bson.json_util import (
    default,
    object_hook
)

def kernelci_json_encoder(obj):
    """Custom JSON serialization function.

    :param obj: The object to serialize.
    :type obj: dict
    :return A unicode string.
    """
    return json.dumps(
        obj, default=default, ensure_ascii=False, separators=(",", ":"))

def kernelci_json_decoder(obj):
    """Custome JSON deserialization function.

    :param obj: The object to deserialize.
    :type obj: string or unicode
    :return A JSON object.
    """
    return json.loads(obj, object_hook=object_hook)
<|endoftext|>


<commit_before>import numpy as np
from scipy.integrate import quad
from scipy.io import loadmat
import pylab as plt
import time

from openmdao.api import Component, Problem, Group

class GaussianWake(Component):

    def __init__(self, nTurbines, direction_id=0, options=None):
        super(GaussianWake, self).__init__()

        self.deriv_options['type'] = 'fd'
        self.deriv_options['form'] = 'central'
        self.deriv_options['step_size'] = 1.0e-12
        self.deriv_options['step_calc'] = 'relative'

        self.nTurbines = nTurbines
        self.direction_id = direction_id

        if options is None:
            self.radius_multiplier = 1.0
            self.nSamples = nSamples = 0
        else:
            # self.radius_multiplier = options['radius multiplier']
            self.nSamples = nSamples = options['nSamples']

        # unused but required for compatibility
        self.add_param('hubHeight', np.zeros(nTurbines), units='m')
        self.add_param('wakeCentersYT', np.zeros(nTurbines*nTurbines), units='m')
        self.add_param('wakeDiametersT', np.zeros(nTurbines*nTurbines), units='m')
        self.add_param('wakeOverlapTRel', np.zeros(nTurbines*nTurbines))

        # used
        self.add_param('turbineXw', val=np.zeros(nTurbines), units='m')
        self.add_param('turbineYw', val=np.zeros(nTurbines), units='m')
        self.add_param('turbineZ', val=np.zeros(nTurbines), units='m')
        self.add_param('yaw%i' % direction_id, np.zeros(nTurbines), units='deg')
        self.add_param('rotorDiameter', val=np.zeros(nTurbines)+126.4, units='m')
        self.add_param('Ct', np.zeros(nTurbines), desc='Turbine thrust coefficients')
        self.add_param('wind_speed', val=8.0, units='m/s')
        self.add_param('axialInduction', val=np.zeros(nTurbines)+1./3.)

        # params for Bastankhah with yaw
        self.add_param('model_params:ky', val=0.022, pass_by_object=True)
        self.add_param('model_params:kz', val=0.022, pass_by_object=True)
        self.add_param('model_params:alpha', val=2.32, pass_by_object=True)
        self.add_param('model_params:beta', val=0.154, pass_by_object=True)
        self.add_param('model_params:I', val=0.075, pass_by_object=True, desc='turbulence intensity')\

        self.add_output('wtVelocity%i' % direction_id, val=np.zeros(nTurbines), units='m/s')

        if nSamples > 0:
            # visualization input
            self.add_param('wsPositionXw', np.zeros(nSamples), units='m', pass_by_object=True,
                           desc='downwind position of desired measurements in wind ref. frame')
            self.add_param('wsPositionYw', np.zeros(nSamples), units='m', pass_by_object=True,
                           desc='crosswind position of desired measurements in wind ref. frame')
            self.add_param('wsPositionZ', np.zeros(nSamples), units='m', pass_by_object=True,
                           desc='position of desired measurements in wind ref. frame')

            # visualization output
            self.add_output('wsArray%i' % direction_id, np.zeros(nSamples), units='m/s', pass_by_object=True,
                            desc='wind speed at measurement locations')

    def solve_nonlinear(self, params, unknowns, resids):

        nTurbines = self.nTurbines
        direction_id = self.direction_id
        nSamples = self.nSamples

        # params for Bastankhah model with yaw
        ky = params['model_params:ky']
        kz = params['model_params:kz']
        alpha = params['model_params:alpha']
        beta = params['model_params:beta']
        I = params['model_params:I']

        # rename inputs and outputs
        turbineXw = params['turbineXw']
        turbineYw = params['turbineYw']
        turbineZ = params['turbineZ']
        yaw = params['yaw%i' % direction_id]
        rotorDiameter = params['rotorDiameter']
        Ct = params['Ct']
        axialInduction = params['axialInduction']
        wind_speed = params['wind_speed']

        # print(turbineXw, turbineYw)

        # Dw0[0] = Dw0[1]
        # m[0] = m[1]

        # rotorDiameter *= 2

        for i in range(0, nTurbines):
            if (Ct[i] > 0.96): # Glauert condition
                axialInduction[i] = 0.143 + np.sqrt(0.0203-0.6427*(0.889 - Ct[i]))
            else:
                axialInduction[i] = 0.5*(1.0-np.sqrt(1.0-Ct[i]))

        print(yaw)
        # NOTE: Bastankhah and Porte Agel 2016 defines yaw as positive clockwise, the negative below accounts for this
        yaw *= -np.pi/180.

        if self.nSamples > 0:
            velX = params['wsPositionXw']
            velY = params['wsPositionYw']
            velZ = params['wsPositionZ']
        else:
            velX = np.zeros([0, 0])
            velY = np.zeros([0, 0])
            velZ = np.zeros([0, 0])

        velocitiesTurbines = np.tile(wind_speed, nTurbines)
        ws_array = np.tile(wind_speed, nSamples)

        for turb in range(0, nTurbines):
            x0 = rotorDiameter[turb] * (np.cos(yaw[turb]) * (1.0 + np.sqrt(1.0 - Ct[turb])) /
                                         (np.sqrt(2.0) * (alpha * I + beta * (1.0 - np.sqrt(1.0 - Ct[turb])))))
            theta_c_0 = 0.3 * yaw[turb] * (1.0 - np.sqrt(1.0 - Ct[turb] * np.cos(yaw[turb]))) / np.cos(yaw[turb])

            for loc in range(0, nSamples):  # at velX-locations
                deltax0 = velX[loc] - (turbineXw[turb] + x0)
                if deltax0 + x0 > 0.0:
                    sigmay = rotorDiameter[turb] * (ky * deltax0 / rotorDiameter[turb]
                                                    + np.cos(yaw[turb]) / np.sqrt(8.0))
                    sigmaz = rotorDiameter[turb] * (kz * deltax0 / rotorDiameter[turb]
                                                    + 1.0 / np.sqrt(8.0))
                    wake_offset = rotorDiameter[turb] * (
                        theta_c_0 * x0 / rotorDiameter[turb] +
                        (theta_c_0 / 14.7) * np.sqrt(np.cos(yaw[turb]) / (ky * kz * Ct[turb])) *
                        (2.9 + 1.3 * np.sqrt(1.0 - Ct[turb]) - Ct[turb]) *
                        np.log(
                            ((1.6 + np.sqrt(Ct[turb])) *
                             (1.6 * np.sqrt(8.0 * sigmay * sigmaz /
                                            (np.cos(yaw[turb]) * rotorDiameter[turb] ** 2))
                              - np.sqrt(Ct[turb]))) /
                            ((1.6 - np.sqrt(Ct[turb])) *
                             (1.6 * np.sqrt(8.0 * sigmay * sigmaz /
                                            (np.cos(yaw[turb]) * rotorDiameter[turb] ** 2))
                              + np.sqrt(Ct[turb])))
                        )
                    )

                    deltay = velY[loc] - (turbineYw[turb] + wake_offset)

                    deltav = wind_speed * (
                        (1.0 - np.sqrt(1.0 - Ct[turb] *
                                       np.cos(yaw[turb]) / (8.0 * sigmay * sigmaz /
                                                            (rotorDiameter[turb] ** 2)))) *
                        np.exp(-0.5 * ((deltay) / sigmay) ** 2) *
                        np.exp(-0.5 * ((velZ[loc] - turbineZ[turb]) / sigmaz) ** 2)
                    )

                    ws_array[loc] -= deltav

            for turbI in range(0, nTurbines):  # at turbineX-locations

                deltax0 = turbineXw[turbI] - (turbineXw[turb] + x0)

                if deltax0 + x0 > 0.0:
                    sigmay = rotorDiameter[turb] * (ky * deltax0 / rotorDiameter[turb]
                                                     + np.cos(yaw[turb]) / np.sqrt(8.0))
                    sigmaz = rotorDiameter[turb] * (kz * deltax0 / rotorDiameter[turb]
                                                     + 1.0 / np.sqrt(8.0))
                    wake_offset = rotorDiameter[turb] * (
                        theta_c_0 * x0 / rotorDiameter[turb] +
                        (theta_c_0 / 14.7) * np.sqrt(np.cos(yaw[turb]) / (ky * kz * Ct[turb])) *
                        (2.9 + 1.3 * np.sqrt(1.0 - Ct[turb]) - Ct[turb]) *
                        np.log(
                            ((1.6 + np.sqrt(Ct[turb])) *
                             (1.6 * np.sqrt(8.0 * sigmay * sigmaz /
                                            (np.cos(yaw[turb]) * rotorDiameter[turb] ** 2))
                              - np.sqrt(Ct[turb]))) /
                            ((1.6 - np.sqrt(Ct[turb])) *
                             (1.6 * np.sqrt(8.0 * sigmay * sigmaz /
                                            (np.cos(yaw[turb]) * rotorDiameter[turb] ** 2))
                              + np.sqrt(Ct[turb])))
                        )
                    )
                    # print wake_offset, turbineYw[turbI]

                    deltay = turbineYw[turbI] - (turbineYw[turb] + wake_offset)

                    deltav = wind_speed * (
                        (1.0 - np.sqrt(1.0 - Ct[turb] *
                                       np.cos(yaw[turb]) / (8.0 * sigmay * sigmaz /
                                                            (rotorDiameter[turb] ** 2)))) *
                        np.exp(-0.5 * ((deltay) / sigmay) ** 2) *
                        np.exp(-0.5 * ((turbineZ[turbI] - turbineZ[turb]) / sigmaz) ** 2)
                    )

                    velocitiesTurbines[turbI] -= deltav

        unknowns['wtVelocity%i' % direction_id] = velocitiesTurbines

        if nSamples > 0.0:
            print nSamples
            unknowns['wsArray%i' % direction_id] = ws_array
            print unknowns['wsArray%i' % direction_id]

if __name__ == "__main__":

    nTurbines = 2
    nDirections = 1

    rotor_diameter = 126.4
    rotorArea = np.pi*rotor_diameter*rotor_diameter/4.0
    axialInduction = 1.0/3.0
    CP = 0.7737/0.944 * 4.0 * 1.0/3.0 * np.power((1 - 1.0/3.0), 2)
    # CP =0.768 * 4.0 * 1.0/3.0 * np.power((1 - 1.0/3.0), 2)
    CT = 4.0*axialInduction*(1.0-axialInduction)
    generator_efficiency = 0.944

    # Define turbine characteristics
    axialInduction = np.array([axialInduction, axialInduction])
    rotorDiameter = np.array([rotor_diameter, rotor_diameter])
    # rotorDiameter = np.array([rotorDiameter, 0.0001*rotorDiameter])
    yaw = np.array([0., 0.])

    # Define site measurements
    wind_direction = 30.
    wind_speed = 8.    # m/s
    air_density = 1.1716

    Ct = np.array([CT, CT])
    Cp = np.array([CP, CP])

    turbineX = np.array([0.0, 7.*rotor_diameter])
    turbineY = np.array([0.0, 0.0])

    prob = Problem()
    prob.root = Group()
    prob.root.add('model', GaussianWake(nTurbines), promotes=['*'])

    prob.setup()

    prob['turbineXw'] = turbineX
    prob['turbineYw'] = turbineY

    GaussianWakeVelocity = list()

    yawrange = np.linspace(-40., 40., 400)

    for yaw1 in yawrange:

        prob['yaw0'] = np.array([yaw1, 0.0])
        prob['Ct'] = Ct*np.cos(prob['yaw0']*np.pi/180.)**2

        prob.run()

        velocitiesTurbines = prob['wtVelocity0']

        GaussianWakeVelocity.append(list(velocitiesTurbines))

    GaussianWakeVelocity = np.array(GaussianWakeVelocity)

    fig, axes = plt.subplots(ncols=2, nrows=2, sharey=False)
    axes[0, 0].plot(yawrange, GaussianWakeVelocity[:, 0]/wind_speed, 'b')
    axes[0, 0].plot(yawrange, GaussianWakeVelocity[:, 1]/wind_speed, 'b')

    axes[0, 0].set_xlabel('yaw angle (deg.)')
    axes[0, 0].set_ylabel('Velcoity ($V_{eff}/V_o$)')

    posrange = np.linspace(-3.*rotor_diameter, 3.*rotor_diameter, 100)

    prob['yaw0'] = np.array([0.0, 0.0])

    GaussianWakeVelocity = list()

    for pos2 in posrange:

        prob['turbineYw'] = np.array([0.0, pos2])

        prob.run()

        velocitiesTurbines = prob['wtVelocity0']

        GaussianWakeVelocity.append(list(velocitiesTurbines))

    GaussianWakeVelocity = np.array(GaussianWakeVelocity)

    wind_speed = 1.0
    axes[0, 1].plot(posrange/rotor_diameter, GaussianWakeVelocity[:, 0]/wind_speed, 'b')
    axes[0, 1].plot(posrange/rotor_diameter, GaussianWakeVelocity[:, 1]/wind_speed, 'b')
    axes[0, 1].set_xlabel('y/D')
    axes[0, 1].set_ylabel('Velocity ($V_{eff}/V_o$)')

    posrange = np.linspace(-3.*rotorDiameter[0], 3.*rotorDiameter[0], num=1000)

    posrange = np.linspace(-1.*rotorDiameter[0], 30.*rotorDiameter[0], num=2000)
    yaw = np.array([0.0, 0.0])
    wind_direction = 0.0

    GaussianWakeVelocity = list()
    for pos2 in posrange:

        prob['turbineXw'] = np.array([0.0, pos2])
        prob['turbineYw'] = np.array([0.0, 0.0])

        prob.run()

        velocitiesTurbines = prob['wtVelocity0']

        GaussianWakeVelocity.append(list(velocitiesTurbines))

    GaussianWakeVelocity = np.array(GaussianWakeVelocity)

    axes[1, 1].plot(posrange/rotor_diameter, GaussianWakeVelocity[:, 1], 'y', label='GH')
    axes[1, 1].plot(np.array([7, 7]), np.array([2, 8]), '--k', label='tuning point')

    plt.xlabel('x/D')
    plt.ylabel('Velocity (m/s)')
    plt.legend(loc=4)

    plt.show()<commit_msg>re-included x0 cutoff<commit_after>import numpy as np
from scipy.integrate import quad
from scipy.io import loadmat
import pylab as plt
import time

from openmdao.api import Component, Problem, Group

class GaussianWake(Component):

    def __init__(self, nTurbines, direction_id=0, options=None):
        super(GaussianWake, self).__init__()

        self.deriv_options['type'] = 'fd'
        self.deriv_options['form'] = 'central'
        self.deriv_options['step_size'] = 1.0e-12
        self.deriv_options['step_calc'] = 'relative'

        self.nTurbines = nTurbines
        self.direction_id = direction_id

        if options is None:
            self.radius_multiplier = 1.0
            self.nSamples = nSamples = 0
        else:
            # self.radius_multiplier = options['radius multiplier']
            self.nSamples = nSamples = options['nSamples']

        # unused but required for compatibility
        self.add_param('hubHeight', np.zeros(nTurbines), units='m')
        self.add_param('wakeCentersYT', np.zeros(nTurbines*nTurbines), units='m')
        self.add_param('wakeDiametersT', np.zeros(nTurbines*nTurbines), units='m')
        self.add_param('wakeOverlapTRel', np.zeros(nTurbines*nTurbines))

        # used
        self.add_param('turbineXw', val=np.zeros(nTurbines), units='m')
        self.add_param('turbineYw', val=np.zeros(nTurbines), units='m')
        self.add_param('turbineZ', val=np.zeros(nTurbines), units='m')
        self.add_param('yaw%i' % direction_id, np.zeros(nTurbines), units='deg')
        self.add_param('rotorDiameter', val=np.zeros(nTurbines)+126.4, units='m')
        self.add_param('Ct', np.zeros(nTurbines), desc='Turbine thrust coefficients')
        self.add_param('wind_speed', val=8.0, units='m/s')
        self.add_param('axialInduction', val=np.zeros(nTurbines)+1./3.)

        # params for Bastankhah with yaw
        self.add_param('model_params:ky', val=0.022, pass_by_object=True)
        self.add_param('model_params:kz', val=0.022, pass_by_object=True)
        self.add_param('model_params:alpha', val=2.32, pass_by_object=True)
        self.add_param('model_params:beta', val=0.154, pass_by_object=True)
        self.add_param('model_params:I', val=0.075, pass_by_object=True, desc='turbulence intensity')\

        self.add_output('wtVelocity%i' % direction_id, val=np.zeros(nTurbines), units='m/s')

        if nSamples > 0:
            # visualization input
            self.add_param('wsPositionXw', np.zeros(nSamples), units='m', pass_by_object=True,
                           desc='downwind position of desired measurements in wind ref. frame')
            self.add_param('wsPositionYw', np.zeros(nSamples), units='m', pass_by_object=True,
                           desc='crosswind position of desired measurements in wind ref. frame')
            self.add_param('wsPositionZ', np.zeros(nSamples), units='m', pass_by_object=True,
                           desc='position of desired measurements in wind ref. frame')

            # visualization output
            self.add_output('wsArray%i' % direction_id, np.zeros(nSamples), units='m/s', pass_by_object=True,
                            desc='wind speed at measurement locations')

    def solve_nonlinear(self, params, unknowns, resids):

        nTurbines = self.nTurbines
        direction_id = self.direction_id
        nSamples = self.nSamples

        # params for Bastankhah model with yaw
        ky = params['model_params:ky']
        kz = params['model_params:kz']
        alpha = params['model_params:alpha']
        beta = params['model_params:beta']
        I = params['model_params:I']

        # rename inputs and outputs
        turbineXw = params['turbineXw']
        turbineYw = params['turbineYw']
        turbineZ = params['turbineZ']
        yaw = params['yaw%i' % direction_id]
        rotorDiameter = params['rotorDiameter']
        Ct = params['Ct']
        axialInduction = params['axialInduction']
        wind_speed = params['wind_speed']

        # print(turbineXw, turbineYw)

        # Dw0[0] = Dw0[1]
        # m[0] = m[1]

        # rotorDiameter *= 2

        for i in range(0, nTurbines):
            if (Ct[i] > 0.96): # Glauert condition
                axialInduction[i] = 0.143 + np.sqrt(0.0203-0.6427*(0.889 - Ct[i]))
            else:
                axialInduction[i] = 0.5*(1.0-np.sqrt(1.0-Ct[i]))

        print(yaw)
        # NOTE: Bastankhah and Porte Agel 2016 defines yaw as positive clockwise, the negative below accounts for this
        yaw *= -np.pi/180.

        if self.nSamples > 0:
            velX = params['wsPositionXw']
            velY = params['wsPositionYw']
            velZ = params['wsPositionZ']
        else:
            velX = np.zeros([0, 0])
            velY = np.zeros([0, 0])
            velZ = np.zeros([0, 0])

        velocitiesTurbines = np.tile(wind_speed, nTurbines)
        ws_array = np.tile(wind_speed, nSamples)

        for turb in range(0, nTurbines):
            x0 = rotorDiameter[turb] * (np.cos(yaw[turb]) * (1.0 + np.sqrt(1.0 - Ct[turb])) /
                                         (np.sqrt(2.0) * (alpha * I + beta * (1.0 - np.sqrt(1.0 - Ct[turb])))))
            theta_c_0 = 0.3 * yaw[turb] * (1.0 - np.sqrt(1.0 - Ct[turb] * np.cos(yaw[turb]))) / np.cos(yaw[turb])

            for loc in range(0, nSamples):  # at velX-locations
                deltax0 = velX[loc] - (turbineXw[turb] + x0)
                if deltax0 > 0.0:
                    sigmay = rotorDiameter[turb] * (ky * deltax0 / rotorDiameter[turb]
                                                    + np.cos(yaw[turb]) / np.sqrt(8.0))
                    sigmaz = rotorDiameter[turb] * (kz * deltax0 / rotorDiameter[turb]
                                                    + 1.0 / np.sqrt(8.0))
                    wake_offset = rotorDiameter[turb] * (
                        theta_c_0 * x0 / rotorDiameter[turb] +
                        (theta_c_0 / 14.7) * np.sqrt(np.cos(yaw[turb]) / (ky * kz * Ct[turb])) *
                        (2.9 + 1.3 * np.sqrt(1.0 - Ct[turb]) - Ct[turb]) *
                        np.log(
                            ((1.6 + np.sqrt(Ct[turb])) *
                             (1.6 * np.sqrt(8.0 * sigmay * sigmaz /
                                            (np.cos(yaw[turb]) * rotorDiameter[turb] ** 2))
                              - np.sqrt(Ct[turb]))) /
                            ((1.6 - np.sqrt(Ct[turb])) *
                             (1.6 * np.sqrt(8.0 * sigmay * sigmaz /
                                            (np.cos(yaw[turb]) * rotorDiameter[turb] ** 2))
                              + np.sqrt(Ct[turb])))
                        )
                    )

                    deltay = velY[loc] - (turbineYw[turb] + wake_offset)

                    deltav = wind_speed * (
                        (1.0 - np.sqrt(1.0 - Ct[turb] *
                                       np.cos(yaw[turb]) / (8.0 * sigmay * sigmaz /
                                                            (rotorDiameter[turb] ** 2)))) *
                        np.exp(-0.5 * ((deltay) / sigmay) ** 2) *
                        np.exp(-0.5 * ((velZ[loc] - turbineZ[turb]) / sigmaz) ** 2)
                    )

                    ws_array[loc] -= deltav

            for turbI in range(0, nTurbines):  # at turbineX-locations

                deltax0 = turbineXw[turbI] - (turbineXw[turb] + x0)

                if deltax0 > 0.0:
                    sigmay = rotorDiameter[turb] * (ky * deltax0 / rotorDiameter[turb]
                                                     + np.cos(yaw[turb]) / np.sqrt(8.0))
                    sigmaz = rotorDiameter[turb] * (kz * deltax0 / rotorDiameter[turb]
                                                     + 1.0 / np.sqrt(8.0))
                    wake_offset = rotorDiameter[turb] * (
                        theta_c_0 * x0 / rotorDiameter[turb] +
                        (theta_c_0 / 14.7) * np.sqrt(np.cos(yaw[turb]) / (ky * kz * Ct[turb])) *
                        (2.9 + 1.3 * np.sqrt(1.0 - Ct[turb]) - Ct[turb]) *
                        np.log(
                            ((1.6 + np.sqrt(Ct[turb])) *
                             (1.6 * np.sqrt(8.0 * sigmay * sigmaz /
                                            (np.cos(yaw[turb]) * rotorDiameter[turb] ** 2))
                              - np.sqrt(Ct[turb]))) /
                            ((1.6 - np.sqrt(Ct[turb])) *
                             (1.6 * np.sqrt(8.0 * sigmay * sigmaz /
                                            (np.cos(yaw[turb]) * rotorDiameter[turb] ** 2))
                              + np.sqrt(Ct[turb])))
                        )
                    )
                    # print wake_offset, turbineYw[turbI]

                    deltay = turbineYw[turbI] - (turbineYw[turb] + wake_offset)

                    deltav = wind_speed * (
                        (1.0 - np.sqrt(1.0 - Ct[turb] *
                                       np.cos(yaw[turb]) / (8.0 * sigmay * sigmaz /
                                                            (rotorDiameter[turb] ** 2)))) *
                        np.exp(-0.5 * ((deltay) / sigmay) ** 2) *
                        np.exp(-0.5 * ((turbineZ[turbI] - turbineZ[turb]) / sigmaz) ** 2)
                    )

                    velocitiesTurbines[turbI] -= deltav

        unknowns['wtVelocity%i' % direction_id] = velocitiesTurbines

        if nSamples > 0.0:
            print nSamples
            unknowns['wsArray%i' % direction_id] = ws_array
            print unknowns['wsArray%i' % direction_id]

if __name__ == "__main__":

    nTurbines = 2
    nDirections = 1

    rotor_diameter = 126.4
    rotorArea = np.pi*rotor_diameter*rotor_diameter/4.0
    axialInduction = 1.0/3.0
    CP = 0.7737/0.944 * 4.0 * 1.0/3.0 * np.power((1 - 1.0/3.0), 2)
    # CP =0.768 * 4.0 * 1.0/3.0 * np.power((1 - 1.0/3.0), 2)
    CT = 4.0*axialInduction*(1.0-axialInduction)
    generator_efficiency = 0.944

    # Define turbine characteristics
    axialInduction = np.array([axialInduction, axialInduction])
    rotorDiameter = np.array([rotor_diameter, rotor_diameter])
    # rotorDiameter = np.array([rotorDiameter, 0.0001*rotorDiameter])
    yaw = np.array([0., 0.])

    # Define site measurements
    wind_direction = 30.
    wind_speed = 8.    # m/s
    air_density = 1.1716

    Ct = np.array([CT, CT])
    Cp = np.array([CP, CP])

    turbineX = np.array([0.0, 7.*rotor_diameter])
    turbineY = np.array([0.0, 0.0])

    prob = Problem()
    prob.root = Group()
    prob.root.add('model', GaussianWake(nTurbines), promotes=['*'])

    prob.setup()

    prob['turbineXw'] = turbineX
    prob['turbineYw'] = turbineY

    GaussianWakeVelocity = list()

    yawrange = np.linspace(-40., 40., 400)

    for yaw1 in yawrange:

        prob['yaw0'] = np.array([yaw1, 0.0])
        prob['Ct'] = Ct*np.cos(prob['yaw0']*np.pi/180.)**2

        prob.run()

        velocitiesTurbines = prob['wtVelocity0']

        GaussianWakeVelocity.append(list(velocitiesTurbines))

    GaussianWakeVelocity = np.array(GaussianWakeVelocity)

    fig, axes = plt.subplots(ncols=2, nrows=2, sharey=False)
    axes[0, 0].plot(yawrange, GaussianWakeVelocity[:, 0]/wind_speed, 'b')
    axes[0, 0].plot(yawrange, GaussianWakeVelocity[:, 1]/wind_speed, 'b')

    axes[0, 0].set_xlabel('yaw angle (deg.)')
    axes[0, 0].set_ylabel('Velcoity ($V_{eff}/V_o$)')

    posrange = np.linspace(-3.*rotor_diameter, 3.*rotor_diameter, 100)

    prob['yaw0'] = np.array([0.0, 0.0])

    GaussianWakeVelocity = list()

    for pos2 in posrange:

        prob['turbineYw'] = np.array([0.0, pos2])

        prob.run()

        velocitiesTurbines = prob['wtVelocity0']

        GaussianWakeVelocity.append(list(velocitiesTurbines))

    GaussianWakeVelocity = np.array(GaussianWakeVelocity)

    wind_speed = 1.0
    axes[0, 1].plot(posrange/rotor_diameter, GaussianWakeVelocity[:, 0]/wind_speed, 'b')
    axes[0, 1].plot(posrange/rotor_diameter, GaussianWakeVelocity[:, 1]/wind_speed, 'b')
    axes[0, 1].set_xlabel('y/D')
    axes[0, 1].set_ylabel('Velocity ($V_{eff}/V_o$)')

    posrange = np.linspace(-3.*rotorDiameter[0], 3.*rotorDiameter[0], num=1000)

    posrange = np.linspace(-1.*rotorDiameter[0], 30.*rotorDiameter[0], num=2000)
    yaw = np.array([0.0, 0.0])
    wind_direction = 0.0

    GaussianWakeVelocity = list()
    for pos2 in posrange:

        prob['turbineXw'] = np.array([0.0, pos2])
        prob['turbineYw'] = np.array([0.0, 0.0])

        prob.run()

        velocitiesTurbines = prob['wtVelocity0']

        GaussianWakeVelocity.append(list(velocitiesTurbines))

    GaussianWakeVelocity = np.array(GaussianWakeVelocity)

    axes[1, 1].plot(posrange/rotor_diameter, GaussianWakeVelocity[:, 1], 'y', label='GH')
    axes[1, 1].plot(np.array([7, 7]), np.array([2, 8]), '--k', label='tuning point')

    plt.xlabel('x/D')
    plt.ylabel('Velocity (m/s)')
    plt.legend(loc=4)

    plt.show()<|endoftext|>