<commit_before>"""Unit tests for labels.py."""

import unittest
import numpy
import pandas
from gewittergefahr.gg_utils import labels
from gewittergefahr.gg_utils import storm_tracking_utils as tracking_utils
from gewittergefahr.gg_utils import link_events_to_storms as events2storms

TOLERANCE = 1e-6

# The following constants are used to test
# _find_storms_near_end_of_tracking_period.
MAX_LEAD_TIME_SEC = 3600

THESE_END_TIMES_UNIX_SEC = numpy.array(
    [10000, 10000, 10000, 10000, 20000, 20000, 20000, 20000], dtype=int)
THESE_VALID_TIMES_UNIX_SEC = numpy.array(
    [2500, 5000, 7500, 10000, 12500, 15000, 16400, 17500], dtype=int)

THIS_DICTIONARY = {
    tracking_utils.TRACKING_END_TIME_COLUMN: THESE_END_TIMES_UNIX_SEC,
    tracking_utils.TIME_COLUMN: THESE_VALID_TIMES_UNIX_SEC
}
STORM_TO_EVENTS_TABLE_WITH_END_OF_PERIOD = pandas.DataFrame.from_dict(
    THIS_DICTIONARY)

INVALID_STORM_OBJECT_INDICES = numpy.array([2, 3, 7], dtype=int)

# The following constants are used to test _find_dead_storms.
MIN_LEAD_TIME_FOR_DEAD_SEC = 3600

THESE_END_TIMES_UNIX_SEC = numpy.array(
    [10000, 10000, 10000, 10000, 20000, 20000, 20000, 20000], dtype=int)
THESE_VALID_TIMES_UNIX_SEC = numpy.array(
    [2500, 5000, 7500, 10000, 12500, 15000, 16400, 17500], dtype=int)

THIS_DICTIONARY = {
    tracking_utils.CELL_END_TIME_COLUMN: THESE_END_TIMES_UNIX_SEC,
    tracking_utils.TIME_COLUMN: THESE_VALID_TIMES_UNIX_SEC
}
STORM_TO_EVENTS_TABLE_WITH_DEAD_STORMS = pandas.DataFrame.from_dict(
    THIS_DICTIONARY)

DEAD_STORM_OBJECT_INDICES = numpy.array([2, 3, 7], dtype=int)

# The following constants are used to test get_column_name_for_regression_label,
# get_column_name_for_num_wind_obs, and
# get_column_name_for_classification_label.
MIN_LEAD_TIME_SEC = 900
MIN_LINK_DISTANCE_METRES = 1.
MAX_LINK_DISTANCE_METRES = 5000.
WIND_SPEED_PERCENTILE_LEVEL = 97.5
WIND_CLASS_CUTOFFS_KT = numpy.array([10., 20., 30., 40., 50.])

REGRESSION_LABEL_COLUMN_NAME = (
    'wind-speed-m-s01_percentile=097.5_lead-time=0900-3600sec_'
    'distance=00001-05000m')
NUM_WIND_OBS_COLUMN_NAME = (
    'num-wind-observations_lead-time=0900-3600sec_distance=00001-05000m')
WIND_CLASSIFICATION_LABEL_COLUMN_NAME = (
    'wind-speed_percentile=097.5_lead-time=0900-3600sec_distance=00001-05000m'
    '_cutoffs=10-20-30-40-50kt')
TORNADO_LABEL_COLUMN_NAME = (
    'tornado_lead-time=0900-3600sec_distance=00001-05000m')

WIND_CLASSIFN_LABEL_NAME_ZERO_LEAD_TIME = (
    'wind-speed_percentile=097.5_lead-time=0000-3600sec_distance=00001-05000m'
    '_cutoffs=10-20-30-40-50kt')

# The following constants are used to test column_name_to_label_params and
# column_name_to_num_classes.
PARAM_DICT_FOR_REGRESSION_LABEL = {
    labels.MIN_LEAD_TIME_KEY: MIN_LEAD_TIME_SEC,
    labels.MAX_LEAD_TIME_KEY: MAX_LEAD_TIME_SEC,
    labels.MIN_LINKAGE_DISTANCE_KEY: MIN_LINK_DISTANCE_METRES,
    labels.MAX_LINKAGE_DISTANCE_KEY: MAX_LINK_DISTANCE_METRES,
    labels.EVENT_TYPE_KEY: events2storms.WIND_EVENT_TYPE_STRING,
    labels.WIND_SPEED_PERCENTILE_LEVEL_KEY: WIND_SPEED_PERCENTILE_LEVEL,
    labels.WIND_SPEED_CLASS_CUTOFFS_KEY: None
}
PARAM_DICT_FOR_WIND_CLASSIFICATION_LABEL = {
    labels.MIN_LEAD_TIME_KEY: MIN_LEAD_TIME_SEC,
    labels.MAX_LEAD_TIME_KEY: MAX_LEAD_TIME_SEC,
    labels.MIN_LINKAGE_DISTANCE_KEY: MIN_LINK_DISTANCE_METRES,
    labels.MAX_LINKAGE_DISTANCE_KEY: MAX_LINK_DISTANCE_METRES,
    labels.EVENT_TYPE_KEY: events2storms.WIND_EVENT_TYPE_STRING,
    labels.WIND_SPEED_PERCENTILE_LEVEL_KEY: WIND_SPEED_PERCENTILE_LEVEL,
    labels.WIND_SPEED_CLASS_CUTOFFS_KEY: WIND_CLASS_CUTOFFS_KT
}
PARAM_DICT_FOR_TORNADO_LABEL = {
    labels.MIN_LEAD_TIME_KEY: MIN_LEAD_TIME_SEC,
    labels.MAX_LEAD_TIME_KEY: MAX_LEAD_TIME_SEC,
    labels.MIN_LINKAGE_DISTANCE_KEY: MIN_LINK_DISTANCE_METRES,
    labels.MAX_LINKAGE_DISTANCE_KEY: MAX_LINK_DISTANCE_METRES,
    labels.EVENT_TYPE_KEY: events2storms.TORNADO_EVENT_TYPE_STRING,
    labels.WIND_SPEED_PERCENTILE_LEVEL_KEY: None,
    labels.WIND_SPEED_CLASS_CUTOFFS_KEY: None
}

# The following constants are used to test get_columns_with_labels,
# get_columns_with_num_wind_obs, and check_label_table.
THIS_DICTIONARY = {
    REGRESSION_LABEL_COLUMN_NAME: numpy.array([]),
    NUM_WIND_OBS_COLUMN_NAME: numpy.array([], dtype=int),
    WIND_CLASSIFICATION_LABEL_COLUMN_NAME: numpy.array([], dtype=int),
    TORNADO_LABEL_COLUMN_NAME: numpy.array([], dtype=int)
}
LABEL_TABLE = pandas.DataFrame.from_dict(THIS_DICTIONARY)

# The following constants are used to test find_label_file.
TOP_DIRECTORY_NAME = 'labels'
FILE_TIME_UNIX_SEC = 1517523991  # 222631 1 Feb 2018
FILE_SPC_DATE_STRING = '20180201'

WIND_LABELS_ONE_TIME_FILE_NAME = (
    'labels/2018/20180201/wind_labels_2018-02-01-222631.p')
WIND_LABELS_SPC_DATE_FILE_NAME = 'labels/2018/wind_labels_20180201.p'
TORNADO_LABELS_ONE_TIME_FILE_NAME = (
    'labels/2018/20180201/tornado_labels_2018-02-01-222631.p')
TORNADO_LABELS_SPC_DATE_FILE_NAME = 'labels/2018/tornado_labels_20180201.p'

class LabelsTests(unittest.TestCase):
    """Each method is a unit test for labels.py."""

    def test_find_storms_near_end_of_tracking_period(self):
        """Ensures correctness of _find_storms_near_end_of_tracking_period."""

        these_indices = labels._find_storms_near_end_of_tracking_period(
            storm_to_events_table=STORM_TO_EVENTS_TABLE_WITH_END_OF_PERIOD,
            max_lead_time_sec=MAX_LEAD_TIME_SEC)

        self.assertTrue(numpy.array_equal(
            these_indices, INVALID_STORM_OBJECT_INDICES))

    def test_find_dead_storms(self):
        """Ensures correct output from _find_dead_storms."""

        these_indices = labels._find_dead_storms(
            storm_to_events_table=STORM_TO_EVENTS_TABLE_WITH_DEAD_STORMS,
            min_lead_time_sec=MIN_LEAD_TIME_FOR_DEAD_SEC)

        self.assertTrue(numpy.array_equal(
            these_indices, DEAD_STORM_OBJECT_INDICES))

    def test_get_column_name_for_regression_label(self):
        """Ensures correct output from get_column_name_for_regression_label."""

        this_column_name = labels.get_column_name_for_regression_label(
            min_lead_time_sec=MIN_LEAD_TIME_SEC,
            max_lead_time_sec=MAX_LEAD_TIME_SEC,
            min_link_distance_metres=MIN_LINK_DISTANCE_METRES,
            max_link_distance_metres=MAX_LINK_DISTANCE_METRES,
            wind_speed_percentile_level=WIND_SPEED_PERCENTILE_LEVEL)
        self.assertTrue(this_column_name == REGRESSION_LABEL_COLUMN_NAME)

    def test_get_column_name_for_num_wind_obs(self):
        """Ensures correct output from get_column_name_for_num_wind_obs."""

        this_column_name = labels.get_column_name_for_num_wind_obs(
            min_lead_time_sec=MIN_LEAD_TIME_SEC,
            max_lead_time_sec=MAX_LEAD_TIME_SEC,
            min_link_distance_metres=MIN_LINK_DISTANCE_METRES,
            max_link_distance_metres=MAX_LINK_DISTANCE_METRES)
        self.assertTrue(this_column_name == NUM_WIND_OBS_COLUMN_NAME)

    def test_get_column_name_for_classification_label_wind(self):
        """Ensures correct output from get_column_name_for_classification_label.

        In this case, event type is wind speed.
        """

        this_column_name = labels.get_column_name_for_classification_label(
            min_lead_time_sec=MIN_LEAD_TIME_SEC,
            max_lead_time_sec=MAX_LEAD_TIME_SEC,
            min_link_distance_metres=MIN_LINK_DISTANCE_METRES,
            max_link_distance_metres=MAX_LINK_DISTANCE_METRES,
            event_type_string=events2storms.WIND_EVENT_TYPE_STRING,
            wind_speed_percentile_level=WIND_SPEED_PERCENTILE_LEVEL,
            wind_speed_class_cutoffs_kt=WIND_CLASS_CUTOFFS_KT)

        self.assertTrue(
            this_column_name == WIND_CLASSIFICATION_LABEL_COLUMN_NAME)

    def test_get_column_name_for_classification_label_tornado(self):
        """Ensures correct output from get_column_name_for_classification_label.

        In this case, event type is tornado occurrence.
        """

        this_column_name = labels.get_column_name_for_classification_label(
            min_lead_time_sec=MIN_LEAD_TIME_SEC,
            max_lead_time_sec=MAX_LEAD_TIME_SEC,
            min_link_distance_metres=MIN_LINK_DISTANCE_METRES,
            max_link_distance_metres=MAX_LINK_DISTANCE_METRES,
            event_type_string=events2storms.TORNADO_EVENT_TYPE_STRING)
        self.assertTrue(this_column_name == TORNADO_LABEL_COLUMN_NAME)

    def test_column_name_to_label_params_regression(self):
        """Ensures correct output from column_name_to_label_params.

        In this case, learning goal is regression.
        """

        this_parameter_dict = labels.column_name_to_label_params(
            REGRESSION_LABEL_COLUMN_NAME)
        self.assertTrue(this_parameter_dict == PARAM_DICT_FOR_REGRESSION_LABEL)

    def test_column_name_to_label_params_wind_classification(self):
        """Ensures correct output from column_name_to_label_params.

        In this case, learning goal is classification and event type is wind
        speed.
        """

        this_parameter_dict = labels.column_name_to_label_params(
            WIND_CLASSIFICATION_LABEL_COLUMN_NAME)

        self.assertTrue(numpy.array_equal(
            this_parameter_dict[labels.WIND_SPEED_CLASS_CUTOFFS_KEY],
            PARAM_DICT_FOR_WIND_CLASSIFICATION_LABEL[
                labels.WIND_SPEED_CLASS_CUTOFFS_KEY]))

        this_parameter_dict.pop(labels.WIND_SPEED_CLASS_CUTOFFS_KEY, None)
        PARAM_DICT_FOR_WIND_CLASSIFICATION_LABEL.pop(
            labels.WIND_SPEED_CLASS_CUTOFFS_KEY, None)

        self.assertTrue(
            this_parameter_dict == PARAM_DICT_FOR_WIND_CLASSIFICATION_LABEL)

    def test_column_name_to_label_params_tornado(self):
        """Ensures correct output from column_name_to_label_params.

        In this case, learning goal is classification and event type is tornado
        occurrence.
        """

        this_parameter_dict = labels.column_name_to_label_params(
            TORNADO_LABEL_COLUMN_NAME)
        self.assertTrue(this_parameter_dict == PARAM_DICT_FOR_TORNADO_LABEL)

    def test_column_name_to_num_classes_regression(self):
        """Ensures correct output from column_name_to_num_classes.

        In this case, learning goal is regression.
        """

        this_num_classes = labels.column_name_to_num_classes(
            REGRESSION_LABEL_COLUMN_NAME)
        self.assertTrue(this_num_classes is None)

    def test_column_name_to_num_classes_wind_with_dead_storms(self):
        """Ensures correct output from column_name_to_num_classes.

        In this case, learning goal is classification; event type is wind speed;
        and the "dead storm" label is included in the number of classes.
        """

        this_num_classes = labels.column_name_to_num_classes(
            WIND_CLASSIFICATION_LABEL_COLUMN_NAME, include_dead_storms=True)
        self.assertTrue(this_num_classes == len(WIND_CLASS_CUTOFFS_KT) + 2)

    def test_column_name_to_num_classes_wind_sans_dead_storms(self):
        """Ensures correct output from column_name_to_num_classes.

        In this case, learning goal is classification; event type is wind speed;
        and the "dead storm" label is *not* included in the number of classes.
        """

        this_num_classes = labels.column_name_to_num_classes(
            WIND_CLASSIFICATION_LABEL_COLUMN_NAME, include_dead_storms=False)
        self.assertTrue(this_num_classes == len(WIND_CLASS_CUTOFFS_KT) + 1)

    def test_column_name_to_num_classes_tornado(self):
        """Ensures correct output from column_name_to_num_classes.

        In this case, learning goal is classification and event type is tornado
        occurrence.
        """

        this_num_classes = labels.column_name_to_num_classes(
            TORNADO_LABEL_COLUMN_NAME)
        self.assertTrue(this_num_classes == 2)

    def test_column_name_to_num_classes_wind_zero_lead_time(self):
        """Ensures correct output from column_name_to_num_classes.

        In this case, learning goal is classification; event type is wind speed;
        and the lead-time range includes 0 seconds, so dead storms are
        impossible.
        """

        this_num_classes = labels.column_name_to_num_classes(
            WIND_CLASSIFN_LABEL_NAME_ZERO_LEAD_TIME, include_dead_storms=True)
        self.assertTrue(this_num_classes == len(WIND_CLASS_CUTOFFS_KT) + 1)

    def test_get_columns_with_labels_regression(self):
        """Ensures correct output from get_columns_with_labels.

        In this case, learning goal is regression.
        """

        these_column_names = labels.get_columns_with_labels(
            label_table=LABEL_TABLE,
            goal_string=labels.REGRESSION_GOAL_STRING,
            event_type_string=events2storms.WIND_EVENT_TYPE_STRING)
        self.assertTrue(these_column_names == [REGRESSION_LABEL_COLUMN_NAME])

    def test_get_columns_with_labels_wind_classification(self):
        """Ensures correct output from get_columns_with_labels.

        In this case, learning goal is classification and event type is wind
        speed.
        """

        these_column_names = labels.get_columns_with_labels(
            label_table=LABEL_TABLE,
            goal_string=labels.CLASSIFICATION_GOAL_STRING,
            event_type_string=events2storms.WIND_EVENT_TYPE_STRING)

        self.assertTrue(
            these_column_names == [WIND_CLASSIFICATION_LABEL_COLUMN_NAME])

    def test_get_columns_with_labels_tornado(self):
        """Ensures correct output from get_columns_with_labels.

        In this case, learning goal is classification and event type is tornado
        occurrence.
        """

        these_column_names = labels.get_columns_with_labels(
            label_table=LABEL_TABLE,
            goal_string=labels.CLASSIFICATION_GOAL_STRING,
            event_type_string=events2storms.TORNADO_EVENT_TYPE_STRING)
        self.assertTrue(these_column_names == [TORNADO_LABEL_COLUMN_NAME])

    def test_get_columns_with_num_wind_obs_regression(self):
        """Ensures correct output from get_columns_with_num_wind_obs.

        In this case, learning goal is regression.
        """

        these_column_names = labels.get_columns_with_num_wind_obs(
            label_table=LABEL_TABLE,
            label_column_names=[REGRESSION_LABEL_COLUMN_NAME])
        self.assertTrue(these_column_names == [NUM_WIND_OBS_COLUMN_NAME])

    def test_get_columns_with_num_wind_obs_classification(self):
        """Ensures correct output from get_columns_with_num_wind_obs.

        In this case, learning goal is classification.
        """

        these_column_names = labels.get_columns_with_num_wind_obs(
            label_table=LABEL_TABLE,
            label_column_names=[WIND_CLASSIFICATION_LABEL_COLUMN_NAME])
        self.assertTrue(these_column_names == [NUM_WIND_OBS_COLUMN_NAME])

    def test_find_label_file_one_time_wind(self):
        """Ensures correct output from find_label_file.

        In this case, event type is damaging wind and file contains data for one
        time step.
        """

        this_file_name = labels.find_label_file(
            top_directory_name=TOP_DIRECTORY_NAME,
            event_type_string=events2storms.WIND_EVENT_TYPE_STRING,
            raise_error_if_missing=False, unix_time_sec=FILE_TIME_UNIX_SEC)
        self.assertTrue(this_file_name == WIND_LABELS_ONE_TIME_FILE_NAME)

    def test_find_label_file_one_time_tornado(self):
        """Ensures correct output from find_label_file.

        In this case, event type is tornado and file contains data for one time
        step.
        """

        this_file_name = labels.find_label_file(
            top_directory_name=TOP_DIRECTORY_NAME,
            event_type_string=events2storms.TORNADO_EVENT_TYPE_STRING,
            raise_error_if_missing=False, unix_time_sec=FILE_TIME_UNIX_SEC)
        self.assertTrue(this_file_name == TORNADO_LABELS_ONE_TIME_FILE_NAME)

    def test_find_label_file_spc_date_wind(self):
        """Ensures correct output from find_label_file.

        In this case, event type is damaging wind and file contains data for one
        SPC date.
        """

        this_file_name = labels.find_label_file(
            top_directory_name=TOP_DIRECTORY_NAME,
            event_type_string=events2storms.WIND_EVENT_TYPE_STRING,
            raise_error_if_missing=False, spc_date_string=FILE_SPC_DATE_STRING)
        self.assertTrue(this_file_name == WIND_LABELS_SPC_DATE_FILE_NAME)

    def test_find_label_file_spc_date_tornado(self):
        """Ensures correct output from find_label_file.

        In this case, event type is tornado and file contains data for one SPC
        date.
        """

        this_file_name = labels.find_label_file(
            top_directory_name=TOP_DIRECTORY_NAME,
            event_type_string=events2storms.TORNADO_EVENT_TYPE_STRING,
            raise_error_if_missing=False, spc_date_string=FILE_SPC_DATE_STRING)
        self.assertTrue(this_file_name == TORNADO_LABELS_SPC_DATE_FILE_NAME)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Removed labels_test.py (dead file).<commit_after><|endoftext|>


<commit_before># fodselsnummer.py - functions for handling Norwegian birth numbers
# coding: utf-8
#
# Copyright (C) 2018 Ilya Vihtinsky
# Copyright (C) 2018 Arthur de Jong
# Copyright (C) 2020 Leon Sandøy
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA

"""Fødselsnummer (Norwegian birth number, the national identity number).

The Fødselsnummer is an eleven-digit number that is built up of the date of
birth of the person, a serial number and two check digits.

More information:

* https://no.wikipedia.org/wiki/F%C3%B8dselsnummer
* https://en.wikipedia.org/wiki/National_identification_number#Norway

>>> validate('151086 95088')
'15108695088'
>>> get_gender('151086-95088')
'F'
>>> validate('15108695077')
Traceback (most recent call last):
    ...
InvalidChecksum: ...
>>> format('15108695077')
'151086 95077'
"""

import datetime

from stdnum.exceptions import *
from stdnum.util import clean, isdigits

def compact(number):
    """Convert the number to the minimal representation. This strips the
    number of any valid separators and removes surrounding whitespace."""
    return clean(number, ' -:')

def calc_check_digit1(number):
    """Calculate the first check digit for the number."""
    weights = (3, 7, 6, 1, 8, 9, 4, 5, 2)
    return str((11 - sum(w * int(n) for w, n in zip(weights, number))) % 11)

def calc_check_digit2(number):
    """Calculate the second check digit for the number."""
    weights = (5, 4, 3, 2, 7, 6, 5, 4, 3, 2)
    return str((11 - sum(w * int(n) for w, n in zip(weights, number))) % 11)

def get_gender(number):
    """Get the person's birth gender ('M' or 'F')."""
    number = compact(number)
    if int(number[8]) % 2:
        return 'M'
    else:
        return 'F'

def get_birth_date(number):
    """Determine and return the birth date."""
    number = compact(number)
    day = int(number[0:2])
    month = int(number[2:4])
    year = int(number[4:6])
    individual_digits = int(number[6:9])
    # Raise a more useful exception for FH numbers
    if day >= 80:
        raise InvalidComponent(
            'This number is an FH-number, and does not contain birth date information by design.')
    # Correct the birth day for D-numbers. These have a modified first digit.
    # https://no.wikipedia.org/wiki/F%C3%B8dselsnummer#D-nummer
    if day > 40:
        day -= 40
    # Correct the birth month for H-numbers. These have a modified third digit.
    # https://no.wikipedia.org/wiki/F%C3%B8dselsnummer#H-nummer
    if month > 40:
        month -= 40
    if individual_digits < 500:
        year += 1900
    elif 500 <= individual_digits < 750 and year >= 54:
        year += 1800
    elif 500 <= individual_digits < 1000 and year < 40:
        year += 2000
    elif 900 <= individual_digits < 1000 and year >= 40:
        year += 1900
    else:
        # The birth century falls outside all defined ranges.
        raise InvalidComponent('The birthdate century cannot be determined')
    try:
        return datetime.date(year, month, day)
    except ValueError:
        raise InvalidComponent('The number does not contain valid birth date information.')

def validate(number):
    """Check if the number is a valid birth number."""
    number = compact(number)
    if len(number) != 11:
        raise InvalidLength()
    if not isdigits(number):
        raise InvalidFormat()
    if number[-2] != calc_check_digit1(number):
        raise InvalidChecksum()
    if number[-1] != calc_check_digit2(number):
        raise InvalidChecksum()
    if get_birth_date(number) > datetime.date.today():
        raise InvalidComponent(
            'The birth date information is valid, but this person has not been born yet.')
    return number

def is_valid(number):
    """Check if the number is a valid birth number."""
    try:
        return bool(validate(number))
    except ValidationError:
        return False

def format(number):
    """Reformat the number to the standard presentation format."""
    number = compact(number)
    return number[:6] + ' ' + number[6:]
<commit_msg>Simplify range checking in Norwegian birth numbers<commit_after># fodselsnummer.py - functions for handling Norwegian birth numbers
# coding: utf-8
#
# Copyright (C) 2018 Ilya Vihtinsky
# Copyright (C) 2018 Arthur de Jong
# Copyright (C) 2020 Leon Sandøy
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA

"""Fødselsnummer (Norwegian birth number, the national identity number).

The Fødselsnummer is an eleven-digit number that is built up of the date of
birth of the person, a serial number and two check digits.

More information:

* https://no.wikipedia.org/wiki/F%C3%B8dselsnummer
* https://en.wikipedia.org/wiki/National_identification_number#Norway

>>> validate('151086 95088')
'15108695088'
>>> get_gender('151086-95088')
'F'
>>> validate('15108695077')
Traceback (most recent call last):
    ...
InvalidChecksum: ...
>>> format('15108695077')
'151086 95077'
"""

import datetime

from stdnum.exceptions import *
from stdnum.util import clean, isdigits

def compact(number):
    """Convert the number to the minimal representation. This strips the
    number of any valid separators and removes surrounding whitespace."""
    return clean(number, ' -:')

def calc_check_digit1(number):
    """Calculate the first check digit for the number."""
    weights = (3, 7, 6, 1, 8, 9, 4, 5, 2)
    return str((11 - sum(w * int(n) for w, n in zip(weights, number))) % 11)

def calc_check_digit2(number):
    """Calculate the second check digit for the number."""
    weights = (5, 4, 3, 2, 7, 6, 5, 4, 3, 2)
    return str((11 - sum(w * int(n) for w, n in zip(weights, number))) % 11)

def get_gender(number):
    """Get the person's birth gender ('M' or 'F')."""
    number = compact(number)
    if int(number[8]) % 2:
        return 'M'
    else:
        return 'F'

def get_birth_date(number):
    """Determine and return the birth date."""
    number = compact(number)
    day = int(number[0:2])
    month = int(number[2:4])
    year = int(number[4:6])
    individual_digits = int(number[6:9])
    # Raise a more useful exception for FH numbers
    if day >= 80:
        raise InvalidComponent(
            'This number is an FH-number, and does not contain birth date information by design.')
    # Correct the birth day for D-numbers. These have a modified first digit.
    # https://no.wikipedia.org/wiki/F%C3%B8dselsnummer#D-nummer
    if day > 40:
        day -= 40
    # Correct the birth month for H-numbers. These have a modified third digit.
    # https://no.wikipedia.org/wiki/F%C3%B8dselsnummer#H-nummer
    if month > 40:
        month -= 40
    if individual_digits < 500:
        year += 1900
    elif individual_digits < 750 and year >= 54:
        year += 1800
    elif individual_digits < 1000 and year < 40:
        year += 2000
    elif 900 <= individual_digits < 1000 and year >= 40:
        year += 1900
    else:
        # The birth century falls outside all defined ranges.
        raise InvalidComponent('The birthdate century cannot be determined')
    try:
        return datetime.date(year, month, day)
    except ValueError:
        raise InvalidComponent('The number does not contain valid birth date information.')

def validate(number):
    """Check if the number is a valid birth number."""
    number = compact(number)
    if len(number) != 11:
        raise InvalidLength()
    if not isdigits(number):
        raise InvalidFormat()
    if number[-2] != calc_check_digit1(number):
        raise InvalidChecksum()
    if number[-1] != calc_check_digit2(number):
        raise InvalidChecksum()
    if get_birth_date(number) > datetime.date.today():
        raise InvalidComponent(
            'The birth date information is valid, but this person has not been born yet.')
    return number

def is_valid(number):
    """Check if the number is a valid birth number."""
    try:
        return bool(validate(number))
    except ValidationError:
        return False

def format(number):
    """Reformat the number to the standard presentation format."""
    number = compact(number)
    return number[:6] + ' ' + number[6:]
<|endoftext|>


<commit_before>#!/usr/bin/env python
#coding: utf-8
import sys,os
esbpath = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
sys.path.append( esbpath )

from servicebus.conf import load_config_from_file

import time

from servicebus.worker.decorators import Task

@Task(name="print_foo")
def print_foo(param):
    print "print_foo", param
    print

@Task(name="do_work")
def do_work(a,b,foo=None, baz=None):
    print "do_work"
    print "params", a,b,foo,baz
    print
    return "hopsasa fikumiku rezultat"

@Task(name="another_task")
def fiku(a,b=None,foo=None, baz=None):
    print "another_task"
    print

@Task(name="messages.mail.send_heavy_spam")
def sendspam(a,b):
    print "sendspam"
    print a,b

@Task(name="long_task")
def long_task(a, x):
    print x, "sleeping:", a
    time.sleep(float(a))
    return "hurra " + str(x)

@Task(name="wyjebka")
def wyjebka(param):
    return param / 0

from servicebus.worker import Daemon

if __name__=="__main__":
    load_config_from_file("../../config.txt")
    daemon = Daemon("fikumiku")
    daemon.run()

#from servicebus.worker.syncclient import SyncClient

#try:
#    import random
#    addr = "1.2.3.4:"+str(random.randint(5000,6000))
#    nsc = SyncClient("fikumiku", addr)
#    nsc.notify_start()
#    import time
#    time.sleep(60)
#finally:
#    nsc.notify_stop()

<commit_msg>bad example fix<commit_after>#!/usr/bin/env python
#coding: utf-8
import sys,os
esbpath = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
sys.path.append( esbpath )

from servicebus.conf import load_config_from_file

import time

from servicebus.worker.decorators import Task

@Task(name="print_foo")
def print_foo(param):
    print "print_foo", param
    print

@Task(name="do_work")
def do_work(a,b,foo=None, baz=None):
    print "do_work"
    print "params", a,b,foo,baz
    print
    return "hopsasa fikumiku rezultat"

@Task(name="another_task")
def fiku(a,b=None,foo=None, baz=None):
    print "another_task"
    print

@Task(name="long_task")
def long_task(a, x):
    print x, "sleeping:", a
    time.sleep(float(a))
    return "hurra " + str(x)

@Task(name="wyjebka")
def wyjebka(param):
    return param / 0

from servicebus.worker import Daemon

if __name__=="__main__":
    load_config_from_file("../../config.txt")
    daemon = Daemon("fikumiku")
    daemon.run()

#from servicebus.worker.syncclient import SyncClient

#try:
#    import random
#    addr = "1.2.3.4:"+str(random.randint(5000,6000))
#    nsc = SyncClient("fikumiku", addr)
#    nsc.notify_start()
#    import time
#    time.sleep(60)
#finally:
#    nsc.notify_stop()

<|endoftext|>


<commit_before>import datetime
import re

from django import template
from django.core.urlresolvers import reverse, NoReverseMatch

from djlotrek import get_host_url
from urllib.parse import urljoin

register = template.Library()

@register.simple_tag
def auto_update_year_range(start=None):
    current_year = datetime.datetime.now().year
    if start == current_year:
        return current_year
    if start:
        return '{0}-{1}'.format(start, current_year)
    return current_year

@register.simple_tag(takes_context=True)
def active(context, pattern_or_urlname):
    try:
        pattern = '^' + reverse(pattern_or_urlname)
    except NoReverseMatch:
        pattern = pattern_or_urlname
    path = context['request'].path
    if re.search(pattern, path):
        return 'active'
    return ''

@register.simple_tag(takes_context=True)
def absolute_url(context, url):
    host = get_host_url(context['request'])
    if url:
        return urljoin(host, url)
    else:
        return ''
<commit_msg>Manage context without request<commit_after>import datetime
import re

from django import template
from django.core.urlresolvers import reverse, NoReverseMatch

from djlotrek import get_host_url
from urllib.parse import urljoin

register = template.Library()

@register.simple_tag
def auto_update_year_range(start=None):
    current_year = datetime.datetime.now().year
    if start == current_year:
        return current_year
    if start:
        return '{0}-{1}'.format(start, current_year)
    return current_year

@register.simple_tag(takes_context=True)
def active(context, pattern_or_urlname):
    try:
        pattern = '^' + reverse(pattern_or_urlname)
    except NoReverseMatch:
        pattern = pattern_or_urlname
    path = context['request'].path
    if re.search(pattern, path):
        return 'active'
    return ''

@register.simple_tag(takes_context=True)
def absolute_url(context, url):
    host = get_host_url('' if 'request' not in context else context['request'])
    if url:
        return urljoin(host, url)
    else:
        return ''
<|endoftext|>


<commit_before>"""
This is the python code for the calculations. To build the workflow, I just
copy & paste this code into Alfred.

Supported Calculations:

1. Percent Change (increase/decrease): `% 3 6` -> 100%
2. Percent of; What is 3 percent of 100: `% 3 of 100` -> 3%
3. Percent Difference; What is 2 percent from 100: `% 100 - 2%`
4. Percent Increase; What is 100 + 2%: `% 100 + 2%`
"""
import sys

def percent_increase(a, b):
    """What is `a` + `b`%?"""
    return str(round(a + (a * (b / 100.0)), 2))

def percent_difference(a, b):
    """What is `a` - `b`%?"""
    return str(round(a - (a * (b / 100.0)), 2))

def percent_of(a, b):
    """What is `a` percent of `b`?"""
    return str(round((a / b) * 100, 2))

def percent_change(a, b):
    """Given two floats, calculate the percent change and return a rounded
    string representation of the change."""
    if a > 0:
        result = ((b - a) / a) * 100
    else:
        result = 0
    return str(round(result, 2))

def parse(input_string):
    """Parses the input string and hands off the values to the correct function."""
    values = input_string.strip().split(' ')
    if len(values) == 2:
        # `% a b`. percent_change.
        return percent_change(float(values[0]), float(values[1]))
    elif len(values) == 3 and 'of' in values:
        # `% a of b`. percent_of
        a = float(values[0])
        b = float(values[-1])
        return percent_of(a, b)
    elif len(values) == 3 and '-' in values:
        # `% a - b%`. percent_difference
        a = float(values[0])
        b = float(values[-1].replace("%", ""))
        return percent_difference(a, b)
    else:
        return "What?"

# -----------------------------------------------------------------------------
# Some Tests. To run these, do:
#
# $ python script.py test
#
# (yes, I reinvented a wheel, here)
# -----------------------------------------------------------------------------
FAILURES = []

def _eq(a, b):
    global FAILURES
    try:
        assert a == b
    except AssertionError:
        sys.stderr.write("F")
        FAILURES.append("{0} is not equal to {1}".format(a, b))
    else:
        sys.stdout.write(".")

def test_percent_increase():
    _eq(percent_increase(100.0, 2.0), "102.0")
    _eq(percent_increase(12.34, 2.0), "12.59")
    _eq(percent_increase(12.34, 2.5), "12.65")
    _eq(percent_increase(14.0, 2.0), "14.28")

def test_percent_difference():
    _eq(percent_difference(100.0, 2.0), "98.0")
    _eq(percent_difference(12.34, 2.0), "12.09")
    _eq(percent_difference(12.34, 2.5), "12.03")
    _eq(percent_difference(14.0, 2.0), "13.72")

def test_percent_of():
    _eq(percent_of(3.0, 100.00), "3.0")
    _eq(percent_of(2.0, 5.0), "40.0")
    _eq(percent_of(5.0, 3.0), "166.67")

def test_percent_change():
    _eq(percent_change(3.0, 6.0), "100.0")
    _eq(percent_change(5.0, 2.0), "-60.0")
    _eq(percent_change(3.5, 3.52), "0.57")

def test_parse():
    _eq(parse("3 6"), "100.0")  # percent_change
    _eq(parse("2 of 5"), "40.0")  # percent_of

if __name__ == "__main__":
    if sys.argv[0] == "-c":  # Getting run by alfred.
        sys.stdout.write(parse('{query}'))

    elif len(sys.argv) == 2 and sys.argv[1] == "test":
        print("Running Tests:\n")
        test_percent_increase()
        test_percent_difference()
        test_percent_of()
        test_percent_change()
        print("\n\nDone.")
        if len(FAILURES):
            print("ERRORS:")
            print("\n{0}\n".format("\n".join(FAILURES)))
<commit_msg>more tests for parse<commit_after>"""
This is the python code for the calculations. To build the workflow, I just
copy & paste this code into Alfred.

Supported Calculations:

1. Percent Change (increase/decrease): `% 3 6` -> 100%
2. Percent of; What is 3 percent of 100: `% 3 of 100` -> 3%
3. Percent Difference; What is 2 percent from 100: `% 100 - 2%`
4. Percent Increase; What is 100 + 2%: `% 100 + 2%`
"""
import sys

def percent_increase(a, b):
    """What is `a` + `b`%?"""
    return str(round(a + (a * (b / 100.0)), 2))

def percent_difference(a, b):
    """What is `a` - `b`%?"""
    return str(round(a - (a * (b / 100.0)), 2))

def percent_of(a, b):
    """What is `a` percent of `b`?"""
    return str(round((a / b) * 100, 2))

def percent_change(a, b):
    """Given two floats, calculate the percent change and return a rounded
    string representation of the change."""
    if a > 0:
        result = ((b - a) / a) * 100
    else:
        result = 0
    return str(round(result, 2))

def parse(input_string):
    """Parses the input string and hands off the values to the correct function."""
    values = input_string.strip().split(' ')
    if len(values) == 2:
        # `% a b`. percent_change.
        return percent_change(float(values[0]), float(values[1]))
    elif len(values) == 3 and 'of' in values:
        # `% a of b`. percent_of
        a = float(values[0])
        b = float(values[-1])
        return percent_of(a, b)
    elif len(values) == 3 and '-' in values:
        # `% a - b%`. percent_difference
        a = float(values[0])
        b = float(values[-1].replace("%", ""))
        return percent_difference(a, b)
    else:
        return "What?"

# -----------------------------------------------------------------------------
# Some Tests. To run these, do:
#
# $ python script.py test
#
# (yes, I reinvented a wheel, here)
# -----------------------------------------------------------------------------
FAILURES = []

def _eq(a, b):
    global FAILURES
    try:
        assert a == b
    except AssertionError:
        sys.stderr.write("F")
        FAILURES.append("{0} is not equal to {1}".format(a, b))
    else:
        sys.stdout.write(".")

def test_percent_increase():
    _eq(percent_increase(100.0, 2.0), "102.0")
    _eq(percent_increase(12.34, 2.0), "12.59")
    _eq(percent_increase(12.34, 2.5), "12.65")
    _eq(percent_increase(14.0, 2.0), "14.28")

def test_percent_difference():
    _eq(percent_difference(100.0, 2.0), "98.0")
    _eq(percent_difference(12.34, 2.0), "12.09")
    _eq(percent_difference(12.34, 2.5), "12.03")
    _eq(percent_difference(14.0, 2.0), "13.72")

def test_percent_of():
    _eq(percent_of(3.0, 100.00), "3.0")
    _eq(percent_of(2.0, 5.0), "40.0")
    _eq(percent_of(5.0, 3.0), "166.67")

def test_percent_change():
    _eq(percent_change(3.0, 6.0), "100.0")
    _eq(percent_change(5.0, 2.0), "-60.0")
    _eq(percent_change(3.5, 3.52), "0.57")

def test_parse():
    _eq(parse("3 6"), "100.0")  # percent_change
    _eq(parse("2 of 5"), "40.0")  # percent_of
    _eq(parse("100 - 2%"), "98.0")  # percent_decrease
    _eq(parse("100 + 2%"), "102.0")  # percent_increase

if __name__ == "__main__":
    if sys.argv[0] == "-c":  # Getting run by alfred.
        sys.stdout.write(parse('{query}'))

    elif len(sys.argv) == 2 and sys.argv[1] == "test":
        print("Running Tests:\n")
        test_percent_increase()
        test_percent_difference()
        test_percent_of()
        test_percent_change()
        print("\n\nDone.")
        if len(FAILURES):
            print("ERRORS:")
            print("\n{0}\n".format("\n".join(FAILURES)))
<|endoftext|>


<commit_before># Takes lists of objects returned by the zkclient module, and
# consolidates the information for display.

import logging
logger = logging.getLogger('kafka.codec').addHandler(logging.NullHandler())

import struct
import socket
from collections import namedtuple
from kafka.client import KafkaClient, OffsetRequest

class ProcessorError(Exception):
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return self.msg

PartitionState = namedtuple('PartitionState',
    [
        'broker',           # Broker host
        'topic',            # Topic on broker
        'partition',        # The partition
        'earliest',         # Earliest offset within partition on broker
        'latest',           # Current offset within partition on broker
        'depth',            # Depth of partition on broker.
        'spout',            # The Spout consuming this partition
        'current',          # Current offset for Spout
        'delta'             # Difference between latest and current
    ])
PartitionsSummary = namedtuple('PartitionsSummary',
    [
        'total_depth',      # Total queue depth.
        'total_delta',      # Total delta across all spout tasks.
        'num_partitions',   # Number of partitions.
        'num_brokers',      # Number of Kafka Brokers.
        'partitions'        # Tuple of PartitionStates
    ])

def process(spouts):
    '''
    Returns a named tuple of type PartitionsSummary.
    '''
    results = []
    total_depth = 0
    total_delta = 0
    brokers = []
    for s in spouts:
        for p in s.partitions:
            try:
                k = KafkaClient(p['broker']['host'], p['broker']['port'])
            except socket.gaierror, e:
                raise ProcessorError('Failed to contact Kafka broker %s (%s)' %
                                     (p['broker']['host'], str(e)))
            earliest_off = OffsetRequest(p['topic'], p['partition'], -2, 1)
            latest_off = OffsetRequest(p['topic'], p['partition'], -1, 1)

            earliest = k.get_offsets(earliest_off)[0]
            latest = k.get_offsets(latest_off)[0]
            current = p['offset']

            brokers.append(p['broker']['host'])
            total_depth = total_depth + (latest - earliest)
            total_delta = total_delta + (latest - current)

            results.append(PartitionState._make([
                p['broker']['host'],
                p['topic'],
                p['partition'],
                earliest,
                latest,
                latest - earliest,
                s.id,
                current,
                latest - current]))
    return PartitionsSummary(total_depth=total_depth,
                             total_delta=total_delta,
                             num_partitions=len(results),
                             num_brokers=len(set(brokers)),
                             partitions=tuple(results))
<commit_msg>include a NullHandler, now seems to be python 2.6 compatible<commit_after># Takes lists of objects returned by the zkclient module, and
# consolidates the information for display.

import logging

class NullHandler(logging.Handler):
    def emit(self, record):
        pass

logger = logging.getLogger('kafka.codec').addHandler(NullHandler())

import struct
import socket
from collections import namedtuple
from kafka.client import KafkaClient, OffsetRequest

class ProcessorError(Exception):
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return self.msg

PartitionState = namedtuple('PartitionState',
    [
        'broker',           # Broker host
        'topic',            # Topic on broker
        'partition',        # The partition
        'earliest',         # Earliest offset within partition on broker
        'latest',           # Current offset within partition on broker
        'depth',            # Depth of partition on broker.
        'spout',            # The Spout consuming this partition
        'current',          # Current offset for Spout
        'delta'             # Difference between latest and current
    ])
PartitionsSummary = namedtuple('PartitionsSummary',
    [
        'total_depth',      # Total queue depth.
        'total_delta',      # Total delta across all spout tasks.
        'num_partitions',   # Number of partitions.
        'num_brokers',      # Number of Kafka Brokers.
        'partitions'        # Tuple of PartitionStates
    ])

def process(spouts):
    '''
    Returns a named tuple of type PartitionsSummary.
    '''
    results = []
    total_depth = 0
    total_delta = 0
    brokers = []
    for s in spouts:
        for p in s.partitions:
            try:
                k = KafkaClient(p['broker']['host'], p['broker']['port'])
            except socket.gaierror, e:
                raise ProcessorError('Failed to contact Kafka broker %s (%s)' %
                                     (p['broker']['host'], str(e)))
            earliest_off = OffsetRequest(p['topic'], p['partition'], -2, 1)
            latest_off = OffsetRequest(p['topic'], p['partition'], -1, 1)

            earliest = k.get_offsets(earliest_off)[0]
            latest = k.get_offsets(latest_off)[0]
            current = p['offset']

            brokers.append(p['broker']['host'])
            total_depth = total_depth + (latest - earliest)
            total_delta = total_delta + (latest - current)

            results.append(PartitionState._make([
                p['broker']['host'],
                p['topic'],
                p['partition'],
                earliest,
                latest,
                latest - earliest,
                s.id,
                current,
                latest - current]))
    return PartitionsSummary(total_depth=total_depth,
                             total_delta=total_delta,
                             num_partitions=len(results),
                             num_brokers=len(set(brokers)),
                             partitions=tuple(results))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import math
from simpleai.machine_learning import ClassificationProblem, is_attribute

from yalign.datatypes import ScoreFunction, SentencePair
from yalign.sequencealigner import SequenceAligner
from yalign.svm import SVMClassifier

class SentencePairScore(ScoreFunction):
    def __init__(self):
        super(SentencePairScore, self).__init__(0, 1)
        self.classifier = None
        self.sign = 1

    # FIXME: Consider giving the word_aligner instead of word_score_function
    def train(self, pairs, word_score_function):
        """
        Trains the sentence pair likelihood score using examples.
        `pairs` is an interable of `SentencePair` instances.
        `word_score_function` is an instance of ScoreFunction, perhaps even an
        instance of `WordPairScore`.
        """
        pairs = list(pairs)
        self.problem = SentencePairScoreProblem(word_score_function)
        self.classifier = SVMClassifier(pairs, self.problem)
        class_ = None
        for a, b in pairs:
            sent = a = SentencePair(a, b)
            score = self.classifier.score(sent)
            if score != 0:
                class_ = bool(self.classifier.classify(sent)[0])
                if (score > 0 and class_ is True) or \
                   (score < 0 and class_ is False):
                    self.sign = -1
                break
        if class_ is None:
            raise ValueError("Cannot infer sign with this data")

    def load(self, filepath):
        self.classifier = SVMClassifier.load(filepath)

    def save(self, filepath):
        self.classifier.save(filepath)

    def __call__(self, a, b):
        """
        Returns the score of a sentence.
        """
        if self.classifier is None:
            raise LookupError("Score not trained or loaded yet")
        a = SentencePair(a, b)
        score = self.classifier.score(a) * self.sign
        result = logistic_function(score * 3)
        # FIXME: Consider moving this to a test
        assert self.min_bound <= result <= self.max_bound
        return result

    @property
    def sentence_pair_aligner(self):
        return self.classifier.problem.aligner

    @property
    def word_pair_score(self):
        return self.classifier.problem.word_pair_score

class SentencePairScoreProblem(ClassificationProblem):
    def __init__(self, word_pair_score):
        super(SentencePairScoreProblem, self).__init__()
        # If gap > 0.5 then the returned value could be > 1.
        self.word_pair_score = word_pair_score
        self.aligner = SequenceAligner(word_pair_score, 0.4999)
        self.aligner = CacheOfSizeOne(self.aligner)

    def word_score(self, alignment):
        raise Exception
        aligns = self.aligner(alignment.a, alignment.b)
        N = max(len(alignment.a), len(alignment.b))
        word_score = sum(x[2] for x in aligns) / float(N)

        # FIXME: Consider moving this to a test
        assert 0 <= word_score <= 1
        return word_score

    def amount_of_alignments(self, alignment):
        raise Exception
        aligns = self.aligner(alignment.a, alignment.b)
        aligns = [x for x in aligns if x[0] is not None and x[1] is not None]
        N = max(len(alignment.a), len(alignment.b))
        value = len(aligns) / float(N)
        return value

    @is_attribute
    def position_difference(self, alignment):
        d = alignment.a.position - alignment.b.position
        return abs(d)

    @is_attribute
    def word_length_difference(self, alignment):
        a = len(alignment.a)
        b = len(alignment.b)
        return ratio(a, b)

    @is_attribute
    def uppercase_words_difference(self, alignment):
        a = len([x for x in alignment.a if x.isupper()])
        b = len([x for x in alignment.b if x.isupper()])
        return ratio(a, b)

    @is_attribute
    def capitalized_words_difference(self, alignment):
        a = len([x for x in alignment.a if x.istitle()])
        b = len([x for x in alignment.b if x.istitle()])
        return ratio(a, b)

    @is_attribute
    def word_match(self, alignment):
        a, b = list(alignment.a), list(alignment.b)
        word_pairs = self._word_pairs(alignment)
        score = 0
        for cost, word_a, word_b in word_pairs:
            if word_a in a and word_b in b:
                score += cost
                a.remove(word_a)
                b.remove(word_b)
        score += len(a) + len(b)
        return float(score) / (len(alignment.a) + len(alignment.b))

    def _word_pairs(self, alignment):
        a, b = alignment
        pairs = []
        for word_a in a:
            for word_b in b:
                item = self.word_pair_score(word_a, word_b), word_a, word_b
                pairs.append(item)
        pairs.sort()
        return pairs

    def target(self, alignment):
        return alignment.aligned

def ratio(a, b):
    if max(a, b) == 0:
        return 1.0
    return min(a, b) / float(max(a, b))

def logistic_function(x):
    """
    See: http://en.wikipedia.org/wiki/Logistic_function
    """
    return 1 / (1 + math.e ** (-x))

class CacheOfSizeOne(object):
    f = None

    def __init__(self, f):
        self.f = f
        self.args = None
        self.kwargs = None

    def __call__(self, *args, **kwargs):
        if args != self.args or kwargs != self.kwargs:
            self.result = self.f(*args, **kwargs)
            self.args = args
            self.kwargs = kwargs
        return self.result

    def __getattr__(self, name):
        return getattr(self.f, name)
<commit_msg>Added more features to the SVM<commit_after># -*- coding: utf-8 -*-

import math
from simpleai.machine_learning import ClassificationProblem, is_attribute

from yalign.datatypes import ScoreFunction, SentencePair
from yalign.sequencealigner import SequenceAligner
from yalign.svm import SVMClassifier

class SentencePairScore(ScoreFunction):
    def __init__(self):
        super(SentencePairScore, self).__init__(0, 1)
        self.classifier = None
        self.sign = 1

    # FIXME: Consider giving the word_aligner instead of word_score_function
    def train(self, pairs, word_score_function):
        """
        Trains the sentence pair likelihood score using examples.
        `pairs` is an interable of `SentencePair` instances.
        `word_score_function` is an instance of ScoreFunction, perhaps even an
        instance of `WordPairScore`.
        """
        pairs = list(pairs)
        self.problem = SentencePairScoreProblem(word_score_function)
        self.classifier = SVMClassifier(pairs, self.problem)
        class_ = None
        for a, b in pairs:
            sent = a = SentencePair(a, b)
            score = self.classifier.score(sent)
            if score != 0:
                class_ = bool(self.classifier.classify(sent)[0])
                if (score > 0 and class_ is True) or \
                   (score < 0 and class_ is False):
                    self.sign = -1
                break
        if class_ is None:
            raise ValueError("Cannot infer sign with this data")

    def load(self, filepath):
        self.classifier = SVMClassifier.load(filepath)

    def save(self, filepath):
        self.classifier.save(filepath)

    def __call__(self, a, b):
        """
        Returns the score of a sentence.
        """
        if self.classifier is None:
            raise LookupError("Score not trained or loaded yet")
        a = SentencePair(a, b)
        score = self.classifier.score(a) * self.sign
        result = logistic_function(score * 3)
        # FIXME: Consider moving this to a test
        assert self.min_bound <= result <= self.max_bound
        return result

    @property
    def sentence_pair_aligner(self):
        return self.classifier.problem.aligner

    @property
    def word_pair_score(self):
        return self.classifier.problem.word_pair_score

class SentencePairScoreProblem(ClassificationProblem):
    def __init__(self, word_pair_score):
        super(SentencePairScoreProblem, self).__init__()
        # If gap > 0.5 then the returned value could be > 1.
        self.word_pair_score = word_pair_score
        self.aligner = SequenceAligner(word_pair_score, 0.4999)
        self.aligner = CacheOfSizeOne(self.aligner)

    def word_score(self, alignment):
        raise Exception
        aligns = self.aligner(alignment.a, alignment.b)
        N = max(len(alignment.a), len(alignment.b))
        word_score = sum(x[2] for x in aligns) / float(N)

        # FIXME: Consider moving this to a test
        assert 0 <= word_score <= 1
        return word_score

    def amount_of_alignments(self, alignment):
        raise Exception
        aligns = self.aligner(alignment.a, alignment.b)
        aligns = [x for x in aligns if x[0] is not None and x[1] is not None]
        N = max(len(alignment.a), len(alignment.b))
        value = len(aligns) / float(N)
        return value

    @is_attribute
    def position_difference(self, alignment):
        d = alignment.a.position - alignment.b.position
        return abs(d)

    @is_attribute
    def word_length_difference(self, alignment):
        a = len(alignment.a)
        b = len(alignment.b)
        return ratio(a, b)

    @is_attribute
    def uppercase_words_difference(self, alignment):
        a = len([x for x in alignment.a if x.isupper()])
        b = len([x for x in alignment.b if x.isupper()])
        return ratio(a, b)

    @is_attribute
    def capitalized_words_difference(self, alignment):
        a = len([x for x in alignment.a if x.istitle()])
        b = len([x for x in alignment.b if x.istitle()])
        return ratio(a, b)

    @is_attribute
    def commas(self, alignment):
        a = len([x for x in alignment.a if x == ","])
        b = len([x for x in alignment.b if x == ","])
        return ratio(a, b)

    @is_attribute
    def question_marks(self, alignment):
        a = len([x for x in alignment.a if x == "?"])
        b = len([x for x in alignment.b if x == "?"])
        return ratio(a, b)

    @is_attribute
    def punctuation(self, alignment):
        punc = ",.-?+<>_\\/"
        a = len([x for x in alignment.a if x in punc])
        b = len([x for x in alignment.b if x in punc])
        return ratio(a, b)

    @is_attribute
    def digits(self, alignment):
        a = len([x for x in alignment.a if x.isdigit()])
        b = len([x for x in alignment.b if x.isdigit()])
        return ratio(a, b)

    @is_attribute
    def word_match(self, alignment):
        a, b = list(alignment.a), list(alignment.b)
        word_pairs = self._word_pairs(alignment)
        score = 0
        for cost, word_a, word_b in word_pairs:
            if word_a in a and word_b in b:
                score += cost
                a.remove(word_a)
                b.remove(word_b)
        score += len(a) + len(b)
        return float(score) / (len(alignment.a) + len(alignment.b))

    def _word_pairs(self, alignment):
        a, b = alignment
        pairs = []
        for word_a in a:
            for word_b in b:
                item = self.word_pair_score(word_a, word_b), word_a, word_b
                pairs.append(item)
        pairs.sort()
        return pairs

    def target(self, alignment):
        return alignment.aligned

def ratio(a, b):
    if max(a, b) == 0:
        return 0.0
    return min(a, b) / float(max(a, b))

def logistic_function(x):
    """
    See: http://en.wikipedia.org/wiki/Logistic_function
    """
    return 1 / (1 + math.e ** (-x))

class CacheOfSizeOne(object):
    f = None

    def __init__(self, f):
        self.f = f
        self.args = None
        self.kwargs = None

    def __call__(self, *args, **kwargs):
        if args != self.args or kwargs != self.kwargs:
            self.result = self.f(*args, **kwargs)
            self.args = args
            self.kwargs = kwargs
        return self.result

    def __getattr__(self, name):
        return getattr(self.f, name)
<|endoftext|>


<commit_before>from .base import *
from django.core.exceptions import ImproperlyConfigured

def get_env_variable(var_name):
    try:
        return os.environ[var_name]
    except KeyError:
        error_msg = "Set the %s environment variable" % var_name
        raise ImproperlyConfigured(error_msg)

SECRET_KEY = get_env_variable('SECRET_KEY')

# STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'<commit_msg>回復設定whitenoise storage<commit_after>from .base import *
from django.core.exceptions import ImproperlyConfigured

def get_env_variable(var_name):
    try:
        return os.environ[var_name]
    except KeyError:
        error_msg = "Set the %s environment variable" % var_name
        raise ImproperlyConfigured(error_msg)

SECRET_KEY = get_env_variable('SECRET_KEY')

STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'<|endoftext|>


<commit_before>import logging
from discord import Embed, Color

logger = logging.getLogger('gradiusbot')

logger.info("[Public Plugin] <trickcord_patches.py> Patches to the Trickcord game for more fun.")

TRICKCORD_ID = 755580145078632508
trickcord_state = 'STANDBY'
trickcord_time = 0
allowed_delay = None

async def action(**kwargs):
    """
    """
    message = kwargs['message']
    config = kwargs['config']
    client = kwargs['client']
    global trickcord_state
    global trickcord_time
    global allowed_delay

    split_message = [s.lower() for s in message.content.split()]

    if allowed_delay is None:
        allowed_delay = config.getint('trickcord', 'allowed_delay')

    spooked_role_id = config.getint('trickcord', 'spooked_role_id')
    spooked_role = message.guild.get_role(spooked_role_id)

    if message.author.id == TRICKCORD_ID:

        # a trickortreater has appeared, check what command is supposed to be said
        if len(message.embeds) > 0 and message.embeds[0].title == 'A trick-or-treater has stopped by!':
            if 'h!tricks' in message.embeds[0].description:
                trickcord_state = 'TRICK'
                trickcord_time = message.created_at

            if 'h!treats' in message.embeds[0].description:
                trickcord_state = 'TREAT'
                trickcord_time = message.created_at

    # if the author wasn't the trickortreater bot
    else:
        # debug tools
        if message.author.id == 101103243991465984 and message.content == 'showstate':
            await message.channel.send(trickcord_state)
        if message.author.id == 101103243991465984 and split_message[0] == 'h!set-delay':
            allowed_delay = split_message[1]
            await message.channel.send(f"Allowed delay has been changed to {allowed_delay}")

        # add the author to the spooked role
        elif trickcord_state == 'TRICK':
            if message.content == 'h!treat':
                await message.author.add_roles(spooked_role, reason='User was spooked.')
                await message.channel.send(f"<@{message.author.id}> sent the wrong command and was spooked by the trick-or-treater! 👻")
            elif message.content == 'h!trick' and (message.created_at - trickcord_time).microseconds/1000 < allowed_delay:
                await message.author.add_roles(spooked_role, reason='User was spooked.')
                await message.channel.send(f"<@{message.author.id}> replied too fast and was spooked by the trick-or-treater! 👻")

        elif trickcord_state == 'TREAT':
            if message.content == 'h!trick':
                await message.author.add_roles(spooked_role, reason='User was spooked.')
                await message.channel.send(f"<@{message.author.id}> sent the wrong command and was spooked by the trick-or-treater! 👻")
            elif message.content == 'h!treat' and (message.created_at - trickcord_time).microseconds/1000 < allowed_delay:
                await message.author.add_roles(spooked_role, reason='User was spooked.')
                await message.channel.send(f"<@{message.author.id}> replied too fast and was spooked by the trick-or-treater! 👻")

<commit_msg>stats<commit_after>import logging
from discord import Embed, Color

logger = logging.getLogger('gradiusbot')

logger.info("[Public Plugin] <trickcord_patches.py> Patches to the Trickcord game for more fun.")

TRICKCORD_ID = 755580145078632508
trickcord_state = 'STANDBY'
trickcord_time = 0
allowed_delay = None

async def action(**kwargs):
    """
    """
    message = kwargs['message']
    config = kwargs['config']
    client = kwargs['client']
    global trickcord_state
    global trickcord_time
    global allowed_delay

    split_message = [s.lower() for s in message.content.split()]

    if allowed_delay is None:
        allowed_delay = config.getint('trickcord', 'allowed_delay')

    spooked_role_id = config.getint('trickcord', 'spooked_role_id')
    spooked_role = message.guild.get_role(spooked_role_id)

    if message.author.id == TRICKCORD_ID:

        # a trickortreater has appeared, check what command is supposed to be said
        if len(message.embeds) > 0 and message.embeds[0].title == 'A trick-or-treater has stopped by!':
            if 'h!tricks' in message.embeds[0].description:
                trickcord_state = 'TRICK'
                trickcord_time = message.created_at

            if 'h!treats' in message.embeds[0].description:
                trickcord_state = 'TREAT'
                trickcord_time = message.created_at

    # if the author wasn't the trickortreater bot
    else:
        # debug tools
        if message.author.id == 101103243991465984 and message.content == 'showstate':
            await message.channel.send(trickcord_state)
        if message.author.id == 101103243991465984 and split_message[0] == 'h!set-delay':
            allowed_delay = split_message[1]
            await message.channel.send(f"Allowed delay has been changed to {allowed_delay}")

        # add the author to the spooked role
        elif trickcord_state == 'TRICK':
            response_time = (message.created_at - trickcord_time).microseconds / 1000
            if message.content == 'h!treat':
                await message.author.add_roles(spooked_role, reason='User was spooked.')
                await message.channel.send(f"<@{message.author.id}> sent the wrong command and was spooked by the trick-or-treater! 👻")
            elif message.content == 'h!trick' and response_time < allowed_delay:
                await message.author.add_roles(spooked_role, reason='User was spooked.')
                await message.channel.send(f"<@{message.author.id}> replied too fast and was spooked by the trick-or-treater! 👻")
            elif message.content == 'h!trick' and response_time >= allowed_delay:
                await message.channel.send(f"<@{message.author.id}> replied in {response_time} ms! So fast!")

        elif trickcord_state == 'TREAT':
            response_time = (message.created_at - trickcord_time).microseconds / 1000
            if message.content == 'h!trick':
                await message.author.add_roles(spooked_role, reason='User was spooked.')
                await message.channel.send(f"<@{message.author.id}> sent the wrong command and was spooked by the trick-or-treater! 👻")
            elif message.content == 'h!treat' and response_time < allowed_delay:
                await message.author.add_roles(spooked_role, reason='User was spooked.')
                await message.channel.send(f"<@{message.author.id}> replied too fast and was spooked by the trick-or-treater! 👻")
            elif message.content == 'h!treat' and response_time >= allowed_delay:
                await message.channel.send(f"<@{message.author.id}> replied in {response_time} ms! So fast!")

<|endoftext|>


<commit_before>import os
from itertools import tee, izip
import tarfile
import ctypes

import numpy as np
from scipy.ndimage.filters import maximum_filter, minimum_filter
from scipy.interpolate import RectSphereBivariateSpline

from c_wrapper import cextrema

EARTH_RADIUS = 6371

def pairwise(iterable):
    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
    a, b = tee(iterable)
    next(b, None)
    return izip(a, b)

def find_extrema_slow(array, print_warning=True):
    '''
    Takes an array and finds its local extrema.

    Returns an array with 0s for not an extrema, 1s for maxs and -1 for mins
    and a list of the indices of all maximums and minimums
    '''
    if print_warning:
        print('Warning, this function is quite slow. Consider using find_extrema(array) instead')
    extrema = np.zeros_like(array)
    maximums = []
    minimums = []
    for i in range(1, array.shape[0] - 1):
        for j in range(0, array.shape[1]):
            val = array[i, j]

            is_max, is_min = True, True
            for ii in range(i - 1, i + 2):
                for jj in range(j - 1, j + 2):
                    if val < array[ii, jj % array.shape[1]]:
                        is_max = False
                    elif val > array[ii, jj % array.shape[1]]:
                        is_min = False
            if is_max:
                extrema[i, j] = 1
                maximums.append((i, j))
            elif is_min:
                extrema[i, j] = -1
                minimums.append((i, j))
    return extrema, maximums, minimums

def find_extrema(array):
    '''
    Takes an array and finds its local extrema.

    Returns an array with 0s for not an extrema, 1s for maxs and -1 for mins
    and a list of the indices of all maximums and minimums

    N.B. this function is much faster than the above.
    '''
    extrema = np.zeros_like(array)
    maximums = []
    minimums = []

    local_max = maximum_filter(array, size=(3, 3)) == array
    local_min = minimum_filter(array, size=(3, 3)) == array
    extrema += local_max
    extrema -= local_min

    where_max = np.where(local_max)
    where_min = np.where(local_min)

    for max_point in zip(where_max[0], where_max[1]):
        if max_point[0] != 0 and max_point[0] != array.shape[0] - 1:
            maximums.append(max_point)

    for min_point in zip(where_min[0], where_min[1]):
        if min_point[0] != 0 and min_point[0] != array.shape[0] - 1:
            minimums.append(min_point)

    return extrema, maximums, minimums

MAX_MAX_MINS = 1000
def cfind_extrema(array):
    extrema = np.zeros_like(array)

    max_x = np.zeros(MAX_MAX_MINS, dtype=np.int32)
    max_y = np.zeros(MAX_MAX_MINS, dtype=np.int32)
    min_x = np.zeros(MAX_MAX_MINS, dtype=np.int32)
    min_y = np.zeros(MAX_MAX_MINS, dtype=np.int32)
    max_length = ctypes.c_int();
    min_length = ctypes.c_int();

    cextrema(array, array.shape[0], array.shape[1], extrema, MAX_MAX_MINS, max_x, max_y, min_x,
            min_y, ctypes.byref(max_length), ctypes.byref(min_length))
    # return max_length, min_length

    return extrema, zip(max_x[:max_length.value], max_y[:max_length.value]), zip(min_x[:min_length.value], min_y[:min_length.value])

def upscale_field(lons, lats, field, x_scale=2, y_scale=2, is_degrees=True):
    '''
    Takes a field defined on a sphere using lons/lats and returns an upscaled
    version, using cubic spline interpolation.
    '''
    if is_degrees:
        lons = lons * np.pi / 180.
        lats = (lats + 90) * np.pi / 180.

    d_lon = lons[1] - lons[0]
    d_lat = lats[1] - lats[0]

    new_lon = np.linspace(lons[0], lons[-1], len(lons) * x_scale)
    new_lat = np.linspace(lats[0], lats[-1], len(lats) * x_scale)

    mesh_new_lat, mesh_new_lon = np.meshgrid(new_lat, new_lon)

    if True:
        lut = RectSphereBivariateSpline(lats[1:-1], lons[1:-1], field[1:-1, 1:-1])

        interp_field = lut.ev(mesh_new_lat[1:-1, 1:-1].ravel(),
                              mesh_new_lon[1:-1, 1:-1].ravel()).reshape(mesh_new_lon.shape[0] - 2,
                                                                        mesh_new_lon.shape[1] - 2).T
    else:
        pass
    if is_degrees:
        new_lon = new_lon * 180. / np.pi
        new_lat = (new_lat * 180. / np.pi) - 90

    return new_lon[1:-1], new_lat[1:-1], interp_field

def geo_dist(p1, p2):
    '''Returns the geodesic distance between two points

    p1, p2 should be of the form (lon, lat) in degrees
    '''
    lon1, lat1 = np.pi * p1[0] / 180., np.pi * p1[1] / 180.
    lon2, lat2 = np.pi * p2[0] / 180., np.pi * p2[1] / 180.
    return np.arccos(
        np.sin(lat1) * np.sin(lat2) +
        np.cos(lat1) * np.cos(lat2) * np.cos(lon2 - lon1)) * EARTH_RADIUS

def dist(p1, p2):
    '''Returns the cartesian distance between two points'''
    return ((p1[0] - p2[0]) ** 2 + (p1[1] - p2[1]) ** 2) ** 0.5

def raster_voronoi(extrema, maximums, minimums):
    '''
    Takes a 2D array and points of max/mins, and returns a 2D array
    with the voronoi sections filled with different values
    '''
    voronoi_arr = np.zeros_like(extrema)
    for i in range(1, voronoi_arr.shape[0] - 1):
        for j in range(0, voronoi_arr.shape[1]):
            min_dist = 1e9
            for k, extrema_point in enumerate(minimums + maximums):
                test_dist = dist((i, j), extrema_point)
                if test_dist < min_dist:
                    min_dist = test_dist
                    voronoi_arr[i, j] = k

    for k, extrema_point in enumerate(minimums + maximums):
        voronoi[extrema_point[0], extrema_point[1]] = 0
    voronoi[voronoi > len(minimums)] = -1

    return voronoi

def compress_dir(data_dir):
    '''Compresses a given data_dir, taking care to get file names correct'''
    curr_dir = os.getcwd()

    parent_dir = os.path.dirname(data_dir)
    os.chdir(parent_dir)
    compressed_filename = data_dir + '.bz2'
    tar = tarfile.open(compressed_filename, 'w:bz2')
    for root, dirs, files in os.walk(data_dir):
        for file in files:
            tar.add(os.path.relpath(os.path.join(root, file), start=parent_dir))
    tar.close()

    os.chdir(curr_dir)
    return compressed_filename

def decompress_file(compressed_filename):
    '''Decompresses a given tarball, taking care to get file names correct'''
    tar = tarfile.open(compressed_filename)
    tar.extractall(os.path.dirname(compressed_filename))
    tar.close()
<commit_msg>Try to get two extrema functions producing same results.<commit_after>import os
from itertools import tee, izip
import tarfile
import ctypes

import numpy as np
from scipy.ndimage.filters import maximum_filter, minimum_filter
from scipy.interpolate import RectSphereBivariateSpline

from c_wrapper import cextrema

EARTH_RADIUS = 6371

def pairwise(iterable):
    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
    a, b = tee(iterable)
    next(b, None)
    return izip(a, b)

def find_extrema_slow(array, print_warning=True):
    '''
    Takes an array and finds its local extrema.

    Returns an array with 0s for not an extrema, 1s for maxs and -1 for mins
    and a list of the indices of all maximums and minimums
    '''
    if print_warning:
        print('Warning, this function is quite slow. Consider using find_extrema(array) instead')
    extrema = np.zeros_like(array)
    maximums = []
    minimums = []
    for i in range(1, array.shape[0] - 1):
        for j in range(0, array.shape[1]):
            val = array[i, j]

            is_max, is_min = True, True
            for ii in range(i - 1, i + 2):
                for jj in range(j - 1, j + 2):
                    if val < array[ii, jj % array.shape[1]]:
                        is_max = False
                    elif val > array[ii, jj % array.shape[1]]:
                        is_min = False
            if is_max:
                extrema[i, j] = 1
                maximums.append((i, j))
            elif is_min:
                extrema[i, j] = -1
                minimums.append((i, j))
    return extrema, maximums, minimums

def find_extrema(array):
    '''
    Takes an array and finds its local extrema.

    Returns an array with 0s for not an extrema, 1s for maxs and -1 for mins
    and a list of the indices of all maximums and minimums

    N.B. this function is much faster than the above.
    '''
    extrema = np.zeros_like(array)
    maximums = []
    minimums = []

    local_max = maximum_filter(array, size=(3, 3)) == array
    local_min = minimum_filter(array, size=(3, 3)) == array
    extrema += local_max
    extrema -= local_min

    where_max = np.where(local_max)
    where_min = np.where(local_min)

    for max_point in zip(where_max[0], where_max[1]):
        if max_point[0] != 0 and max_point[0] != array.shape[0] - 1 and
           max_point[1] != 0 and max_point[1] != array.shape[1] - 1:
            maximums.append(max_point)

    for min_point in zip(where_min[0], where_min[1]):
        if min_point[0] != 0 and min_point[0] != array.shape[0] - 1 and
           min_point[1] != 0 and min_point[1] != array.shape[1] - 1:
            minimums.append(min_point)

    return extrema, maximums, minimums

MAX_MAX_MINS = 1000
def cfind_extrema(array):
    extrema = np.zeros_like(array)

    max_x = np.zeros(MAX_MAX_MINS, dtype=np.int32)
    max_y = np.zeros(MAX_MAX_MINS, dtype=np.int32)
    min_x = np.zeros(MAX_MAX_MINS, dtype=np.int32)
    min_y = np.zeros(MAX_MAX_MINS, dtype=np.int32)

    max_length = ctypes.c_int();
    min_length = ctypes.c_int();

    cextrema(array, array.shape[0], array.shape[1], extrema, MAX_MAX_MINS, max_x, max_y, min_x,
            min_y, ctypes.byref(max_length), ctypes.byref(min_length))

    return extrema, zip(max_x[:max_length.value], max_y[:max_length.value]), zip(min_x[:min_length.value], min_y[:min_length.value])

def upscale_field(lons, lats, field, x_scale=2, y_scale=2, is_degrees=True):
    '''
    Takes a field defined on a sphere using lons/lats and returns an upscaled
    version, using cubic spline interpolation.
    '''
    if is_degrees:
        lons = lons * np.pi / 180.
        lats = (lats + 90) * np.pi / 180.

    d_lon = lons[1] - lons[0]
    d_lat = lats[1] - lats[0]

    new_lon = np.linspace(lons[0], lons[-1], len(lons) * x_scale)
    new_lat = np.linspace(lats[0], lats[-1], len(lats) * x_scale)

    mesh_new_lat, mesh_new_lon = np.meshgrid(new_lat, new_lon)

    if True:
        lut = RectSphereBivariateSpline(lats[1:-1], lons[1:-1], field[1:-1, 1:-1])

        interp_field = lut.ev(mesh_new_lat[1:-1, 1:-1].ravel(),
                              mesh_new_lon[1:-1, 1:-1].ravel()).reshape(mesh_new_lon.shape[0] - 2,
                                                                        mesh_new_lon.shape[1] - 2).T
    else:
        pass
    if is_degrees:
        new_lon = new_lon * 180. / np.pi
        new_lat = (new_lat * 180. / np.pi) - 90

    return new_lon[1:-1], new_lat[1:-1], interp_field

def geo_dist(p1, p2):
    '''Returns the geodesic distance between two points

    p1, p2 should be of the form (lon, lat) in degrees
    '''
    lon1, lat1 = np.pi * p1[0] / 180., np.pi * p1[1] / 180.
    lon2, lat2 = np.pi * p2[0] / 180., np.pi * p2[1] / 180.
    return np.arccos(
        np.sin(lat1) * np.sin(lat2) +
        np.cos(lat1) * np.cos(lat2) * np.cos(lon2 - lon1)) * EARTH_RADIUS

def dist(p1, p2):
    '''Returns the cartesian distance between two points'''
    return ((p1[0] - p2[0]) ** 2 + (p1[1] - p2[1]) ** 2) ** 0.5

def raster_voronoi(extrema, maximums, minimums):
    '''
    Takes a 2D array and points of max/mins, and returns a 2D array
    with the voronoi sections filled with different values
    '''
    voronoi_arr = np.zeros_like(extrema)
    for i in range(1, voronoi_arr.shape[0] - 1):
        for j in range(0, voronoi_arr.shape[1]):
            min_dist = 1e9
            for k, extrema_point in enumerate(minimums + maximums):
                test_dist = dist((i, j), extrema_point)
                if test_dist < min_dist:
                    min_dist = test_dist
                    voronoi_arr[i, j] = k

    for k, extrema_point in enumerate(minimums + maximums):
        voronoi[extrema_point[0], extrema_point[1]] = 0
    voronoi[voronoi > len(minimums)] = -1

    return voronoi

def compress_dir(data_dir):
    '''Compresses a given data_dir, taking care to get file names correct'''
    curr_dir = os.getcwd()

    parent_dir = os.path.dirname(data_dir)
    os.chdir(parent_dir)
    compressed_filename = data_dir + '.bz2'
    tar = tarfile.open(compressed_filename, 'w:bz2')
    for root, dirs, files in os.walk(data_dir):
        for file in files:
            tar.add(os.path.relpath(os.path.join(root, file), start=parent_dir))
    tar.close()

    os.chdir(curr_dir)
    return compressed_filename

def decompress_file(compressed_filename):
    '''Decompresses a given tarball, taking care to get file names correct'''
    tar = tarfile.open(compressed_filename)
    tar.extractall(os.path.dirname(compressed_filename))
    tar.close()
<|endoftext|>


<commit_before># -*-coding:Utf-8 -*

# Copyright (c) 2010-2016 LE GOFF Vincent
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
# * Neither the name of the copyright holder nor the names of its contributors
#   may be used to endorse or promote products derived from this software
#   without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

"""Fichier contenant le module secondaire tags."""

from textwrap import dedent

from abstraits.module import *
from corps.fonctions import valider_cle
from primaires.format.fonctions import format_nb
from secondaires.tags import commandes
from secondaires.tags.editeurs.selection_tags import SelectionTags
from secondaires.tags.tag import Tag
from secondaires.tags.tags import Tags

class Module(BaseModule):

    """Module gérant les tags de différentes natures.

    Un tag représente une fonctionnalité que l'on peut appliquer, par exemple, sur un objet ou un PNJ. Les tags facilitent aussi la copie de scripts.

    Par exemple, ce module permettrait de créer un tag 'marchand'
    pour les PNJ. Les PNJ vendeurs de magasin pourraient avoir ce
    tag, ce qui simplifierait la recherche dans plusieurs cas. En
    outre, les scripts sélectionnés seraient copiés, permettant
    d'avoir une réplique des fonctionnalités sur plusieurs PNJ. Bien
    que les types de tag soient définis dans le code, créer et appliquer
    un tag peut se faire dans l'univers sans difficulté.

    """

    def __init__(self, importeur):
        """Constructeur du module"""
        BaseModule.__init__(self, importeur, "tags", "secondaire")
        self.tags = {}
        self.configuration = None
        self.types = ["objet", "pnj"]
        self.cles = {}
        self.logger = self.importeur.man_logs.creer_logger("tags", "tags")

    def init(self):
        """Chargement des objets du module."""
        # Abonnement aux hooks
        self.importeur.hook["editeur:etendre"].ajouter_evenement(
                self.ajouter_tags)

        # Charge les tags
        self.configuration = self.importeur.supenr.charger_unique(Tags)
        if self.configuration is None:
            self.configuration = Tags()

        tags = self.importeur.supenr.charger_groupe(Tag)
        groupes = {}
        for tag in tags:
            self.ajouter_tag(tag)
            if tag.type not in groupes:
                groupes[tag.type] = []

            groupe = groupes[tag.type]
            groupe.append(tag)

        self.logger.info(format_nb(len(tags),
                "{nb} tag{s} récupéré{s}"))

        for type, groupe in groupes.items():
            self.logger.info(format_nb(len(groupe),
                    "Dont {nb} tag{s} du type " + type))

        # Réferencement des clés
        self.cles = {
                "objet": importeur.objet._prototypes,
                "pnj": importeur.pnj._prototypes,
        }

        # Abbonnement aux hooks
        self.importeur.hook["recherche:filtres"].ajouter_evenement(
                self.recherche_tags)

        BaseModule.init(self)

    def ajouter_commandes(self):
        """Ajout des commandes dans l'interpréteur"""
        self.commandes = [
            commandes.tags.CmdTags(),
        ]

        for cmd in self.commandes:
            self.importeur.interpreteur.ajouter_commande(cmd)

    def creer_tag(self, cle, type):
        """Crée un nouveau tag."""
        valider_cle(cle)

        if type not in self.types:
            raise ValueError("le type de tag {} n'existe pas".format(
                    repr(type)))

        if cle in self.tags:
            raise ValueError("le tag {} existe déjà".format(
                    repr(cle)))

        tag = Tag(cle, type)
        self.ajouter_tag(tag)
        return tag

    def ajouter_tag(self, tag):
        """Ajoute le tag."""
        if tag.cle in self.tags:
            raise ValueError("le tag de clé {} est " \
                    "déjà défini".format(repr(tag.cle)))

        self.tags[tag.cle] = tag

    def supprimer_tag(self, cle):
        """Supprime un tag."""
        if cle not in self.tags:
            raise ValueError("le tag {} n'existe pas".format(
                    repr(cle)))

        self.tags.pop(cle).detruire()

    def ajouter_tags(self, editeur, presentation, objet):
        """Ajoute l'éditeur de tags aux éditeurs."""
        print("Extension de", editeur)
        if editeur.lower() in self.types:
            liste = [t.cle for t in self.tags.values() if t.type == \
                    editeur.lower()]
            liste.sort()
            lst_tags = self.configuration[objet]
            tags = presentation.ajouter_choix("tags", None,
                    SelectionTags, lst_tags,
                    "tags", liste, objet)
            tags.parent = presentation
            tags.apercu = "{valeur}"
            tags.aide_courte = dedent("""
                Entrez |cmd|/|ff| pour revenir à la fenêtre parente.

                Vous pouvez ici ajouter et supprimer des tags pour '{nom}'.
                Si vous ajoutez un tag, en entrant son nom, les scripts
                associés à ce tag seront automatiquement copiés dans
                '{nom}'.

                Tags possibles : {liste}

                Tags actuels : {{valeur}}""".format(nom=str(objet),
                liste=liste).lstrip("\n"))

    def recherche_tags(self, cherchable):
        """Ajout de la recherche par tags."""
        if cherchable.nom_cherchable in ("probjet", "prpnj"):
            cherchable.ajouter_filtre("", "tags", self.filtrer_tags, "chaine")

    def filtrer_tags(self, objet, tags):
        """Filtre les données possédant le ou les tags indiqués.

        Vous pouvez préciser en argument un seul tag, ou bien une
        liste de tags séparés par un espace. Si l'un des tags est
        présent dans la donnée, elle sera affichée.

        """
        o_tags = importeur.tags.configuration[objet].tags
        return any(tag.lower() in o_tags for tag in tags.split(" "))
<commit_msg>Ajout des tags de salle<commit_after># -*-coding:Utf-8 -*

# Copyright (c) 2010-2016 LE GOFF Vincent
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
# * Neither the name of the copyright holder nor the names of its contributors
#   may be used to endorse or promote products derived from this software
#   without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

"""Fichier contenant le module secondaire tags."""

from textwrap import dedent

from abstraits.module import *
from corps.fonctions import valider_cle
from primaires.format.fonctions import format_nb
from secondaires.tags import commandes
from secondaires.tags.editeurs.selection_tags import SelectionTags
from secondaires.tags.tag import Tag
from secondaires.tags.tags import Tags

class Module(BaseModule):

    """Module gérant les tags de différentes natures.

    Un tag représente une fonctionnalité que l'on peut appliquer, par exemple, sur un objet ou un PNJ. Les tags facilitent aussi la copie de scripts.

    Par exemple, ce module permettrait de créer un tag 'marchand'
    pour les PNJ. Les PNJ vendeurs de magasin pourraient avoir ce
    tag, ce qui simplifierait la recherche dans plusieurs cas. En
    outre, les scripts sélectionnés seraient copiés, permettant
    d'avoir une réplique des fonctionnalités sur plusieurs PNJ. Bien
    que les types de tag soient définis dans le code, créer et appliquer
    un tag peut se faire dans l'univers sans difficulté.

    """

    def __init__(self, importeur):
        """Constructeur du module"""
        BaseModule.__init__(self, importeur, "tags", "secondaire")
        self.tags = {}
        self.configuration = None
        self.types = ["objet", "pnj", "salle"]
        self.cles = {}
        self.logger = self.importeur.man_logs.creer_logger("tags", "tags")

    def init(self):
        """Chargement des objets du module."""
        # Abonnement aux hooks
        self.importeur.hook["editeur:etendre"].ajouter_evenement(
                self.ajouter_tags)

        # Charge les tags
        self.configuration = self.importeur.supenr.charger_unique(Tags)
        if self.configuration is None:
            self.configuration = Tags()

        tags = self.importeur.supenr.charger_groupe(Tag)
        groupes = {}
        for tag in tags:
            self.ajouter_tag(tag)
            if tag.type not in groupes:
                groupes[tag.type] = []

            groupe = groupes[tag.type]
            groupe.append(tag)

        self.logger.info(format_nb(len(tags),
                "{nb} tag{s} récupéré{s}"))

        for type, groupe in groupes.items():
            self.logger.info(format_nb(len(groupe),
                    "Dont {nb} tag{s} du type " + type))

        # Réferencement des clés
        self.cles = {
                "objet": importeur.objet._prototypes,
                "pnj": importeur.pnj._prototypes,
                "salle": importeur.salle._salles,
        }

        # Abbonnement aux hooks
        self.importeur.hook["recherche:filtres"].ajouter_evenement(
                self.recherche_tags)

        BaseModule.init(self)

    def ajouter_commandes(self):
        """Ajout des commandes dans l'interpréteur"""
        self.commandes = [
            commandes.tags.CmdTags(),
        ]

        for cmd in self.commandes:
            self.importeur.interpreteur.ajouter_commande(cmd)

    def creer_tag(self, cle, type):
        """Crée un nouveau tag."""
        valider_cle(cle)

        if type not in self.types:
            raise ValueError("le type de tag {} n'existe pas".format(
                    repr(type)))

        if cle in self.tags:
            raise ValueError("le tag {} existe déjà".format(
                    repr(cle)))

        tag = Tag(cle, type)
        self.ajouter_tag(tag)
        return tag

    def ajouter_tag(self, tag):
        """Ajoute le tag."""
        if tag.cle in self.tags:
            raise ValueError("le tag de clé {} est " \
                    "déjà défini".format(repr(tag.cle)))

        self.tags[tag.cle] = tag

    def supprimer_tag(self, cle):
        """Supprime un tag."""
        if cle not in self.tags:
            raise ValueError("le tag {} n'existe pas".format(
                    repr(cle)))

        self.tags.pop(cle).detruire()

    def ajouter_tags(self, editeur, presentation, objet):
        """Ajoute l'éditeur de tags aux éditeurs."""
        if editeur.lower() in self.types:
            liste = [t.cle for t in self.tags.values() if t.type == \
                    editeur.lower()]
            liste.sort()
            lst_tags = self.configuration[objet]
            tags = presentation.ajouter_choix("tags", None,
                    SelectionTags, lst_tags,
                    "tags", liste, objet)
            tags.parent = presentation
            tags.apercu = "{valeur}"
            tags.aide_courte = dedent("""
                Entrez |cmd|/|ff| pour revenir à la fenêtre parente.

                Vous pouvez ici ajouter et supprimer des tags pour '{nom}'.
                Si vous ajoutez un tag, en entrant son nom, les scripts
                associés à ce tag seront automatiquement copiés dans
                '{nom}'.

                Tags possibles : {liste}

                Tags actuels : {{valeur}}""".format(nom=str(objet),
                liste=liste).lstrip("\n"))

    def recherche_tags(self, cherchable):
        """Ajout de la recherche par tags."""
        if cherchable.nom_cherchable in ("probjet", "prpnj", "salle"):
            cherchable.ajouter_filtre("", "tags", self.filtrer_tags, "chaine")

    def filtrer_tags(self, objet, tags):
        """Filtre les données possédant le ou les tags indiqués.

        Vous pouvez préciser en argument un seul tag, ou bien une
        liste de tags séparés par un espace. Si l'un des tags est
        présent dans la donnée, elle sera affichée.

        """
        o_tags = importeur.tags.configuration[objet].tags
        return any(tag.lower() in o_tags for tag in tags.split(" "))
<|endoftext|>