<commit_before>"""
Support for cftime axis in matplotlib.

"""

from __future__ import (absolute_import, division, print_function)
from six.moves import (filter, input, map, range, zip)  # noqa

from collections import namedtuple

import matplotlib.dates as mdates
import matplotlib.ticker as mticker
import matplotlib.transforms as mtransforms
import matplotlib.units as munits
import cftime
import numpy as np

# Define __version__ based on versioneer's interpretation.
from ._version import get_versions
__version__ = get_versions()['version']
del get_versions

# Lower and upper are in number of days.
FormatOption = namedtuple('FormatOption', ['lower', 'upper', 'format_string'])

class CalendarDateTime(object):
    """
    Container for :class:`cftime.datetime` object and calendar.

    """
    def __init__(self, datetime, calendar):
        self.datetime = datetime
        self.calendar = calendar

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
                self.datetime == other.datetime and
                self.calendar == other.calendar)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        msg = '<{}: datetime={}, calendar={}>'
        return msg.format(type(self).__name__, self.datetime, self.calendar)

class NetCDFTimeDateFormatter(mticker.Formatter):
    """
    Formatter for cftime.datetime data.

    """
    # Some magic numbers. These seem to work pretty well.
    format_options = [FormatOption(0.0, 0.2, '%H:%M:%S'),
                      FormatOption(0.2, 0.8, '%H:%M'),
                      FormatOption(0.8, 15, '%Y-%m-%d %H:%M'),
                      FormatOption(15, 90, '%Y-%m-%d'),
                      FormatOption(90, 900, '%Y-%m'),
                      FormatOption(900, 6000000, '%Y')]

    def __init__(self, locator, calendar, time_units):
        #: The locator associated with this formatter. This is used to get hold
        #: of the scaling information.
        self.locator = locator
        self.calendar = calendar
        self.time_units = time_units

    def pick_format(self, ndays):
        """
        Returns a format string for an interval of the given number of days.

        """
        for option in self.format_options:
            if option.lower < ndays <= option.upper:
                return option.format_string
        else:
            msg = 'No formatter found for an interval of {} days.'
            raise ValueError(msg.format(ndays))

    def __call__(self, x, pos=0):
        format_string = self.pick_format(ndays=self.locator.ndays)
        dt = cftime.utime(self.time_units, self.calendar).num2date(x)
        return dt.strftime(format_string)

class NetCDFTimeDateLocator(mticker.Locator):
    """
    Determines tick locations when plotting cftime.datetime data.

    """
    def __init__(self, max_n_ticks, calendar, date_unit, min_n_ticks=3):
        # The date unit must be in the form of days since ...

        self.max_n_ticks = max_n_ticks
        self.min_n_ticks = min_n_ticks
        self._max_n_locator = mticker.MaxNLocator(max_n_ticks, integer=True)
        self._max_n_locator_days = mticker.MaxNLocator(
            max_n_ticks, integer=True, steps=[1, 2, 4, 7, 10])
        self.calendar = calendar
        self.date_unit = date_unit
        if not self.date_unit.lower().startswith('days since'):
            msg = 'The date unit must be days since for a NetCDF time locator.'
            raise ValueError(msg)

        self._cached_resolution = {}

    def compute_resolution(self, num1, num2, date1, date2):
        """
        Returns the resolution of the dates (hourly, minutely, yearly), and
        an **approximate** number of those units.

        """
        num_days = float(np.abs(num1 - num2))
        resolution = 'SECONDLY'
        n = mdates.SEC_PER_DAY
        if num_days * mdates.MINUTES_PER_DAY > self.max_n_ticks:
            resolution = 'MINUTELY'
            n = int(num_days / mdates.MINUTES_PER_DAY)
        if num_days * mdates.HOURS_PER_DAY > self.max_n_ticks:
            resolution = 'HOURLY'
            n = int(num_days / mdates.HOURS_PER_DAY)
        if num_days > self.max_n_ticks:
            resolution = 'DAILY'
            n = int(num_days)
        if num_days > 30 * self.max_n_ticks:
            resolution = 'MONTHLY'
            n = num_days // 30
        if num_days > 365 * self.max_n_ticks:
            resolution = 'YEARLY'
            n = abs(date1.year - date2.year)

        return resolution, n

    def __call__(self):
        vmin, vmax = self.axis.get_view_interval()
        return self.tick_values(vmin, vmax)

    def tick_values(self, vmin, vmax):
        vmin, vmax = mtransforms.nonsingular(vmin, vmax, expander=1e-7,
                                             tiny=1e-13)

        self.ndays = float(abs(vmax - vmin))

        utime = cftime.utime(self.date_unit, self.calendar)
        lower = utime.num2date(vmin)
        upper = utime.num2date(vmax)

        resolution, n = self.compute_resolution(vmin, vmax, lower, upper)

        if resolution == 'YEARLY':
            # TODO START AT THE BEGINNING OF A DECADE/CENTURY/MILLENIUM as
            # appropriate.
            years = self._max_n_locator.tick_values(lower.year, upper.year)
            ticks = [cftime.datetime(int(year), 1, 1) for year in years]
        elif resolution == 'MONTHLY':
            # TODO START AT THE BEGINNING OF A DECADE/CENTURY/MILLENIUM as
            # appropriate.
            months_offset = self._max_n_locator.tick_values(0, n)
            ticks = []
            for offset in months_offset:
                year = lower.year + np.floor((lower.month + offset) / 12)
                month = ((lower.month + offset) % 12) + 1
                ticks.append(cftime.datetime(int(year), int(month), 1))
        elif resolution == 'DAILY':
            # TODO: It would be great if this favoured multiples of 7.
            days = self._max_n_locator_days.tick_values(vmin, vmax)
            ticks = [utime.num2date(dt) for dt in days]
        elif resolution == 'HOURLY':
            hour_unit = 'hours since 2000-01-01'
            hour_utime = cftime.utime(hour_unit, self.calendar)
            in_hours = hour_utime.date2num([lower, upper])
            hours = self._max_n_locator.tick_values(in_hours[0], in_hours[1])
            ticks = [hour_utime.num2date(dt) for dt in hours]
        elif resolution == 'MINUTELY':
            minute_unit = 'minutes since 2000-01-01'
            minute_utime = cftime.utime(minute_unit, self.calendar)
            in_minutes = minute_utime.date2num([lower, upper])
            minutes = self._max_n_locator.tick_values(in_minutes[0],
                                                      in_minutes[1])
            ticks = [minute_utime.num2date(dt) for dt in minutes]
        elif resolution == 'SECONDLY':
            second_unit = 'seconds since 2000-01-01'
            second_utime = cftime.utime(second_unit, self.calendar)
            in_seconds = second_utime.date2num([lower, upper])
            seconds = self._max_n_locator.tick_values(in_seconds[0],
                                                      in_seconds[1])
            ticks = [second_utime.num2date(dt) for dt in seconds]
        else:
            msg = 'Resolution {} not implemented yet.'.format(resolution)
            raise ValueError(msg)
        # Some calenders do not allow a year 0.
        # Remove ticks to avoid raising an error.
        if self.calendar in ["proleptic_gregorian", "gregorian", "julian"]:
            ticks = [t for t in ticks if t.year != 0]
        return utime.date2num(ticks)

class NetCDFTimeConverter(mdates.DateConverter):
    """
    Converter for cftime.datetime data.

    """
    standard_unit = 'days since 2000-01-01'

    @staticmethod
    def axisinfo(unit, axis):
        """
        Returns the :class:`~matplotlib.units.AxisInfo` for *unit*.

        *unit* is a tzinfo instance or None.
        The *axis* argument is required but not used.
        """
        calendar, date_unit, date_type = unit

        majloc = NetCDFTimeDateLocator(4, calendar=calendar,
                                       date_unit=date_unit)
        majfmt = NetCDFTimeDateFormatter(majloc, calendar=calendar,
                                         time_units=date_unit)
        if date_type is CalendarDateTime:
            datemin = CalendarDateTime(cftime.datetime(2000, 1, 1),
                                       calendar=calendar)
            datemax = CalendarDateTime(cftime.datetime(2010, 1, 1),
                                       calendar=calendar)
        else:
            datemin = date_type(2000, 1, 1)
            datemax = date_type(2010, 1, 1)
        return munits.AxisInfo(majloc=majloc, majfmt=majfmt, label='',
                               default_limits=(datemin, datemax))

    @classmethod
    def default_units(cls, sample_point, axis):
        """
        Computes some units for the given data point.

        """
        if hasattr(sample_point, '__iter__'):
            # Deal with nD `sample_point` arrays.
            if isinstance(sample_point, np.ndarray):
                sample_point = sample_point.reshape(-1)
            calendars = np.array([point.calendar for point in sample_point])
            if np.all(calendars == calendars[0]):
                calendar = calendars[0]
            else:
                raise ValueError('Calendar units are not all equal.')
            date_type = type(sample_point[0])
        else:
            # Deal with a single `sample_point` value.
            if not hasattr(sample_point, 'calendar'):
                msg = ('Expecting cftimes with an extra '
                       '"calendar" attribute.')
                raise ValueError(msg)
            else:
                calendar = sample_point.calendar
            date_type = type(sample_point)
        return calendar, cls.standard_unit, date_type

    @classmethod
    def convert(cls, value, unit, axis):
        """
        Converts value, if it is not already a number or sequence of numbers,
        with :func:`cftime.utime().date2num`.

        """
        shape = None
        if isinstance(value, np.ndarray):
            # Don't do anything with numeric types.
            if value.dtype != np.object:
                return value
            shape = value.shape
            value = value.reshape(-1)
            first_value = value[0]
        else:
            # Don't do anything with numeric types.
            if munits.ConversionInterface.is_numlike(value):
                return value
            first_value = value

        if not isinstance(first_value, (CalendarDateTime, cftime.datetime)):
            raise ValueError('The values must be numbers or instances of '
                             '"nc_time_axis.CalendarDateTime" or '
                             '"cftime.datetime".')

        if isinstance(first_value, CalendarDateTime):
            if not isinstance(first_value.datetime, cftime.datetime):
                raise ValueError('The datetime attribute of the '
                                 'CalendarDateTime object must be of type '
                                 '`cftime.datetime`.')

        ut = cftime.utime(cls.standard_unit, calendar=first_value.calendar)

        if isinstance(value, (CalendarDateTime, cftime.datetime)):
            value = [value]

        if isinstance(first_value, CalendarDateTime):
            result = ut.date2num([v.datetime for v in value])
        else:
            result = ut.date2num(value)

        if shape is not None:
            result = result.reshape(shape)

        return result

# Automatically register NetCDFTimeConverter with matplotlib.unit's converter
# dictionary.
if CalendarDateTime not in munits.registry:
    munits.registry[CalendarDateTime] = NetCDFTimeConverter()

CFTIME_TYPES = [cftime.DatetimeNoLeap, cftime.DatetimeAllLeap,
                cftime.DatetimeProlepticGregorian, cftime.DatetimeGregorian,
                cftime.Datetime360Day, cftime.DatetimeJulian]
for date_type in CFTIME_TYPES:
    if date_type not in munits.registry:
        munits.registry[date_type] = NetCDFTimeConverter()
<commit_msg>Update nc_time_axis/__init__.py<commit_after>"""
Support for cftime axis in matplotlib.

"""

from __future__ import (absolute_import, division, print_function)
from six.moves import (filter, input, map, range, zip)  # noqa

from collections import namedtuple

import matplotlib.dates as mdates
import matplotlib.ticker as mticker
import matplotlib.transforms as mtransforms
import matplotlib.units as munits
import cftime
import numpy as np

# Define __version__ based on versioneer's interpretation.
from ._version import get_versions
__version__ = get_versions()['version']
del get_versions

# Lower and upper are in number of days.
FormatOption = namedtuple('FormatOption', ['lower', 'upper', 'format_string'])

class CalendarDateTime(object):
    """
    Container for :class:`cftime.datetime` object and calendar.

    """
    def __init__(self, datetime, calendar):
        self.datetime = datetime
        self.calendar = calendar

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
                self.datetime == other.datetime and
                self.calendar == other.calendar)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        msg = '<{}: datetime={}, calendar={}>'
        return msg.format(type(self).__name__, self.datetime, self.calendar)

class NetCDFTimeDateFormatter(mticker.Formatter):
    """
    Formatter for cftime.datetime data.

    """
    # Some magic numbers. These seem to work pretty well.
    format_options = [FormatOption(0.0, 0.2, '%H:%M:%S'),
                      FormatOption(0.2, 0.8, '%H:%M'),
                      FormatOption(0.8, 15, '%Y-%m-%d %H:%M'),
                      FormatOption(15, 90, '%Y-%m-%d'),
                      FormatOption(90, 900, '%Y-%m'),
                      FormatOption(900, 6000000, '%Y')]

    def __init__(self, locator, calendar, time_units):
        #: The locator associated with this formatter. This is used to get hold
        #: of the scaling information.
        self.locator = locator
        self.calendar = calendar
        self.time_units = time_units

    def pick_format(self, ndays):
        """
        Returns a format string for an interval of the given number of days.

        """
        for option in self.format_options:
            if option.lower < ndays <= option.upper:
                return option.format_string
        else:
            msg = 'No formatter found for an interval of {} days.'
            raise ValueError(msg.format(ndays))

    def __call__(self, x, pos=0):
        format_string = self.pick_format(ndays=self.locator.ndays)
        dt = cftime.utime(self.time_units, self.calendar).num2date(x)
        return dt.strftime(format_string)

class NetCDFTimeDateLocator(mticker.Locator):
    """
    Determines tick locations when plotting cftime.datetime data.

    """
    def __init__(self, max_n_ticks, calendar, date_unit, min_n_ticks=3):
        # The date unit must be in the form of days since ...

        self.max_n_ticks = max_n_ticks
        self.min_n_ticks = min_n_ticks
        self._max_n_locator = mticker.MaxNLocator(max_n_ticks, integer=True)
        self._max_n_locator_days = mticker.MaxNLocator(
            max_n_ticks, integer=True, steps=[1, 2, 4, 7, 10])
        self.calendar = calendar
        self.date_unit = date_unit
        if not self.date_unit.lower().startswith('days since'):
            msg = 'The date unit must be days since for a NetCDF time locator.'
            raise ValueError(msg)

        self._cached_resolution = {}

    def compute_resolution(self, num1, num2, date1, date2):
        """
        Returns the resolution of the dates (hourly, minutely, yearly), and
        an **approximate** number of those units.

        """
        num_days = float(np.abs(num1 - num2))
        resolution = 'SECONDLY'
        n = mdates.SEC_PER_DAY
        if num_days * mdates.MINUTES_PER_DAY > self.max_n_ticks:
            resolution = 'MINUTELY'
            n = int(num_days / mdates.MINUTES_PER_DAY)
        if num_days * mdates.HOURS_PER_DAY > self.max_n_ticks:
            resolution = 'HOURLY'
            n = int(num_days / mdates.HOURS_PER_DAY)
        if num_days > self.max_n_ticks:
            resolution = 'DAILY'
            n = int(num_days)
        if num_days > 30 * self.max_n_ticks:
            resolution = 'MONTHLY'
            n = num_days // 30
        if num_days > 365 * self.max_n_ticks:
            resolution = 'YEARLY'
            n = abs(date1.year - date2.year)

        return resolution, n

    def __call__(self):
        vmin, vmax = self.axis.get_view_interval()
        return self.tick_values(vmin, vmax)

    def tick_values(self, vmin, vmax):
        vmin, vmax = mtransforms.nonsingular(vmin, vmax, expander=1e-7,
                                             tiny=1e-13)

        self.ndays = float(abs(vmax - vmin))

        utime = cftime.utime(self.date_unit, self.calendar)
        lower = utime.num2date(vmin)
        upper = utime.num2date(vmax)

        resolution, n = self.compute_resolution(vmin, vmax, lower, upper)

        if resolution == 'YEARLY':
            # TODO START AT THE BEGINNING OF A DECADE/CENTURY/MILLENIUM as
            # appropriate.
            years = self._max_n_locator.tick_values(lower.year, upper.year)
            ticks = [cftime.datetime(int(year), 1, 1) for year in years]
        elif resolution == 'MONTHLY':
            # TODO START AT THE BEGINNING OF A DECADE/CENTURY/MILLENIUM as
            # appropriate.
            months_offset = self._max_n_locator.tick_values(0, n)
            ticks = []
            for offset in months_offset:
                year = lower.year + np.floor((lower.month + offset) / 12)
                month = ((lower.month + offset) % 12) + 1
                ticks.append(cftime.datetime(int(year), int(month), 1))
        elif resolution == 'DAILY':
            # TODO: It would be great if this favoured multiples of 7.
            days = self._max_n_locator_days.tick_values(vmin, vmax)
            ticks = [utime.num2date(dt) for dt in days]
        elif resolution == 'HOURLY':
            hour_unit = 'hours since 2000-01-01'
            hour_utime = cftime.utime(hour_unit, self.calendar)
            in_hours = hour_utime.date2num([lower, upper])
            hours = self._max_n_locator.tick_values(in_hours[0], in_hours[1])
            ticks = [hour_utime.num2date(dt) for dt in hours]
        elif resolution == 'MINUTELY':
            minute_unit = 'minutes since 2000-01-01'
            minute_utime = cftime.utime(minute_unit, self.calendar)
            in_minutes = minute_utime.date2num([lower, upper])
            minutes = self._max_n_locator.tick_values(in_minutes[0],
                                                      in_minutes[1])
            ticks = [minute_utime.num2date(dt) for dt in minutes]
        elif resolution == 'SECONDLY':
            second_unit = 'seconds since 2000-01-01'
            second_utime = cftime.utime(second_unit, self.calendar)
            in_seconds = second_utime.date2num([lower, upper])
            seconds = self._max_n_locator.tick_values(in_seconds[0],
                                                      in_seconds[1])
            ticks = [second_utime.num2date(dt) for dt in seconds]
        else:
            msg = 'Resolution {} not implemented yet.'.format(resolution)
            raise ValueError(msg)
        # Some calenders do not allow a year 0.
        # Remove ticks to avoid raising an error.
        if self.calendar in ["proleptic_gregorian", "gregorian", "julian",
                                     "standard"]:
            ticks = [t for t in ticks if t.year != 0]
        return utime.date2num(ticks)

class NetCDFTimeConverter(mdates.DateConverter):
    """
    Converter for cftime.datetime data.

    """
    standard_unit = 'days since 2000-01-01'

    @staticmethod
    def axisinfo(unit, axis):
        """
        Returns the :class:`~matplotlib.units.AxisInfo` for *unit*.

        *unit* is a tzinfo instance or None.
        The *axis* argument is required but not used.
        """
        calendar, date_unit, date_type = unit

        majloc = NetCDFTimeDateLocator(4, calendar=calendar,
                                       date_unit=date_unit)
        majfmt = NetCDFTimeDateFormatter(majloc, calendar=calendar,
                                         time_units=date_unit)
        if date_type is CalendarDateTime:
            datemin = CalendarDateTime(cftime.datetime(2000, 1, 1),
                                       calendar=calendar)
            datemax = CalendarDateTime(cftime.datetime(2010, 1, 1),
                                       calendar=calendar)
        else:
            datemin = date_type(2000, 1, 1)
            datemax = date_type(2010, 1, 1)
        return munits.AxisInfo(majloc=majloc, majfmt=majfmt, label='',
                               default_limits=(datemin, datemax))

    @classmethod
    def default_units(cls, sample_point, axis):
        """
        Computes some units for the given data point.

        """
        if hasattr(sample_point, '__iter__'):
            # Deal with nD `sample_point` arrays.
            if isinstance(sample_point, np.ndarray):
                sample_point = sample_point.reshape(-1)
            calendars = np.array([point.calendar for point in sample_point])
            if np.all(calendars == calendars[0]):
                calendar = calendars[0]
            else:
                raise ValueError('Calendar units are not all equal.')
            date_type = type(sample_point[0])
        else:
            # Deal with a single `sample_point` value.
            if not hasattr(sample_point, 'calendar'):
                msg = ('Expecting cftimes with an extra '
                       '"calendar" attribute.')
                raise ValueError(msg)
            else:
                calendar = sample_point.calendar
            date_type = type(sample_point)
        return calendar, cls.standard_unit, date_type

    @classmethod
    def convert(cls, value, unit, axis):
        """
        Converts value, if it is not already a number or sequence of numbers,
        with :func:`cftime.utime().date2num`.

        """
        shape = None
        if isinstance(value, np.ndarray):
            # Don't do anything with numeric types.
            if value.dtype != np.object:
                return value
            shape = value.shape
            value = value.reshape(-1)
            first_value = value[0]
        else:
            # Don't do anything with numeric types.
            if munits.ConversionInterface.is_numlike(value):
                return value
            first_value = value

        if not isinstance(first_value, (CalendarDateTime, cftime.datetime)):
            raise ValueError('The values must be numbers or instances of '
                             '"nc_time_axis.CalendarDateTime" or '
                             '"cftime.datetime".')

        if isinstance(first_value, CalendarDateTime):
            if not isinstance(first_value.datetime, cftime.datetime):
                raise ValueError('The datetime attribute of the '
                                 'CalendarDateTime object must be of type '
                                 '`cftime.datetime`.')

        ut = cftime.utime(cls.standard_unit, calendar=first_value.calendar)

        if isinstance(value, (CalendarDateTime, cftime.datetime)):
            value = [value]

        if isinstance(first_value, CalendarDateTime):
            result = ut.date2num([v.datetime for v in value])
        else:
            result = ut.date2num(value)

        if shape is not None:
            result = result.reshape(shape)

        return result

# Automatically register NetCDFTimeConverter with matplotlib.unit's converter
# dictionary.
if CalendarDateTime not in munits.registry:
    munits.registry[CalendarDateTime] = NetCDFTimeConverter()

CFTIME_TYPES = [cftime.DatetimeNoLeap, cftime.DatetimeAllLeap,
                cftime.DatetimeProlepticGregorian, cftime.DatetimeGregorian,
                cftime.Datetime360Day, cftime.DatetimeJulian]
for date_type in CFTIME_TYPES:
    if date_type not in munits.registry:
        munits.registry[date_type] = NetCDFTimeConverter()
<|endoftext|>


<commit_before># Copyright 2009 Shikhar Bhushan
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import six

def _abbreviate(uri):
    if uri.startswith("urn:ietf:params") and ":netconf:" in uri:
        splitted = uri.split(":")
        if ":capability:" in uri:
            if uri.startswith("urn:ietf:params:xml:ns:netconf"):
                name, version = splitted[7], splitted[8]
            else:
                name, version = splitted[5], splitted[6]
            return [ ":" + name, ":" + name + ":" + version ]
        elif ":base:" in uri:
            if uri.startswith("urn:ietf:params:xml:ns:netconf"):
                return [ ":base", ":base" + ":" + splitted[7] ]
            else:
                return [ ":base", ":base" + ":" + splitted[5] ]
    return []

def schemes(url_uri):
    "Given a URI that has a *scheme* query string (i.e. `:url` capability URI), will return a list of supported schemes."
    return url_uri.partition("?scheme=")[2].split(",")

class Capabilities(object):

    "Represents the set of capabilities available to a NETCONF client or server. It is initialized with a list of capability URI's."

    def __init__(self, capabilities):
        self._dict = {}
        for uri in capabilities:
            self.add(uri)

    def __contains__(self, key):
        try:
            self.__getitem__(key)
        except KeyError:
            return False
        else:
            return True

    def __getitem__(self, key):
        try:
            return self._dict[key]
        except KeyError:
            for capability in six.itervalues(self._dict):
                if key in capability.get_abbreviations():
                    return capability

        raise KeyError(key)

    def __len__(self):
        return len(self._dict)

    # python 2 and 3 compatible
    def __iter__(self):
        return six.iterkeys(self._dict)

    def __repr__(self):
        return repr(six.iterkeys(self._dict))

    def add(self, uri):
        "Add a capability."
        self._dict[uri] = Capability.from_uri(uri)

    def remove(self, uri):
        "Remove a capability."
        if uri in self._dict:
            del self._dict[uri]

class Capability(object):

    """Represents a single capability"""

    def __init__(self, namespace_uri, parameters=None):
        self.namespace_uri = namespace_uri
        self.parameters = parameters or {}

    @classmethod
    def from_uri(cls, uri):
        split_uri = uri.split('?')
        namespace_uri = split_uri[0]
        capability = cls(namespace_uri)

        try:
            param_string = split_uri[1]
        except IndexError:
            return capability

        capability.parameters = {
            param.key: param.value
            for param in _parse_parameter_string(param_string)
        }

        return capability

    def get_abbreviations(self):
        return _abbreviate(self.namespace_uri)

def _parse_parameter_string(string):
    for param_string in string.split('&'):
        yield _Parameter.from_string(param_string)

class _Parameter(object):

    """Represents a parameter to a capability"""

    def __init__(self, key, value):
        self.key = key
        self.value = value

    @classmethod
    def from_string(cls, string):
        try:
            key, value = string.split('=')
        except ValueError:
            # TODO raise custom error
            raise

        return cls(key, value)
<commit_msg>Added equality meta-method to Capability class for tests that compare them<commit_after># Copyright 2009 Shikhar Bhushan
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import six

def _abbreviate(uri):
    if uri.startswith("urn:ietf:params") and ":netconf:" in uri:
        splitted = uri.split(":")
        if ":capability:" in uri:
            if uri.startswith("urn:ietf:params:xml:ns:netconf"):
                name, version = splitted[7], splitted[8]
            else:
                name, version = splitted[5], splitted[6]
            return [ ":" + name, ":" + name + ":" + version ]
        elif ":base:" in uri:
            if uri.startswith("urn:ietf:params:xml:ns:netconf"):
                return [ ":base", ":base" + ":" + splitted[7] ]
            else:
                return [ ":base", ":base" + ":" + splitted[5] ]
    return []

def schemes(url_uri):
    "Given a URI that has a *scheme* query string (i.e. `:url` capability URI), will return a list of supported schemes."
    return url_uri.partition("?scheme=")[2].split(",")

class Capabilities(object):

    "Represents the set of capabilities available to a NETCONF client or server. It is initialized with a list of capability URI's."

    def __init__(self, capabilities):
        self._dict = {}
        for uri in capabilities:
            self.add(uri)

    def __contains__(self, key):
        try:
            self.__getitem__(key)
        except KeyError:
            return False
        else:
            return True

    def __getitem__(self, key):
        try:
            return self._dict[key]
        except KeyError:
            for capability in six.itervalues(self._dict):
                if key in capability.get_abbreviations():
                    return capability

        raise KeyError(key)

    def __len__(self):
        return len(self._dict)

    # python 2 and 3 compatible
    def __iter__(self):
        return six.iterkeys(self._dict)

    def __repr__(self):
        return repr(six.iterkeys(self._dict))

    def add(self, uri):
        "Add a capability."
        self._dict[uri] = Capability.from_uri(uri)

    def remove(self, uri):
        "Remove a capability."
        if uri in self._dict:
            del self._dict[uri]

class Capability(object):

    """Represents a single capability"""

    def __init__(self, namespace_uri, parameters=None):
        self.namespace_uri = namespace_uri
        self.parameters = parameters or {}

    @classmethod
    def from_uri(cls, uri):
        split_uri = uri.split('?')
        namespace_uri = split_uri[0]
        capability = cls(namespace_uri)

        try:
            param_string = split_uri[1]
        except IndexError:
            return capability

        capability.parameters = {
            param.key: param.value
            for param in _parse_parameter_string(param_string)
        }

        return capability

    def __eq__(self, other):
        return (
            self.namespace_uri == other.namespace_uri and
            self.parameters == other.parameters
        )

    def get_abbreviations(self):
        return _abbreviate(self.namespace_uri)

def _parse_parameter_string(string):
    for param_string in string.split('&'):
        yield _Parameter.from_string(param_string)

class _Parameter(object):

    """Represents a parameter to a capability"""

    def __init__(self, key, value):
        self.key = key
        self.value = value

    @classmethod
    def from_string(cls, string):
        try:
            key, value = string.split('=')
        except ValueError:
            # TODO raise custom error
            raise

        return cls(key, value)
<|endoftext|>


<commit_before>from py2neo import neo4j
import sys
from .exception import DoesNotExist, NotConnected
from .util import camel_to_upper

OUTGOING = neo4j.Direction.OUTGOING
INCOMING = neo4j.Direction.INCOMING
EITHER = neo4j.Direction.EITHER

def _dir_2_str(direction):
    if direction == OUTGOING:
        return 'a outgoing'
    elif direction == INCOMING:
        return 'a incoming'
    return 'either'

def _related(direction):
    if direction == OUTGOING:
        return '-[:{0}]->'
    elif direction == INCOMING:
        return '<-[:{0}]-'
    return '-[:{0}]-'

def _properties(ident, **kwargs):
    props = [ident + '.' + k + '! = {' + k + '}' for k in kwargs]
    return '(' + ', '.join(props) + ')'

class RelationshipManager(object):
    def __init__(self, direction, relation_type, node_classes, origin):
        self.direction = direction
        self.relation_type = relation_type
        if isinstance(node_classes, list):
            self.node_classes = node_classes
            self.class_map = dict(zip([camel_to_upper(c.__name__)
                for c in node_classes], node_classes))
        else:
            self.node_class = node_classes
        self.origin = origin

    def __str__(self):
        return "{0} in {1} direction of type {2} on node ({3}) of class '{4}'".format(
            self.description, _dir_2_str(self.direction),
            self.relation_type, self.origin.__node__.id, self.origin.__class__.__name__)

    def __bool__(self):
        return self.__len__() > 0

    def __nonezero__(self):
        return self.__len__() > 0

    def __len__(self):
        query = "START a=node({self}) MATCH (a)"
        query += _related(self.direction).format(self.relation_type)
        query += "(x) RETURN COUNT(x)"
        return int(self.origin.cypher(query)[0][0][0])

    @property
    def client(self):
        return self.origin.client

    def count(self):
        return self.__len__()

    def all(self):
        if hasattr(self, 'node_classes'):
            return self._all_multi_class()
        else:
            return self._all_single_class()

    def _inflate_nodes_by_rel(self, results):
        """With resultset containing [node, rel] pairs
        wrap each node in correct neomodel class based on rel.type"""
        nodes = [row[0] for row in results]
        classes = [self.class_map[row[1].type] for row in results]
        return [cls.inflate(node) for node, cls in zip(nodes, classes)]

    def _all_multi_class(self):
        cat_types = "|".join([camel_to_upper(c.__name__) for c in self.node_classes])
        query = "START a=node({self}) MATCH (a)"
        query += _related(self.direction).format(self.relation_type)
        query += "(x)<-[r:{0}]-() WHERE r.__instance__! = true RETURN x, r".format(cat_types)
        results = self.origin.cypher(query)[0]
        return self._inflate_nodes_by_rel(results)

    def _all_single_class(self):
        query = "START a=node({self}) MATCH (a)"
        query += _related(self.direction).format(self.relation_type) + "(x) RETURN x"
        results = self.origin.cypher(query)
        return [self.node_class.inflate(n[0]) for n in results[0]] if results else []

    def get(self, **kwargs):
        result = self.search(**kwargs)
        if len(result) == 1:
            return result[0]
        if len(result) > 1:
            raise Exception("Multiple items returned")
        if not result:
            if hasattr(self, 'node_class'):
                raise self.node_class.DoesNotExist("No items exist for the specified arguments")
            else:
                raise DoesNotExist("No items exist for the specified arguments")

    def search(self, **kwargs):
        if not kwargs:
            if hasattr(self, 'node_classes'):
                return self._all_multi_class()
            else:
                return self._all_single_class()

        if hasattr(self, 'node_classes'):
            return self._search_multi_class(**kwargs)
        else:
            return self._search_single_class(**kwargs)

    def _search_single_class(self, **kwargs):
        query = "START a=node({self}) MATCH (a)"
        query += _related(self.direction).format(self.relation_type)
        query += "(b) WHERE " + _properties('b', **kwargs)
        query += " RETURN b"

        results = self.origin.cypher(query, kwargs)[0]
        return [self.node_class.inflate(row[0]) for row in results]

    def _search_multi_class(self, **kwargs):
        cat_types = "|".join([camel_to_upper(c.__name__) for c in self.node_classes])
        query = "START a=node({self}) MATCH (a)"
        query += _related(self.direction).format(self.relation_type)
        query += "(b)<-[r:{0}]-() ".format(cat_types)
        query += "WHERE r.__instance__! = true AND " + _properties('b', **kwargs)
        query += " RETURN b, r"
        results = self.origin.cypher(query, kwargs)[0]
        return self._inflate_nodes_by_rel(results)

    def is_connected(self, obj):
        return self.origin.__node__.has_relationship_with(obj.__node__, self.direction, self.relation_type)

    def connect(self, obj):
        if self.direction == EITHER:
            raise Exception("Cannot connect with direction EITHER")
        # is single class rel
        if hasattr(self, 'node_class'):
            if not self.node_class.__subclasscheck__(obj.__class__):
                raise Exception("Expected object of class (or a subclass of) {0} got {1}".format(self.node_class, obj.__class__))
        # or is multi class rel
        elif hasattr(self, 'node_classes'):
            for cls in self.node_classes:
                if cls.__subclasscheck__(obj.__class__):
                    node_class = cls
            if not node_class:
                allowed_cls = ", ".join([c.__name__ for c in self.node_classes])
                raise Exception("Expected object of class of "
                        + allowed_cls + " got " + obj.__class__.__name__)

        if not obj.__node__:
            raise Exception("Can't create relationship to unsaved node")
        if self.direction == OUTGOING:
            self.client.get_or_create_relationships((self.origin.__node__, self.relation_type, obj.__node__))
        elif self.direction == INCOMING:
            self.client.get_or_create_relationships((obj.__node__, self.relation_type, self.origin.__node__))
        else:
            raise Exception("Unknown relationship direction {0}".format(self.direction))

    def reconnect(self, old_obj, new_obj):
        if self.is_connected(old_obj):
            rels = self.origin.__node__.get_relationships_with(
                old_obj.__node__, self.direction, self.relation_type)
            for r in rels:
                r.delete()
        else:
            raise NotConnected('reconnect', self.origin, old_obj)

        self.client.get_or_create_relationships((self.origin.__node__, self.relation_type, new_obj.__node__),)

    def disconnect(self, obj):
        rels = self.origin.__node__.get_relationships_with(obj.__node__, self.direction, self.relation_type)
        if not rels:
            raise NotConnected('disconnect', self.origin, obj)
        if len(rels) > 1:
            raise Exception("Expected single relationship got {0}".format(rels))
        rels[0].delete()

    def single(self):
        nodes = self.all()
        if nodes:
            return nodes[0]
        else:
            return None

class RelationshipDefinition(object):
    def __init__(self, relation_type, cls_name, direction, manager=RelationshipManager):
        self.module_name = sys._getframe(4).f_globals['__name__']
        self.relation_type = relation_type
        self.node_class = cls_name
        self.manager = manager
        self.direction = direction

    def lookup_classes(self):
        if isinstance(self.node_class, list):
            return [self._lookup(name) for name in self.node_class]
        else:
            return self._lookup(self.node_class)

    def _lookup(self, name):
        if name.find('.') is -1:
            module = self.module_name
        else:
            module, _, name = name.rpartition('.')

        if not module in sys.modules:
            __import__(module)
        return getattr(sys.modules[module], name)

    def build_manager(self, origin, name):
        rel = self.manager(
            self.direction,
            self.relation_type,
            self.lookup_classes(),
            origin)
        rel.name = name
        return rel

class ZeroOrMore(RelationshipManager):
    description = "zero or more relationships"

def _relate(cls_name, direction, rel_type, cardinality=None):
    if not isinstance(cls_name, (str, unicode, list)):
        raise Exception('Expected class name or list of class names, got ' + repr(cls_name))
    return RelationshipDefinition(rel_type, cls_name, direction, cardinality)

def RelationshipTo(cls_name, rel_type, cardinality=ZeroOrMore):
    return _relate(cls_name, OUTGOING, rel_type, cardinality)

def RelationshipFrom(cls_name, rel_type, cardinality=ZeroOrMore):
    return _relate(cls_name, INCOMING, rel_type, cardinality)

def Relationship(cls_name, rel_type, cardinality=ZeroOrMore):
    return _relate(cls_name, EITHER, rel_type, cardinality)
<commit_msg>Clean up<commit_after>from py2neo import neo4j
import sys
from .exception import DoesNotExist, NotConnected
from .util import camel_to_upper

OUTGOING = neo4j.Direction.OUTGOING
INCOMING = neo4j.Direction.INCOMING
EITHER = neo4j.Direction.EITHER

def _dir_2_str(direction):
    if direction == OUTGOING:
        return 'a outgoing'
    elif direction == INCOMING:
        return 'a incoming'
    return 'either'

def _related(direction):
    if direction == OUTGOING:
        return '-[:{0}]->'
    elif direction == INCOMING:
        return '<-[:{0}]-'
    return '-[:{0}]-'

def _properties(ident, **kwargs):
    props = [ident + '.' + k + '! = {' + k + '}' for k in kwargs]
    return '(' + ', '.join(props) + ')'

class RelationshipManager(object):
    def __init__(self, direction, relation_type, node_classes, origin):
        self.direction = direction
        self.relation_type = relation_type
        if isinstance(node_classes, list):
            self.node_classes = node_classes
            self.class_map = dict(zip([camel_to_upper(c.__name__)
                for c in node_classes], node_classes))
        else:
            self.node_class = node_classes
        self.origin = origin

    def __str__(self):
        return "{0} in {1} direction of type {2} on node ({3}) of class '{4}'".format(
            self.description, _dir_2_str(self.direction),
            self.relation_type, self.origin.__node__.id, self.origin.__class__.__name__)

    def __bool__(self):
        return self.__len__() > 0

    def __nonezero__(self):
        return self.__len__() > 0

    def __len__(self):
        query = "START a=node({self}) MATCH (a)"
        query += _related(self.direction).format(self.relation_type)
        query += "(x) RETURN COUNT(x)"
        return int(self.origin.cypher(query)[0][0][0])

    @property
    def client(self):
        return self.origin.client

    def count(self):
        return self.__len__()

    def all(self):
        if hasattr(self, 'node_classes'):
            return self._all_multi_class()
        else:
            return self._all_single_class()

    def _inflate_nodes_by_rel(self, results):
        """With resultset containing [node, rel] pairs
        wrap each node in correct neomodel class based on rel.type"""
        nodes = [row[0] for row in results]
        classes = [self.class_map[row[1].type] for row in results]
        return [cls.inflate(node) for node, cls in zip(nodes, classes)]

    def _all_multi_class(self):
        cat_types = "|".join([camel_to_upper(c.__name__) for c in self.node_classes])
        query = "START a=node({self}) MATCH (a)"
        query += _related(self.direction).format(self.relation_type)
        query += "(x)<-[r:{0}]-() WHERE r.__instance__! = true RETURN x, r".format(cat_types)
        results = self.origin.cypher(query)[0]
        return self._inflate_nodes_by_rel(results)

    def _all_single_class(self):
        query = "START a=node({self}) MATCH (a)"
        query += _related(self.direction).format(self.relation_type) + "(x) RETURN x"
        results = self.origin.cypher(query)
        return [self.node_class.inflate(n[0]) for n in results[0]] if results else []

    def get(self, **kwargs):
        result = self.search(**kwargs)
        if len(result) == 1:
            return result[0]
        if len(result) > 1:
            raise Exception("Multiple items returned")
        if not result:
            if hasattr(self, 'node_class'):
                raise self.node_class.DoesNotExist("No items exist for the specified arguments")
            else:
                raise DoesNotExist("No items exist for the specified arguments")

    def search(self, **kwargs):
        if not kwargs:
            if hasattr(self, 'node_classes'):
                return self._all_multi_class()
            else:
                return self._all_single_class()

        if hasattr(self, 'node_classes'):
            return self._search_multi_class(**kwargs)
        else:
            return self._search_single_class(**kwargs)

    def _search_single_class(self, **kwargs):
        query = "START a=node({self}) MATCH (a)"
        query += _related(self.direction).format(self.relation_type)
        query += "(b) WHERE " + _properties('b', **kwargs)
        query += " RETURN b"

        results = self.origin.cypher(query, kwargs)[0]
        return [self.node_class.inflate(row[0]) for row in results]

    def _search_multi_class(self, **kwargs):
        cat_types = "|".join([camel_to_upper(c.__name__) for c in self.node_classes])
        query = "START a=node({self}) MATCH (a)"
        query += _related(self.direction).format(self.relation_type)
        query += "(b)<-[r:{0}]-() ".format(cat_types)
        query += "WHERE r.__instance__! = true AND " + _properties('b', **kwargs)
        query += " RETURN b, r"
        results = self.origin.cypher(query, kwargs)[0]
        return self._inflate_nodes_by_rel(results)

    def is_connected(self, obj):
        return self.origin.__node__.has_relationship_with(obj.__node__, self.direction, self.relation_type)

    def connect(self, obj):
        if self.direction == EITHER:
            raise Exception("Cannot connect with direction EITHER")
        # is single class rel
        if hasattr(self, 'node_class'):
            if not self.node_class.__subclasscheck__(obj.__class__):
                raise Exception("Expected object of class (or a subclass of) {0} got {1}".format(self.node_class, obj.__class__))
        # or is multi class rel
        elif hasattr(self, 'node_classes'):
            for cls in self.node_classes:
                if cls.__subclasscheck__(obj.__class__):
                    node_class = cls
            if not node_class:
                allowed_cls = ", ".join([c.__name__ for c in self.node_classes])
                raise Exception("Expected object of class of "
                        + allowed_cls + " got " + obj.__class__.__name__)

        if not obj.__node__:
            raise Exception("Can't create relationship to unsaved node")
        if self.direction == OUTGOING:
            self.client.get_or_create_relationships((self.origin.__node__, self.relation_type, obj.__node__))
        elif self.direction == INCOMING:
            self.client.get_or_create_relationships((obj.__node__, self.relation_type, self.origin.__node__))
        else:
            raise Exception("Unknown relationship direction {0}".format(self.direction))

    def reconnect(self, old_obj, new_obj):
        if self.is_connected(old_obj):
            rels = self.origin.__node__.get_relationships_with(
                old_obj.__node__, self.direction, self.relation_type)
            for r in rels:
                r.delete()
        else:
            raise NotConnected('reconnect', self.origin, old_obj)

        self.client.get_or_create_relationships((self.origin.__node__, self.relation_type, new_obj.__node__),)

    def disconnect(self, obj):
        rels = self.origin.__node__.get_relationships_with(obj.__node__, self.direction, self.relation_type)
        if not rels:
            raise NotConnected('disconnect', self.origin, obj)
        if len(rels) > 1:
            raise Exception("Expected single relationship got {0}".format(rels))
        rels[0].delete()

    def single(self): # TODO should limit this query
        nodes = self.all()
        return nodes[0] if nodes else None

class RelationshipDefinition(object):
    def __init__(self, relation_type, cls_name, direction, manager=RelationshipManager):
        self.module_name = sys._getframe(4).f_globals['__name__']
        self.relation_type = relation_type
        self.node_class = cls_name
        self.manager = manager
        self.direction = direction

    def lookup_classes(self):
        if isinstance(self.node_class, list):
            return [self._lookup(name) for name in self.node_class]
        else:
            return self._lookup(self.node_class)

    def _lookup(self, name):
        if name.find('.') is -1:
            module = self.module_name
        else:
            module, _, name = name.rpartition('.')

        if not module in sys.modules:
            __import__(module)
        return getattr(sys.modules[module], name)

    def build_manager(self, origin, name):
        rel = self.manager(
            self.direction,
            self.relation_type,
            self.lookup_classes(),
            origin)
        rel.name = name
        return rel

class ZeroOrMore(RelationshipManager):
    description = "zero or more relationships"

def _relate(cls_name, direction, rel_type, cardinality=None):
    if not isinstance(cls_name, (str, unicode, list)):
        raise Exception('Expected class name or list of class names, got ' + repr(cls_name))
    return RelationshipDefinition(rel_type, cls_name, direction, cardinality)

def RelationshipTo(cls_name, rel_type, cardinality=ZeroOrMore):
    return _relate(cls_name, OUTGOING, rel_type, cardinality)

def RelationshipFrom(cls_name, rel_type, cardinality=ZeroOrMore):
    return _relate(cls_name, INCOMING, rel_type, cardinality)

def Relationship(cls_name, rel_type, cardinality=ZeroOrMore):
    return _relate(cls_name, EITHER, rel_type, cardinality)
<|endoftext|>


<commit_before># Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
Functional tests for ``flocker.node.agents.cinder`` using a real OpenStack
cluster.
"""

# A helper to check the environment for username and API key
# Create an authenticated cinder API instance
# Supply that to make_iblockdeviceapi_tests

from uuid import uuid4

from ..cinder import cinder_api, authenticated_cinder_client
from ..testtools import (
    make_iblockdeviceapi_tests, require_cinder_credentials, todo_except
)

@require_cinder_credentials
def cinderblockdeviceapi_for_test(
        test_case, OPENSTACK_API_USER, OPENSTACK_API_KEY
):
    """

    """
    cinder_client = authenticated_cinder_client(
        username=OPENSTACK_API_USER,
        api_key=OPENSTACK_API_KEY,
        region='DFW',
    )
    return cinder_api(
        cinder_client,
        cluster_id=unicode(uuid4()),
    )

@todo_except(
    supported_tests=[
        'test_interface',
        'test_created_is_listed',
        'test_created_volume_attributes',
        'test_list_volume_empty',
        'test_listed_volume_attributes',
    ]
)
class CinderBlockDeviceAPITests(
        make_iblockdeviceapi_tests(
            blockdevice_api_factory=cinderblockdeviceapi_for_test
        )
):
    """
    Interface adherence Tests for ``CinderBlockDeviceAPI``.
    """
<commit_msg>I want to get to a place where I can build verified cinder client that's capable of cleaning up the volumes that it created in its lifetime....this is another step in that direction<commit_after># Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
Functional tests for ``flocker.node.agents.cinder`` using a real OpenStack
cluster.
"""

# A helper to check the environment for username and API key
# Create an authenticated cinder API instance
# Supply that to make_iblockdeviceapi_tests

from uuid import uuid4

from ..cinder import cinder_api, authenticated_cinder_client
from ..testtools import (
    make_iblockdeviceapi_tests, require_cinder_credentials, todo_except
)

@require_cinder_credentials
def cinder_client_for_test(
        test_case, OPENSTACK_API_USER, OPENSTACK_API_KEY
):
    client = authenticated_cinder_client(
        username=OPENSTACK_API_USER,
        api_key=OPENSTACK_API_KEY,
        region='DFW',
    )
    return client

def cinderblockdeviceapi_for_test(test_case):
    """
    """
    return cinder_api(
        cinder_client=cinder_client_for_test(test_case),
        cluster_id=unicode(uuid4()),
    )

@todo_except(
    supported_tests=[
        'test_interface',
        'test_created_is_listed',
        'test_created_volume_attributes',
        'test_list_volume_empty',
        'test_listed_volume_attributes',
    ]
)
class CinderBlockDeviceAPITests(
        make_iblockdeviceapi_tests(
            blockdevice_api_factory=cinderblockdeviceapi_for_test
        )
):
    """
    Interface adherence Tests for ``CinderBlockDeviceAPI``.
    """
<|endoftext|>


<commit_before><commit_msg>Linux should use Unix makefiles. Duh me. :D<commit_after><|endoftext|>


<commit_before>import pytest
import numpy as np
import tensorflow as tf

from foolbox.models import TensorFlowEagerModel

@pytest.mark.parametrize('num_classes', [10, 1000])
def test_eager_model(num_classes):
    bounds = (0, 255)
    channels = num_classes

    def mean_brightness_net(images):
        logits = tf.reduce_mean(images, axis=(1, 2))
        return logits

    tf.enable_eager_execution()

    model = TensorFlowEagerModel(
        mean_brightness_net,
        bounds=bounds,
        num_classes=num_classes)

    test_images = np.random.rand(2, channels, 5, 5).astype(np.float32)
    test_label = 7

    assert model.batch_predictions(test_images).shape \
        == (2, num_classes)

    test_logits = model.predictions(test_images[0])
    assert test_logits.shape == (num_classes,)

    test_gradient = model.gradient(test_images[0], test_label)
    assert test_gradient.shape == test_images[0].shape

    np.testing.assert_almost_equal(
        model.predictions_and_gradient(test_images[0], test_label)[0],
        test_logits)
    np.testing.assert_almost_equal(
        model.predictions_and_gradient(test_images[0], test_label)[1],
        test_gradient)

    assert model.num_classes() == num_classes

def test_eager_model_preprocessing():
    num_classes = 1000
    bounds = (0, 255)
    channels = num_classes

    def mean_brightness_net(images):
        logits = tf.reduce_mean(images, axis=(1, 2))
        return logits

    model = mean_brightness_net

    q = (np.arange(num_classes)[None, None],
         np.random.uniform(size=(5, 5, channels)) + 1)

    model1 = TensorFlowEagerModel(
        model,
        bounds=bounds,
        num_classes=num_classes)

    model2 = TensorFlowEagerModel(
        model,
        bounds=bounds,
        num_classes=num_classes,
        preprocessing=q)

    model3 = TensorFlowEagerModel(
        model,
        bounds=bounds,
        num_classes=num_classes)

    np.random.seed(22)
    test_images = np.random.rand(2, channels, 5, 5).astype(np.float32)
    test_images_copy = test_images.copy()

    p1 = model1.batch_predictions(test_images)
    p2 = model2.batch_predictions(test_images)

    # make sure the images have not been changed by
    # the in-place preprocessing
    assert np.all(test_images == test_images_copy)

    p3 = model3.batch_predictions(test_images)

    assert p1.shape == p2.shape == p3.shape == (2, num_classes)

    np.testing.assert_array_almost_equal(
        p1 - p1.max(),
        p3 - p3.max(),
        decimal=5)

def test_eager_model_gradient():
    num_classes = 1000
    bounds = (0, 255)
    channels = num_classes

    def mean_brightness_net(images):
        logits = tf.reduce_mean(images, axis=(1, 2))
        return logits

    model = mean_brightness_net

    q = (np.arange(num_classes)[None, None],
         np.random.uniform(size=(5, 5, channels)) + 1)

    model = TensorFlowEagerModel(
        model,
        bounds=bounds,
        num_classes=num_classes,
        preprocessing=q)

    epsilon = 1e-2

    np.random.seed(23)
    test_image = np.random.rand(channels, 5, 5).astype(np.float32)
    test_label = 7

    _, g1 = model.predictions_and_gradient(test_image, test_label)

    l1 = model._loss_fn(test_image - epsilon / 2 * g1, test_label)
    l2 = model._loss_fn(test_image + epsilon / 2 * g1, test_label)

    assert 1e4 * (l2 - l1) > 1

    # make sure that gradient is numerically correct
    np.testing.assert_array_almost_equal(
        1e4 * (l2 - l1),
        1e4 * epsilon * np.linalg.norm(g1)**2,
        decimal=1)

@pytest.mark.parametrize('num_classes', [10, 1000])
def test_eager_backward(num_classes):
    bounds = (0, 255)
    channels = num_classes

    def mean_brightness_net(images):
        logits = tf.reduce_mean(images, axis=(1, 2))
        return logits

    model = mean_brightness_net
    model = TensorFlowEagerModel(
        model,
        bounds=bounds,
        num_classes=num_classes)

    test_image = np.random.rand(channels, 5, 5).astype(np.float32)
    test_grad_pre = np.random.rand(num_classes).astype(np.float32)

    test_grad = model.backward(test_grad_pre, test_image)
    assert test_grad.shape == test_image.shape

    manual_grad = np.repeat(np.repeat(
        (test_grad_pre / 25.).reshape((-1, 1, 1)),
        5, axis=1), 5, axis=2)

    np.testing.assert_almost_equal(
        test_grad,
        manual_grad)
<commit_msg>fixed tf eager tests<commit_after>import pytest
import numpy as np
import tensorflow as tf
tf.enable_eager_execution()

from foolbox.models import TensorFlowEagerModel

@pytest.mark.parametrize('num_classes', [10, 1000])
def test_eager_model(num_classes):
    bounds = (0, 255)
    channels = num_classes

    def mean_brightness_net(images):
        logits = tf.reduce_mean(images, axis=(1, 2))
        return logits

    model = TensorFlowEagerModel(
        mean_brightness_net,
        bounds=bounds,
        num_classes=num_classes)

    test_images = np.random.rand(2, 5, 5, channels).astype(np.float32)
    test_label = 7

    assert model.batch_predictions(test_images).shape \
        == (2, num_classes)

    test_logits = model.predictions(test_images[0])
    assert test_logits.shape == (num_classes,)

    test_gradient = model.gradient(test_images[0], test_label)
    assert test_gradient.shape == test_images[0].shape

    np.testing.assert_almost_equal(
        model.predictions_and_gradient(test_images[0], test_label)[0],
        test_logits)
    np.testing.assert_almost_equal(
        model.predictions_and_gradient(test_images[0], test_label)[1],
        test_gradient)

    assert model.num_classes() == num_classes

def test_eager_model_preprocessing():
    num_classes = 1000
    bounds = (0, 255)
    channels = num_classes

    def mean_brightness_net(images):
        logits = tf.reduce_mean(images, axis=(1, 2))
        return logits

    model = mean_brightness_net

    q = (np.arange(num_classes)[None, None],
         np.random.uniform(size=(5, 5, channels)) + 1)

    model1 = TensorFlowEagerModel(
        model,
        bounds=bounds,
        num_classes=num_classes)

    model2 = TensorFlowEagerModel(
        model,
        bounds=bounds,
        num_classes=num_classes,
        preprocessing=q)

    model3 = TensorFlowEagerModel(
        model,
        bounds=bounds,
        num_classes=num_classes)

    np.random.seed(22)
    test_images = np.random.rand(2, 5, 5, channels).astype(np.float32)
    test_images_copy = test_images.copy()

    p1 = model1.batch_predictions(test_images)
    p2 = model2.batch_predictions(test_images)

    # make sure the images have not been changed by
    # the in-place preprocessing
    assert np.all(test_images == test_images_copy)

    p3 = model3.batch_predictions(test_images)

    assert p1.shape == p2.shape == p3.shape == (2, num_classes)

    np.testing.assert_array_almost_equal(
        p1 - p1.max(),
        p3 - p3.max(),
        decimal=5)

def test_eager_model_gradient():
    num_classes = 1000
    bounds = (0, 255)
    channels = num_classes

    def mean_brightness_net(images):
        logits = tf.reduce_mean(images, axis=(1, 2))
        return logits

    model = mean_brightness_net

    q = (np.arange(num_classes)[None, None],
         np.random.uniform(size=(5, 5, channels)) + 1)

    model = TensorFlowEagerModel(
        model,
        bounds=bounds,
        num_classes=num_classes,
        preprocessing=q)

    epsilon = 1e-2

    np.random.seed(23)
    test_image = np.random.rand(5, 5, channels).astype(np.float32)
    test_label = 7

    _, g1 = model.predictions_and_gradient(test_image, test_label)

    l1 = model._loss_fn(test_image - epsilon / 2 * g1, test_label)
    l2 = model._loss_fn(test_image + epsilon / 2 * g1, test_label)

    assert 1e4 * (l2 - l1) > 1

    # make sure that gradient is numerically correct
    np.testing.assert_array_almost_equal(
        1e4 * (l2 - l1),
        1e4 * epsilon * np.linalg.norm(g1)**2,
        decimal=1)

@pytest.mark.parametrize('num_classes', [10, 1000])
def test_eager_backward(num_classes):
    bounds = (0, 255)
    channels = num_classes

    def mean_brightness_net(images):
        logits = tf.reduce_mean(images, axis=(1, 2))
        return logits

    model = mean_brightness_net
    model = TensorFlowEagerModel(
        model,
        bounds=bounds,
        num_classes=num_classes)

    test_image = np.random.rand(5, 5, channels).astype(np.float32)
    test_grad_pre = np.random.rand(num_classes).astype(np.float32)

    test_grad = model.backward(test_grad_pre, test_image)
    assert test_grad.shape == test_image.shape

    manual_grad = np.repeat(np.repeat(
        (test_grad_pre / 25.).reshape((1, 1, -1)),
        5, axis=0), 5, axis=1)

    np.testing.assert_almost_equal(
        test_grad,
        manual_grad)
<|endoftext|>


<commit_before>
from argparse import ArgumentParser

import socket
import struct
import sys
import threading
import time

from ._fakeds import FakeDS

__all__ = ['Netconsole', 'main', 'run']

def _output_fn(s):
    sys.stdout.write(s.encode(sys.stdout.encoding, errors='replace').decode(sys.stdout.encoding))
    sys.stdout.write('\n')

class StreamEOF(IOError):
    pass

class Netconsole:
    '''
        Implements the 2018+ netconsole protocol
    '''

    TAG_ERROR = 11
    TAG_INFO = 12

    def __init__(self, printfn=_output_fn):

        self.frames = {
            self.TAG_ERROR: self._onError,
            self.TAG_INFO: self._onInfo
        }

        self.cond = threading.Condition()
        self.sock = None
        self.sockrfp = None
        self.sockwfp = None

        self.sockaddr = None
        self.running = False

        self.printfn = printfn

    def start(self, address, port=1741, connect_event=None, block=True):
        with self.cond:
            if self.running:
                raise ValueError("Cannot start without stopping first")

            self.sockaddr = (address, port)
            self.connect_event = connect_event

            self.running = True

        self._rt = threading.Thread(target=self._readThread,
                                    name='nc-read-thread',
                                    daemon=True)
        self._rt.start()

        if block:
            self._keepAlive()
        else:
            self._kt = threading.Thread(target=self._keepAlive,
                                        name='nc-keepalive-thread',
                                        daemon=True)
            self._kt.start()

    @property
    def connected(self):
        return self.sockrfp is not None

    def stop(self):
        with self.cond:
            self.running = False
            self.cond.notifyAll()
            self.sock.close()

    def _connectionDropped(self):
        print(".. connection dropped", file=sys.stderr)
        self.sock.close()

        with self.cond:
            self.sockrfp = None
            self.cond.notifyAll()

    def _keepAliveReady(self):
        if not self.running:
            return -1
        elif not self.connected:
            return -2

    def _keepAlive(self):
        while self.running:
            with self.cond:
                ret = self.cond.wait_for(self._keepAliveReady, timeout=2.0)

            if ret == -1:
                return
            elif ret == -2:
                self._reconnect()
            else:
                try:
                    self.sockwfp.write(b'\x00\x00')
                    self.sockwfp.flush()
                except IOError:
                    self._connectionDropped()

    def _readThreadReady(self):
        if not self.running:
            return -1
        return self.sockrfp

    def _readThread(self):
        while True:
            with self.cond:
                sockrfp = self.cond.wait_for(self._readThreadReady)
                if sockrfp == -1:
                    return

            try:
                data = sockrfp.read(self._headerSz)
            except IOError:
                data = ''

            if len(data) != self._headerSz:
                self._connectionDropped()
                continue

            blen, tag = self._header.unpack(data)
            blen -= 1

            try:
                buf = sockrfp.read(blen)
            except IOError:
                buf = ''

            if len(buf) != blen:
                self._connectionDropped()
                continue

            # process the frame
            fn = self.frames.get(tag)
            if fn:
                fn(buf)
            else:
                print("ERROR: Unknown tag %s; Ignoring..." % tag, file=sys.stderr)

    def _reconnect(self):
        # returns once the socket is connected or an exit is requested

        while self.running:
            sys.stderr.write("Connecting to %s:%s..." % self.sockaddr)

            try:
                sock = socket.create_connection(self.sockaddr, timeout=3.0)
            except IOError:
                sys.stderr.write(" :(\n")
                # don't busywait, just in case
                time.sleep(1.0)
                continue
            else:
                sys.stderr.write("OK\n")

            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)

            sockrfp = sock.makefile('rb')
            sockwfp = sock.makefile('wb')

            if self.connect_event:
                self.connect_event.set()

            with self.cond:
                self.sock = sock
                self.sockrfp = sockrfp
                self.sockwfp = sockwfp
                self.cond.notifyAll()

            break

    #
    # Message
    #

    _header = struct.Struct('>Hb')
    _headerSz = _header.size

    _errorFrame = struct.Struct('>fHHiB')
    _errorFrameSz = _errorFrame.size

    _infoFrame = struct.Struct('>fH')
    _infoFrameSz = _infoFrame.size

    _slen = struct.Struct('>H')
    _slenSz = _slen.size

    def _onError(self, b):
        ts, _seq, _numOcc, errorCode, flags = self._errorFrame.unpack_from(b, 0)
        details, nidx = self._getStr(b, self._errorFrameSz)
        location, nidx = self._getStr(b, nidx)
        callStack, _ = self._getStr(b, nidx)

        self.printfn('[%0.2f] %d %s %s %s' % (ts, errorCode, details, location, callStack))

    def _getStr(self, b, idx):
        sidx = idx + self._slenSz
        blen, = self._slen.unpack_from(b, idx)
        nextidx = sidx + blen
        return b[sidx:nextidx].decode('utf-8', errors='replace'), nextidx

    def _onInfo(self, b):
        ts, _seq = self._infoFrame.unpack_from(b, 0)
        msg = b[self._infoFrameSz:].decode('utf-8', errors='replace')
        self.printfn('[%0.2f] %s' % (ts, msg))

def run(address, connect_event=None, fakeds=False):
    '''
        Starts the netconsole loop. Note that netconsole will only send output
        if the DS is connected. If you don't have a DS available, the 'fakeds'
        flag can be specified to fake a DS connection.

        :param address: Address of the netconsole server
        :param connect_event: a threading.event object, upon which the 'set'
                              function will be called when the connection has
                              succeeded.
        :param fakeds: Fake a driver station connection
    '''

    if fakeds:
        raise NotImplementedError("This will be implemented later in the season; for now, use a real DS")

        ds = FakeDS()
        ds.start(address)

    nc = Netconsole()
    nc.start(address, connect_event=connect_event)

def main():

    parser = ArgumentParser()
    parser.add_argument('address',
                        help='Address of Robot')
    parser.add_argument('-f', '--fakeds', action='store_true', default=False,
                        help='Fake a driver station connection to the robot')

    args = parser.parse_args()

    run(args.address, fakeds=args.fakeds)

<commit_msg>Get rid of socket timeout after connection is made<commit_after>
from argparse import ArgumentParser

import socket
import struct
import sys
import threading
import time

from ._fakeds import FakeDS

__all__ = ['Netconsole', 'main', 'run']

def _output_fn(s):
    sys.stdout.write(s.encode(sys.stdout.encoding, errors='replace').decode(sys.stdout.encoding))
    sys.stdout.write('\n')

class StreamEOF(IOError):
    pass

class Netconsole:
    '''
        Implements the 2018+ netconsole protocol
    '''

    TAG_ERROR = 11
    TAG_INFO = 12

    def __init__(self, printfn=_output_fn):

        self.frames = {
            self.TAG_ERROR: self._onError,
            self.TAG_INFO: self._onInfo
        }

        self.cond = threading.Condition()
        self.sock = None
        self.sockrfp = None
        self.sockwfp = None

        self.sockaddr = None
        self.running = False

        self.printfn = printfn

    def start(self, address, port=1741, connect_event=None, block=True):
        with self.cond:
            if self.running:
                raise ValueError("Cannot start without stopping first")

            self.sockaddr = (address, port)
            self.connect_event = connect_event

            self.running = True

        self._rt = threading.Thread(target=self._readThread,
                                    name='nc-read-thread',
                                    daemon=True)
        self._rt.start()

        if block:
            self._keepAlive()
        else:
            self._kt = threading.Thread(target=self._keepAlive,
                                        name='nc-keepalive-thread',
                                        daemon=True)
            self._kt.start()

    @property
    def connected(self):
        return self.sockrfp is not None

    def stop(self):
        with self.cond:
            self.running = False
            self.cond.notifyAll()
            self.sock.close()

    def _connectionDropped(self):
        print(".. connection dropped", file=sys.stderr)
        self.sock.close()

        with self.cond:
            self.sockrfp = None
            self.cond.notifyAll()

    def _keepAliveReady(self):
        if not self.running:
            return -1
        elif not self.connected:
            return -2

    def _keepAlive(self):
        while self.running:
            with self.cond:
                ret = self.cond.wait_for(self._keepAliveReady, timeout=2.0)

            if ret == -1:
                return
            elif ret == -2:
                self._reconnect()
            else:
                try:
                    self.sockwfp.write(b'\x00\x00')
                    self.sockwfp.flush()
                except IOError:
                    self._connectionDropped()

    def _readThreadReady(self):
        if not self.running:
            return -1
        return self.sockrfp

    def _readThread(self):
        while True:
            with self.cond:
                sockrfp = self.cond.wait_for(self._readThreadReady)
                if sockrfp == -1:
                    return

            try:
                data = sockrfp.read(self._headerSz)
            except IOError:
                data = ''

            if len(data) != self._headerSz:
                self._connectionDropped()
                continue

            blen, tag = self._header.unpack(data)
            blen -= 1

            try:
                buf = sockrfp.read(blen)
            except IOError:
                buf = ''

            if len(buf) != blen:
                self._connectionDropped()
                continue

            # process the frame
            fn = self.frames.get(tag)
            if fn:
                fn(buf)
            else:
                print("ERROR: Unknown tag %s; Ignoring..." % tag, file=sys.stderr)

    def _reconnect(self):
        # returns once the socket is connected or an exit is requested

        while self.running:
            sys.stderr.write("Connecting to %s:%s..." % self.sockaddr)

            try:
                sock = socket.create_connection(self.sockaddr, timeout=3.0)
            except IOError:
                sys.stderr.write(" :(\n")
                # don't busywait, just in case
                time.sleep(1.0)
                continue
            else:
                sys.stderr.write("OK\n")

            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
            sock.settimeout(None)

            sockrfp = sock.makefile('rb')
            sockwfp = sock.makefile('wb')

            if self.connect_event:
                self.connect_event.set()

            with self.cond:
                self.sock = sock
                self.sockrfp = sockrfp
                self.sockwfp = sockwfp
                self.cond.notifyAll()

            break

    #
    # Message
    #

    _header = struct.Struct('>Hb')
    _headerSz = _header.size

    _errorFrame = struct.Struct('>fHHiB')
    _errorFrameSz = _errorFrame.size

    _infoFrame = struct.Struct('>fH')
    _infoFrameSz = _infoFrame.size

    _slen = struct.Struct('>H')
    _slenSz = _slen.size

    def _onError(self, b):
        ts, _seq, _numOcc, errorCode, flags = self._errorFrame.unpack_from(b, 0)
        details, nidx = self._getStr(b, self._errorFrameSz)
        location, nidx = self._getStr(b, nidx)
        callStack, _ = self._getStr(b, nidx)

        self.printfn('[%0.2f] %d %s %s %s' % (ts, errorCode, details, location, callStack))

    def _getStr(self, b, idx):
        sidx = idx + self._slenSz
        blen, = self._slen.unpack_from(b, idx)
        nextidx = sidx + blen
        return b[sidx:nextidx].decode('utf-8', errors='replace'), nextidx

    def _onInfo(self, b):
        ts, _seq = self._infoFrame.unpack_from(b, 0)
        msg = b[self._infoFrameSz:].decode('utf-8', errors='replace')
        self.printfn('[%0.2f] %s' % (ts, msg))

def run(address, connect_event=None, fakeds=False):
    '''
        Starts the netconsole loop. Note that netconsole will only send output
        if the DS is connected. If you don't have a DS available, the 'fakeds'
        flag can be specified to fake a DS connection.

        :param address: Address of the netconsole server
        :param connect_event: a threading.event object, upon which the 'set'
                              function will be called when the connection has
                              succeeded.
        :param fakeds: Fake a driver station connection
    '''

    if fakeds:
        raise NotImplementedError("This will be implemented later in the season; for now, use a real DS")

        ds = FakeDS()
        ds.start(address)

    nc = Netconsole()
    nc.start(address, connect_event=connect_event)

def main():

    parser = ArgumentParser()
    parser.add_argument('address',
                        help='Address of Robot')
    parser.add_argument('-f', '--fakeds', action='store_true', default=False,
                        help='Fake a driver station connection to the robot')

    args = parser.parse_args()

    run(args.address, fakeds=args.fakeds)

<|endoftext|>


<commit_before>import os
from framework.Targets import ApacheTarget

class Target(ApacheTarget):

    def get_path(filename):
        return os.path.dirname(os.path.realpath(__file__)) + '/' + filename

    name = "Wordpress 2.8.3"
    application_dir_mapping = [get_path("application"), "/var/www"]
    database_filename = get_path("database.sql")
    database_name = "wordpress_2_8_3"
    plugins = [["Relevanssi 2.7.2",
                get_path("plugins/relevanssi_2_7_2"),
                get_path("plugins/relevanssi_2_7_2/database.sql"),
                "var/www/wordpress/wp-content/plugins/relevanssi_2_7_2"]]
    chroot_environment = "Debian7"
<commit_msg>relevanssi plugin<commit_after>import os
from framework.Targets import ApacheTarget

class Target(ApacheTarget):

    def get_path(filename):
        return os.path.dirname(os.path.realpath(__file__)) + '/' + filename

    name = "Wordpress 2.8.3"
    application_dir_mapping = [get_path("application"), "/var/www"]
    database_filename = get_path("database.sql")
    database_name = "wordpress_2_8_3"
    plugins = [["Relevanssi 2.7.2",
                get_path("plugins/relevanssi_2_7_2"),
                get_path("plugins/relevanssi_2_7_2/database.sql"),
                "var/www/wordpress/wp-content/plugins/relevanssi"]]
    chroot_environment = "Debian7"
<|endoftext|>


<commit_before>import requests
# import hashlib
import json

class HttpRequest(object):
    """通过抓取客户端的包得到的API，相对较旧，新版客户端使用POST发包并有所加密，不知道加密形式无法进行请求。
       2015年时候的API。函数名居然aBbbC形式与a_bbb_c形式同时写在一个类里。

    """
    # 使用keep-alive，
    # keep-alive保持持久连接，没有必要开启很多个TCP链接，浪费资源。
    # 使用会话(session)来保持持久连接。
    sessions = requests.session()

    default_timeout = 10
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Connection': 'keep-alive',
        'Pragma': 'no-cache',
        'Cache-Control': 'no-cache',
        'Accept-Encoding': 'gzip,deflate,sdch',
        'Accept-Language': 'zh-CN,zh;q=0.8',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36'
    }

    cookies = {}

    def httpRequest(self, action, method="GET", add=None, data=None, headers=None, cookies='',\
                    timeout=default_timeout, urlencode='utf-8'):
        """
            默认以get方式请求，
            GET方式附加内容用add参数，POST方式提交内容用data参数。
            编码用urlencode参数，默认utf-8。
            GET方式返回json形式请求的内容。
            POST方式返回cookies和json形式的内容。(0,1)
            默认cookies为空。
        """
        if not headers:
            headers = self.headers

        if method.upper() == 'GET':
            if add:
                html = self.sessions.get(action, params=add, headers=headers, cookies=cookies, timeout=timeout)
            else:
                html = self.sessions.get(action, headers=headers, cookies=cookies, timeout=timeout)
            html.encoding = urlencode
            return json.loads(html.text)
        elif method.upper() == 'POST':
            if data:
                html = self.sessions.post(action, data=data, headers=headers, cookies=cookies, timeout=timeout)
            else:
                html = self.sessions.post(action, headers=headers, cookies=cookies, timeout=timeout)
            html.encoding = urlencode
            return html.cookies, json.loads(html.text)<commit_msg>del some files<commit_after><|endoftext|>


<commit_before>#encoding=utf8

from __future__ import division
from abc import ABCMeta, abstractmethod
import numpy as np
import FukuML.LinearRegression as linear_regression

class BlendingClassifier(object):

    def add_model(self, model):

        self.models.append(model)

        return self.models

    @abstractmethod
    def prediction(self, input_data='', mode='test_data'):
        return

    def calculate_avg_error(self, input_data_file=''):

        data_num = 0
        error_num = 0
        avg_error = 0

        with open(input_data_file) as f:
            for line in f:
                data_num = data_num+1
                data = line.split()
                answer = data[-1]
                prediction = self.prediction(line)
                if float(prediction['prediction']) != float(answer):
                    error_num = error_num+1

        avg_error = float(error_num/data_num)

        return avg_error

class UniformBlendingClassifier(BlendingClassifier):

    def __init__(self):

        self.models = []

    def add_model(self, model):

        return super(UniformBlendingClassifier, self).add_model(model)

    def prediction(self, input_data='', mode='test_data'):

        prediction = {}
        vote = []

        for model in self.models:

            prediction = model.prediction(input_data, mode)
            vote.append(prediction['prediction'])

        prediction_return = max(set(vote), key=vote.count)

        if mode == 'future_data':
            data = input_data.split()
            input_data_x = [float(v) for v in data]
            input_data_x = np.ravel(input_data_x)
            return {"input_data_x": input_data_x, "input_data_y": None, "prediction": prediction_return}
        else:
            data = input_data.split()
            input_data_x = [float(v) for v in data[:-1]]
            input_data_x = np.ravel(input_data_x)
            input_data_y = float(data[-1])
            return {"input_data_x": input_data_x, "input_data_y": input_data_y, "prediction": prediction_return}

    def calculate_avg_error(self, input_data_file=''):

        return super(UniformBlendingClassifier, self).calculate_avg_error(input_data_file)

class LinearBlendingClassifier(BlendingClassifier):

    def __init__(self):

        self.models = []
        self.linear_mc = None

    def add_model(self, model):

        return super(LinearBlendingClassifier, self).add_model(model)

    def feature_transform(self, x_string):

        transform_x = []

        for model in self.models:
            prediction = model.prediction(x_string, 'future_data')
            transform_x.append(prediction['prediction'])

        return transform_x

    def train(self):

        predict_X = []

        for x in self.models[0].train_X:
            x_string = np.array(map(str, x))
            x_string = ' '.join(x_string[1:])
            transform_x = self.feature_transform(x_string)
            transform_x.insert(0, 1)
            predict_X.append(transform_x)

        self.linear_mc = linear_regression.MultiClassifier()
        self.linear_mc.load_train_data()
        self.linear_mc.load_test_data()
        self.linear_mc.train_X = np.array(predict_X)
        self.linear_mc.train_Y = self.models[0].train_Y
        self.linear_mc.set_param()
        self.linear_mc.init_W()
        self.linear_mc.train()

    def prediction(self, input_data='', mode='test_data'):

        transform_input_data = ''

        data = input_data.split()
        x_string = ' '.join(data[0:-1])
        transform_input_data = self.feature_transform(x_string)

        transform_input_data_string = np.array(map(str, transform_input_data))
        transform_input_data_string = ' '.join(transform_input_data_string[:])

        if mode == 'test_data':
            transform_input_data_string = transform_input_data_string + ' ' + data[-1]

        prediction = self.linear_mc.prediction(transform_input_data_string, mode)

        return prediction

    def calculate_avg_error(self, input_data_file=''):

        return super(LinearBlendingClassifier, self).calculate_avg_error(input_data_file)
<commit_msg>fix python3 bug<commit_after>#encoding=utf8

from __future__ import division
from abc import ABCMeta, abstractmethod
import numpy as np
import FukuML.LinearRegression as linear_regression

class BlendingClassifier(object):

    def add_model(self, model):

        self.models.append(model)

        return self.models

    @abstractmethod
    def prediction(self, input_data='', mode='test_data'):
        return

    def calculate_avg_error(self, input_data_file=''):

        data_num = 0
        error_num = 0
        avg_error = 0

        with open(input_data_file) as f:
            for line in f:
                data_num = data_num+1
                data = line.split()
                answer = data[-1]
                prediction = self.prediction(line)
                if float(prediction['prediction']) != float(answer):
                    error_num = error_num+1

        avg_error = float(error_num/data_num)

        return avg_error

class UniformBlendingClassifier(BlendingClassifier):

    def __init__(self):

        self.models = []

    def add_model(self, model):

        return super(UniformBlendingClassifier, self).add_model(model)

    def prediction(self, input_data='', mode='test_data'):

        prediction = {}
        vote = []

        for model in self.models:

            prediction = model.prediction(input_data, mode)
            vote.append(prediction['prediction'])

        prediction_return = max(set(vote), key=vote.count)

        if mode == 'future_data':
            data = input_data.split()
            input_data_x = [float(v) for v in data]
            input_data_x = np.ravel(input_data_x)
            return {"input_data_x": input_data_x, "input_data_y": None, "prediction": prediction_return}
        else:
            data = input_data.split()
            input_data_x = [float(v) for v in data[:-1]]
            input_data_x = np.ravel(input_data_x)
            input_data_y = float(data[-1])
            return {"input_data_x": input_data_x, "input_data_y": input_data_y, "prediction": prediction_return}

    def calculate_avg_error(self, input_data_file=''):

        return super(UniformBlendingClassifier, self).calculate_avg_error(input_data_file)

class LinearBlendingClassifier(BlendingClassifier):

    def __init__(self):

        self.models = []
        self.linear_mc = None

    def add_model(self, model):

        return super(LinearBlendingClassifier, self).add_model(model)

    def feature_transform(self, x_string):

        transform_x = []

        for model in self.models:
            prediction = model.prediction(x_string, 'future_data')
            transform_x.append(prediction['prediction'])

        return transform_x

    def train(self):

        predict_X = []

        for x in self.models[0].train_X:
            x_string = ' '.join(map(str, x.tolist()[1:]))
            transform_x = self.feature_transform(x_string)
            transform_x.insert(0, 1)
            predict_X.append(transform_x)

        self.linear_mc = linear_regression.MultiClassifier()
        self.linear_mc.load_train_data()
        self.linear_mc.load_test_data()
        self.linear_mc.train_X = np.array(predict_X)
        self.linear_mc.train_Y = self.models[0].train_Y
        self.linear_mc.set_param()
        self.linear_mc.init_W()
        self.linear_mc.train()

    def prediction(self, input_data='', mode='test_data'):

        transform_input_data = ''

        data = input_data.split()
        x_string = ' '.join(data[0:-1])
        transform_input_data = self.feature_transform(x_string)
        transform_input_data_string = ' '.join(map(str, transform_input_data))

        if mode == 'test_data':
            transform_input_data_string = transform_input_data_string + ' ' + data[-1]

        prediction = self.linear_mc.prediction(transform_input_data_string, mode)

        return prediction

    def calculate_avg_error(self, input_data_file=''):

        return super(LinearBlendingClassifier, self).calculate_avg_error(input_data_file)
<|endoftext|>


<commit_before>from IRCMessage import IRCMessage
from IRCResponse import IRCResponse, ResponseType
from Function import Function
import WebUtils

import re

class Instantiate(Function):
    Help = "googl/shorten <url> - Gives you a shortened version of a url, via Goo.gl"

    def GetResponse(self, message):
        if message.Type != 'PRIVMSG':
            return

        match = re.search('(goo\.?gl|shorten)', message.Command, re.IGNORECASE)
        if not match:
            return

        if len(message.ParameterList) == 0:
            return IRCResponse(ResponseType.Say, "You didn't give a URL to shorten!", message.ReplyTo)

        url = WebUtils.ShortenGoogl(message.Parameters)

        return IRCResponse(ResponseType.Say, url, message.ReplyTo)

<commit_msg>Fix Googl call pattern<commit_after>from IRCMessage import IRCMessage
from IRCResponse import IRCResponse, ResponseType
from Function import Function
import WebUtils

import re

class Instantiate(Function):
    Help = "googl/shorten <url> - Gives you a shortened version of a url, via Goo.gl"

    def GetResponse(self, message):
        if message.Type != 'PRIVMSG':
            return

        match = re.search('^(goo\.?gl|shorten)$', message.Command, re.IGNORECASE)
        if not match:
            return

        if len(message.ParameterList) == 0:
            return IRCResponse(ResponseType.Say, "You didn't give a URL to shorten!", message.ReplyTo)

        url = WebUtils.ShortenGoogl(message.Parameters)

        return IRCResponse(ResponseType.Say, url, message.ReplyTo)

<|endoftext|>


<commit_before>########################################
# Mako template handlers
########################################

import os

from mako import util, lexer
from mako.lookup import TemplateLookup

lookup = None

def get_deps(filename):
    text = util.read_file(filename)
    lex = lexer.Lexer(text=text, filename=filename)
    lex.parse()

    deps = []
    for n in lex.template.nodes:
        if getattr(n, 'keyword', None) == "inherit":
            deps.append(n.attributes['file'])
    return deps

def get_template_lookup(directories):
    return TemplateLookup(
        directories=directories,
        module_directory='tmp',
        output_encoding='utf-8',
        )

def render_template(template_name, output_name, context, global_context):
    template = lookup.get_template(template_name)
    context.update(global_context)
    try:
        os.makedirs(os.path.dirname(output_name))
    except:
        pass
    with open(output_name, 'w+') as output:
        output.write(template.render(**context))

def template_deps(template_name):
    template = lookup.get_template(template_name)
    dep_filenames = get_deps(template.filename)
    deps = [template.filename]
    for fname in dep_filenames:
        deps += template_deps(fname)
    return deps
<commit_msg>added comment about missing functionality<commit_after>########################################
# Mako template handlers
########################################

import os

from mako import util, lexer
from mako.lookup import TemplateLookup

lookup = None

def get_deps(filename):
    text = util.read_file(filename)
    lex = lexer.Lexer(text=text, filename=filename)
    lex.parse()

    deps = []
    for n in lex.template.nodes:
        if getattr(n, 'keyword', None) == "inherit":
            deps.append(n.attributes['file'])
        # TODO: include tags are not handled
    return deps

def get_template_lookup(directories):
    return TemplateLookup(
        directories=directories,
        module_directory='tmp',
        output_encoding='utf-8',
        )

def render_template(template_name, output_name, context, global_context):
    template = lookup.get_template(template_name)
    context.update(global_context)
    try:
        os.makedirs(os.path.dirname(output_name))
    except:
        pass
    with open(output_name, 'w+') as output:
        output.write(template.render(**context))

def template_deps(template_name):
    template = lookup.get_template(template_name)
    dep_filenames = get_deps(template.filename)
    deps = [template.filename]
    for fname in dep_filenames:
        deps += template_deps(fname)
    return deps
<|endoftext|>


<commit_before>import csv
import math
import os
from matplotlib.pyplot import show, ion, savefig
import pandas
from pandas.util.testing import DataFrame
from util import check_dir_exists
import numpy as np

class PlotOutput:

    @staticmethod
    def plot_output(name, infile_path, model_names, filter, export_pdf):
        graphs = {}
        graphs['SSE'] = {}
        graphs['NLPD'] = {}
        for m in model_names:
            data_config = PlotOutput.read_config(infile_path + m + '/' + 'config_' + '.csv')
            if filter is None or filter(data_config):
                data_test = pandas.read_csv(infile_path + m + '/' + 'test_' +  '.csv')
                data_train = pandas.read_csv(infile_path + m + '/' + 'train_' + '.csv')
                Y_mean = data_train['Y0'].mean()
                Ypred = data_test['Ypred__0']
                Ytrue = data_test['Ytrue0']
                Yvar = data_test['Yvar_pred__0']

                if data_config['ll'] in ['UnivariateGaussian']:
                    graphs['SSE'][str(data_config)] = (Ypred - Ytrue)**2 / ((Y_mean - Ytrue) **2).mean()
                    graphs['NLPD'][str(data_config)] = 0.5*(Ytrue-Ypred) ** 2./Yvar+np.log(2*math.pi*Yvar)

        for n, g in graphs.iteritems():
            g= DataFrame(g)
            ion()
            ax = g.plot(kind='box', title=n)
            if export_pdf:
                check_dir_exists(infile_path + name + '/graphs/')
                savefig(infile_path + name + '/graphs/'+'n' + '.pdf')
            show(block=True)

    @staticmethod
    def plot_output_all(name, path, filter, export_pdf):
        dir = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))]
        PlotOutput.plot_output(name, path, dir, filter, export_pdf)

    @staticmethod
    def read_config(path):
        with open(path) as csvfile:
            reader = csv.DictReader(csvfile)
            return reader.next()

if __name__ == '__main__':
    PlotOutput.plot_output('boston', '../../results/', ['boston_full', 'boston_mix1', 'boston_mix2', 'boston_gp'], None, True)

<commit_msg>added plot for CCR<commit_after>import csv
import math
import os
from matplotlib.pyplot import show, ion, savefig
import pandas
from pandas.util.testing import DataFrame
from util import check_dir_exists
import numpy as np

class PlotOutput:

    @staticmethod
    def plot_output(name, infile_path, model_names, filter, export_pdf):
        graphs = {}
        graphs['SSE'] = {}
        graphs['NLPD'] = {}
        graphs['CCR'] = {}
        for m in model_names:
            data_config = PlotOutput.read_config(infile_path + m + '/' + 'config_' + '.csv')
            if filter is None or filter(data_config):
                data_test = pandas.read_csv(infile_path + m + '/' + 'test_' +  '.csv')
                data_train = pandas.read_csv(infile_path + m + '/' + 'train_' + '.csv')
                Y_mean = data_train['Y0'].mean()
                Ypred = data_test['Ypred__0']
                Ytrue = data_test['Ytrue0']
                Yvar = data_test['Yvar_pred__0']

                if data_config['ll'] in ['UnivariateGaussian']:
                    graphs['SSE'][str(data_config)] = (Ypred - Ytrue)**2 / ((Y_mean - Ytrue) **2).mean()
                    graphs['NLPD'][str(data_config)] = 0.5*(Ytrue-Ypred) ** 2./Yvar+np.log(2*math.pi*Yvar)

                if data_config['ll'] in ['LogisticLL']:
                    graphs['CCR'][str(data_config)] = 1. * ((Ypred > 0.5) & (Ytrue == 1))
                    graphs['NLPD'][str(data_config)] = 0.5*(Ytrue-Ypred) ** 2./Yvar+np.log(2*math.pi*Yvar)

        for n, g in graphs.iteritems():
            if g:
                g= DataFrame(g)
                ion()
                ax = g.plot(kind='box', title=n)
                if export_pdf:
                    check_dir_exists(infile_path + name + '/graphs/')
                    savefig(infile_path + name + '/graphs/'+'n' + '.pdf')
                show(block=True)

    @staticmethod
    def plot_output_all(name, path, filter, export_pdf):
        dir = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))]
        PlotOutput.plot_output(name, path, dir, filter, export_pdf)

    @staticmethod
    def read_config(path):
        with open(path) as csvfile:
            reader = csv.DictReader(csvfile)
            return reader.next()

if __name__ == '__main__':
    PlotOutput.plot_output('boston', '../../results/', ['boston_full', 'boston_mix1', 'boston_mix2', 'boston_gp'], None, True)

<|endoftext|>