<commit_before>import serial
import numpy as np
from scipy.signal import butter, lfilter
from collections import deque
from Logger import Logger

class QRSDetector(object):
    """QRS complex detector."""

    def __init__(self, port, baud_rate):
        """Variables initialization and start reading ECG measurements."""

        # Parameters.
        self.cycling_window = 200  # samples

        # Detection description.
        self.detected_beat_indicator = 0
        self.qrs_interval = 0  # samples

        # Received the most recent measurement.
        self.timestamp = 0.0
        self.measurement = None

        # Most recent measurements array.
        self.most_recent_measurements = deque([], self.cycling_window)

        # Run the detector.
        self.connect_to_ecg(port=port, baud_rate=baud_rate)

        # General params.
        self.signal_freq = 255  # signal frequency

        # Filtering parameters.
        self.filter_lowcut = 0.0  # band pass filter low cut value
        self.filter_highcut = 15.0  # band pass filter high cut value

        # Integration window.
        self.integration_window = 15  # signal integration window length in samples

        # Realtime params.
        self.refractory_period = 120  # samples
        self.buffer_detection_window = 40  # samples

        self.peak_timestamp = 100000000000  # seconds
        self.last_peak_timestamp = 0

        # Signal processing variables.
        self.filtered = np.array([])
        self.differentiated_signal = np.array([])
        self.squared_signal = np.array([])
        self.integrated_signal = np.array([])

        # Integrated signal detection and thresholding params.
        self.spk_i = 0.0
        self.npk_i = 0.0
        self.threshold_i = 0.0
        self.spk_i_measurement_weight = 0.125
        self.npk_i_measurement_weight = 0.125
        self.threshold_i_diff_weight = 0.25

        # Peak thresholding variables.
        self.qrs_peak_i = np.array([])
        self.noise_peak_i = np.array([])

        # Data logger set up.
        self.logger = Logger("QRS", " ", "timestamp", "ecg", "beat_detected", "ibi")

    # ECG interfacing methods.
    def connect_to_ecg(self, port, baud_rate):
        try:
            serial_port = serial.Serial(port, baud_rate)
            print("Connected! Starting reading ECG measurements.")
        except:
            print("Cannot connect to provided port!")
            return

        while True:
            raw_measurement = serial_port.readline()
            self.process_measurement(raw_measurement=raw_measurement)

    # Data processing methods.
    def process_measurement(self, raw_measurement):
        """Parsing raw data line."""

        # TODO: Refactor detected_beat_indicator - move it from here.
        self.detected_beat_indicator = 0

        # TODO: Add encoding for Python 3.
        raw_measurement_split = raw_measurement.rstrip().split(';')

        # TODO: Change it to == 2 - need to test it with real data.
        if len(raw_measurement_split) < 2:
            return
        try:
            timestamp = float(raw_measurement_split[0])
            measurement = float(raw_measurement_split[1])
        except Exception:
            return

        # Not physiologically possible ECG error measurements rejection.
        if measurement > 10:
            return

        # TODO: Check whether it can be done differently or move somewhere.
        self.qrs_interval += 1

        # TODO: Remove it - it is used only for logging. Find different way.
        self.timestamp = timestamp
        self.measurement = measurement

        self.most_recent_measurements.append(self.measurement)

        if len(self.most_recent_measurements) == 1:
            return

        self.extract_peaks()

    def extract_peaks(self):
        """Proceses received data."""
        # Signal filtering - band pass 0-15 Hz.
        self.filtered_signal = self.bandpass_filter(self.most_recent_measurements, lowcut=self.filter_lowcut,
                                                    highcut=self.filter_highcut, signal_freq=self.signal_freq,
                                                    filter_order=1)

        # Derivative - provides QRS slope info.
        self.differentiated_signal = np.ediff1d(self.filtered_signal)

        # Squaring.
        self.squared_signal = self.differentiated_signal ** 2

        # Moving-window integration.
        self.integrated_signal = np.convolve(self.squared_signal, np.ones(self.integration_window))

        # Fiducial mark - peak detection - integrated signal.
        self.peaks_indices = self.findpeaks(self.integrated_signal, limit=0.40, spacing=50)

        ## Peak detection variables.
        self.fiducial_mark_idx_i = np.array([])
        self.fiducial_mark_val_i = np.array([])

        for peak_index in self.peaks_indices:
            self.fiducial_mark_idx_i = np.append(self.fiducial_mark_idx_i, peak_index)
            self.fiducial_mark_val_i = np.append(self.fiducial_mark_val_i, self.integrated_signal[peak_index])

        self.detect_qrs()

    # Detection methods.
    def detect_qrs(self):
        """Thresholding detected peaks - integrated - signal."""
        # Check whether refractory period has passed.
        # After a valid QRS complex detection, there is a 200 ms refractory period before the next one can be detected.
        if self.qrs_interval > self.refractory_period:

            # Check whether any peak was detected in analysed samples window.
            if len(self.fiducial_mark_idx_i) > 0:

                # Take the last one detected in analysed samples window.
                peak_idx_i, peak_val_i = self.fiducial_mark_idx_i[-1], self.fiducial_mark_val_i[-1]

                # Check whether detected peak occured within defined window from end of samples buffer - making sure that the same peak will not be detected twice.
                if peak_idx_i > self.cycling_window - self.buffer_detection_window:

                    # Peak must be classified as a noise peak or a signal peak. To be a signal peak it must exceed threshold_i_1.
                    if peak_val_i > self.threshold_i:
                        self.detected_beat_indicator = 1
                        self.qrs_interval = 0
                        self.handle_detection()
                        self.logger.log(str(self.timestamp), str(self.measurement), str(self.detected_beat_indicator),
                                        str(self.timestamp - self.peak_timestamp))
                        self.peak_timestamp = self.timestamp

                        self.spk_i = self.spk_i_measurement_weight * peak_val_i + (
                                                                                1 - self.spk_i_measurement_weight) * self.spk_i
                        self.qrs_peak_i = np.append(self.qrs_peak_i, peak_idx_i)
                    else:
                        print "NOISE detected!"
                        self.npk_i = self.npk_i_measurement_weight * peak_val_i + (
                                                                                1 - self.npk_i_measurement_weight) * self.npk_i
                        self.noise_peak_i = np.append(self.noise_peak_i, peak_idx_i)
                        self.logger.log(str(self.timestamp), str(self.measurement), str(self.detected_beat_indicator),
                                        str(self.timestamp - self.peak_timestamp))

                    self.threshold_i = self.npk_i + self.threshold_i_diff_weight * (self.spk_i - self.npk_i)

                else:
                    self.logger.log(str(self.timestamp), str(self.measurement), str(self.detected_beat_indicator),
                                    str(self.timestamp - self.peak_timestamp))

            else:
                self.logger.log(str(self.timestamp), str(self.measurement), str(self.detected_beat_indicator),
                                str(self.timestamp - self.peak_timestamp))

        else:
            self.logger.log(str(self.timestamp), str(self.measurement), str(self.detected_beat_indicator),
                            str(self.timestamp - self.peak_timestamp))

    def handle_detection(self):
        print "Pulse"
        # with open("flag.txt", "w") as fout:
        #     fout.write("%s %s %s %s" % (str(self.timestamp), str(self.measurement), str(self.detected_beat_indicator),
        #                                 str(self.timestamp - self.peak_timestamp)))

    # Tools methods.
    def bandpass_filter(self, data, lowcut, highcut, signal_freq, filter_order):
        """Constructs signal filter and uses it to given dataset."""
        nyquist_freq = 0.5 * signal_freq
        low = lowcut / nyquist_freq
        high = highcut / nyquist_freq
        b, a = butter(filter_order, [low, high], btype="band")
        y = lfilter(b, a, data)
        return y

    # Janko SLavic peak detection algorithm and implementation.
    # https://github.com/jankoslavic/py-tools/tree/master/findpeaks
    def findpeaks(self, data, spacing=1, limit=None):
        """Finds peaks in `data` which are of `spacing` width and >=`limit`.
        :param data: values
        :param spacing: minimum spacing to the next peak (should be 1 or more)
        :param limit: peaks should have value greater or equal
        :return:
        """
        len = data.size
        x = np.zeros(len + 2 * spacing)
        x[:spacing] = data[0] - 1.e-6
        x[-spacing:] = data[-1] - 1.e-6
        x[spacing:spacing + len] = data
        peak_candidate = np.zeros(len)
        peak_candidate[:] = True
        for s in range(spacing):
            start = spacing - s - 1
            h_b = x[start: start + len]  # before
            start = spacing
            h_c = x[start: start + len]  # central
            start = spacing + s + 1
            h_a = x[start: start + len]  # after
            peak_candidate = np.logical_and(peak_candidate, np.logical_and(h_c > h_b, h_c > h_a))

        ind = np.argwhere(peak_candidate)
        ind = ind.reshape(ind.size)
        if limit is not None:
            ind = ind[data[ind] > limit]
        return ind

if __name__ == "__main__":
    qrs_detector = QRSDetector(port="/dev/cu.usbmodem1411", baud_rate="115200")
    qrs_detector.connect_to_ecg()
<commit_msg>Refactored extract_peaks.<commit_after>import serial
import numpy as np
from scipy.signal import butter, lfilter
from collections import deque
from Logger import Logger

class QRSDetector(object):
    """QRS complex detector."""

    def __init__(self, port, baud_rate):
        """Variables initialization and start reading ECG measurements."""

        # Parameters.
        self.cycling_window = 200  # samples
        self.filter_lowcut = 0.0  # band pass filter low cut value
        self.filter_highcut = 15.0  # band pass filter high cut value
        self.signal_freq = 255  # signal frequency
        self.integration_window = 15  # signal integration window length in samples

        # Detection description.
        self.detected_beat_indicator = 0
        self.qrs_interval = 0  # samples

        # Received the most recent measurement.
        self.timestamp = 0.0
        self.measurement = None

        # Most recent measurements array.
        self.most_recent_measurements = deque([], self.cycling_window)

        # Run the detector.
        self.connect_to_ecg(port=port, baud_rate=baud_rate)

        # Realtime params.
        self.refractory_period = 120  # samples
        self.buffer_detection_window = 40  # samples

        self.peak_timestamp = 100000000000  # seconds
        self.last_peak_timestamp = 0

        # Integrated signal detection and thresholding params.
        self.spk_i = 0.0
        self.npk_i = 0.0
        self.threshold_i = 0.0
        self.spk_i_measurement_weight = 0.125
        self.npk_i_measurement_weight = 0.125
        self.threshold_i_diff_weight = 0.25

        # Peak thresholding variables.
        self.qrs_peak_i = np.array([])
        self.noise_peak_i = np.array([])

        # Data logger set up.
        self.logger = Logger("QRS", " ", "timestamp", "ecg", "beat_detected", "ibi")

    # ECG interfacing methods.
    def connect_to_ecg(self, port, baud_rate):
        try:
            serial_port = serial.Serial(port, baud_rate)
            print("Connected! Starting reading ECG measurements.")
        except:
            print("Cannot connect to provided port!")
            return

        while True:
            raw_measurement = serial_port.readline()
            self.process_measurement(raw_measurement=raw_measurement)

    # Data processing methods.
    def process_measurement(self, raw_measurement):
        """Parsing raw data line."""

        # TODO: Refactor detected_beat_indicator - move it from here.
        self.detected_beat_indicator = 0

        # TODO: Add encoding for Python 3.
        raw_measurement_split = raw_measurement.rstrip().split(';')

        # TODO: Change it to == 2 - need to test it with real data.
        if len(raw_measurement_split) < 2:
            return
        try:
            timestamp = float(raw_measurement_split[0])
            measurement = float(raw_measurement_split[1])
        except Exception:
            return

        # Not physiologically possible ECG error measurements rejection.
        if measurement > 10:
            return

        # TODO: Check whether it can be done differently or move somewhere.
        self.qrs_interval += 1

        # TODO: Remove it - it is used only for logging. Find different way.
        self.timestamp = timestamp
        self.measurement = measurement

        self.most_recent_measurements.append(self.measurement)

        if len(self.most_recent_measurements) == 1:
            return

        self.extract_peaks()

    def extract_peaks(self):
        """Proceses received data."""

        # TODO: All parameters to go to init.
        # Signal filtering - band pass 0-15 Hz.
        filtered_signal = self.bandpass_filter(self.most_recent_measurements, lowcut=self.filter_lowcut,
                                                    highcut=self.filter_highcut, signal_freq=self.signal_freq,
                                                    filter_order=1)

        # Derivative - provides QRS slope info.
        differentiated_signal = np.ediff1d(filtered_signal)

        # Squaring.
        squared_signal = differentiated_signal ** 2

        # Moving-window integration.
        integrated_signal = np.convolve(squared_signal, np.ones(self.integration_window))

        # Fiducial mark - peak detection - integrated signal.
        # TODO: All parameters to go to init.
        detected_peaks_indices = self.findpeaks(integrated_signal, limit=0.40, spacing=50)

        # TODO: Check whether detected_peaks_indices == fiducial_mark_idx_i - if yes use fiducial_mark_idx_i to create np.array()
        # TODO: Refactor fiducial_mark_idx_i to detected_peaks_indices and detected_peaks_values.
        # TODO: Refactor it to one 2D array with idx and vals in one array.
        fiducial_mark_idx_i = np.array([])
        fiducial_mark_val_i = np.array([])
        for detected_peak_index in detected_peaks_indices:
            fiducial_mark_idx_i = np.append(fiducial_mark_idx_i, detected_peak_index)
            fiducial_mark_val_i = np.append(fiducial_mark_val_i, integrated_signal[detected_peak_index])

        self.detect_qrs(detected_peaks_indices=fiducial_mark_idx_i, detected_peaks_values=fiducial_mark_val_i)

    # TODO: Refactor detection.
    # Detection methods.
    def detect_qrs(self, detected_peaks_indices, detected_peaks_values):
        """Thresholding detected peaks - integrated - signal."""
        # Check whether refractory period has passed.
        # After a valid QRS complex detection, there is a 200 ms refractory period before the next one can be detected.
        if self.qrs_interval > self.refractory_period:

            # Check whether any peak was detected in analysed samples window.
            if len(detected_peaks_indices) > 0:

                # Take the last one detected in analysed samples window.
                peak_idx_i, peak_val_i = detected_peaks_indices[-1], detected_peaks_values[-1]

                # Check whether detected peak occured within defined window from end of samples buffer - making sure that the same peak will not be detected twice.
                if peak_idx_i > self.cycling_window - self.buffer_detection_window:

                    # Peak must be classified as a noise peak or a signal peak. To be a signal peak it must exceed threshold_i_1.
                    if peak_val_i > self.threshold_i:
                        self.detected_beat_indicator = 1
                        self.qrs_interval = 0
                        self.handle_detection()
                        self.logger.log(str(self.timestamp), str(self.measurement), str(self.detected_beat_indicator),
                                        str(self.timestamp - self.peak_timestamp))
                        self.peak_timestamp = self.timestamp

                        self.spk_i = self.spk_i_measurement_weight * peak_val_i + (
                                                                                1 - self.spk_i_measurement_weight) * self.spk_i
                        self.qrs_peak_i = np.append(self.qrs_peak_i, peak_idx_i)
                    else:
                        print "NOISE detected!"
                        self.npk_i = self.npk_i_measurement_weight * peak_val_i + (
                                                                                1 - self.npk_i_measurement_weight) * self.npk_i
                        self.noise_peak_i = np.append(self.noise_peak_i, peak_idx_i)
                        self.logger.log(str(self.timestamp), str(self.measurement), str(self.detected_beat_indicator),
                                        str(self.timestamp - self.peak_timestamp))

                    self.threshold_i = self.npk_i + self.threshold_i_diff_weight * (self.spk_i - self.npk_i)

                else:
                    self.logger.log(str(self.timestamp), str(self.measurement), str(self.detected_beat_indicator),
                                    str(self.timestamp - self.peak_timestamp))

            else:
                self.logger.log(str(self.timestamp), str(self.measurement), str(self.detected_beat_indicator),
                                str(self.timestamp - self.peak_timestamp))

        else:
            self.logger.log(str(self.timestamp), str(self.measurement), str(self.detected_beat_indicator),
                            str(self.timestamp - self.peak_timestamp))

    def handle_detection(self):
        print "Pulse"
        # with open("flag.txt", "w") as fout:
        #     fout.write("%s %s %s %s" % (str(self.timestamp), str(self.measurement), str(self.detected_beat_indicator),
        #                                 str(self.timestamp - self.peak_timestamp)))

    # Tools methods.
    def bandpass_filter(self, data, lowcut, highcut, signal_freq, filter_order):
        """Constructs signal filter and uses it to given dataset."""
        nyquist_freq = 0.5 * signal_freq
        low = lowcut / nyquist_freq
        high = highcut / nyquist_freq
        b, a = butter(filter_order, [low, high], btype="band")
        y = lfilter(b, a, data)
        return y

    # Janko SLavic peak detection algorithm and implementation.
    # https://github.com/jankoslavic/py-tools/tree/master/findpeaks
    def findpeaks(self, data, spacing=1, limit=None):
        """Finds peaks in `data` which are of `spacing` width and >=`limit`.
        :param data: values
        :param spacing: minimum spacing to the next peak (should be 1 or more)
        :param limit: peaks should have value greater or equal
        :return:
        """
        len = data.size
        x = np.zeros(len + 2 * spacing)
        x[:spacing] = data[0] - 1.e-6
        x[-spacing:] = data[-1] - 1.e-6
        x[spacing:spacing + len] = data
        peak_candidate = np.zeros(len)
        peak_candidate[:] = True
        for s in range(spacing):
            start = spacing - s - 1
            h_b = x[start: start + len]  # before
            start = spacing
            h_c = x[start: start + len]  # central
            start = spacing + s + 1
            h_a = x[start: start + len]  # after
            peak_candidate = np.logical_and(peak_candidate, np.logical_and(h_c > h_b, h_c > h_a))

        ind = np.argwhere(peak_candidate)
        ind = ind.reshape(ind.size)
        if limit is not None:
            ind = ind[data[ind] > limit]
        return ind

if __name__ == "__main__":
    qrs_detector = QRSDetector(port="/dev/cu.usbmodem1411", baud_rate="115200")
    qrs_detector.connect_to_ecg()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import collections
from datetime import (
    datetime,
    timedelta,
)

from py3oauth2 import (
    authorizationcodegrant,
    implicitgrant,
    message,
    refreshtokengrant,
    utils,
)
from py3oauth2.exceptions import (
    AccessDenied,
    ErrorResponse,
    ValidationError,
)
from py3oauth2.interfaces import ClientType

__all__ = ['AuthorizationProvider', 'ResourceProvider']

class AuthorizationProvider:
    authz_handlers = {
        ('token', ): implicitgrant.Request,
        ('code', ): authorizationcodegrant.AuthorizationRequest,
    }

    token_handlers = {
        'refresh_token': refreshtokengrant.Request,
        'authorization_code': authorizationcodegrant.AccessTokenRequest,
    }

    def __init__(self, store, **kwargs):
        self.store = store

    @staticmethod
    def normalize_scope(scope):
        if isinstance(scope, str):
            return set(scope.split())

        return set()

    @staticmethod
    def normalize_response_type(response_type):
        assert isinstance(response_type, collections.Iterable)

        response_type = sorted(response_type)
        response_type = tuple(type for i, type in enumerate(response_type)
                              if response_type not in response_type[:i])
        return response_type

    @classmethod
    def add_authorization_handler(cls, response_type, handler):
        assert isinstance(response_type, tuple)
        assert issubclass(handler, message.Request)

        cls.authz_handlers[cls.normalize_response_type(response_type)]\
            = handler

    @classmethod
    def add_token_handler(cls, grant_type, handler):
        assert isinstance(grant_type, str)
        assert issubclass(handler, message.Request)

        cls.token_handlers[grant_type] = handler

    def authorize_client(self, client):
        raise NotImplementedError

    def validate_redirect_uri(self, client, redirect_uri):
        if not client.get_redirect_uri():
            return client.get_type() is ClientType.CONFIDENTIAL

        authorized_url = utils.normalize_url(client.get_redirect_uri())
        if '?' in authorized_url:
            authorized_url = authorized_url.split('?', 1)[0]

        redirect_uri = utils.normalize_url(redirect_uri)
        if '?' in redirect_uri:
            redirect_uri = authorized_url.split('?', 1)[0]

        return authorized_url == redirect_uri

    def _decode_request(self, registry, key, request_dict, err_kind, state):
        assert isinstance(registry, dict)
        assert isinstance(request_dict, dict)
        assert isinstance(err_kind, str)

        try:
            handler = registry[key]
        except KeyError as why:
            raise ErrorResponse(
                self._err_response(request_dict, err_kind, state)) from why

        request = handler.from_dict(request_dict)
        try:
            request.validate()
            return request
        except BaseException as why:
            if isinstance(why, ValidationError):
                why = message.InvalidRequest()
            elif not isinstance(why, message.RequestError):
                why = message.ServerError()

            raise ErrorResponse(
                self._err_response(request, why.kind, state)) from why

    def _err_response(self, request, err_kind, state):
        return message.ErrorResponse.from_dict(request, {
            'error': err_kind,
            'state': state,
        })

    def decode_authorize_request(self, request_dict):
        err_kind = 'unsupported_response_type'
        state = request_dict.get('state')

        response_type = request_dict.get('response_type')
        if not isinstance(response_type, str):
            raise ErrorResponse(self._err_response(
                request_dict, err_kind, state))
        response_type = self.normalize_response_type(response_type.split())

        return self._decode_request(self.authz_handlers,
                                    response_type,
                                    request_dict,
                                    err_kind,
                                    state)

    def decode_token_request(self, request_dict):
        err_kind = 'unsupported_grant_type'
        state = request_dict.get('state')

        grant_type = request_dict.get('grant_type')
        if not isinstance(grant_type, str):
            raise ErrorResponse(self._err_response(
                request_dict, err_kind, state))

        return self._decode_request(self.token_handlers,
                                    grant_type,
                                    request_dict,
                                    err_kind,
                                    state)

    def handle_request(self, request, owner=None):
        try:
            resp = request.answer(self, owner)
            resp.validate()

            return resp
        except BaseException as why:
            if not isinstance(why, message.RequestError):
                why = message.ServerError()

            resp = request.err_response(request)
            resp.error = why.kind
            if hasattr(resp, 'state') and request.state:
                resp.state = request.state

            raise ErrorResponse(resp) from why

class ResourceProvider:

    def __init__(self, store):
        self.store = store

    def get_access_token(self):
        raise NotImplementedError

    def validate_access_token(self, token, token_type):
        if token.get_type() != token_type:
            return False

        expires_at =\
            token.get_issued_at() + timedelta(seconds=token.get_expires_in())
        return datetime.utcnow() <= expires_at

    def authorize(self, required_scope):
        assert isinstance(required_scope, set)
        try:
            token, token_type = self.get_access_token()
            tokenobj = self.store.get_access_token(token)
            if tokenobj is None or\
                    not self.validate_access_token(tokenobj, token_type):
                raise AccessDenied()

            authorized_scopes = set(tokenobj.get_scope().split())
            if not required_scope.issubset(authorized_scopes):
                raise AccessDenied()
        except AccessDenied:
            raise
        else:
            return tokenobj
<commit_msg>remove impossible condition<commit_after># -*- coding: utf-8 -*-

import collections
from datetime import (
    datetime,
    timedelta,
)

from py3oauth2 import (
    authorizationcodegrant,
    implicitgrant,
    message,
    refreshtokengrant,
    utils,
)
from py3oauth2.exceptions import (
    AccessDenied,
    ErrorResponse,
    ValidationError,
)
from py3oauth2.interfaces import ClientType

__all__ = ['AuthorizationProvider', 'ResourceProvider']

class AuthorizationProvider:
    authz_handlers = {
        ('token', ): implicitgrant.Request,
        ('code', ): authorizationcodegrant.AuthorizationRequest,
    }

    token_handlers = {
        'refresh_token': refreshtokengrant.Request,
        'authorization_code': authorizationcodegrant.AccessTokenRequest,
    }

    def __init__(self, store, **kwargs):
        self.store = store

    @staticmethod
    def normalize_scope(scope):
        if isinstance(scope, str):
            return set(scope.split())

        return set()

    @staticmethod
    def normalize_response_type(response_type):
        assert isinstance(response_type, collections.Iterable)

        response_type = sorted(response_type)
        response_type = tuple(type for i, type in enumerate(response_type)
                              if response_type not in response_type[:i])
        return response_type

    @classmethod
    def add_authorization_handler(cls, response_type, handler):
        assert isinstance(response_type, tuple)
        assert issubclass(handler, message.Request)

        cls.authz_handlers[cls.normalize_response_type(response_type)]\
            = handler

    @classmethod
    def add_token_handler(cls, grant_type, handler):
        assert isinstance(grant_type, str)
        assert issubclass(handler, message.Request)

        cls.token_handlers[grant_type] = handler

    def authorize_client(self, client):
        raise NotImplementedError

    def validate_redirect_uri(self, client, redirect_uri):
        if not client.get_redirect_uri():
            return client.get_type() is ClientType.CONFIDENTIAL

        authorized_url = utils.normalize_url(client.get_redirect_uri())
        if '?' in authorized_url:
            authorized_url = authorized_url.split('?', 1)[0]

        redirect_uri = utils.normalize_url(redirect_uri)
        if '?' in redirect_uri:
            redirect_uri = authorized_url.split('?', 1)[0]

        return authorized_url == redirect_uri

    def _decode_request(self, registry, key, request_dict, err_kind, state):
        assert isinstance(registry, dict)
        assert isinstance(request_dict, dict)
        assert isinstance(err_kind, str)

        try:
            handler = registry[key]
        except KeyError as why:
            raise ErrorResponse(
                self._err_response(request_dict, err_kind, state)) from why

        request = handler.from_dict(request_dict)
        try:
            request.validate()
            return request
        except BaseException as why:
            if isinstance(why, ValidationError):
                why = message.InvalidRequest()
            else:
                why = message.ServerError()

            raise ErrorResponse(
                self._err_response(request, why.kind, state)) from why

    def _err_response(self, request, err_kind, state):
        return message.ErrorResponse.from_dict(request, {
            'error': err_kind,
            'state': state,
        })

    def decode_authorize_request(self, request_dict):
        err_kind = 'unsupported_response_type'
        state = request_dict.get('state')

        response_type = request_dict.get('response_type')
        if not isinstance(response_type, str):
            raise ErrorResponse(self._err_response(
                request_dict, err_kind, state))
        response_type = self.normalize_response_type(response_type.split())

        return self._decode_request(self.authz_handlers,
                                    response_type,
                                    request_dict,
                                    err_kind,
                                    state)

    def decode_token_request(self, request_dict):
        err_kind = 'unsupported_grant_type'
        state = request_dict.get('state')

        grant_type = request_dict.get('grant_type')
        if not isinstance(grant_type, str):
            raise ErrorResponse(self._err_response(
                request_dict, err_kind, state))

        return self._decode_request(self.token_handlers,
                                    grant_type,
                                    request_dict,
                                    err_kind,
                                    state)

    def handle_request(self, request, owner=None):
        try:
            resp = request.answer(self, owner)
            resp.validate()

            return resp
        except BaseException as why:
            if not isinstance(why, message.RequestError):
                why = message.ServerError()

            resp = request.err_response(request)
            resp.error = why.kind
            if hasattr(resp, 'state') and request.state:
                resp.state = request.state

            raise ErrorResponse(resp) from why

class ResourceProvider:

    def __init__(self, store):
        self.store = store

    def get_access_token(self):
        raise NotImplementedError

    def validate_access_token(self, token, token_type):
        if token.get_type() != token_type:
            return False

        expires_at =\
            token.get_issued_at() + timedelta(seconds=token.get_expires_in())
        return datetime.utcnow() <= expires_at

    def authorize(self, required_scope):
        assert isinstance(required_scope, set)
        try:
            token, token_type = self.get_access_token()
            tokenobj = self.store.get_access_token(token)
            if tokenobj is None or\
                    not self.validate_access_token(tokenobj, token_type):
                raise AccessDenied()

            authorized_scopes = set(tokenobj.get_scope().split())
            if not required_scope.issubset(authorized_scopes):
                raise AccessDenied()
        except AccessDenied:
            raise
        else:
            return tokenobj
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-

from flask import render_template, request, flash, session, url_for, redirect, g
from wtforms import TextField, PasswordField, SubmitField, validators
from flask.ext.mail import Message, Mail
from flask.ext.login import LoginManager, login_user, logout_user, login_required, current_user, AnonymousUserMixin

from collections import defaultdict

from forms import SigninForm, AddFeedForm
from pyaggr3g470r import app, db

import feedgetter
import models

mail = Mail()

login_manager = LoginManager()
login_manager.init_app(app)

@app.before_request
def before_request():
    g.user = current_user
    if g.user.is_authenticated():
        pass
        #g.user.last_seen = datetime.utcnow()
        #db.session.add(g.user)
        #db.session.commit()

@app.errorhandler(403)
def authentication_failed(e):
    flash('Authenticated failed.')
    return redirect(url_for('login'))

@app.errorhandler(401)
def authentication_failed(e):
    flash('Authenticated required.')
    return redirect(url_for('login'))

@login_manager.user_loader
def load_user(email):
    # Return an instance of the User model
    return models.User.objects(email=email).first()

@app.route('/login/', methods=['GET', 'POST'])
def login():
    g.user = AnonymousUserMixin()
    form = SigninForm()

    if form.validate_on_submit():
        user = models.User.objects(email=form.email.data).first()
        login_user(user)
        g.user = user
        flash("Logged in successfully.")
        return redirect(url_for('home'))
    return render_template('login.html', form=form)

@app.route('/logout/')
@login_required
def logout():
    """
    Remove the user information from the session.
    """
    logout_user()
    return redirect(url_for('home'))

@app.route('/')
@login_required
def home():
    user = g.user
    feeds = models.User.objects(email=g.user.email).fields(slice__feeds__articles=9).first().feeds
    return render_template('home.html', user=user, feeds=feeds)

@app.route('/fetch/', methods=['GET'])
@login_required
def fetch():
    feed_getter = feedgetter.FeedGetter(g.user.email)
    feed_getter.retrieve_feed()
    return redirect(url_for('home'))

@app.route('/about/', methods=['GET'])
@login_required
def about():
    return render_template('about.html')

@app.route('/feeds/', methods=['GET'])
@login_required
def feeds():
    feeds = models.User.objects(email=g.user.email).first().feeds
    return render_template('feeds.html', feeds=feeds)

@app.route('/feed/<feed_id>', methods=['GET'])
@login_required
def feed(feed_id=None):
    user = models.User.objects(email=g.user.email, feeds__oid=feed_id).first()
    for feed in user.feeds:
        if str(feed.oid) == feed_id:
            return render_template('feed.html', feed=feed)

@app.route('/article/<article_id>', methods=['GET'])
@login_required
def article(article_id=None):
    #user = models.User.objects(email=g.user.email, feeds__oid=feed_id).first()
    article = models.Article.objects(id=article_id).first()
    if not article.readed:
        article.readed = True
        article.save()
    return render_template('article.html', article=article)

@app.route('/delete/<article_id>', methods=['GET'])
@login_required
def delete(article_id=None):
    user = models.User.objects(email=g.user.email).first()
    # delete the article
    for feed in user.feeds:
        for article in feed.articles:
            if str(article.id) == article_id:
                feed.articles.remove(article)
                article.delete()
                user.save()
                return redirect(url_for('home'))

@app.route('/delete_feed/<feed_id>', methods=['GET'])
@login_required
def delete_feed(feed_id=None):
    user = models.User.objects(email=g.user.email).first()
    # delete all articles (Document objects)
    for feed in user.feeds:
        if str(feed.oid) == feed_id:
            for article in feed.articles:
                article.delete()
            feed.articles = []
            # delete the feed (EmbeddedDocument object)
            user.feeds.remove(feed)
            user.save()
            return redirect(url_for('home'))

@app.route('/articles/<feed_id>', methods=['GET'])
@login_required
def articles(feed_id=None):
    user = models.User.objects(email=g.user.email, feeds__oid=feed_id).first()
    for feed in user.feeds:
        if str(feed.oid) == feed_id:
            return render_template('articles.html', feed=user.feeds)

@app.route('/favorites/', methods=['GET'])
@login_required
def favorites():
    user = models.User.objects(email=g.user.email).first()
    result = []
    for feed in user.feeds:
        feed.articles = [article for article in feed.articles if article.like]
        if len(feed.articles) != 0:
            result.append(feed)
    return render_template('favorites.html', feeds=result)

@app.route('/unread/', methods=['GET'])
@login_required
def unread():
    user = models.User.objects(email=g.user.email).first()
    result = []
    for feed in user.feeds:
        feed.articles = [article for article in feed.articles if not article.readed]
        if len(feed.articles) != 0:
            result.append(feed)
    return render_template('unread.html', feeds=result)

@app.route('/management/', methods=['GET'])
@login_required
def management():
    form = AddFeedForm()
    user = models.User.objects(email=g.user.email).first()
    nb_feeds = len(user.feeds)
    nb_articles = sum([len(feed.articles) for feed in user.feeds])
    nb_unread_articles = sum([len([article for article in feed.articles if not article.readed]) for feed in user.feeds])
    return render_template('management.html', form=form, \
                            nb_feeds=nb_feeds, nb_articles=nb_articles, nb_unread_articles=nb_unread_articles)

@app.route('/add_feed/', methods=['GET', 'POST'])
@login_required
def add_feed():
    user = models.User.objects(email=g.user.email).first()
    form = AddFeedForm()

    if request.method == 'POST':
        if form.validate() == False:
            return render_template('management.html', form=form)
        new_feed = models.Feed(title=form.title.data, link=form.link.data, site_link=form.site_link.data)

        user.feeds.append(new_feed)
        user.feeds = sorted(user.feeds, key=lambda t: t.title.lower())
        user.save()
        return redirect(url_for('home'))
    return render_template('management.html', form=form)<commit_msg>Minor bugfix<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-

from flask import render_template, request, flash, session, url_for, redirect, g
from wtforms import TextField, PasswordField, SubmitField, validators
from flask.ext.mail import Message, Mail
from flask.ext.login import LoginManager, login_user, logout_user, login_required, current_user, AnonymousUserMixin

from collections import defaultdict

from forms import SigninForm, AddFeedForm
from pyaggr3g470r import app, db

import feedgetter
import models

mail = Mail()

login_manager = LoginManager()
login_manager.init_app(app)

@app.before_request
def before_request():
    g.user = current_user
    if g.user.is_authenticated():
        pass
        #g.user.last_seen = datetime.utcnow()
        #db.session.add(g.user)
        #db.session.commit()

@app.errorhandler(403)
def authentication_failed(e):
    flash('Authenticated failed.')
    return redirect(url_for('login'))

@app.errorhandler(401)
def authentication_failed(e):
    flash('Authenticated required.')
    return redirect(url_for('login'))

@login_manager.user_loader
def load_user(email):
    # Return an instance of the User model
    return models.User.objects(email=email).first()

@app.route('/login/', methods=['GET', 'POST'])
def login():
    g.user = AnonymousUserMixin()
    form = SigninForm()

    if form.validate_on_submit():
        user = models.User.objects(email=form.email.data).first()
        login_user(user)
        g.user = user
        flash("Logged in successfully.")
        return redirect(url_for('home'))
    return render_template('login.html', form=form)

@app.route('/logout/')
@login_required
def logout():
    """
    Remove the user information from the session.
    """
    logout_user()
    return redirect(url_for('home'))

@app.route('/')
@login_required
def home():
    user = g.user
    feeds = models.User.objects(email=g.user.email).fields(slice__feeds__articles=9).first().feeds
    return render_template('home.html', user=user, feeds=feeds)

@app.route('/fetch/', methods=['GET'])
@login_required
def fetch():
    feed_getter = feedgetter.FeedGetter(g.user.email)
    feed_getter.retrieve_feed()
    return redirect(url_for('home'))

@app.route('/about/', methods=['GET'])
@login_required
def about():
    return render_template('about.html')

@app.route('/feeds/', methods=['GET'])
@login_required
def feeds():
    feeds = models.User.objects(email=g.user.email).first().feeds
    return render_template('feeds.html', feeds=feeds)

@app.route('/feed/<feed_id>', methods=['GET'])
@login_required
def feed(feed_id=None):
    user = models.User.objects(email=g.user.email, feeds__oid=feed_id).first()
    for feed in user.feeds:
        if str(feed.oid) == feed_id:
            return render_template('feed.html', feed=feed)

@app.route('/article/<article_id>', methods=['GET'])
@login_required
def article(article_id=None):
    #user = models.User.objects(email=g.user.email, feeds__oid=feed_id).first()
    article = models.Article.objects(id=article_id).first()
    if not article.readed:
        article.readed = True
        article.save()
    return render_template('article.html', article=article)

@app.route('/delete/<article_id>', methods=['GET'])
@login_required
def delete(article_id=None):
    user = models.User.objects(email=g.user.email).first()
    # delete the article
    for feed in user.feeds:
        for article in feed.articles:
            if str(article.id) == article_id:
                feed.articles.remove(article)
                article.delete()
                user.save()
                return redirect(url_for('home'))

@app.route('/delete_feed/<feed_id>', methods=['GET'])
@login_required
def delete_feed(feed_id=None):
    user = models.User.objects(email=g.user.email).first()
    # delete all articles (Document objects)
    for feed in user.feeds:
        if str(feed.oid) == feed_id:
            for article in feed.articles:
                article.delete()
            feed.articles = []
            # delete the feed (EmbeddedDocument object)
            user.feeds.remove(feed)
            user.save()
            return redirect(url_for('home'))

@app.route('/articles/<feed_id>', methods=['GET'])
@login_required
def articles(feed_id=None):
    user = models.User.objects(email=g.user.email, feeds__oid=feed_id).first()
    for feed in user.feeds:
        if str(feed.oid) == feed_id:
            return render_template('articles.html', feed=feed)

@app.route('/favorites/', methods=['GET'])
@login_required
def favorites():
    user = models.User.objects(email=g.user.email).first()
    result = []
    for feed in user.feeds:
        feed.articles = [article for article in feed.articles if article.like]
        if len(feed.articles) != 0:
            result.append(feed)
    return render_template('favorites.html', feeds=result)

@app.route('/unread/', methods=['GET'])
@login_required
def unread():
    user = models.User.objects(email=g.user.email).first()
    result = []
    for feed in user.feeds:
        feed.articles = [article for article in feed.articles if not article.readed]
        if len(feed.articles) != 0:
            result.append(feed)
    return render_template('unread.html', feeds=result)

@app.route('/management/', methods=['GET'])
@login_required
def management():
    form = AddFeedForm()
    user = models.User.objects(email=g.user.email).first()
    nb_feeds = len(user.feeds)
    nb_articles = sum([len(feed.articles) for feed in user.feeds])
    nb_unread_articles = sum([len([article for article in feed.articles if not article.readed]) for feed in user.feeds])
    return render_template('management.html', form=form, \
                            nb_feeds=nb_feeds, nb_articles=nb_articles, nb_unread_articles=nb_unread_articles)

@app.route('/add_feed/', methods=['GET', 'POST'])
@login_required
def add_feed():
    user = models.User.objects(email=g.user.email).first()
    form = AddFeedForm()

    if request.method == 'POST':
        if form.validate() == False:
            return render_template('management.html', form=form)
        new_feed = models.Feed(title=form.title.data, link=form.link.data, site_link=form.site_link.data)

        user.feeds.append(new_feed)
        user.feeds = sorted(user.feeds, key=lambda t: t.title.lower())
        user.save()
        return redirect(url_for('home'))
    return render_template('management.html', form=form)<|endoftext|>


<commit_before>"""YANG usage guidelines plugin
See RFC 6087
Other plugins can derive from this and make it more specific, e.g.,
ietf.py derives from this and sets the namespace and module name prefixes
to IETF-specific values.
"""

import optparse
import sys

from pyang import plugin
from pyang import statements
from pyang import error
from pyang.error import err_add

def pyang_plugin_init():
    plugin.register_plugin(LintPlugin())

class LintPlugin(plugin.PyangPlugin):
    def __init__(self):
        plugin.PyangPlugin.__init__(self)
        ## Other plugins (e.g., ietf.py) can derive from this plugin
        ## and override these variables.

        # Set this to a list of allowed namespace prefixes.
        # The code checks that the namespace is on the form:
        #   <prefix><modulename>
        # If some other convention is used, the derived plugin can
        # define its own checks.
        self.namespace_prefixes = []

        # Set this to a list of allowed module name prefixes.
        # The code checks that the module name is on the form:
        #   <prefix>-...
        # If some other convention is used, the derived plugin can
        # define its own checks.
        self.modulename_prefixes = []

    def add_opts(self, optparser):
        optlist = [
            optparse.make_option("--lint",
                                 dest="lint",
                                 action="store_true",
                                 help="Validate the module(s) according to " \
                                 "RFC 6087 rules."),
            optparse.make_option("--lint-namespace-prefix",
                                 dest="namespace_prefixes",
                                 default=[],
                                 action="append",
                                 help="Validate that the module's namespace " \
                                     "matches one of the given prefixes."),
            optparse.make_option("--lint-modulename-prefix",
                                 dest="modulename_prefixes",
                                 default=[],
                                 action="append",
                                 help="Validate that the module's name " \
                                     "matches one of the given prefixes."),
            ]
        optparser.add_options(optlist)

    def setup_ctx(self, ctx):
        if not ctx.opts.lint:
            return
        self._setup_ctx(ctx)

    def _setup_ctx(self, ctx):
        "Should be called by any derived plugin's setup_ctx() function."

        ctx.strict = True
        ctx.canonical = True
        ctx.max_identifier_len = 64
        ctx.implicit_errors = False

        # always add additional prefixes given on the command line
        self.namespace_prefixes.extend(ctx.opts.namespace_prefixes)
        self.modulename_prefixes.extend(ctx.opts.modulename_prefixes)

        # register our grammar validation funs

        statements.add_validation_var(
            '$chk_default',
            lambda keyword: keyword in _keyword_with_default)
        statements.add_validation_var(
            '$chk_required',
            lambda keyword: keyword in _required_substatements)

        statements.add_validation_var(
            '$chk_recommended',
            lambda keyword: keyword in _recommended_substatements)

        statements.add_validation_fun(
            'grammar', ['$chk_default'],
            lambda ctx, s: v_chk_default(ctx, s))
        statements.add_validation_fun(
            'grammar', ['$chk_required'],
            lambda ctx, s: v_chk_required_substmt(ctx, s))
        statements.add_validation_fun(
            'grammar', ['$chk_recommended'],
            lambda ctx, s: v_chk_recommended_substmt(ctx, s))

        statements.add_validation_fun(
            'grammar', ['namespace'],
            lambda ctx, s: v_chk_namespace(ctx, s, self.namespace_prefixes))

        statements.add_validation_fun(
            'grammar', ['module', 'submodule'],
            lambda ctx, s: v_chk_module_name(ctx, s, self.modulename_prefixes))

        # register our error codes
        error.add_error_code(
            'LINT_EXPLICIT_DEFAULT', 4,
            'RFC 6087: 4.3: '
            + 'statement "%s" is given with its default value "%s"')
        error.add_error_code(
            'LINT_MISSING_REQUIRED_SUBSTMT', 3,
            '%s: '
            + 'statement "%s" must have a "%s" substatement')
        error.add_error_code(
            'LINT_MISSING_RECOMMENDED_SUBSTMT', 4,
            '%s: '
            + 'statement "%s" should have a "%s" substatement')
        error.add_error_code(
            'LINT_BAD_NAMESPACE_VALUE', 4,
            'RFC 6087: 4.8: namespace value should be "%s"')
        error.add_error_code(
            'LINT_BAD_MODULENAME_PREFIX', 4,
            'RFC 6087: 4.1: '
            + 'no module name prefix used, suggest %s-%s')
        error.add_error_code(
            'LINT_NO_MODULENAME_PREFIX', 4,
            'RFC 6087: 4.1: '
            + 'no module name prefix used')

        # override std error string
        error.add_error_code(
            'LONG_IDENTIFIER', 3,
            'RFC 6087: 4.2: identifier %s exceeds %s characters')

_keyword_with_default = {
    'status': 'current',
    'mandatory': 'false',
    'min-elements': '0',
    'max-elements': 'unbounded',
    'config': 'true',
    'yin-element': 'false',
    }

_required_substatements = {
    'module': (('contact', 'organization', 'description', 'revision'),
               "RFC 6087: 4.7"),
    'submodule': (('contact', 'organization', 'description', 'revision'),
                  "RFC 6087: 4.7"),
    'revision':(('reference',), "RFC 6087: 4.7"),
    'extension':(('description',), "RFC 6087: 4.12"),
    'feature':(('description',), "RFC 6087: 4.12"),
    'identity':(('description',), "RFC 6087: 4.12"),
    'typedef':(('description',), "RFC 6087: 4.11,4.12"),
    'grouping':(('description',), "RFC 6087: 4.12"),
    'augment':(('description',), "RFC 6087: 4.12"),
    'rpc':(('description',), "RFC 6087: 4.12"),
    'notification':(('description',), "RFC 6087: 4.12,4.14"),
    'container':(('description',), "RFC 6087: 4.12"),
    'leaf':(('description',), "RFC 6087: 4.12"),
    'leaf-list':(('description',), "RFC 6087: 4.12"),
    'list':(('description',), "RFC 6087: 4.12"),
    'choice':(('description',), "RFC 6087: 4.12"),
    'anyxml':(('description',), "RFC 6087: 4.12"),
    }

_recommended_substatements = {
    'must':(('description',), "RFC 6087: 4.12"),
    'when':(('description',), "RFC 6087: 4.12"),
    'enum':(('description',), "RFC 6087: 4.10,4.12"),
    'bit':(('description',), "RFC 6087: 4.10,4.12"),
    }

_ietf_namespace_prefix = 'urn:ietf:params:xml:ns:yang:'

def v_chk_default(ctx, stmt):
    if (stmt.arg == _keyword_with_default[stmt.keyword] and
        stmt.parent.keyword != 'refine'):
        err_add(ctx.errors, stmt.pos, 'LINT_EXPLICIT_DEFAULT',
                (stmt.keyword, stmt.arg))

def v_chk_required_substmt(ctx, stmt):
    if stmt.keyword in _required_substatements:
        (required, s) = _required_substatements[stmt.keyword]
        for r in required:
            if stmt.search_one(r) is None:
                err_add(ctx.errors, stmt.pos,
                        'LINT_MISSING_REQUIRED_SUBSTMT',
                        (s, stmt.keyword, r))

def v_chk_recommended_substmt(ctx, stmt):
    if stmt.keyword in _recommended_substatements:
        (recommended, s) = _recommended_substatements[stmt.keyword]
        for r in recommended:
            if stmt.search_one(r) is None:
                err_add(ctx.errors, stmt.pos,
                        'LINT_MISSING_RECOMMENDED_SUBSTMT',
                        (s, stmt.keyword, r))

def v_chk_namespace(ctx, stmt, namespace_prefixes):
    if namespace_prefixes != []:
        for prefix in namespace_prefixes:
            if stmt.arg == prefix + stmt.i_module.arg:
                return
        err_add(ctx.errors, stmt.pos, 'LINT_BAD_NAMESPACE_VALUE',
                namespace_prefixes[0] + stmt.i_module.arg)

def v_chk_module_name(ctx, stmt, modulename_prefixes):
    if modulename_prefixes != []:
        for prefix in modulename_prefixes:
            if stmt.arg.find(prefix + '-') == 0:
                return
        err_add(ctx.errors, stmt.pos, 'LINT_BAD_MODULENAME_PREFIX',
                (modulename_prefixes[0], stmt.arg))
    elif stmt.arg.find('-') == -1:
        # can't check much, but we can check that a prefix is used
        err_add(ctx.errors, stmt.pos, 'LINT_NO_MODULENAME_PREFIX', ())
<commit_msg>fixes #179 - better error message<commit_after>"""YANG usage guidelines plugin
See RFC 6087
Other plugins can derive from this and make it more specific, e.g.,
ietf.py derives from this and sets the namespace and module name prefixes
to IETF-specific values.
"""

import optparse
import sys

from pyang import plugin
from pyang import statements
from pyang import error
from pyang.error import err_add

def pyang_plugin_init():
    plugin.register_plugin(LintPlugin())

class LintPlugin(plugin.PyangPlugin):
    def __init__(self):
        plugin.PyangPlugin.__init__(self)
        ## Other plugins (e.g., ietf.py) can derive from this plugin
        ## and override these variables.

        # Set this to a list of allowed namespace prefixes.
        # The code checks that the namespace is on the form:
        #   <prefix><modulename>
        # If some other convention is used, the derived plugin can
        # define its own checks.
        self.namespace_prefixes = []

        # Set this to a list of allowed module name prefixes.
        # The code checks that the module name is on the form:
        #   <prefix>-...
        # If some other convention is used, the derived plugin can
        # define its own checks.
        self.modulename_prefixes = []

    def add_opts(self, optparser):
        optlist = [
            optparse.make_option("--lint",
                                 dest="lint",
                                 action="store_true",
                                 help="Validate the module(s) according to " \
                                 "RFC 6087 rules."),
            optparse.make_option("--lint-namespace-prefix",
                                 dest="namespace_prefixes",
                                 default=[],
                                 action="append",
                                 help="Validate that the module's namespace " \
                                     "matches one of the given prefixes."),
            optparse.make_option("--lint-modulename-prefix",
                                 dest="modulename_prefixes",
                                 default=[],
                                 action="append",
                                 help="Validate that the module's name " \
                                     "matches one of the given prefixes."),
            ]
        optparser.add_options(optlist)

    def setup_ctx(self, ctx):
        if not ctx.opts.lint:
            return
        self._setup_ctx(ctx)

    def _setup_ctx(self, ctx):
        "Should be called by any derived plugin's setup_ctx() function."

        ctx.strict = True
        ctx.canonical = True
        ctx.max_identifier_len = 64
        ctx.implicit_errors = False

        # always add additional prefixes given on the command line
        self.namespace_prefixes.extend(ctx.opts.namespace_prefixes)
        self.modulename_prefixes.extend(ctx.opts.modulename_prefixes)

        # register our grammar validation funs

        statements.add_validation_var(
            '$chk_default',
            lambda keyword: keyword in _keyword_with_default)
        statements.add_validation_var(
            '$chk_required',
            lambda keyword: keyword in _required_substatements)

        statements.add_validation_var(
            '$chk_recommended',
            lambda keyword: keyword in _recommended_substatements)

        statements.add_validation_fun(
            'grammar', ['$chk_default'],
            lambda ctx, s: v_chk_default(ctx, s))
        statements.add_validation_fun(
            'grammar', ['$chk_required'],
            lambda ctx, s: v_chk_required_substmt(ctx, s))
        statements.add_validation_fun(
            'grammar', ['$chk_recommended'],
            lambda ctx, s: v_chk_recommended_substmt(ctx, s))

        statements.add_validation_fun(
            'grammar', ['namespace'],
            lambda ctx, s: v_chk_namespace(ctx, s, self.namespace_prefixes))

        statements.add_validation_fun(
            'grammar', ['module', 'submodule'],
            lambda ctx, s: v_chk_module_name(ctx, s, self.modulename_prefixes))

        # register our error codes
        error.add_error_code(
            'LINT_EXPLICIT_DEFAULT', 4,
            'RFC 6087: 4.3: '
            + 'statement "%s" is given with its default value "%s"')
        error.add_error_code(
            'LINT_MISSING_REQUIRED_SUBSTMT', 3,
            '%s: '
            + 'statement "%s" must have a "%s" substatement')
        error.add_error_code(
            'LINT_MISSING_RECOMMENDED_SUBSTMT', 4,
            '%s: '
            + 'statement "%s" should have a "%s" substatement')
        error.add_error_code(
            'LINT_BAD_NAMESPACE_VALUE', 4,
            'RFC 6087: 4.8: namespace value should be "%s"')
        error.add_error_code(
            'LINT_BAD_MODULENAME_PREFIX_1', 4,
            'RFC 6087: 4.1: '
            + 'the module name should start with the string %s')
        error.add_error_code(
            'LINT_BAD_MODULENAME_PREFIX_N', 4,
            'RFC 6087: 4.1: '
            + 'the module name should start with one of the strings %s')
        error.add_error_code(
            'LINT_NO_MODULENAME_PREFIX', 4,
            'RFC 6087: 4.1: '
            + 'no module name prefix string used')

        # override std error string
        error.add_error_code(
            'LONG_IDENTIFIER', 3,
            'RFC 6087: 4.2: identifier %s exceeds %s characters')

_keyword_with_default = {
    'status': 'current',
    'mandatory': 'false',
    'min-elements': '0',
    'max-elements': 'unbounded',
    'config': 'true',
    'yin-element': 'false',
    }

_required_substatements = {
    'module': (('contact', 'organization', 'description', 'revision'),
               "RFC 6087: 4.7"),
    'submodule': (('contact', 'organization', 'description', 'revision'),
                  "RFC 6087: 4.7"),
    'revision':(('reference',), "RFC 6087: 4.7"),
    'extension':(('description',), "RFC 6087: 4.12"),
    'feature':(('description',), "RFC 6087: 4.12"),
    'identity':(('description',), "RFC 6087: 4.12"),
    'typedef':(('description',), "RFC 6087: 4.11,4.12"),
    'grouping':(('description',), "RFC 6087: 4.12"),
    'augment':(('description',), "RFC 6087: 4.12"),
    'rpc':(('description',), "RFC 6087: 4.12"),
    'notification':(('description',), "RFC 6087: 4.12,4.14"),
    'container':(('description',), "RFC 6087: 4.12"),
    'leaf':(('description',), "RFC 6087: 4.12"),
    'leaf-list':(('description',), "RFC 6087: 4.12"),
    'list':(('description',), "RFC 6087: 4.12"),
    'choice':(('description',), "RFC 6087: 4.12"),
    'anyxml':(('description',), "RFC 6087: 4.12"),
    }

_recommended_substatements = {
    'must':(('description',), "RFC 6087: 4.12"),
    'when':(('description',), "RFC 6087: 4.12"),
    'enum':(('description',), "RFC 6087: 4.10,4.12"),
    'bit':(('description',), "RFC 6087: 4.10,4.12"),
    }

_ietf_namespace_prefix = 'urn:ietf:params:xml:ns:yang:'

def v_chk_default(ctx, stmt):
    if (stmt.arg == _keyword_with_default[stmt.keyword] and
        stmt.parent.keyword != 'refine'):
        err_add(ctx.errors, stmt.pos, 'LINT_EXPLICIT_DEFAULT',
                (stmt.keyword, stmt.arg))

def v_chk_required_substmt(ctx, stmt):
    if stmt.keyword in _required_substatements:
        (required, s) = _required_substatements[stmt.keyword]
        for r in required:
            if stmt.search_one(r) is None:
                err_add(ctx.errors, stmt.pos,
                        'LINT_MISSING_REQUIRED_SUBSTMT',
                        (s, stmt.keyword, r))

def v_chk_recommended_substmt(ctx, stmt):
    if stmt.keyword in _recommended_substatements:
        (recommended, s) = _recommended_substatements[stmt.keyword]
        for r in recommended:
            if stmt.search_one(r) is None:
                err_add(ctx.errors, stmt.pos,
                        'LINT_MISSING_RECOMMENDED_SUBSTMT',
                        (s, stmt.keyword, r))

def v_chk_namespace(ctx, stmt, namespace_prefixes):
    if namespace_prefixes != []:
        for prefix in namespace_prefixes:
            if stmt.arg == prefix + stmt.i_module.arg:
                return
        err_add(ctx.errors, stmt.pos, 'LINT_BAD_NAMESPACE_VALUE',
                namespace_prefixes[0] + stmt.i_module.arg)

def v_chk_module_name(ctx, stmt, modulename_prefixes):
    if modulename_prefixes != []:
        for prefix in modulename_prefixes:
            if stmt.arg.find(prefix + '-') == 0:
                return
        if len(modulename_prefixes) == 1:
            err_add(ctx.errors, stmt.pos, 'LINT_BAD_MODULENAME_PREFIX_1',
                    '"' + modulename_prefixes[0] + '-"')
        elif len(modulename_prefixes) == 2:
            s = " or ".join(['"' + p + '-"' for p in modulename_prefixes])
            err_add(ctx.errors, stmt.pos, 'LINT_BAD_MODULENAME_PREFIX_N', s)
        else:
            s = ", ".join(['"' + p + '-"' for p in modulename_prefixes[:-1]]) +\
            ', or "' + modulename_prefixes[-1] + '-"'
            err_add(ctx.errors, stmt.pos, 'LINT_BAD_MODULENAME_PREFIX_N', s)
    elif stmt.arg.find('-') == -1:
        # can't check much, but we can check that a prefix is used
        err_add(ctx.errors, stmt.pos, 'LINT_NO_MODULENAME_PREFIX', ())
<|endoftext|>


<commit_before># -*- coding: utf8 -*-
# This file is part of PYBOSSA.
#
# Copyright (C) 2015 Scifabric LTD.
#
# PYBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PYBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PYBOSSA.  If not, see <http://www.gnu.org/licenses/>.
"""
This module exports all the extensions used by PYBOSSA.

The objects are:
    * sentinel: for caching data, ratelimiting, etc.
    * signer: for signing emails, cookies, etc.
    * mail: for sending emails,
    * login_manager: to handle account sigin/signout
    * facebook: for Facebook signin
    * twitter: for Twitter signin
    * google: for Google signin
    * misaka: for app.long_description markdown support,
    * babel: for i18n support,
    * uploader: for file uploads support,
    * csrf: for CSRF protection
    * newsletter: for subscribing users to Mailchimp newsletter
    * assets: for assets management (SASS, etc.)

"""
__all__ = ['sentinel', 'db', 'signer', 'mail', 'login_manager', 'facebook',
           'twitter', 'google', 'misaka', 'babel', 'uploader', 'debug_toolbar',
           'csrf', 'timeouts', 'ratelimits', 'user_repo', 'project_repo',
           'task_repo', 'blog_repo', 'auditlog_repo', 'webhook_repo',
           'result_repo', 'newsletter', 'importer', 'flickr',
           'plugin_manager', 'assets', 'JSONEncoder']

# CACHE
from pybossa.sentinel import Sentinel
sentinel = Sentinel()

# DB
from flask.ext.sqlalchemy import SQLAlchemy
db = SQLAlchemy()
db.slave_session = db.session

# Repositories
user_repo = None
project_repo = None
blog_repo = None
task_repo = None
auditlog_repo = None
webhook_repo = None
result_repo = None

# Signer
from pybossa.signer import Signer
signer = Signer()

# Mail
from flask.ext.mail import Mail
mail = Mail()

# Login Manager
from flask.ext.login import LoginManager
login_manager = LoginManager()

# Debug Toolbar
from flask.ext.debugtoolbar import DebugToolbarExtension
debug_toolbar = DebugToolbarExtension()

# OAuth providers
from pybossa.oauth_providers import Facebook
facebook = Facebook()

from pybossa.oauth_providers import Twitter
twitter = Twitter()

from pybossa.oauth_providers import Google
google = Google()

from pybossa.oauth_providers import Flickr
flickr = Flickr()

# Markdown support
from flask.ext.misaka import Misaka
misaka = Misaka()

# Babel
from flask.ext.babel import Babel
babel = Babel()

# Uploader
uploader = None

# Exporters
json_exporter = None
csv_exporter = None

# CSRF protection
from flask_wtf.csrf import CsrfProtect
csrf = CsrfProtect()

# Timeouts
timeouts = dict()

# Ratelimits
ratelimits = dict()

# Newsletter
from newsletter import Newsletter
newsletter = Newsletter()

# Importer
from importers import Importer
importer = Importer()

from flask.ext.plugins import PluginManager
plugin_manager = PluginManager()

from flask.ext.assets import Environment
assets = Environment()

from flask.json import JSONEncoder as BaseEncoder
from speaklater import _LazyString

class JSONEncoder(BaseEncoder):
    def default(self, o):
        if isinstance(o, _LazyString):
            return str(o)

        return BaseEncoder.default(self, o)
<commit_msg>Add docstring.<commit_after># -*- coding: utf8 -*-
# This file is part of PYBOSSA.
#
# Copyright (C) 2015 Scifabric LTD.
#
# PYBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PYBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PYBOSSA.  If not, see <http://www.gnu.org/licenses/>.
"""
This module exports all the extensions used by PYBOSSA.

The objects are:
    * sentinel: for caching data, ratelimiting, etc.
    * signer: for signing emails, cookies, etc.
    * mail: for sending emails,
    * login_manager: to handle account sigin/signout
    * facebook: for Facebook signin
    * twitter: for Twitter signin
    * google: for Google signin
    * misaka: for app.long_description markdown support,
    * babel: for i18n support,
    * uploader: for file uploads support,
    * csrf: for CSRF protection
    * newsletter: for subscribing users to Mailchimp newsletter
    * assets: for assets management (SASS, etc.)

"""
__all__ = ['sentinel', 'db', 'signer', 'mail', 'login_manager', 'facebook',
           'twitter', 'google', 'misaka', 'babel', 'uploader', 'debug_toolbar',
           'csrf', 'timeouts', 'ratelimits', 'user_repo', 'project_repo',
           'task_repo', 'blog_repo', 'auditlog_repo', 'webhook_repo',
           'result_repo', 'newsletter', 'importer', 'flickr',
           'plugin_manager', 'assets', 'JSONEncoder']

# CACHE
from pybossa.sentinel import Sentinel
sentinel = Sentinel()

# DB
from flask.ext.sqlalchemy import SQLAlchemy
db = SQLAlchemy()
db.slave_session = db.session

# Repositories
user_repo = None
project_repo = None
blog_repo = None
task_repo = None
auditlog_repo = None
webhook_repo = None
result_repo = None

# Signer
from pybossa.signer import Signer
signer = Signer()

# Mail
from flask.ext.mail import Mail
mail = Mail()

# Login Manager
from flask.ext.login import LoginManager
login_manager = LoginManager()

# Debug Toolbar
from flask.ext.debugtoolbar import DebugToolbarExtension
debug_toolbar = DebugToolbarExtension()

# OAuth providers
from pybossa.oauth_providers import Facebook
facebook = Facebook()

from pybossa.oauth_providers import Twitter
twitter = Twitter()

from pybossa.oauth_providers import Google
google = Google()

from pybossa.oauth_providers import Flickr
flickr = Flickr()

# Markdown support
from flask.ext.misaka import Misaka
misaka = Misaka()

# Babel
from flask.ext.babel import Babel
babel = Babel()

# Uploader
uploader = None

# Exporters
json_exporter = None
csv_exporter = None

# CSRF protection
from flask_wtf.csrf import CsrfProtect
csrf = CsrfProtect()

# Timeouts
timeouts = dict()

# Ratelimits
ratelimits = dict()

# Newsletter
from newsletter import Newsletter
newsletter = Newsletter()

# Importer
from importers import Importer
importer = Importer()

from flask.ext.plugins import PluginManager
plugin_manager = PluginManager()

from flask.ext.assets import Environment
assets = Environment()

from flask.json import JSONEncoder as BaseEncoder
from speaklater import _LazyString

class JSONEncoder(BaseEncoder): # pragma: no cover
    """JSON Encoder to deal with Babel lazy strings."""
    def default(self, o):
        if isinstance(o, _LazyString):
            return str(o)

        return BaseEncoder.default(self, o)
<|endoftext|>


<commit_before>import os
import unittest

from pychron.canvas.canvas2D.tests.calibration_item import CalibrationObjectTestCase
from pychron.core.helpers.tests.floatfmt import SigFigStdFmtTestCase
from pychron.core.stats.tests.mswd_tests import MSWDTestCase

# # Core
from pychron.core.stats.tests.peak_detection_test import MultiPeakDetectionTestCase
from pychron.core.tests.spell_correct import SpellCorrectTestCase
from pychron.core.tests.filtering_tests import FilteringTestCase

from pychron.core.helpers.tests.floatfmt import FloatfmtTestCase
from pychron.core.helpers.tests.strtools import CamelCaseTestCase

from pychron.core.xml.tests.xml_parser import XMLParserTestCase
from pychron.core.regression.tests.regression import OLSRegressionTest, MeanRegressionTest, \
    FilterOLSRegressionTest, OLSRegressionTest2, TruncateRegressionTest
from pychron.core.tests.alpha_tests import AlphaTestCase
from pychron.experiment.tests.backup import BackupTestCase
from pychron.experiment.tests.comment_template import CommentTemplaterTestCase
from pychron.experiment.tests.conditionals import ConditionalsTestCase, ParseConditionalsTestCase
from pychron.experiment.tests.duration_tracker import DurationTrackerTestCase
from pychron.experiment.tests.frequency_test import FrequencyTestCase, FrequencyTemplateTestCase
from pychron.experiment.tests.identifier import IdentifierTestCase
from pychron.experiment.tests.peak_hop_parse import PeakHopYamlCase1, PeakHopTxtCase
from pychron.experiment.tests.peak_hop_parse import PeakHopYamlCase2
from pychron.experiment.tests.position_regex_test import XYTestCase
from pychron.experiment.tests.renumber_aliquot_test import RenumberAliquotTestCase
from pychron.external_pipette.tests.external_pipette import ExternalPipetteTestCase
from pychron.processing.tests.age_converter import AgeConverterTestCase
from pychron.processing.tests.plateau import PlateauTestCase
from pychron.processing.tests.ratio import RatioTestCase
from pychron.stage.tests.stage_map import StageMapTestCase, TransformTestCase
#
# os.environ['MassSpecDBVersion'] = '16'
#
# from pychron.paths import paths
#
# paths.build('_dev')
#
# use_logger = False
#
#
def suite():
    # set env. variables
    os.environ["MassSpecDBVersion"] = "16"

    from pychron.paths import paths

    paths.build("_dev")

    # if use_logger:
    #     from pychron.core.helpers.logger_setup import logging_setup
    #     logging_setup('unittests')

    loader = unittest.TestLoader()
    suite = unittest.TestSuite()

    tests = (
        # Canvas
        CalibrationObjectTestCase,
        # Core
        AlphaTestCase,
        SpellCorrectTestCase,
        FilteringTestCase,
        MultiPeakDetectionTestCase,
        FloatfmtTestCase,
        SigFigStdFmtTestCase,
        CamelCaseTestCase,
        RatioTestCase,
        XMLParserTestCase,
        OLSRegressionTest,
        MeanRegressionTest,
        FilterOLSRegressionTest,
        OLSRegressionTest2,
        TruncateRegressionTest,
        MSWDTestCase,
        # old
        # ExpoRegressionTest,
        # ExpoRegressionTest2,

        # DataMapper
        # USGSVSCFileSourceUnittest,
        # USGSVSCIrradiationSourceUnittest,
        # NMGRLLegacySourceUnittest,

        # Experiment
        PeakHopYamlCase1,
        PeakHopYamlCase2,
        BackupTestCase,
        PeakHopTxtCase,
        DurationTrackerTestCase,
        FrequencyTestCase,
        FrequencyTemplateTestCase,
        XYTestCase,
        RenumberAliquotTestCase,
        ConditionalsTestCase,
        ParseConditionalsTestCase,
        IdentifierTestCase,
        CommentTemplaterTestCase,
        # ExternalPipette
        ExternalPipetteTestCase,
        # Processing
        PlateauTestCase,
        RatioTestCase,
        AgeConverterTestCase,
        # Pyscripts
        # WaitForTestCase,
        # InterpolationTestCase,
        # DocstrContextTestCase,
        # Spectrometer
        # MFTableTestCase,
        # DiscreteMFTableTestCase,
        # IntegrationTimeTestCase,

        # Stage
        StageMapTestCase,
        TransformTestCase
    )

    for t in tests:
        suite.addTest(loader.loadTestsFromTestCase(t))

    return suite

if __name__ == "__main__":
    unittest.TextTestRunner(verbosity=2).run(suite())
<commit_msg>updated test suite<commit_after>import os
import unittest

from pychron.canvas.canvas2D.tests.calibration_item import CalibrationObjectTestCase
from pychron.core.helpers.tests.floatfmt import SigFigStdFmtTestCase
from pychron.core.stats.tests.mswd_tests import MSWDTestCase

# # Core
from pychron.core.stats.tests.peak_detection_test import MultiPeakDetectionTestCase
from pychron.core.tests.spell_correct import SpellCorrectTestCase
from pychron.core.tests.filtering_tests import FilteringTestCase

from pychron.core.helpers.tests.floatfmt import FloatfmtTestCase
from pychron.core.helpers.tests.strtools import CamelCaseTestCase

from pychron.core.xml.tests.xml_parser import XMLParserTestCase
from pychron.core.regression.tests.regression import OLSRegressionTest, MeanRegressionTest, \
    FilterOLSRegressionTest, OLSRegressionTest2, TruncateRegressionTest
from pychron.core.tests.alpha_tests import AlphaTestCase
from pychron.experiment.tests.backup import BackupTestCase
from pychron.experiment.tests.comment_template import CommentTemplaterTestCase
from pychron.experiment.tests.conditionals import ConditionalsTestCase, ParseConditionalsTestCase
from pychron.experiment.tests.duration_tracker import DurationTrackerTestCase
from pychron.experiment.tests.frequency_test import FrequencyTestCase, FrequencyTemplateTestCase
from pychron.experiment.tests.identifier import IdentifierTestCase
from pychron.experiment.tests.peak_hop_parse import PeakHopYamlCase1, PeakHopTxtCase
from pychron.experiment.tests.peak_hop_parse import PeakHopYamlCase2
from pychron.experiment.tests.position_regex_test import XYTestCase
from pychron.experiment.tests.renumber_aliquot_test import RenumberAliquotTestCase
from pychron.external_pipette.tests.external_pipette import ExternalPipetteTestCase
from pychron.processing.tests.age_converter import AgeConverterTestCase
from pychron.processing.tests.plateau import PlateauTestCase
from pychron.processing.tests.ratio import RatioTestCase
from pychron.spectrometer.tests.mftable import MFTableTestCase
from pychron.stage.tests.stage_map import StageMapTestCase, TransformTestCase
#
# os.environ['MassSpecDBVersion'] = '16'
#
# from pychron.paths import paths
#
# paths.build('_dev')
#
# use_logger = False
#
#
def suite():
    # set env. variables
    os.environ["MassSpecDBVersion"] = "16"

    from pychron.paths import paths

    paths.build("_dev")

    # if use_logger:
    #     from pychron.core.helpers.logger_setup import logging_setup
    #     logging_setup('unittests')

    loader = unittest.TestLoader()
    suite = unittest.TestSuite()

    tests = (
        # Canvas
        # CalibrationObjectTestCase,
        # Core
        AlphaTestCase,
        SpellCorrectTestCase,
        FilteringTestCase,
        MultiPeakDetectionTestCase,
        FloatfmtTestCase,
        SigFigStdFmtTestCase,
        CamelCaseTestCase,
        RatioTestCase,
        XMLParserTestCase,
        OLSRegressionTest,
        MeanRegressionTest,
        FilterOLSRegressionTest,
        OLSRegressionTest2,
        TruncateRegressionTest,
        MSWDTestCase,
        # old
        # ExpoRegressionTest,
        # ExpoRegressionTest2,

        # DataMapper
        # USGSVSCFileSourceUnittest,
        # USGSVSCIrradiationSourceUnittest,
        # NMGRLLegacySourceUnittest,

        # Experiment
        PeakHopYamlCase1,
        PeakHopYamlCase2,
        BackupTestCase,
        PeakHopTxtCase,
        DurationTrackerTestCase,
        FrequencyTestCase,
        FrequencyTemplateTestCase,
        XYTestCase,
        RenumberAliquotTestCase,
        ConditionalsTestCase,
        ParseConditionalsTestCase,
        IdentifierTestCase,
        CommentTemplaterTestCase,
        # ExternalPipette
        ExternalPipetteTestCase,
        # Processing
        PlateauTestCase,
        RatioTestCase,
        AgeConverterTestCase,
        # Pyscripts
        # WaitForTestCase,
        # InterpolationTestCase,
        # DocstrContextTestCase,
        # Spectrometer
        # MFTableTestCase,
        # DiscreteMFTableTestCase,
        # IntegrationTimeTestCase,

        # Stage
        # StageMapTestCase,
        # TransformTestCase
    )

    for t in tests:
        suite.addTest(loader.loadTestsFromTestCase(t))

    return suite

if __name__ == "__main__":
    unittest.TextTestRunner(verbosity=2).run(suite())
<|endoftext|>


<commit_before>import numpy as np
import h5py

CHUNK_0_MAX=32
CHUNK_1_MAX=128

class Element(object):
    def append(self, *args, **kwargs):
        raise NotImplementedError

class FixedElement(h5py.Dataset, Element):
    def __init__(self, loc, name, **kwargs):
        if name in loc:
            d = loc[name]
        else:
            d = loc.create_dataset(name, **kwargs)
        super(FixedElement, self).__init__(d._id)
        self.step = None
        self.step_offset = None
        self.time = None
        self.time_offset = None
    def append(self, v, step=None, time=None):
        pass
    def get_by_idx(self, idx):
        return self
    @property
    def value(self):
        return h5py.Dataset(self._id)
    @property
    def element_type(self):
        return 'FixedElement'
    def __repr__(self):
        return 'H5MD FixedElement'

class LinearElement(h5py.Group, Element):
    def __init__(self, loc, name, **kwargs):
        is_new = name not in loc
        g = loc.require_group(name)
        if is_new:
            step = kwargs.pop('step')
            step_offset = kwargs.pop('step_offset', None)
            time = kwargs.pop('time', None)
            time_offset = kwargs.pop('time_offset', None)
        super(LinearElement, self).__init__(g._id)
        if is_new:
            self.value = g.create_dataset('value', **kwargs)
            g.create_dataset('step', data=int(step))
            self.step = int(step)
            if step_offset is not None:
                g['step'].attrs['offset'] = int(step_offset)
                self.step_offset = int(step_offset)
            self.step_offset = None
            if time is not None:
                g.create_dataset('time', data=time)
                self.time = time
                if time_offset is not None:
                    g['time'].attrs['offset'] = time_offset
                self.time_offset = time_offset
        else:
            self.step = g['step'][()]
            if 'offset' in self['step'].attrs:
                self.step_offset = self['step'].attrs['offset']
            else:
                self.step_offset = None
            self.value = self['value']
            if 'time' in self:
                self.time = self['time'][()]
                if 'offset' in self['time'].attrs:
                    self.time_offset = self['time'].attrs['offset']
                else:
                    self.time_offset = None
            else:
                self.time = None
                self.time_offset = None
    @property
    def element_type(self):
        return 'LinearElement'
    def append(self, v, step=None, time=None, region=None, collective=False):
        self.value.resize(self.value.shape[0]+1, axis=0)
        if region is not None:
            if collective:
                with self.value.collective:
                    self.value[-1,region[0]:region[1],...] = v
            else:
                self.value[-1,region[0]:region[1],...] = v
        else:
            self.value[-1] = v
    def get_by_idx(self, idx):
        return self['value'][idx]
    def __repr__(self):
        return 'H5MD LinearElement'

class TimeElement(h5py.Group, Element):
    def __init__(self, loc, name, **kwargs):
        is_new = name not in loc
        g = loc.require_group(name)
        if is_new:
            step_from = kwargs.pop('step_from', None)
            if step_from is not None:
                g['step'] = step_from.step
                self.step = step_from.step
                self.own_step = False
            else:
                self.step = g.create_dataset('step', dtype=int, shape=(0,), maxshape=(None,))
                self.own_step = True
            time = kwargs.pop('time', None)
            if time is not None:
                if self.own_step:
                    if time==True:
                        self.time = g.create_dataset('time', dtype=float, shape=(0,), maxshape=(None,))
                    else:
                        self.time = g.create_dataset('time', data=time)
                else:
                    g['time'] = self.time = step_from.time
            else:
                self.time = None
            self.value = g.create_dataset('value', **kwargs)
        else:
            self.step = g['step']
            if 'offset' in g['step'].attrs:
                self.step_offset = g['step'].attrs['offset']
            else:
                self.step_offset = None
            if 'time' in g:
                self.time = g['time']
                if 'offset' in g['time'].attrs:
                    self.time_offset = g['time'].attrs['offset']
                else:
                    self.time_offset = None
            else:
                self.time = None
                self.time_offset = None
            self.value = g['value']
        super(TimeElement, self).__init__(g._id)
    def append(self, v, step, time=None, region=None, collective=False):
        self.value.resize(self.value.shape[0]+1, axis=0)
        if region is not None:
            if collective:
                with self.value.collective:
                    self.value[-1,region[0]:region[1],...] = v
            else:
                self.value[-1,region[0]:region[1],...] = v
        else:
            self.value[-1] = v
        if self.own_step:
            self.step.resize(self.step.shape[0]+1, axis=0)
            self.step[-1] = step
            if self.time and len(self.time.shape)==1:
                self.time.resize(self.time.shape[0]+1, axis=0)
                self.time[-1] = time
    def get_by_idx(self, idx):
        return self['value'][idx]
    @property
    def element_type(self):
        return 'TimeElement'
    def __repr__(self):
        return 'H5MD TimeElement'

def default_chunks(shape):
    result = list(shape)
    if len(shape)==0:
        raise ValueError
    if shape==(0,):
        return (64,)
    if shape[0]==0:
        result[0] = CHUNK_0_MAX
        result[1] = min(CHUNK_1_MAX, shape[1])
        return tuple(result)
    else:
        result[:2] = CHUNK_0_MAX, CHUNK_1_MAX
        return tuple(np.minimum(result,shape))

def element(loc, name, **kwargs):
    if name in loc:
        tmp_element = loc[name]
        if isinstance(tmp_element,h5py.Group):
            assert 'value' in tmp_element
            assert 'step' in tmp_element
            if tmp_element['step'].shape==():
                return LinearElement(loc, name, **kwargs)
            else:
                return TimeElement(loc, name, **kwargs)
        elif isinstance(tmp_element, h5py.Dataset):
            return FixedElement(loc, name, **kwargs)
        else:
            return None
    store = kwargs.pop('store')
    if store=='fixed':
        return FixedElement(loc, name, **kwargs)
    if 'shape' in kwargs:
        assert 'data' not in kwargs
        kwargs['shape'] = (0,) + kwargs['shape']
        kwargs['chunks'] = default_chunks(kwargs['shape'])
        if 'maxshape' in kwargs:
            kwargs['maxshape'] = (None,) + kwargs['maxshape']
        else:
            kwargs['maxshape'] = (None,) + kwargs['shape'][1:]
    data = kwargs.pop('data', None)
    if data is not None:
        if type(data) is not np.ndarray:
            data = np.asarray(data)
        kwargs['shape'] = (0,) + data.shape
        if 'maxshape' in kwargs:
            kwargs['maxshape'] = (None,) + kwargs['maxshape']
        else:
            kwargs['maxshape'] = (None,) + data.shape
        kwargs['dtype'] = data.dtype
    if store=='linear':
        return LinearElement(loc, name, **kwargs)
    elif store=='time':
        return TimeElement(loc, name, **kwargs)
    else:
        raise ValueError

class File(h5py.File):
    def __init__(self, name, mode=None, *args, **kwargs):
        if mode=='w':
            author = kwargs.pop('author', 'N/A')
            author_email = kwargs.pop('author_email', None)
            creator = kwargs.pop('creator', 'N/A')
            creator_version = kwargs.pop('creator_version', 'N/A')
        super(File, self).__init__(name, mode, *args, **kwargs)
        if mode=='w':
            g = self.create_group('h5md')
            g.attrs['version'] = np.array([1,1])
            g = self.create_group('h5md/author')
            g.attrs['name'] = author
            if author_email is not None:
                g.attrs['email'] = author_email
            g = self.create_group('h5md/creator')
            g.attrs['name'] = creator
            g.attrs['version'] = creator_version

    def particles_group(self, name):
        return ParticlesGroup(self, name)

class ParticlesGroup(h5py.Group):
    """Represents a particles group within a H5MD file."""
    def __init__(self, parent, name):
        """Create a new ParticlesGroup object."""
        super(ParticlesGroup, self).__init__(parent.require_group('particles').require_group(name)._id)

    def create_box(self, dimension=None, boundary=None, **kwargs):
        self.box = self.require_group('box')
        if dimension is not None and boundary is not None:
            assert len(boundary)==dimension
            self.box.attrs['dimension'] = dimension
            self.box.attrs['boundary'] = np.string_(boundary)
        if len(kwargs)>0:
            self.box.edges = element(self.box, 'edges', **kwargs)
<commit_msg>add error on illegal arguments to TimeElement<commit_after>import numpy as np
import h5py

CHUNK_0_MAX=32
CHUNK_1_MAX=128

class Element(object):
    def append(self, *args, **kwargs):
        raise NotImplementedError

class FixedElement(h5py.Dataset, Element):
    def __init__(self, loc, name, **kwargs):
        if name in loc:
            d = loc[name]
        else:
            d = loc.create_dataset(name, **kwargs)
        super(FixedElement, self).__init__(d._id)
        self.step = None
        self.step_offset = None
        self.time = None
        self.time_offset = None
    def append(self, v, step=None, time=None):
        pass
    def get_by_idx(self, idx):
        return self
    @property
    def value(self):
        return h5py.Dataset(self._id)
    @property
    def element_type(self):
        return 'FixedElement'
    def __repr__(self):
        return 'H5MD FixedElement'

class LinearElement(h5py.Group, Element):
    def __init__(self, loc, name, **kwargs):
        is_new = name not in loc
        g = loc.require_group(name)
        if is_new:
            step = kwargs.pop('step')
            step_offset = kwargs.pop('step_offset', None)
            time = kwargs.pop('time', None)
            time_offset = kwargs.pop('time_offset', None)
        super(LinearElement, self).__init__(g._id)
        if is_new:
            self.value = g.create_dataset('value', **kwargs)
            g.create_dataset('step', data=int(step))
            self.step = int(step)
            if step_offset is not None:
                g['step'].attrs['offset'] = int(step_offset)
                self.step_offset = int(step_offset)
            self.step_offset = None
            if time is not None:
                g.create_dataset('time', data=time)
                self.time = time
                if time_offset is not None:
                    g['time'].attrs['offset'] = time_offset
                self.time_offset = time_offset
        else:
            self.step = g['step'][()]
            if 'offset' in self['step'].attrs:
                self.step_offset = self['step'].attrs['offset']
            else:
                self.step_offset = None
            self.value = self['value']
            if 'time' in self:
                self.time = self['time'][()]
                if 'offset' in self['time'].attrs:
                    self.time_offset = self['time'].attrs['offset']
                else:
                    self.time_offset = None
            else:
                self.time = None
                self.time_offset = None
    @property
    def element_type(self):
        return 'LinearElement'
    def append(self, v, step=None, time=None, region=None, collective=False):
        self.value.resize(self.value.shape[0]+1, axis=0)
        if region is not None:
            if collective:
                with self.value.collective:
                    self.value[-1,region[0]:region[1],...] = v
            else:
                self.value[-1,region[0]:region[1],...] = v
        else:
            self.value[-1] = v
    def get_by_idx(self, idx):
        return self['value'][idx]
    def __repr__(self):
        return 'H5MD LinearElement'

class TimeElement(h5py.Group, Element):
    def __init__(self, loc, name, **kwargs):
        is_new = name not in loc
        g = loc.require_group(name)
        if is_new:
            step_from = kwargs.pop('step_from', None)
            if step_from is not None:
                g['step'] = step_from.step
                self.step = step_from.step
                self.own_step = False
            else:
                self.step = g.create_dataset('step', dtype=int, shape=(0,), maxshape=(None,))
                self.own_step = True
            time = kwargs.pop('time', None)
            if time is not None:
                if self.own_step:
                    if time==True:
                        self.time = g.create_dataset('time', dtype=float, shape=(0,), maxshape=(None,))
                    else:
                        raise ValueError("Time must be True or None for TimeElement")
                else:
                    g['time'] = self.time = step_from.time
            else:
                self.time = None
            self.value = g.create_dataset('value', **kwargs)
        else:
            self.step = g['step']
            if 'offset' in g['step'].attrs:
                self.step_offset = g['step'].attrs['offset']
            else:
                self.step_offset = None
            if 'time' in g:
                self.time = g['time']
                if 'offset' in g['time'].attrs:
                    self.time_offset = g['time'].attrs['offset']
                else:
                    self.time_offset = None
            else:
                self.time = None
                self.time_offset = None
            self.value = g['value']
        super(TimeElement, self).__init__(g._id)
    def append(self, v, step, time=None, region=None, collective=False):
        self.value.resize(self.value.shape[0]+1, axis=0)
        if region is not None:
            if collective:
                with self.value.collective:
                    self.value[-1,region[0]:region[1],...] = v
            else:
                self.value[-1,region[0]:region[1],...] = v
        else:
            self.value[-1] = v
        if self.own_step:
            self.step.resize(self.step.shape[0]+1, axis=0)
            self.step[-1] = step
            if self.time and len(self.time.shape)==1:
                self.time.resize(self.time.shape[0]+1, axis=0)
                self.time[-1] = time
    def get_by_idx(self, idx):
        return self['value'][idx]
    @property
    def element_type(self):
        return 'TimeElement'
    def __repr__(self):
        return 'H5MD TimeElement'

def default_chunks(shape):
    result = list(shape)
    if len(shape)==0:
        raise ValueError
    if shape==(0,):
        return (64,)
    if shape[0]==0:
        result[0] = CHUNK_0_MAX
        result[1] = min(CHUNK_1_MAX, shape[1])
        return tuple(result)
    else:
        result[:2] = CHUNK_0_MAX, CHUNK_1_MAX
        return tuple(np.minimum(result,shape))

def element(loc, name, **kwargs):
    if name in loc:
        tmp_element = loc[name]
        if isinstance(tmp_element,h5py.Group):
            assert 'value' in tmp_element
            assert 'step' in tmp_element
            if tmp_element['step'].shape==():
                return LinearElement(loc, name, **kwargs)
            else:
                return TimeElement(loc, name, **kwargs)
        elif isinstance(tmp_element, h5py.Dataset):
            return FixedElement(loc, name, **kwargs)
        else:
            return None
    store = kwargs.pop('store')
    if store=='fixed':
        return FixedElement(loc, name, **kwargs)
    if 'shape' in kwargs:
        assert 'data' not in kwargs
        kwargs['shape'] = (0,) + kwargs['shape']
        kwargs['chunks'] = default_chunks(kwargs['shape'])
        if 'maxshape' in kwargs:
            kwargs['maxshape'] = (None,) + kwargs['maxshape']
        else:
            kwargs['maxshape'] = (None,) + kwargs['shape'][1:]
    data = kwargs.pop('data', None)
    if data is not None:
        if type(data) is not np.ndarray:
            data = np.asarray(data)
        kwargs['shape'] = (0,) + data.shape
        if 'maxshape' in kwargs:
            kwargs['maxshape'] = (None,) + kwargs['maxshape']
        else:
            kwargs['maxshape'] = (None,) + data.shape
        kwargs['dtype'] = data.dtype
    if store=='linear':
        return LinearElement(loc, name, **kwargs)
    elif store=='time':
        return TimeElement(loc, name, **kwargs)
    else:
        raise ValueError

class File(h5py.File):
    def __init__(self, name, mode=None, *args, **kwargs):
        if mode=='w':
            author = kwargs.pop('author', 'N/A')
            author_email = kwargs.pop('author_email', None)
            creator = kwargs.pop('creator', 'N/A')
            creator_version = kwargs.pop('creator_version', 'N/A')
        super(File, self).__init__(name, mode, *args, **kwargs)
        if mode=='w':
            g = self.create_group('h5md')
            g.attrs['version'] = np.array([1,1])
            g = self.create_group('h5md/author')
            g.attrs['name'] = author
            if author_email is not None:
                g.attrs['email'] = author_email
            g = self.create_group('h5md/creator')
            g.attrs['name'] = creator
            g.attrs['version'] = creator_version

    def particles_group(self, name):
        return ParticlesGroup(self, name)

class ParticlesGroup(h5py.Group):
    """Represents a particles group within a H5MD file."""
    def __init__(self, parent, name):
        """Create a new ParticlesGroup object."""
        super(ParticlesGroup, self).__init__(parent.require_group('particles').require_group(name)._id)

    def create_box(self, dimension=None, boundary=None, **kwargs):
        self.box = self.require_group('box')
        if dimension is not None and boundary is not None:
            assert len(boundary)==dimension
            self.box.attrs['dimension'] = dimension
            self.box.attrs['boundary'] = np.string_(boundary)
        if len(kwargs)>0:
            self.box.edges = element(self.box, 'edges', **kwargs)
<|endoftext|>


<commit_before>import sys
sys.path.append('../')
from pyhparser.pyhparser import *
from pyhparser.cleaner import *
from pyhparser.utils import *
from pyhparser.classParser import *<commit_msg>__initi__ fixes<commit_after>import sys
sys.path.append('../')
from pyhparser.pyhparser import *
from pyhparser.grammar import *
from pyhparser.utils import *
from pyhparser.classParser import *<|endoftext|>