<commit_before>#! /usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2013, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import os
import sys
import shutil
import gzip

#########################################################
debug = "NTA_DATA_DEBUG" in os.environ

#########################################################
def findDataset(datasetPath):
  """Returns the absolute path to a dataset (if not found, rasises Exception)
  The search rules for the dataset are:
    1. NTA_DATA_PATH  (":" separated value similar to PATH)
    2. ./data
    3. ./
    4. NTA_ROOTDIR
    5. ~/nta/current/share/prediction/data (unix)

  If the dataset is compressed (ends in .gz) returns the name of the compressed
  file.
  """

  if os.path.isabs(datasetPath) and os.path.isfile(datasetPath):
    if debug:
      print "Data found at absolute path '%s'" % datasetPath
    return datasetPath

  dataDirs = _getDataDirs()
  for d, location in dataDirs:
    fullPath = os.path.abspath(os.path.join(d, datasetPath))
    if os.path.isfile(fullPath):
      if debug:
        print "Data '%s' found at '%s' %s" % (datasetPath, fullPath, location)
      return fullPath

  # Not found. Look for a compressed dataset but don't uncompress is
  for d, location in dataDirs:
    fullPath = os.path.abspath(os.path.join(d, datasetPath))
    gzPath = fullPath + '.gz'
    if os.path.isfile(gzPath):
      return gzPath

  # Not found
  if debug:
    print "Data '%s' not found. " % datasetPath,
    print "dataDirs: %s" % dataDirs
    if "NTA_DATA_PATH" in os.environ:
      print "NTA_DATA_PATH: %s" % os.environ["NTA_DATA_PATH"]
    else:
      print "NTA_DATA_PATH is not set."

  raise Exception('Unable to locate: %s using NTA_DATA_PATH of %s' % \
                  (datasetPath, os.environ.get('NTA_DATA_PATH', '')))

def prependDatasetSearchPath(datasetDir):
  """ Prepend a dataset search path to the locagtions that findDataset() will
  search.  This is useful for test code that needs to
  copy the desired datasets to a temporary directory in order to avoid
  leaving generated results (e.g. from aggregation) in the source tree.
  """
  oldNtaDataPaths =  os.environ.get('NTA_DATA_PATH')
  if oldNtaDataPaths is None:
    newNtaDataPaths = datasetDir
  else:
    if oldNtaDataPaths.startswith(datasetDir):
      return

    newNtaDataPaths = datasetDir + ":" + oldNtaDataPaths

  os.environ['NTA_DATA_PATH'] = newNtaDataPaths

  return

def _getDataDirs():
  """Return all possible data directories in order

  1. NTA_DATA_PATH  (":" separated value similar to PATH)
  2. ./data
  3. ./
  4. NTA_ROOTDIR
  5. ~/nta/current/share/prediction/data (unix)

  """
  dataDirs = []
  if "NTA_DATA_PATH" in os.environ:
    dirs = os.environ["NTA_DATA_PATH"].split(":")
    for d in dirs:
      dataDirs.append((d, 'via NTA_DATA_PATH'))

  #  Local data dir
  dataDirs.append(('data', "in local directory 'data'"))
  dataDirs.append((".", "in current working directory"))

  if "NTA_ROOTDIR" in os.environ:
    d = os.path.join(os.environ['NTA_ROOTDIR'], 'share/prediction/data')
    if os.path.isdir(d):
      if d not in zip(*dataDirs)[0]:
        dataDirs.append((d, 'in $NTA_ROOTDIR/share/prediction/data'))

  if sys.platform != 'win32' and 'HOME' in os.environ:
    d = os.path.join(os.environ['HOME'], 'nta/current/share/prediction/data')
    if os.path.isdir(d):
      if d not in zip(*dataDirs)[0]:
        dataDirs.append((d, 'in $NTA_ROOTDIR/share/prediction/data'))

  return dataDirs

#########################################################
def uncompressAndCopyDataset(path, destDir=None, overwrite=True):
  """Uncompress a data file. If destDir is specified, uncompressed it
  into this directory. Otherwise uncompress it into the original directory.

  If the file is not compressed (does not end in .gz) copy it to the
  destination directory if needed.

  If overwrite is True, we overwrite any existing file. Otherwise,
  we just return without copying

  The uncompress and copy operations are combined to minimize
  the work, since we can uncompress directly into the destination
  directory.

  """
  if not os.path.isfile(path):
    raise RuntimeError("Dataset '%s' not found" % dataset)

  if destDir is not None and not os.path.isdir(destDir):
        raise RuntimeError("Destination directory '%s' not found" % destDir)

  if not path.endswith(".gz"):
    # File is not compressed. Copy it if requested
    currentDir = os.path.dirname(path)
    if destDir is None or currentDir == destDir:
      uncompressedPath = path
    else:
      uncompressedPath = os.path.join(destDir, os.path.basename(path))
      if not os.path.exists(uncompressedPath) or overwrite:
        shutil.copy2(path, uncompressedPath)

  else:
    # File is compressed. Uncompress it.
    uncompressedPath = path[:-3]
    if destDir is not None:
      basename = os.path.basename(uncompressedPath)
      uncompressedPath = os.path.join(destDir, basename)
    if not os.path.exists(uncompressedPath) or overwrite:
      f = gzip.GzipFile(path)
      o = open(uncompressedPath, 'w')
      for line in f.readlines():
        o.write(line)
      o.close()
    assert os.path.isfile(uncompressedPath)
  return uncompressedPath
<commit_msg>Update datasethelpers.py<commit_after>#! /usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2013, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import os
import sys
import shutil
import gzip

#########################################################
debug = "NTA_DATA_DEBUG" in os.environ

#########################################################
def findDataset(datasetPath):
  """Returns the absolute path to a dataset (if not found, rasises Exception)
  The search rules for the dataset are:
    1. NTA_DATA_PATH  (":" separated value similar to PATH)
    2. ./data
    3. ./
    4. NTA
    5. ~/nta/current/share/prediction/data (unix)

  If the dataset is compressed (ends in .gz) returns the name of the compressed
  file.
  """

  if os.path.isabs(datasetPath) and os.path.isfile(datasetPath):
    if debug:
      print "Data found at absolute path '%s'" % datasetPath
    return datasetPath

  dataDirs = _getDataDirs()
  for d, location in dataDirs:
    fullPath = os.path.abspath(os.path.join(d, datasetPath))
    if os.path.isfile(fullPath):
      if debug:
        print "Data '%s' found at '%s' %s" % (datasetPath, fullPath, location)
      return fullPath

  # Not found. Look for a compressed dataset but don't uncompress is
  for d, location in dataDirs:
    fullPath = os.path.abspath(os.path.join(d, datasetPath))
    gzPath = fullPath + '.gz'
    if os.path.isfile(gzPath):
      return gzPath

  # Not found
  if debug:
    print "Data '%s' not found. " % datasetPath,
    print "dataDirs: %s" % dataDirs
    if "NTA_DATA_PATH" in os.environ:
      print "NTA_DATA_PATH: %s" % os.environ["NTA_DATA_PATH"]
    else:
      print "NTA_DATA_PATH is not set."

  raise Exception('Unable to locate: %s using NTA_DATA_PATH of %s' % \
                  (datasetPath, os.environ.get('NTA_DATA_PATH', '')))

def prependDatasetSearchPath(datasetDir):
  """ Prepend a dataset search path to the locagtions that findDataset() will
  search.  This is useful for test code that needs to
  copy the desired datasets to a temporary directory in order to avoid
  leaving generated results (e.g. from aggregation) in the source tree.
  """
  oldNtaDataPaths =  os.environ.get('NTA_DATA_PATH')
  if oldNtaDataPaths is None:
    newNtaDataPaths = datasetDir
  else:
    if oldNtaDataPaths.startswith(datasetDir):
      return

    newNtaDataPaths = datasetDir + ":" + oldNtaDataPaths

  os.environ['NTA_DATA_PATH'] = newNtaDataPaths

  return

def _getDataDirs():
  """Return all possible data directories in order

  1. NTA_DATA_PATH  (":" separated value similar to PATH)
  2. ./data
  3. ./
  4. NTA
  5. ~/nta/current/share/prediction/data (unix)

  """
  dataDirs = []
  if "NTA_DATA_PATH" in os.environ:
    dirs = os.environ["NTA_DATA_PATH"].split(":")
    for d in dirs:
      dataDirs.append((d, 'via NTA_DATA_PATH'))

  #  Local data dir
  dataDirs.append(('data', "in local directory 'data'"))
  dataDirs.append((".", "in current working directory"))

  if "NTA" in os.environ:
    d = os.path.join(os.environ['NTA'], 'share/prediction/data')
    if os.path.isdir(d):
      if d not in zip(*dataDirs)[0]:
        dataDirs.append((d, 'in $NTA/share/prediction/data'))

  if sys.platform != 'win32' and 'HOME' in os.environ:
    d = os.path.join(os.environ['HOME'], 'nta/current/share/prediction/data')
    if os.path.isdir(d):
      if d not in zip(*dataDirs)[0]:
        dataDirs.append((d, 'in $NTA/share/prediction/data'))

  return dataDirs

#########################################################
def uncompressAndCopyDataset(path, destDir=None, overwrite=True):
  """Uncompress a data file. If destDir is specified, uncompressed it
  into this directory. Otherwise uncompress it into the original directory.

  If the file is not compressed (does not end in .gz) copy it to the
  destination directory if needed.

  If overwrite is True, we overwrite any existing file. Otherwise,
  we just return without copying

  The uncompress and copy operations are combined to minimize
  the work, since we can uncompress directly into the destination
  directory.

  """
  if not os.path.isfile(path):
    raise RuntimeError("Dataset '%s' not found" % dataset)

  if destDir is not None and not os.path.isdir(destDir):
        raise RuntimeError("Destination directory '%s' not found" % destDir)

  if not path.endswith(".gz"):
    # File is not compressed. Copy it if requested
    currentDir = os.path.dirname(path)
    if destDir is None or currentDir == destDir:
      uncompressedPath = path
    else:
      uncompressedPath = os.path.join(destDir, os.path.basename(path))
      if not os.path.exists(uncompressedPath) or overwrite:
        shutil.copy2(path, uncompressedPath)

  else:
    # File is compressed. Uncompress it.
    uncompressedPath = path[:-3]
    if destDir is not None:
      basename = os.path.basename(uncompressedPath)
      uncompressedPath = os.path.join(destDir, basename)
    if not os.path.exists(uncompressedPath) or overwrite:
      f = gzip.GzipFile(path)
      o = open(uncompressedPath, 'w')
      for line in f.readlines():
        o.write(line)
      o.close()
    assert os.path.isfile(uncompressedPath)
  return uncompressedPath
<|endoftext|>


<commit_before>#!/usr/bin/env python
import time
import logging
from evento import Event
from py2030.base_component import BaseComponent

try:
    # import pysimpledmx
    from py2030.dependencies import pysimpledmx
except ImportError:
    pysimpledmx = None

class DmxOutput(BaseComponent):
    config_name = 'dmx_outputs'

    def __init__(self, options = {}):
        self.options = options
        self.event_manager = None
        self.output_events = None

        self.num_channels = self.options['num_channels'] if 'num_channels' in self.options else 10
        self.channel_event_prefix = self.options['channel_event_prefix'] if 'channel_event_prefix' in self.options else 'ch'
        self.deviceName = self.options['deviceName'] if 'deviceName' in self.options else None
        self.deviceNumber = int(self.options['deviceNumber']) if 'deviceNumber' in self.options else None

        self.fps = self.options['fps'] if 'fps' in self.options else 4.0
        self.frameTime = 1.0 / self.fps

        self.logger = logging.getLogger(__name__)
        if 'verbose' in options and options['verbose']:
            self.logger.setLevel(logging.DEBUG)

        # events
        self.messageEvent = Event()
        self.dirty = True;
        self.nextFrameTime = 0.0

    def setup(self, event_manager=None):
        self.event_manager = event_manager

        if not pysimpledmx:
            self.logger.warn("pysimpledmx lib not loaded")
            self.dmx = None
        else:
            try:
                if self.deviceName != None:
                    self.dmx = pysimpledmx.DMXConnection(self.deviceName)
                elif self.deviceNumber != None:
                    self.dmx = pysimpledmx.DMXConnection(self.deviceNumber)
            except RuntimeError as err:
                self.logger.warn("Failed to open DMX serial device:")
                self.logger.warn(str(err))
                self.dmx = None

        if self.dmx:
            self.dmx.clear()

        for i in range(self.num_channels):
            evt = self.event_manager.get(self.channel_event_prefix+str(i+1))
            evt += lambda val, idx=i: self._setChannel(idx, val)

        # self.logger.debug('registered event listeners for '+str(self.num_channels)+' channels')

    def update(self):
        t = time.time()
        if self.dirty and t >= self.nextFrameTime:
            # dmx update
            if self.dmx != None:
                self.dmx.render()

            # self.logger.debug('DMX update')
            self.nextFrameTime = t + self.frameTime
            self.dirty = False

    def destroy(self):
        if self.dmx:
            self.dmx.clear()
            self.dmx.render()
            self.dmx.close()
            self.dmx = None

        # remove events from event manager, good idea?
        for i in range(self.num_channels):
            self.event_manager.remove(self.channel_event_prefix+str(i))

        self.event_manager = None
        self.output_events = None

    def _setChannel(self, idx, val):
        # self.logger.debug('_setChannel: '+str(idx+1)+' with: '+str(val)+' ('+str(int(val * 255.0))+')')
        dmxVal = int(val * 255.0)

        if self.dmx != None:
            self.dmx.setChannel(idx+1, dmxVal) # +2?!
        self.dirty = True
        self.logger.debug('new value for channel '+str(idx+1)+": "+str(dmxVal))
<commit_msg>restore +2 (still don't know why)<commit_after>#!/usr/bin/env python
import time
import logging
from evento import Event
from py2030.base_component import BaseComponent

try:
    # import pysimpledmx
    from py2030.dependencies import pysimpledmx
except ImportError:
    pysimpledmx = None

class DmxOutput(BaseComponent):
    config_name = 'dmx_outputs'

    def __init__(self, options = {}):
        self.options = options
        self.event_manager = None
        self.output_events = None

        self.num_channels = self.options['num_channels'] if 'num_channels' in self.options else 10
        self.channel_event_prefix = self.options['channel_event_prefix'] if 'channel_event_prefix' in self.options else 'ch'
        self.deviceName = self.options['deviceName'] if 'deviceName' in self.options else None
        self.deviceNumber = int(self.options['deviceNumber']) if 'deviceNumber' in self.options else None

        self.fps = self.options['fps'] if 'fps' in self.options else 4.0
        self.frameTime = 1.0 / self.fps

        self.logger = logging.getLogger(__name__)
        if 'verbose' in options and options['verbose']:
            self.logger.setLevel(logging.DEBUG)

        # events
        self.messageEvent = Event()
        self.dirty = True;
        self.nextFrameTime = 0.0

    def setup(self, event_manager=None):
        self.event_manager = event_manager

        if not pysimpledmx:
            self.logger.warn("pysimpledmx lib not loaded")
            self.dmx = None
        else:
            try:
                if self.deviceName != None:
                    self.dmx = pysimpledmx.DMXConnection(self.deviceName)
                elif self.deviceNumber != None:
                    self.dmx = pysimpledmx.DMXConnection(self.deviceNumber)
            except RuntimeError as err:
                self.logger.warn("Failed to open DMX serial device:")
                self.logger.warn(str(err))
                self.dmx = None

        if self.dmx:
            self.dmx.clear()

        for i in range(self.num_channels):
            evt = self.event_manager.get(self.channel_event_prefix+str(i+1))
            evt += lambda val, idx=i: self._setChannel(idx, val)

        # self.logger.debug('registered event listeners for '+str(self.num_channels)+' channels')

    def update(self):
        t = time.time()
        if self.dirty and t >= self.nextFrameTime:
            # dmx update
            if self.dmx != None:
                self.dmx.render()

            # self.logger.debug('DMX update')
            self.nextFrameTime = t + self.frameTime
            self.dirty = False

    def destroy(self):
        if self.dmx:
            self.dmx.clear()
            self.dmx.render()
            self.dmx.close()
            self.dmx = None

        # remove events from event manager, good idea?
        for i in range(self.num_channels):
            self.event_manager.remove(self.channel_event_prefix+str(i))

        self.event_manager = None
        self.output_events = None

    def _setChannel(self, idx, val):
        # self.logger.debug('_setChannel: '+str(idx+1)+' with: '+str(val)+' ('+str(int(val * 255.0))+')')
        dmxVal = int(val * 255.0)

        if self.dmx != None:
            self.dmx.setChannel(idx+2, dmxVal) # +2?!
        self.dirty = True
        self.logger.debug('new value for channel '+str(idx+1)+": "+str(dmxVal))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Support i3pystatus modules in py3status.

i3pystatus, https://github.com/enkore/i3pystatus, is an alternative to
py3status and provides a variety of modules. This py3status module allows
these modules to run and be display inside py3status.

Configuration parameters:
    module: specifiy i3pystatus module to use (default None)

Requires:
    i3pystatus: i3status replacement written in python

Examples:
```
# the modules parameters are provided as such
i3pystatus clock {
    module = 'clock'
    format = [('%a %b %-d %b %X', 'America/New_York'), ('%X', 'Etc/GMT+9')]
}

# if backend(s) are provided they should be given as a dict with the key being
# the backend name and the value being a dict of the backend settings
i3pystatus weather {
    module = 'weather'
    format = '{condition} {current_temp}{temp_unit}[ {icon}]'
    format += '[ Hi: {high_temp}][ Lo: {low_temp}][ {update_error}]'
    backend = {
        'weathercom.Weathercom': {
            'location_code': '94107:4:US',
            'units': 'imperial',
        }
    }
}

# backends that have no configuration should be defined as shown here
i3pystatus updates{
    module = 'updates'
    backends = {'dnf.Dnf': {}}
}
```

@author tobes

SAMPLE OUTPUT
{'full_text': 'i3pystatus module'}
"""

import os

from importlib import import_module
from threading import Timer

try:
    import i3pystatus
except ImportError:
    i3pystatus = None

MIN_CHECK_INTERVAL = 15  # the minimum time between checking the modules output
MAX_AUTO_TIMEOUT = 30  # the longest we'll leave it without checking the output

CLICK_EVENTS = [
    "on_leftclick",
    "on_middleclick",
    "on_rightclick",
    "on_upscroll",
    "on_downscroll",
    "on_unhandledclick",
]

DBL_CLICK_EVENTS = [
    "on_doubleleftclick",
    "on_doublemiddleclick",
    "on_doublerightclick",
    "on_doubleupscroll",
    "on_doubledownscroll",
    "on_doubleunhandledclick",
]

# we can only have one status running ever
status = None
backends = None

def get_backends():
    """
    Get backends info so that we can find the correct one.
    We just look in the directory structure to find modules.
    """
    IGNORE_DIRS = ["core", "tools", "utils"]

    global backends
    if backends is None:
        backends = {}
        i3pystatus_dir = os.path.dirname(i3pystatus.__file__)
        subdirs = [
            x
            for x in next(os.walk(i3pystatus_dir))[1]
            if not x.startswith("_") and x not in IGNORE_DIRS
        ]
        for subdir in subdirs:
            dirs = next(os.walk(os.path.join(i3pystatus_dir, subdir)))[2]
            backends.update(
                {
                    x[:-3]: "i3pystatus.%s.%s" % (subdir, x[:-3])
                    for x in dirs
                    if not x.startswith("_") and x.endswith(".py")
                }
            )
    return backends

class ClickTimer:
    """
    i3pystatus support double clicks that py3status does not
    therefore we have to add this functionality.

    Although double clicks are a simple concept double clicks on the mouse
    wheel do not make much sense as when we are scrolling it could easily be
    interpreted as multiple double clicks.  For this reason we only worry about
    double clicks if the mouse button has a double click event.  If it dies not
    then we just count the click as a single click. more than 2 clicks are also
    viewed as a double click.

    Additionally all buttons 6 and more use the unhandled click event but each
    button is treated independently.
    """

    def __init__(self, parent, callbacks):
        self.callbacks = callbacks
        self.parent = parent
        self.last_button = None
        self.clicks = 0
        self.click_time = getattr(self.parent.module, "multi_click_timeout", 0.25)
        self.timer = None

    def trigger(self):
        """
        Actually trigger the event
        """
        if self.last_button:
            button_index = min(self.last_button, len(self.callbacks)) - 1
            click_style = 0 if self.clicks == 1 else 1
            callback = self.callbacks[button_index][click_style]
            if callback:
                # we can do test for str as only python > 3 is supported
                if isinstance(callback, str):
                    # no parameters
                    callback_name = callback
                    callback_args = []
                else:
                    # has parameters
                    callback_name = callback[0]
                    callback_args = callback[1:]
                callback_function = getattr(self.parent.module, callback_name)
                callback_function(*callback_args)
                # We want to ensure that the module has the latest output.  The
                # best way is to call the run method of the module
                self.parent.module.run()
            else:
                self.parent.py3.prevent_refresh()
        self.last_button = None
        self.clicks = 0

    def event(self, button):
        """
        button has been clicked
        """
        # cancel any pending timer
        if self.timer:
            self.timer.cancel()
        if self.last_button != button:
            if self.last_button:
                # new button clicked process the one before.
                self.trigger()
            self.clicks = 1
        else:
            self.clicks += 1
        # do we accept double clicks on this button?
        # if not then just process the click
        button_index = min(button, len(self.callbacks)) - 1
        if not self.callbacks[button_index][1]:
            # set things up correctly for the trigger
            self.clicks = 1
            self.last_button = button
            self.trigger()
        else:
            # set a timeout to trigger the click
            # this will be cancelled if we click again before it runs
            self.last_button = button
            self.timer = Timer(self.click_time, self.trigger)
            self.timer.start()

SKIP_ATTRS = ["on_click", "run", "post_config_hook", "module", "py3"]

class Py3status:
    """
    """

    # available configuration parameters
    module = None

    def post_config_hook(self):
        if i3pystatus is None:
            raise Exception("i3pystatus is not installed")

        if self.py3.is_python_2():
            raise Exception("Python 2 not supported by i3pystatus :(")

        if not self.module:
            raise Exception("No module selected")

        settings = {}
        for attribute in dir(self):
            if attribute.startswith("__") or attribute in SKIP_ATTRS:
                continue
            settings[attribute] = getattr(self, attribute)

        # backends
        backend_type = None
        if "backend" in settings:
            backend_type = "backend"
        elif "backends" in settings:
            backend_type = "backends"

        if backend_type:
            backends = settings[backend_type]
            backends_initiated = []
            for key, value in backends.items():
                mod_info = key.split(".")
                backend_module = import_module(get_backends().get(mod_info[0]))
                try:
                    backend = getattr(backend_module, mod_info[1])(**value)
                except i3pystatus.core.exceptions.ConfigMissingError as e:
                    msg = e.message
                    msg = (
                        "i3pystatus module `{}` backend `{}`"
                        "missing configuration options {}"
                    ).format(self.module, key, msg[msg.index("{") :])
                    self.py3.notify_user(msg)
                    raise Exception("Missing configuration options")
                backends_initiated.append(backend)
            if backend_type == "backend":
                settings["backend"] = backends_initiated[0]
            else:
                settings["backends"] = backends_initiated

        # i3pystatus.Status can only exist once
        # so create it and share
        global status

        if status is None:
            status = i3pystatus.Status(standalone=False)

        # create the module and register it
        finder = status.modules.finder
        try:
            module = finder.instanciate_class_from_module(self.module, **settings)
        except i3pystatus.core.exceptions.ConfigMissingError as e:
            msg = e.message
            msg = "i3pystatus module `{}` missing configuration options {}".format(
                self.module, msg[msg.index("{") :]
            )
            self.py3.notify_user(msg)
            raise Exception("Missing configuration options")
        status.register(module)
        self.module = module

        self.is_interval_module = isinstance(module, i3pystatus.IntervalModule)
        # modules update their output independently so we need to periodically
        # check if it has been updated.  For modules with long intervals it is
        # important to do this output check much more regularly thank the
        # interval.
        self._cache_timeout = min(
            getattr(module, "interval", MIN_CHECK_INTERVAL), MIN_CHECK_INTERVAL
        )

        # get callbacks available, useful for deciding if double clicks exist
        callbacks = []
        for i in range(len(CLICK_EVENTS)):
            click = getattr(module, CLICK_EVENTS[i], None)
            dbclick = getattr(module, DBL_CLICK_EVENTS[i], None)
            callbacks.append((click, dbclick))

        self._click_timer = ClickTimer(self, callbacks)
        self._last_content = None
        self._timeout = 1

    def run(self):
        output = self.module.output or {"full_text": ""}

        if self._last_content != output:
            self._last_content = output
            self._timeout = 1
        else:
            self._timeout *= 2
            if self._timeout > MAX_AUTO_TIMEOUT:
                self._timeout = MAX_AUTO_TIMEOUT

        # some modules return tuples
        if isinstance(output["full_text"], tuple):
            output["full_text"] = output["full_text"][0]

        if self.is_interval_module:
            output["cached_until"] = self.py3.time_in(
                sync_to=min(self._cache_timeout, self._timeout)
            )
        else:
            output["cached_until"] = self.py3.time_in(sync_to=self._timeout)

        return output

    def on_click(self, event):
        self._click_timer.event(event["button"])
<commit_msg>i3pystatus: check python first before telling python2 users to install (#1760)<commit_after># -*- coding: utf-8 -*-
"""
Support i3pystatus modules in py3status.

i3pystatus, https://github.com/enkore/i3pystatus, is an alternative to
py3status and provides a variety of modules. This py3status module allows
these modules to run and be display inside py3status.

Configuration parameters:
    module: specifiy i3pystatus module to use (default None)

Requires:
    i3pystatus: i3status replacement written in python

Examples:
```
# the modules parameters are provided as such
i3pystatus clock {
    module = 'clock'
    format = [('%a %b %-d %b %X', 'America/New_York'), ('%X', 'Etc/GMT+9')]
}

# if backend(s) are provided they should be given as a dict with the key being
# the backend name and the value being a dict of the backend settings
i3pystatus weather {
    module = 'weather'
    format = '{condition} {current_temp}{temp_unit}[ {icon}]'
    format += '[ Hi: {high_temp}][ Lo: {low_temp}][ {update_error}]'
    backend = {
        'weathercom.Weathercom': {
            'location_code': '94107:4:US',
            'units': 'imperial',
        }
    }
}

# backends that have no configuration should be defined as shown here
i3pystatus updates{
    module = 'updates'
    backends = {'dnf.Dnf': {}}
}
```

@author tobes

SAMPLE OUTPUT
{'full_text': 'i3pystatus module'}
"""

import os

from importlib import import_module
from threading import Timer

try:
    import i3pystatus
except ImportError:
    i3pystatus = None

MIN_CHECK_INTERVAL = 15  # the minimum time between checking the modules output
MAX_AUTO_TIMEOUT = 30  # the longest we'll leave it without checking the output

CLICK_EVENTS = [
    "on_leftclick",
    "on_middleclick",
    "on_rightclick",
    "on_upscroll",
    "on_downscroll",
    "on_unhandledclick",
]

DBL_CLICK_EVENTS = [
    "on_doubleleftclick",
    "on_doublemiddleclick",
    "on_doublerightclick",
    "on_doubleupscroll",
    "on_doubledownscroll",
    "on_doubleunhandledclick",
]

# we can only have one status running ever
status = None
backends = None

def get_backends():
    """
    Get backends info so that we can find the correct one.
    We just look in the directory structure to find modules.
    """
    IGNORE_DIRS = ["core", "tools", "utils"]

    global backends
    if backends is None:
        backends = {}
        i3pystatus_dir = os.path.dirname(i3pystatus.__file__)
        subdirs = [
            x
            for x in next(os.walk(i3pystatus_dir))[1]
            if not x.startswith("_") and x not in IGNORE_DIRS
        ]
        for subdir in subdirs:
            dirs = next(os.walk(os.path.join(i3pystatus_dir, subdir)))[2]
            backends.update(
                {
                    x[:-3]: "i3pystatus.%s.%s" % (subdir, x[:-3])
                    for x in dirs
                    if not x.startswith("_") and x.endswith(".py")
                }
            )
    return backends

class ClickTimer:
    """
    i3pystatus support double clicks that py3status does not
    therefore we have to add this functionality.

    Although double clicks are a simple concept double clicks on the mouse
    wheel do not make much sense as when we are scrolling it could easily be
    interpreted as multiple double clicks.  For this reason we only worry about
    double clicks if the mouse button has a double click event.  If it dies not
    then we just count the click as a single click. more than 2 clicks are also
    viewed as a double click.

    Additionally all buttons 6 and more use the unhandled click event but each
    button is treated independently.
    """

    def __init__(self, parent, callbacks):
        self.callbacks = callbacks
        self.parent = parent
        self.last_button = None
        self.clicks = 0
        self.click_time = getattr(self.parent.module, "multi_click_timeout", 0.25)
        self.timer = None

    def trigger(self):
        """
        Actually trigger the event
        """
        if self.last_button:
            button_index = min(self.last_button, len(self.callbacks)) - 1
            click_style = 0 if self.clicks == 1 else 1
            callback = self.callbacks[button_index][click_style]
            if callback:
                # we can do test for str as only python > 3 is supported
                if isinstance(callback, str):
                    # no parameters
                    callback_name = callback
                    callback_args = []
                else:
                    # has parameters
                    callback_name = callback[0]
                    callback_args = callback[1:]
                callback_function = getattr(self.parent.module, callback_name)
                callback_function(*callback_args)
                # We want to ensure that the module has the latest output.  The
                # best way is to call the run method of the module
                self.parent.module.run()
            else:
                self.parent.py3.prevent_refresh()
        self.last_button = None
        self.clicks = 0

    def event(self, button):
        """
        button has been clicked
        """
        # cancel any pending timer
        if self.timer:
            self.timer.cancel()
        if self.last_button != button:
            if self.last_button:
                # new button clicked process the one before.
                self.trigger()
            self.clicks = 1
        else:
            self.clicks += 1
        # do we accept double clicks on this button?
        # if not then just process the click
        button_index = min(button, len(self.callbacks)) - 1
        if not self.callbacks[button_index][1]:
            # set things up correctly for the trigger
            self.clicks = 1
            self.last_button = button
            self.trigger()
        else:
            # set a timeout to trigger the click
            # this will be cancelled if we click again before it runs
            self.last_button = button
            self.timer = Timer(self.click_time, self.trigger)
            self.timer.start()

STRING_NOT_SUPPORTED = "python2 not supported"
STRING_NOT_INSTALLED = "not installed"
STRING_MISSING_MODULE = "missing module"
SKIP_ATTRS = ["on_click", "run", "post_config_hook", "module", "py3"]

class Py3status:
    """
    """

    # available configuration parameters
    module = None

    def post_config_hook(self):
        if self.py3.is_python_2():
            raise Exception(STRING_NOT_SUPPORTED)
        elif not i3pystatus:
            raise Exception(STRING_NOT_INSTALLED)
        elif not self.module:
            raise Exception(STRING_MISSING_MODULE)

        settings = {}
        for attribute in dir(self):
            if attribute.startswith("__") or attribute in SKIP_ATTRS:
                continue
            settings[attribute] = getattr(self, attribute)

        # backends
        backend_type = None
        if "backend" in settings:
            backend_type = "backend"
        elif "backends" in settings:
            backend_type = "backends"

        if backend_type:
            backends = settings[backend_type]
            backends_initiated = []
            for key, value in backends.items():
                mod_info = key.split(".")
                backend_module = import_module(get_backends().get(mod_info[0]))
                try:
                    backend = getattr(backend_module, mod_info[1])(**value)
                except i3pystatus.core.exceptions.ConfigMissingError as e:
                    msg = e.message
                    msg = (
                        "i3pystatus module `{}` backend `{}`"
                        "missing configuration options {}"
                    ).format(self.module, key, msg[msg.index("{") :])
                    self.py3.notify_user(msg)
                    raise Exception("Missing configuration options")
                backends_initiated.append(backend)
            if backend_type == "backend":
                settings["backend"] = backends_initiated[0]
            else:
                settings["backends"] = backends_initiated

        # i3pystatus.Status can only exist once
        # so create it and share
        global status

        if status is None:
            status = i3pystatus.Status(standalone=False)

        # create the module and register it
        finder = status.modules.finder
        try:
            module = finder.instanciate_class_from_module(self.module, **settings)
        except i3pystatus.core.exceptions.ConfigMissingError as e:
            msg = e.message
            msg = "i3pystatus module `{}` missing configuration options {}".format(
                self.module, msg[msg.index("{") :]
            )
            self.py3.notify_user(msg)
            raise Exception("Missing configuration options")
        status.register(module)
        self.module = module

        self.is_interval_module = isinstance(module, i3pystatus.IntervalModule)
        # modules update their output independently so we need to periodically
        # check if it has been updated.  For modules with long intervals it is
        # important to do this output check much more regularly thank the
        # interval.
        self._cache_timeout = min(
            getattr(module, "interval", MIN_CHECK_INTERVAL), MIN_CHECK_INTERVAL
        )

        # get callbacks available, useful for deciding if double clicks exist
        callbacks = []
        for i in range(len(CLICK_EVENTS)):
            click = getattr(module, CLICK_EVENTS[i], None)
            dbclick = getattr(module, DBL_CLICK_EVENTS[i], None)
            callbacks.append((click, dbclick))

        self._click_timer = ClickTimer(self, callbacks)
        self._last_content = None
        self._timeout = 1

    def run(self):
        output = self.module.output or {"full_text": ""}

        if self._last_content != output:
            self._last_content = output
            self._timeout = 1
        else:
            self._timeout *= 2
            if self._timeout > MAX_AUTO_TIMEOUT:
                self._timeout = MAX_AUTO_TIMEOUT

        # some modules return tuples
        if isinstance(output["full_text"], tuple):
            output["full_text"] = output["full_text"][0]

        if self.is_interval_module:
            output["cached_until"] = self.py3.time_in(
                sync_to=min(self._cache_timeout, self._timeout)
            )
        else:
            output["cached_until"] = self.py3.time_in(sync_to=self._timeout)

        return output

    def on_click(self, event):
        self._click_timer.event(event["button"])
<|endoftext|>


<commit_before>from unittest import TestCase

__author__ = 'm300028'

from pyCMBS.data import *
from pyCMBS.diagnostic import *
from pyCMBS.plots import *
import scipy as sc
import pylab as pl
import numpy as np
from pyCMBS.region import *
####from pyCMBS import *

class TestData(TestCase):

    def setUp(self):
        #init Data object for testing
        n=1000 #slows down significantly! constraint is percentile  test
        x = sc.randn(n)*100. #generate dummy data
        self.D = Data(None,None)
        d=np.ones((n,1,1))
        self.D.data = d; self.D.data[:,0,0]=x
        self.D.data = np.ma.array(self.D.data,mask=self.D.data != self.D.data)
        self.D.verbose = True; self.D.unit = 'myunit'
        self.D.label = 'testlabel'; self.D.filename = 'testinputfilename.nc'
        self.D.varname = 'testvarname'; self.D.long_name = 'This is the longname'
        self.D.time = np.arange(n) + pl.datestr2num('2001-01-01') - 1
        self.D.time_str = "days since 0001-01-01 00:00:00"
        self.D.calendar = 'gregorian'

    def xxxtest_gleckler_index(self):
        """
        test Reichler index/Gleckler plot
        @return:
        """

        #--- generate sample data ---
        tmp = np.zeros((5,3,1)); tmp[:,0,0] = np.ones(5)*1.
        tmp[:,1,0] = np.ones(5)*2.; tmp[:,2,0] = np.ones(5)*5.

        x = self.D.copy()
        x.data = np.ma.array(tmp,mask=tmp!=tmp)
        x.std  = np.ones(x.data.shape)

        tmp = np.ones((3,1))
        tmp[1,0] = 2.; x.cell_area = tmp*1.

        y = self.D.copy()
        tmp = np.ones(x.data.shape)
        y.data = np.ma.array(tmp,mask=tmp!=tmp)

        #--- diagnostic ---
        D = GlecklerPlot()
        r = D.calc_index(x, y, 'a', 'b')
        self.assertEqual(r,np.sqrt(22.5))

        #... use different std
        x.std = np.ones(x.data.shape)
        x.std[:,2,0] = 0.5
        r=D.calc_index(x, y, 'a', 'b')
        self.assertEqual(r, np.sqrt(42.5))

<commit_msg>fixed tests for gleckler analysis<commit_after>from unittest import TestCase

__author__ = 'm300028'

from pyCMBS.data import *
from pyCMBS.diagnostic import *
from pyCMBS.plots import *
import scipy as sc
import pylab as pl
import numpy as np
from pyCMBS.region import *
####from pyCMBS import *

class TestData(TestCase):

    def setUp(self):
        #init Data object for testing
        n=1000 #slows down significantly! constraint is percentile  test
        x = sc.randn(n)*100. #generate dummy data
        self.D = Data(None, None)
        d=np.ones((n,1,1))
        self.D.data = d; self.D.data[:,0,0]=x
        self.D.data = np.ma.array(self.D.data, mask=self.D.data != self.D.data)
        self.D.verbose = True
        self.D.unit = 'myunit'
        self.D.label = 'testlabel'
        self.D.filename = 'testinputfilename.nc'
        self.D.varname = 'testvarname'
        self.D.long_name = 'This is the longname'
        self.D.time = np.arange(n) + pl.datestr2num('2001-01-01') - 1
        self.D.time_str = "days since 0001-01-01 00:00:00"
        self.D.calendar = 'gregorian'

    def test_gleckler_index(self):
        """
        test Reichler index/Gleckler plot
        @return:
        """

        #--- generate sample data ---
        tmp = np.zeros((5,3,1))
        tmp[:,0,0] = np.ones(5)*1.
        tmp[:,1,0] = np.ones(5)*2.
        tmp[:,2,0] = np.ones(5)*5.

        x = self.D.copy()
        x._temporal_subsetting(0, 4)

        x.data = np.ma.array(tmp, mask=tmp!=tmp)
        x.std = np.ones(x.data.shape)
        x.time[0] = pl.datestr2num('2000-02-15')
        x.time[1] = pl.datestr2num('2000-03-15')
        x.time[2] = pl.datestr2num('2000-04-15')
        x.time[3] = pl.datestr2num('2000-05-15')
        x.time[4] = pl.datestr2num('2000-06-15')

        tmp = np.ones((3, 1))
        tmp[1, 0] = 2.
        x.cell_area = tmp*1.

        y = self.D.copy()
        y._temporal_subsetting(0, 4)
        tmp = np.ones(x.data.shape)
        y.data = np.ma.array(tmp, mask=tmp!=tmp)
        y.time[0] = pl.datestr2num('2000-02-15')
        y.time[1] = pl.datestr2num('2000-03-15')
        y.time[2] = pl.datestr2num('2000-04-15')
        y.time[3] = pl.datestr2num('2000-05-15')
        y.time[4] = pl.datestr2num('2000-06-15')

        #--- diagnostic ---
        D = GlecklerPlot()
        r = D.calc_index(x, y, 'a', 'b')
        self.assertEqual(r, np.sqrt(22.5))

        #... use different std
        x.std = np.ones(x.data.shape)
        x.std[:,2,0] = 0.5
        r=D.calc_index(x, y, 'a', 'b')
        self.assertEqual(r, np.sqrt(42.5))

<|endoftext|>


<commit_before>import numpy as np

from ..core import Node, register_node_type
from pyqtgraph.Qt import QtCore, QtGui
from pyqtgraph.util.mutex import Mutex

import socket
import struct

_dtype_trigger = [('pos', 'int64'),
                ('points', 'int64'),
                ('channel', 'int64'),
                ('type', 'S16'),  # TODO check size
                ('description', 'S16'),  # TODO check size
                ]

def recv_brainamp_frame(brainamp_socket, reqsize):
    buf =b''
    n = 0
    while len(buf) < reqsize:
        newbytes = brainamp_socket.recv(reqsize - n)
        if newbytes == '':
            raise RuntimeError('connection broken')
        buf = buf+newbytes
        n += len(buf)

    if len(buf)>=reqsize:
        buf = buf[:reqsize]
    return buf

class BrainAmpThread(QtCore.QThread):
    def __init__(self, outputs, brainamp_host, brainamp_port, nb_channel, parent=None):
        QtCore.QThread.__init__(self)
        self.outputs = outputs
        self.brainamp_host= brainamp_host
        self.brainamp_port= brainamp_port
        self.nb_channel = nb_channel

        self.lock = Mutex()
        self.running = False

    def run(self):
        brainamp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        brainamp_socket.connect((self.brainamp_host, self.brainamp_port))
        with self.lock:
            self.running = True

        dt = np.dtype('float32')

        head = 0
        head_marker = 0
        while True:
            with self.lock:
                    if not self.running:
                        break

            buf_header = recv_brainamp_frame(brainamp_socket, 24)
            (id1, id2, id3, id4, msgsize, msgtype) = struct.unpack('<llllLL', buf_header)

            rawdata = recv_brainamp_frame(brainamp_socket, msgsize - 24)
            # TODO  msgtype == 3 (msgtype == 1 is header done in Node.configure)
            if msgtype == 4:
                #~ block, chunk, markers = get_signal_and_markers(rawdata, self.nb_channel)
                hs = 12

                # Extract numerical data
                block, points, nb_marker = struct.unpack('<LLL', rawdata[:hs])
                sigsize = dt.itemsize * points * self.nb_channel
                sigs = np.frombuffer(rawdata[hs:hs+sigsize], dtype=dt)
                sigs = sigs.reshape(points, self.nb_channel)
                head += points
                self.outputs['signals'].send(sigs, index=head)

                # Extract markers
                markers = np.empty((nb_marker,), dtype=_dtype_trigger)
                index = hs + sigsize
                for m in range(nb_marker):
                    markersize, = struct.unpack('<L', rawdata[index:index+4])
                    markers['pos'][m], markers['points'][m],markers['channel'][m] = struct.unpack('<LLl', rawdata[index+4:index+16])
                    markers['type'][m], markers['description'][m] = rawdata[index+16:index+markersize].tostring().split('\x00')[:2]
                    index = index + markersize
                head_marker += nb_marker
                self.outputs['triggers'].send(markers, index=nb_marker)

        brainamp_socket.close()

    def stop(self):
        with self.lock:
            self.running = False

class BrainAmpSocket(Node):
    """
    BrainAmp EEG amplifier from Brain Products http://www.brainproducts.com/.

    This class is a bridge between pyacq and the socket-based data streaming
    provided by the Vision recorder acquisition software.
    """
    _output_specs = {'signals': dict(streamtype='analogsignal',dtype='float32',
                                                shape=(-1, 32), compression ='', timeaxis=0,
                                                sample_rate = 512.),
                                'triggers': dict(streamtype = 'event', dtype = _dtype_trigger,
                                                shape = (-1,)),
                                }

    def __init__(self, **kargs):
        Node.__init__(self, **kargs)

    def _configure(self, brainamp_host='localhost', brainamp_port=51244):
        self.brainamp_host = brainamp_host
        self.brainamp_port = brainamp_port

        # recv header from brain amp
        brainamp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        brainamp_socket.connect((self.brainamp_host, self.brainamp_port))
        buf_header = recv_brainamp_frame(brainamp_socket, 24)
        id1, id2, id3, id4, msgsize, msgtype = struct.unpack('<llllLL', buf_header)
        rawdata = recv_brainamp_frame(brainamp_socket, msgsize - 24)
        assert msgtype == 1, 'First message from brainamp is not type 1'
        self.nb_channel, sample_interval = struct.unpack('<Ld', rawdata[:12])
        n = self.nb_channel
        sample_interval = sample_interval*1e-6
        self.sample_rate = 1./sample_interval
        self.resolutions = np.array(struct.unpack('<'+'d'*n, rawdata[12:12+8*n]), dtype='f')
        self.channel_names = rawdata[12+8*n:].decode().split('\x00')[:-1]
        #~ self.channel_indexes = range(nb_channel)
        brainamp_socket.close()

        self.outputs['signals'].spec['shape'] = (-1, self.nb_channel)
        self.outputs['signals'].spec['sample_rate'] = self.sample_rate
        self.outputs['signals'].spec['nb_channel'] = self.nb_channel

    def _initialize(self):
        self._thread = BrainAmpThread(self.outputs, self.brainamp_host, self.brainamp_port,
                             self.nb_channel, parent=self)

    def _start(self):
        self._thread.start()

    def _stop(self):
        self._thread.stop()
        self._thread.wait()

    def _close(self):
        pass

register_node_type(BrainAmpSocket)
<commit_msg>Add _configure doc<commit_after>import numpy as np

from ..core import Node, register_node_type
from pyqtgraph.Qt import QtCore, QtGui
from pyqtgraph.util.mutex import Mutex

import socket
import struct

_dtype_trigger = [('pos', 'int64'),
                ('points', 'int64'),
                ('channel', 'int64'),
                ('type', 'S16'),  # TODO check size
                ('description', 'S16'),  # TODO check size
                ]

def recv_brainamp_frame(brainamp_socket, reqsize):
    buf =b''
    n = 0
    while len(buf) < reqsize:
        newbytes = brainamp_socket.recv(reqsize - n)
        if newbytes == '':
            raise RuntimeError('connection broken')
        buf = buf+newbytes
        n += len(buf)

    if len(buf)>=reqsize:
        buf = buf[:reqsize]
    return buf

class BrainAmpThread(QtCore.QThread):
    def __init__(self, outputs, brainamp_host, brainamp_port, nb_channel, parent=None):
        QtCore.QThread.__init__(self)
        self.outputs = outputs
        self.brainamp_host= brainamp_host
        self.brainamp_port= brainamp_port
        self.nb_channel = nb_channel

        self.lock = Mutex()
        self.running = False

    def run(self):
        brainamp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        brainamp_socket.connect((self.brainamp_host, self.brainamp_port))
        with self.lock:
            self.running = True

        dt = np.dtype('float32')

        head = 0
        head_marker = 0
        while True:
            with self.lock:
                    if not self.running:
                        break

            buf_header = recv_brainamp_frame(brainamp_socket, 24)
            (id1, id2, id3, id4, msgsize, msgtype) = struct.unpack('<llllLL', buf_header)

            rawdata = recv_brainamp_frame(brainamp_socket, msgsize - 24)
            # TODO  msgtype == 3 (msgtype == 1 is header done in Node.configure)
            if msgtype == 4:
                #~ block, chunk, markers = get_signal_and_markers(rawdata, self.nb_channel)
                hs = 12

                # Extract numerical data
                block, points, nb_marker = struct.unpack('<LLL', rawdata[:hs])
                sigsize = dt.itemsize * points * self.nb_channel
                sigs = np.frombuffer(rawdata[hs:hs+sigsize], dtype=dt)
                sigs = sigs.reshape(points, self.nb_channel)
                head += points
                self.outputs['signals'].send(sigs, index=head)

                # Extract markers
                markers = np.empty((nb_marker,), dtype=_dtype_trigger)
                index = hs + sigsize
                for m in range(nb_marker):
                    markersize, = struct.unpack('<L', rawdata[index:index+4])
                    markers['pos'][m], markers['points'][m],markers['channel'][m] = struct.unpack('<LLl', rawdata[index+4:index+16])
                    markers['type'][m], markers['description'][m] = rawdata[index+16:index+markersize].tostring().split('\x00')[:2]
                    index = index + markersize
                head_marker += nb_marker
                self.outputs['triggers'].send(markers, index=nb_marker)

        brainamp_socket.close()

    def stop(self):
        with self.lock:
            self.running = False

class BrainAmpSocket(Node):
    """
    BrainAmp EEG amplifier from Brain Products http://www.brainproducts.com/.

    This class is a bridge between pyacq and the socket-based data streaming
    provided by the Vision recorder acquisition software.
    """
    _output_specs = {'signals': dict(streamtype='analogsignal',dtype='float32',
                                                shape=(-1, 32), compression ='', timeaxis=0,
                                                sample_rate = 512.),
                                'triggers': dict(streamtype = 'event', dtype = _dtype_trigger,
                                                shape = (-1,)),
                                }

    def __init__(self, **kargs):
        Node.__init__(self, **kargs)

    def _configure(self, brainamp_host='localhost', brainamp_port=51244):
        '''
        Parameters
        ----------
        brainamp_host : str
            address used by Vision recorder to send data. Default is 'localhost'.
        brainamp_port : int
            port used by Brain Vision recorder. Default is 51244.
        '''
        self.brainamp_host = brainamp_host
        self.brainamp_port = brainamp_port

        # recv header from brain amp
        brainamp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        brainamp_socket.connect((self.brainamp_host, self.brainamp_port))
        buf_header = recv_brainamp_frame(brainamp_socket, 24)
        id1, id2, id3, id4, msgsize, msgtype = struct.unpack('<llllLL', buf_header)
        rawdata = recv_brainamp_frame(brainamp_socket, msgsize - 24)
        assert msgtype == 1, 'First message from brainamp is not type 1'
        self.nb_channel, sample_interval = struct.unpack('<Ld', rawdata[:12])
        n = self.nb_channel
        sample_interval = sample_interval*1e-6
        self.sample_rate = 1./sample_interval
        self.resolutions = np.array(struct.unpack('<'+'d'*n, rawdata[12:12+8*n]), dtype='f')
        self.channel_names = rawdata[12+8*n:].decode().split('\x00')[:-1]
        #~ self.channel_indexes = range(nb_channel)
        brainamp_socket.close()

        self.outputs['signals'].spec['shape'] = (-1, self.nb_channel)
        self.outputs['signals'].spec['sample_rate'] = self.sample_rate
        self.outputs['signals'].spec['nb_channel'] = self.nb_channel

    def _initialize(self):
        self._thread = BrainAmpThread(self.outputs, self.brainamp_host, self.brainamp_port,
                             self.nb_channel, parent=self)

    def _start(self):
        self._thread.start()

    def _stop(self):
        self._thread.stop()
        self._thread.wait()

    def _close(self):
        pass

register_node_type(BrainAmpSocket)
<|endoftext|>


<commit_before>#
#  Copyright 2015 Jon Turner
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from .vanilla_reader import *
from ._pychro import *
import struct
import os
import mmap

class Appender:
    def __init__(self, chronicle, tid, filenum, pos, utcnow, max_msg_size=64*1024):
        self._tid = tid
        self._utcnow = utcnow
        self._chronicle = chronicle
        self._filenum = filenum
        self._pos = pos
        self._start_pos = self._pos
        self._max_msg_size = max_msg_size
        self._start_date = None
        self._mm = None

    def fill(self, size, ch):
        mm = self._start()
        if self._pos + size >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos:self._pos+size] = ch*size
        self._pos += size

    def advance(self, size):
        self._start()
        if self._pos + size >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        self._pos += size

    def bytes_written(self):
        return self._pos - self._start_pos

    def write_byte(self, val):
        assert val < 256
        mm = self._start()
        if self._pos + 1 >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos] = val
        self._pos += 1

    def write_double(self, val):
        mm = self._start()
        if self._pos + 8 >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos:self._pos+8] = struct.pack('d', val)
        self._pos += 8

    def write_boolean(self, val):
        mm = self._start()
        if self._pos + 1 >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos] = 1 if val else 0
        self._pos += 1

    def write_short(self, val):
        mm = self._start()
        if self._pos + 2 >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos:self._pos+2] = struct.pack('h', val)
        self._pos += 2

    def write_long(self, val):
        mm = self._start()
        if self._pos + 8 >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos:self._pos+8] = struct.pack('q', val)
        self._pos += 8

    def write_int(self, val):
        mm = self._start()
        if self._pos + 4 >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos:self._pos+4] = struct.pack('i', val)
        self._pos += 4

    # will add filler to fixed_size if set
    # else if serialises to larger, is an error
    def write_fixed_string(self, val, size):
        mm = self._start()
        start_pos = self._pos
        encoded = val.encode()
        l = len(encoded)
        self.write_stopbit(l)
        if self._pos - start_pos + l > size:
            raise pychro.InvalidArgumentError
        if self._pos + l >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos:self._pos+l] = encoded
        self._pos = start_pos + size

    def write_string(self, val):
        mm = self._start()
        encoded = val.encode()
        l = len(encoded)
        self.write_stopbit(l)
        if self._pos + l >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos:self._pos+l] = encoded
        self._pos += l

    def write_stopbit(self, val):
        mm = self._start()
        while val < 0 or val > 127:
            mm[self._pos] = 0x80 | (val & 0x7f)
            self._pos += 1
            val = val >> 7
        mm[self._pos] = val
        self._pos += 1

    def _start(self):
        if self._start_date is None:
            self._start_date = self._utcnow().date()
            if self._start_date != self._chronicle._date:
                self._chronicle._day_rollover(self._start_date)
                self._pos = 4
                self._start_pos = self._pos
                self._filenum = 0
                self._mm = None
        if self._mm is None:
            self._mm = self._chronicle._get_data_memory_map(self._filenum, self._tid)
        return self._mm

    def finish(self):
        length = self._pos - self._start_pos
        now_date = self._utcnow().date()
        if now_date != self._chronicle._date:
            # need to rewrite pos-start_pos bytes
            bytes = self._mm(self._filenum, self._tid)[self._start_pos:self._pos]
            if not self._chronicle._day_rollover(now_date):
                raise pychro.PartialWriteLostOnRollover()
            self._pos = length + 4
            self._start_pos = 4
            self._filenum = 0
            self._mm = self._chronicle._get_data_memory_map(self._filenum, self._tid)
            self._mm[self._start_pos:self._pos] = bytes

        self._mm[self._start_pos-4:self._start_pos] = struct.pack('i', ~length)

        self._chronicle._set_index(self._tid, self._filenum, self._start_pos)

        if self._pos + self._max_msg_size > pychro.DATA_FILE_SIZE:
            self._pos = 0
            self._filenum += 1
            self._mm = None
        self._pos += 4
        self._chronicle._set_appender_pos(self._tid, self._filenum, self._pos)
        self._start_pos = self._pos
        self._start_date = None

class VanillaChronicleWriter(VanillaChronicleReader):
    def __init__(self, base_dir, polling_interval=None,
                 max_mapped_memory=pychro.DEFAULT_MAX_MAPPED_MEMORY_PER_READER,
                 thread_id_bits=None, utcnow=datetime.datetime.utcnow):
        try:
            os.makedirs(base_dir)
        except FileExistsError:
            pass
        super().__init__(base_dir=base_dir, polling_interval=polling_interval,
                         max_mapped_memory=max_mapped_memory, thread_id_bits=thread_id_bits,
                         utcnow=utcnow)
        self._positions = dict()
        self._update_date_and_index_base(self._utcnow().date())
        todays_dir = os.path.join(self._base_dir, '%4d%02d%02d' % (self._date.year, self._date.month, self._date.day))
        if self._cycle_dir != todays_dir:
            self._cycle_dir = todays_dir
            try:
                os.makedirs(todays_dir)
            except FileExistsError:
                pass
        self._open_next_index()
        self._open_next_index()
        while True:
            try:
                filenum, pos, tid = self._next_position()
                self._positions[tid] = (filenum+1, 4)
            except pychro.NoData:
                break

    def _set_appender_pos(self, tid, filenum, pos):
        self._positions[tid] = (filenum, pos)

    #Returns whether rollover succeeded or not
    def _day_rollover(self, new_date):
        todays_dir = os.path.join(self._base_dir, '%4d%02d%02d'
                                  % (new_date.year, new_date.month, new_date.day))
        try:
            os.makedirs(todays_dir)
            ret = True
        except FileExistsError:
            # todo: wait here for rollover initiated by another to complete
            ret = False
        self.close()
        self._positions = dict()
        self._cycle_dir = todays_dir
        self._open_next_index()
        self._open_next_index()
        self._update_date_and_index_base(new_date)
        return ret

    def _set_index(self, tid, data_filenum, offset):
        assert self._date == self._utcnow().date()

        index_val = (tid << (64-self._thread_id_bits)) | (data_filenum << pychro.FILENUM_FROM_POS_SHIFT) | offset
        self.set_end_index_today()
        while True:
            index_filenum, index_offset = divmod(self._index*8, pychro.INDEX_FILE_SIZE)
            # keep an extra one open
            if len(self._index_mm) <= index_filenum+1:
                self._open_next_index()
            prev_index_val = pychro.read_mmap(self._index_mm[index_filenum], index_offset)
            if prev_index_val != 0:
                self._index += 1
                continue
            if prev_index_val != pychro.try_atomic_write_mmap(self._index_mm[index_filenum],
                                                              index_offset, prev_index_val, index_val):
                continue
            break

    def _get_tid(self):
        return get_thread_id() & self._thread_id_mask
        #thread_id_bits not large enough? have to live with this..
        #assert get_thread_id() == tid

    def _open_next_index(self):
        file_num = len(self._index_fh)
        fn = os.path.join(self._cycle_dir, 'index-%s' % file_num)
        if os.path.isfile(fn):
            fh = open(fn, 'r+b')
        else:
            fh = open(fn, 'w+b')
            fh.truncate(pychro.INDEX_FILE_SIZE)
            fh.flush()
        self._index_fh += [fh]
        self._index_mm += [pychro.open_write_mmap(fh, pychro.INDEX_FILE_SIZE)]

    def _open_data_file(self, filenum, thread):
        fn = os.path.join(self._cycle_dir, 'data-%s-%s' % (thread, filenum))
        if os.path.isfile(fn):
            fh = open(fn, 'r+b')
        else:
            fh = open(fn, 'w+b')
            fh.truncate(pychro.DATA_FILE_SIZE)
        return fh

    def _open_data_memory_map(self, filenum, thread):
        fh = self._data_fhs.get((filenum, thread))
        if not fh:
            fh = self._open_data_file(filenum, thread)
            self._data_fhs[(filenum, thread)] = fh

        if pychro.PLATFORM_WINDOWS:
            return mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_WRITE)
        while True:
            try:
                return mmap.mmap(fh.fileno(), 0, prot=mmap.PROT_READ | mmap.PROT_WRITE)
            except ValueError:
                pass
                # Alternative to this ugliness appears to be os.fsync(), but is much slower.
                # A thread lazily creating new data and index files would be a good optimisation.

    def __str__(self):
        return '<VanillaChronicleWriter dir:%s idx:%s tid:%s>' % (self._cycle_dir, self._index, self._get_tid())

    def get_appender(self):
        tid = self._get_tid()
        filenum, pos = self._positions.get(tid, (0, 4))
        return Appender(self, tid, filenum, pos, self._utcnow)

<commit_msg>Optimised to call set_end_index_today less when writing, expensive when we're already at the end.<commit_after>#
#  Copyright 2015 Jon Turner
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from .vanilla_reader import *
from ._pychro import *
import struct
import os
import mmap

class Appender:
    def __init__(self, chronicle, tid, filenum, pos, utcnow, max_msg_size=64*1024):
        self._tid = tid
        self._utcnow = utcnow
        self._chronicle = chronicle
        self._filenum = filenum
        self._pos = pos
        self._start_pos = self._pos
        self._max_msg_size = max_msg_size
        self._start_date = None
        self._mm = None

    def fill(self, size, ch):
        mm = self._start()
        if self._pos + size >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos:self._pos+size] = ch*size
        self._pos += size

    def advance(self, size):
        self._start()
        if self._pos + size >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        self._pos += size

    def bytes_written(self):
        return self._pos - self._start_pos

    def write_byte(self, val):
        assert val < 256
        mm = self._start()
        if self._pos + 1 >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos] = val
        self._pos += 1

    def write_double(self, val):
        mm = self._start()
        if self._pos + 8 >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos:self._pos+8] = struct.pack('d', val)
        self._pos += 8

    def write_boolean(self, val):
        mm = self._start()
        if self._pos + 1 >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos] = 1 if val else 0
        self._pos += 1

    def write_short(self, val):
        mm = self._start()
        if self._pos + 2 >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos:self._pos+2] = struct.pack('h', val)
        self._pos += 2

    def write_long(self, val):
        mm = self._start()
        if self._pos + 8 >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos:self._pos+8] = struct.pack('q', val)
        self._pos += 8

    def write_int(self, val):
        mm = self._start()
        if self._pos + 4 >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos:self._pos+4] = struct.pack('i', val)
        self._pos += 4

    # will add filler to fixed_size if set
    # else if serialises to larger, is an error
    def write_fixed_string(self, val, size):
        mm = self._start()
        start_pos = self._pos
        encoded = val.encode()
        l = len(encoded)
        self.write_stopbit(l)
        if self._pos - start_pos + l > size:
            raise pychro.InvalidArgumentError
        if self._pos + l >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos:self._pos+l] = encoded
        self._pos = start_pos + size

    def write_string(self, val):
        mm = self._start()
        encoded = val.encode()
        l = len(encoded)
        self.write_stopbit(l)
        if self._pos + l >= pychro.DATA_FILE_SIZE:
            raise pychro.NoSpace
        mm[self._pos:self._pos+l] = encoded
        self._pos += l

    def write_stopbit(self, val):
        mm = self._start()
        while val < 0 or val > 127:
            mm[self._pos] = 0x80 | (val & 0x7f)
            self._pos += 1
            val = val >> 7
        mm[self._pos] = val
        self._pos += 1

    def _start(self):
        if self._start_date is None:
            self._start_date = self._utcnow().date()
            if self._start_date != self._chronicle._date:
                self._chronicle._day_rollover(self._start_date)
                self._pos = 4
                self._start_pos = self._pos
                self._filenum = 0
                self._mm = None
        if self._mm is None:
            self._mm = self._chronicle._get_data_memory_map(self._filenum, self._tid)
        return self._mm

    def finish(self):
        length = self._pos - self._start_pos
        now_date = self._utcnow().date()
        if now_date != self._chronicle._date:
            # need to rewrite pos-start_pos bytes
            bytes = self._mm(self._filenum, self._tid)[self._start_pos:self._pos]
            if not self._chronicle._day_rollover(now_date):
                raise pychro.PartialWriteLostOnRollover()
            self._pos = length + 4
            self._start_pos = 4
            self._filenum = 0
            self._mm = self._chronicle._get_data_memory_map(self._filenum, self._tid)
            self._mm[self._start_pos:self._pos] = bytes

        self._mm[self._start_pos-4:self._start_pos] = struct.pack('i', ~length)

        self._chronicle._set_index(self._tid, self._filenum, self._start_pos)

        if self._pos + self._max_msg_size > pychro.DATA_FILE_SIZE:
            self._pos = 0
            self._filenum += 1
            self._mm = None
        self._pos += 4
        self._chronicle._set_appender_pos(self._tid, self._filenum, self._pos)
        self._start_pos = self._pos
        self._start_date = None

class VanillaChronicleWriter(VanillaChronicleReader):
    def __init__(self, base_dir, polling_interval=None,
                 max_mapped_memory=pychro.DEFAULT_MAX_MAPPED_MEMORY_PER_READER,
                 thread_id_bits=None, utcnow=datetime.datetime.utcnow):
        try:
            os.makedirs(base_dir)
        except FileExistsError:
            pass
        super().__init__(base_dir=base_dir, polling_interval=polling_interval,
                         max_mapped_memory=max_mapped_memory, thread_id_bits=thread_id_bits,
                         utcnow=utcnow)
        self._positions = dict()
        self._update_date_and_index_base(self._utcnow().date())
        todays_dir = os.path.join(self._base_dir, '%4d%02d%02d' % (self._date.year, self._date.month, self._date.day))
        if self._cycle_dir != todays_dir:
            self._cycle_dir = todays_dir
            try:
                os.makedirs(todays_dir)
            except FileExistsError:
                pass
        self._open_next_index()
        self._open_next_index()
        while True:
            try:
                filenum, pos, tid = self._next_position()
                self._positions[tid] = (filenum+1, 4)
            except pychro.NoData:
                break

    def _set_appender_pos(self, tid, filenum, pos):
        self._positions[tid] = (filenum, pos)

    #Returns whether rollover succeeded or not
    def _day_rollover(self, new_date):
        todays_dir = os.path.join(self._base_dir, '%4d%02d%02d'
                                  % (new_date.year, new_date.month, new_date.day))
        try:
            os.makedirs(todays_dir)
            ret = True
        except FileExistsError:
            # todo: wait here for rollover initiated by another to complete
            ret = False
        self.close()
        self._positions = dict()
        self._cycle_dir = todays_dir
        self._open_next_index()
        self._open_next_index()
        self._update_date_and_index_base(new_date)
        return ret

    def _set_index(self, tid, data_filenum, offset):
        assert self._date == self._utcnow().date()

        index_val = (tid << (64-self._thread_id_bits)) | (data_filenum << pychro.FILENUM_FROM_POS_SHIFT) | offset

        if self._index == 0:
            self.set_end_index_today()

        while True:
            index_filenum, index_offset = divmod(self._index*8, pychro.INDEX_FILE_SIZE)
            # keep an extra one open
            if len(self._index_mm) <= index_filenum+1:
                self._open_next_index()
            prev_index_val = pychro.read_mmap(self._index_mm[index_filenum], index_offset)
            if prev_index_val != 0:
                self._index += 1
                continue
            if prev_index_val != pychro.try_atomic_write_mmap(self._index_mm[index_filenum],
                                                              index_offset, prev_index_val, index_val):
                continue
            break

    def _get_tid(self):
        return get_thread_id() & self._thread_id_mask
        #thread_id_bits not large enough? have to live with this..
        #assert get_thread_id() == tid

    def _open_next_index(self):
        file_num = len(self._index_fh)
        fn = os.path.join(self._cycle_dir, 'index-%s' % file_num)
        if os.path.isfile(fn):
            fh = open(fn, 'r+b')
        else:
            fh = open(fn, 'w+b')
            fh.truncate(pychro.INDEX_FILE_SIZE)
            fh.flush()
        self._index_fh += [fh]
        self._index_mm += [pychro.open_write_mmap(fh, pychro.INDEX_FILE_SIZE)]

    def _open_data_file(self, filenum, thread):
        fn = os.path.join(self._cycle_dir, 'data-%s-%s' % (thread, filenum))
        if os.path.isfile(fn):
            fh = open(fn, 'r+b')
        else:
            fh = open(fn, 'w+b')
            fh.truncate(pychro.DATA_FILE_SIZE)
        return fh

    def _open_data_memory_map(self, filenum, thread):
        fh = self._data_fhs.get((filenum, thread))
        if not fh:
            fh = self._open_data_file(filenum, thread)
            self._data_fhs[(filenum, thread)] = fh

        if pychro.PLATFORM_WINDOWS:
            return mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_WRITE)
        while True:
            try:
                return mmap.mmap(fh.fileno(), 0, prot=mmap.PROT_READ | mmap.PROT_WRITE)
            except ValueError:
                pass
                # Alternative to this ugliness appears to be os.fsync(), but is much slower.
                # A thread lazily creating new data and index files would be a good optimisation.

    def __str__(self):
        return '<VanillaChronicleWriter dir:%s idx:%s tid:%s>' % (self._cycle_dir, self._index, self._get_tid())

    def get_appender(self):
        tid = self._get_tid()
        filenum, pos = self._positions.get(tid, (0, 4))
        return Appender(self, tid, filenum, pos, self._utcnow)

<|endoftext|>


<commit_before>from pydoctor import nevowhtml, model
import cStringIO, textwrap

import py

if not nevowhtml.EPYTEXT:
    py.test.skip("tests assume epydoc is present")

def getHTMLOf(ob):
    writer = nevowhtml.NevowWriter('')
    f = cStringIO.StringIO()
    writer.writeDocsForOne(ob, f)
    return f.getvalue()

def test_simple():
    src = '''
    def f():
        """This is a docstring."""
    '''
    mod = model.fromText(textwrap.dedent(src))
    v = getHTMLOf(mod.contents['f'])
    assert 'This is a docstring' in v
<commit_msg>bodge a test<commit_after>from pydoctor import nevowhtml, model
import cStringIO, textwrap

import py

if not nevowhtml.EPYTEXT:
    py.test.skip("tests assume epydoc is present")

class System:
    class options:
        htmlusesorttable = False

def getHTMLOf(ob):
    writer = nevowhtml.NevowWriter('')
    writer.system = System
    f = cStringIO.StringIO()
    writer.writeDocsForOne(ob, f)
    return f.getvalue()

def test_simple():
    src = '''
    def f():
        """This is a docstring."""
    '''
    mod = model.fromText(textwrap.dedent(src))
    v = getHTMLOf(mod.contents['f'])
    assert 'This is a docstring' in v
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import sys

import numpy as np
import pytest

from pygam import *

@pytest.fixture
def mcycle_gam(mcycle):
    X, y = mcycle
    gam = LinearGAM().fit(X,y)
    return gam

def test_LinearGAM_pdeps_shape(wage):
    """
    check that we get the expected number of partial dependence functions
    """
    X, y = wage
    gam = LinearGAM().fit(X, y)
    pdeps = gam.partial_dependence(X)
    assert(X.shape == pdeps.shape)

def test_LinearGAM_prediction(mcycle, mcycle_gam):
    """
    check that we the predictions we get are correct shape
    """
    X, y = mcycle
    preds = mcycle_gam.predict(X)
    assert(preds.shape == y.shape)

def test_LogisticGAM_accuracy(default):
    """
    check that we can compute accuracy correctly
    """
    X, y = default
    gam = LogisticGAM().fit(X, y)

    preds = gam.predict(X)
    acc0 = (preds == y).mean()
    acc1 = gam.accuracy(X, y)
    assert(acc0 == acc1)

def test_PoissonGAM_exposure(coal):
    """
    check that we can fit a Poisson GAM on real data
    """
    X, y = coal
    gam = PoissonGAM().fit(X, y, exposure=np.ones_like(y))
    assert(gam._is_fitted)

def test_large_GAM(coal):
    """
    check that we can fit a GAM in py3 when we have more than 50,000 samples
    """
    X = np.linspace(0, 100, 100000)
    y = X**2
    gam = LinearGAM().fit(X, y)
    assert(gam._is_fitted)

def test_summary(mcycle, mcycle_gam):
    """
    check that we can get a summary if we've fitted the model, else not
    """
    X, y = mcycle
    gam = LinearGAM()

    try:
      gam.summary()
    except AttributeError:
      assert(True)

    mcycle_gam.summary()
    assert(True)

def test_more_splines_than_samples(mcycle):
    """
    check that gridsearch returns the expected number of models
    """
    X, y = mcycle
    n = len(X)

    gam = LinearGAM(n_splines=n+1).fit(X, y)
    assert(gam._is_fitted)

def test_deviance_residuals(mcycle, mcycle_gam):
    """
    for linear GAMs, the deviance residuals should be equal to the y - y_pred
    """
    X, y = mcycle
    res = mcycle_gam.deviance_residuals(X, y)
    err = y - mcycle_gam.predict(X)
    assert((res == err).all())

def test_conf_intervals_return_array(mcycle, mcycle_gam):
    """
    make sure that the confidence_intervals method returns an array
    """
    X, y = mcycle
    conf_ints = mcycle_gam.confidence_intervals(X)
    assert(conf_ints.ndim == 2)

def test_conf_intervals_quantiles_width_interchangable(mcycle, mcycle_gam):
    """
    getting confidence_intervals via width or specifying quantiles
    should return the same result
    """
    X, y = mcycle
    conf_ints_a = mcycle_gam.confidence_intervals(X, width=.9)
    conf_ints_b = mcycle_gam.confidence_intervals(X, quantiles=[.05, .95])
    assert(np.allclose(conf_ints_a, conf_ints_b))

def test_conf_intervals_ordered(mcycle, mcycle_gam):
    """
    comfidence intervals returned via width should be ordered
    """
    X, y = mcycle
    conf_ints = mcycle_gam.confidence_intervals(X)
    assert((conf_ints[:,0] <= conf_ints[:,1]).all())

def test_partial_dependence_on_univar_data(mcycle, mcycle_gam):
    """
    partial dependence with univariate data should equal the overall model
    if fit intercept is false
    """
    X, y = mcycle
    gam = LinearGAM(fit_intercept=False).fit(X,y)
    pred = gam.predict(X)
    pdep = gam.partial_dependence(X)
    assert((pred == pdep.ravel()).all())

def test_partial_dependence_on_univar_data2(mcycle, mcycle_gam):
    """
    partial dependence with univariate data should NOT equal the overall model
    if fit intercept is false
    """
    X, y = mcycle
    gam = LinearGAM(fit_intercept=True).fit(X,y)
    pred = gam.predict(X)
    pdep = gam.partial_dependence(X)
    assert((pred != pdep.ravel()).all())

def test_partial_dependence_feature_doesnt_exist(mcycle, mcycle_gam):
    """
    partial dependence should raise ValueError when requesting a nonexistent
    feature
    """
    X, y = mcycle
    try:
        mcycle_gam.partial_dependence(X, feature=10)
    except ValueError:
        assert(True)

def test_summary_returns_12_lines(mcycle_gam):
    """
    check that the summary method works and returns 12 lines like:

    'Model Statistics',
    '-----------------',
    'edof       12.376',
    'AIC      1223.761',
    'AICc     1227.003',
    'GCV       624.359',
    'scale     520.733',
    '',
    'Pseudo-R^2',
    '----------------------------',
    'explained_deviance     0.796',
    ''

    """
    if sys.version_info.major == 2:
        from StringIO import StringIO
    if sys.version_info.major == 3:
        from io import StringIO
    stdout = sys.stdout  #keep a handle on the real standard output
    sys.stdout = StringIO() #Choose a file-like object to write to
    mcycle_gam.summary()
    assert(len(sys.stdout.getvalue().split('\n')) == 12)

def test_is_fitted_predict(mcycle):
    """
    test predict requires fitted model
    """
    X, y = mcycle
    gam = LinearGAM()
    try:
        gam.predict(X)
    except AttributeError:
        assert(True)

def test_is_fitted_predict_mu(mcycle):
    """
    test predict_mu requires fitted model
    """
    X, y = mcycle
    gam = LinearGAM()
    try:
        gam.predict_mu(X)
    except AttributeError:
        assert(True)

def test_is_fitted_dev_resid(mcycle):
    """
    test deviance_residuals requires fitted model
    """
    X, y = mcycle
    gam = LinearGAM()
    try:
        gam.deviance_residuals(X, y)
    except AttributeError:
        assert(True)

def test_is_fitted_conf_intervals(mcycle):
    """
    test confidence_intervals requires fitted model
    """
    X, y = mcycle
    gam = LinearGAM()
    try:
        gam.confidence_intervals(X)
    except AttributeError:
        assert(True)

def test_is_fitted_pdep(mcycle):
    """
    test partial_dependence requires fitted model
    """
    X, y = mcycle
    gam = LinearGAM()
    try:
        gam.partial_dependence(X)
    except AttributeError:
        assert(True)

def test_is_fitted_summary(mcycle):
    """
    test summary requires fitted model
    """
    X, y = mcycle
    gam = LinearGAM()
    try:
        gam.summary()
    except AttributeError:
        assert(True)

def test_set_params_with_external_param():
    """
    test set_params sets a real parameter
    """
    gam = GAM(lam=1)
    gam.set_params(lam=420)
    assert(gam.lam == 420)

def test_set_params_with_hidden_param():
    """
    test set_params should not set any params that are not exposed to the user
    """
    gam = GAM()
    gam.set_params(_lam=420)
    assert(gam._lam != 420)

def test_set_params_with_phony_param():
    """
    test set_params should not set any phony param
    """
    gam = GAM()
    gam.set_params(cat=420)
    assert(not hasattr(gam, 'cat'))

def test_set_params_with_hidden_param_deep():
    """
    test set_params can set hidden params if we use the deep=True
    """
    gam = GAM()
    assert(gam._lam != 420)

    gam.set_params(_lam=420, deep=True)
    assert(gam._lam == 420)

def test_set_params_with_phony_param_force():
    """
    test set_params can set phony params if we use the force=True
    """
    gam = GAM()
    assert(not hasattr(gam, 'cat'))

    gam.set_params(cat=420, force=True)
    assert(gam.cat == 420)

def test_get_params():
    """
    test gam gets our params
    """
    gam = GAM(lam=420)
    params = gam.get_params()
    assert(params['lam'] == 420)

def test_get_params_hidden():
    """
    test gam gets our params only if we do deep=True
    """
    gam = GAM()
    params = gam.get_params()
    assert('_lam' not in list(params.keys()))

    params = gam.get_params(deep=True)
    assert('_lam' in list(params.keys()))

# TODO test linear gam pred intervals
<commit_msg>summary prints more lines<commit_after># -*- coding: utf-8 -*-

import sys

import numpy as np
import pytest

from pygam import *

@pytest.fixture
def mcycle_gam(mcycle):
    X, y = mcycle
    gam = LinearGAM().fit(X,y)
    return gam

def test_LinearGAM_pdeps_shape(wage):
    """
    check that we get the expected number of partial dependence functions
    """
    X, y = wage
    gam = LinearGAM().fit(X, y)
    pdeps = gam.partial_dependence(X)
    assert(X.shape == pdeps.shape)

def test_LinearGAM_prediction(mcycle, mcycle_gam):
    """
    check that we the predictions we get are correct shape
    """
    X, y = mcycle
    preds = mcycle_gam.predict(X)
    assert(preds.shape == y.shape)

def test_LogisticGAM_accuracy(default):
    """
    check that we can compute accuracy correctly
    """
    X, y = default
    gam = LogisticGAM().fit(X, y)

    preds = gam.predict(X)
    acc0 = (preds == y).mean()
    acc1 = gam.accuracy(X, y)
    assert(acc0 == acc1)

def test_PoissonGAM_exposure(coal):
    """
    check that we can fit a Poisson GAM on real data
    """
    X, y = coal
    gam = PoissonGAM().fit(X, y, exposure=np.ones_like(y))
    assert(gam._is_fitted)

def test_large_GAM(coal):
    """
    check that we can fit a GAM in py3 when we have more than 50,000 samples
    """
    X = np.linspace(0, 100, 100000)
    y = X**2
    gam = LinearGAM().fit(X, y)
    assert(gam._is_fitted)

def test_summary(mcycle, mcycle_gam):
    """
    check that we can get a summary if we've fitted the model, else not
    """
    X, y = mcycle
    gam = LinearGAM()

    try:
      gam.summary()
    except AttributeError:
      assert(True)

    mcycle_gam.summary()
    assert(True)

def test_more_splines_than_samples(mcycle):
    """
    check that gridsearch returns the expected number of models
    """
    X, y = mcycle
    n = len(X)

    gam = LinearGAM(n_splines=n+1).fit(X, y)
    assert(gam._is_fitted)

def test_deviance_residuals(mcycle, mcycle_gam):
    """
    for linear GAMs, the deviance residuals should be equal to the y - y_pred
    """
    X, y = mcycle
    res = mcycle_gam.deviance_residuals(X, y)
    err = y - mcycle_gam.predict(X)
    assert((res == err).all())

def test_conf_intervals_return_array(mcycle, mcycle_gam):
    """
    make sure that the confidence_intervals method returns an array
    """
    X, y = mcycle
    conf_ints = mcycle_gam.confidence_intervals(X)
    assert(conf_ints.ndim == 2)

def test_conf_intervals_quantiles_width_interchangable(mcycle, mcycle_gam):
    """
    getting confidence_intervals via width or specifying quantiles
    should return the same result
    """
    X, y = mcycle
    conf_ints_a = mcycle_gam.confidence_intervals(X, width=.9)
    conf_ints_b = mcycle_gam.confidence_intervals(X, quantiles=[.05, .95])
    assert(np.allclose(conf_ints_a, conf_ints_b))

def test_conf_intervals_ordered(mcycle, mcycle_gam):
    """
    comfidence intervals returned via width should be ordered
    """
    X, y = mcycle
    conf_ints = mcycle_gam.confidence_intervals(X)
    assert((conf_ints[:,0] <= conf_ints[:,1]).all())

def test_partial_dependence_on_univar_data(mcycle, mcycle_gam):
    """
    partial dependence with univariate data should equal the overall model
    if fit intercept is false
    """
    X, y = mcycle
    gam = LinearGAM(fit_intercept=False).fit(X,y)
    pred = gam.predict(X)
    pdep = gam.partial_dependence(X)
    assert((pred == pdep.ravel()).all())

def test_partial_dependence_on_univar_data2(mcycle, mcycle_gam):
    """
    partial dependence with univariate data should NOT equal the overall model
    if fit intercept is false
    """
    X, y = mcycle
    gam = LinearGAM(fit_intercept=True).fit(X,y)
    pred = gam.predict(X)
    pdep = gam.partial_dependence(X)
    assert((pred != pdep.ravel()).all())

def test_partial_dependence_feature_doesnt_exist(mcycle, mcycle_gam):
    """
    partial dependence should raise ValueError when requesting a nonexistent
    feature
    """
    X, y = mcycle
    try:
        mcycle_gam.partial_dependence(X, feature=10)
    except ValueError:
        assert(True)

def test_summary_returns_12_lines(mcycle_gam):
    """
    check that the summary method works and returns 16 lines like:

    Model Statistics
    -------------------------
    edof               12.321
    AIC              1221.082
    AICc             1224.297
    GCV               611.627
    loglikelihood     -597.22
    deviance          120.679
    scale             510.561

    Pseudo-R^2
    --------------------------
    explained_deviance     0.8
    McFadden             0.288
    McFadden_adj         0.273

    """
    if sys.version_info.major == 2:
        from StringIO import StringIO
    if sys.version_info.major == 3:
        from io import StringIO
    stdout = sys.stdout  #keep a handle on the real standard output
    sys.stdout = StringIO() #Choose a file-like object to write to
    mcycle_gam.summary()
    assert(len(sys.stdout.getvalue().split('\n')) == 16)

def test_is_fitted_predict(mcycle):
    """
    test predict requires fitted model
    """
    X, y = mcycle
    gam = LinearGAM()
    try:
        gam.predict(X)
    except AttributeError:
        assert(True)

def test_is_fitted_predict_mu(mcycle):
    """
    test predict_mu requires fitted model
    """
    X, y = mcycle
    gam = LinearGAM()
    try:
        gam.predict_mu(X)
    except AttributeError:
        assert(True)

def test_is_fitted_dev_resid(mcycle):
    """
    test deviance_residuals requires fitted model
    """
    X, y = mcycle
    gam = LinearGAM()
    try:
        gam.deviance_residuals(X, y)
    except AttributeError:
        assert(True)

def test_is_fitted_conf_intervals(mcycle):
    """
    test confidence_intervals requires fitted model
    """
    X, y = mcycle
    gam = LinearGAM()
    try:
        gam.confidence_intervals(X)
    except AttributeError:
        assert(True)

def test_is_fitted_pdep(mcycle):
    """
    test partial_dependence requires fitted model
    """
    X, y = mcycle
    gam = LinearGAM()
    try:
        gam.partial_dependence(X)
    except AttributeError:
        assert(True)

def test_is_fitted_summary(mcycle):
    """
    test summary requires fitted model
    """
    X, y = mcycle
    gam = LinearGAM()
    try:
        gam.summary()
    except AttributeError:
        assert(True)

def test_set_params_with_external_param():
    """
    test set_params sets a real parameter
    """
    gam = GAM(lam=1)
    gam.set_params(lam=420)
    assert(gam.lam == 420)

def test_set_params_with_hidden_param():
    """
    test set_params should not set any params that are not exposed to the user
    """
    gam = GAM()
    gam.set_params(_lam=420)
    assert(gam._lam != 420)

def test_set_params_with_phony_param():
    """
    test set_params should not set any phony param
    """
    gam = GAM()
    gam.set_params(cat=420)
    assert(not hasattr(gam, 'cat'))

def test_set_params_with_hidden_param_deep():
    """
    test set_params can set hidden params if we use the deep=True
    """
    gam = GAM()
    assert(gam._lam != 420)

    gam.set_params(_lam=420, deep=True)
    assert(gam._lam == 420)

def test_set_params_with_phony_param_force():
    """
    test set_params can set phony params if we use the force=True
    """
    gam = GAM()
    assert(not hasattr(gam, 'cat'))

    gam.set_params(cat=420, force=True)
    assert(gam.cat == 420)

def test_get_params():
    """
    test gam gets our params
    """
    gam = GAM(lam=420)
    params = gam.get_params()
    assert(params['lam'] == 420)

def test_get_params_hidden():
    """
    test gam gets our params only if we do deep=True
    """
    gam = GAM()
    params = gam.get_params()
    assert('_lam' not in list(params.keys()))

    params = gam.get_params(deep=True)
    assert('_lam' in list(params.keys()))

# TODO test linear gam pred intervals
<|endoftext|>