<commit_before># Copyright (c) 2016 Canonical Ltd
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
import time

import six
from six.moves.urllib import parse
from ws4py.manager import WebSocketManager
from ws4py.client import WebSocketBaseClient

from pylxd import exceptions, managers, mixin
from pylxd.deprecation import deprecated
from pylxd.operation import Operation

class ContainerState():
    def __init__(self, **kwargs):
        for key, value in six.iteritems(kwargs):
            setattr(self, key, value)

class Container(mixin.Waitable, mixin.Marshallable):
    """An LXD Container.

    This class is not intended to be used directly, but rather to be used
    via `Client.containers.create`.
    """
    class FilesManager(object):
        """A pseudo-manager for namespacing file operations."""

        def __init__(self, client, container):
            self._client = client
            self._container = container

        def put(self, filepath, data):
            response = self._client.api.containers[
                self._container.name].files.post(
                params={'path': filepath}, data=data)
            return response.status_code == 200

        def get(self, filepath):
            try:
                response = self._client.api.containers[
                    self._container.name].files.get(
                    params={'path': filepath})
            except exceptions.LXDAPIException as e:
                # LXD 2.0.3+ return 404, not 500,
                if e.response.status_code in (500, 404):
                    raise exceptions.NotFound()
                raise
            return response.content

    __slots__ = [
        '_client',
        'architecture', 'config', 'created_at', 'devices', 'ephemeral',
        'expanded_config', 'expanded_devices', 'name', 'profiles', 'status'
    ]

    @classmethod
    def get(cls, client, name):
        """Get a container by name."""
        try:
            response = client.api.containers[name].get()
        except exceptions.LXDAPIException as e:
            if e.response.status_code == 404:
                raise exceptions.NotFound()
            raise

        container = cls(_client=client, **response.json()['metadata'])
        return container

    @classmethod
    def all(cls, client):
        """Get all containers.

        Containers returned from this method will only have the name
        set, as that is the only property returned from LXD. If more
        information is needed, `Container.fetch` is the method call
        that should be used.
        """
        response = client.api.containers.get()

        containers = []
        for url in response.json()['metadata']:
            name = url.split('/')[-1]
            containers.append(cls(_client=client, name=name))
        return containers

    @classmethod
    def create(cls, client, config, wait=False):
        """Create a new container config."""
        try:
            response = client.api.containers.post(json=config)
        except exceptions.LXDAPIException as e:
            raise exceptions.CreateFailed(e.response)

        if wait:
            Operation.wait_for_operation(client, response.json()['operation'])
        return cls(name=config['name'], _client=client)

    def __init__(self, **kwargs):
        super(Container, self).__init__()
        for key, value in six.iteritems(kwargs):
            setattr(self, key, value)

        self.snapshots = managers.SnapshotManager(self._client, self)
        self.files = self.FilesManager(self._client, self)

    def fetch(self):
        """Reload the container information."""
        try:
            response = self._client.api.containers[self.name].get()
        except exceptions.LXDAPIException as e:
            if e.response.status_code == 404:
                raise exceptions.NotFound()
            raise
        for key, value in six.iteritems(response.json()['metadata']):
            setattr(self, key, value)
    # XXX: rockstar (28 Mar 2016) - This method was named improperly
    # originally. It's being kept here for backwards compatibility.
    reload = deprecated(
        "Container.reload is deprecated. Please use Container.fetch")(
        fetch)

    def update(self, wait=False):
        """Update the container in lxd from local changes."""
        try:
            marshalled = self.marshall()
        except AttributeError:
            raise exceptions.ObjectIncomplete()
        # These two properties are explicitly not allowed.
        del marshalled['name']
        del marshalled['status']

        response = self._client.api.containers[self.name].put(
            json=marshalled)

        if wait:
            self.wait_for_operation(response.json()['operation'])

    def rename(self, name, wait=False):
        """Rename a container."""
        response = self._client.api.containers[
            self.name].post(json={'name': name})

        if wait:
            self.wait_for_operation(response.json()['operation'])
        self.name = name

    def delete(self, wait=False):
        """Delete the container."""
        response = self._client.api.containers[self.name].delete()

        if wait:
            self.wait_for_operation(response.json()['operation'])

    def _set_state(self, state, timeout=30, force=True, wait=False):
        response = self._client.api.containers[self.name].state.put(json={
            'action': state,
            'timeout': timeout,
            'force': force
        })
        if wait:
            self.wait_for_operation(response.json()['operation'])
            self.fetch()

    def state(self):
        response = self._client.api.containers[self.name].state.get()
        state = ContainerState(**response.json()['metadata'])
        return state

    def start(self, timeout=30, force=True, wait=False):
        """Start the container."""
        return self._set_state('start',
                               timeout=timeout,
                               force=force,
                               wait=wait)

    def stop(self, timeout=30, force=True, wait=False):
        """Stop the container."""
        return self._set_state('stop',
                               timeout=timeout,
                               force=force,
                               wait=wait)

    def restart(self, timeout=30, force=True, wait=False):
        """Restart the container."""
        return self._set_state('restart',
                               timeout=timeout,
                               force=force,
                               wait=wait)

    def freeze(self, timeout=30, force=True, wait=False):
        """Freeze the container."""
        return self._set_state('freeze',
                               timeout=timeout,
                               force=force,
                               wait=wait)

    def unfreeze(self, timeout=30, force=True, wait=False):
        """Unfreeze the container."""
        return self._set_state('unfreeze',
                               timeout=timeout,
                               force=force,
                               wait=wait)

    @deprecated('Container.snapshot is deprecated. Please use Container.snapshots.create')  # NOQA
    def snapshot(self, name, stateful=False, wait=False):  # pragma: no cover
        """Take a snapshot of the container."""
        self.snapshots.create(name, stateful, wait)

    @deprecated('Container.list_snapshots is deprecated. Please use Container.snapshots.all')  # NOQA
    def list_snapshots(self):  # pragma: no cover
        """List all container snapshots."""
        return [s.name for s in self.snapshots.all()]

    @deprecated('Container.rename_snapshot is deprecated. Please use Snapshot.rename')  # NOQA
    def rename_snapshot(self, old, new, wait=False):  # pragma: no cover
        """Rename a snapshot."""
        snapshot = self.snapshots.get(old)
        snapshot.rename(new, wait=wait)

    @deprecated('Container.delete_snapshot is deprecated. Please use Snapshot.delete')  # NOQA
    def delete_snapshot(self, name, wait=False):  # pragma: no cover
        """Delete a snapshot."""
        snapshot = self.snapshots.get(name)
        snapshot.delete(wait=wait)

    @deprecated('Container.get_file is deprecated. Please use Container.files.get')  # NOQA
    def get_file(self, filepath):  # pragma: no cover
        """Get a file from the container."""
        return self.files.get(filepath)

    @deprecated('Container.put_file is deprecated. Please use Container.files.put')  # NOQA
    def put_file(self, filepath, data):  # pragma: no cover
        """Put a file on the container."""
        return self.files.put(filepath, data)

    def execute(self, commands, environment={}):
        """Execute a command on the container."""
        if isinstance(commands, six.string_types):
            raise TypeError("First argument must be a list.")
        response = self._client.api.containers[self.name]['exec'].post(json={
            'command': commands,
            'environment': environment,
            'wait-for-websocket': True,
            'interactive': False,
        })

        fds = response.json()['metadata']['metadata']['fds']
        operation_id = response.json()['operation'].split('/')[-1]
        parsed = parse.urlparse(self._client.api.operations[operation_id].websocket._api_endpoint)

        manager = WebSocketManager()

        stdin = _StdinWebsocket(manager, self._client.websocket_url)
        stdin.resource = '{}?secret={}'.format(parsed.path, fds['0'])
        stdin.connect()
        stdout = _CommandWebsocketClient(manager, self._client.websocket_url)
        stdout.resource = '{}?secret={}'.format(parsed.path, fds['1'])
        stdout.connect()
        stderr = _CommandWebsocketClient(manager, self._client.websocket_url)
        stderr.resource = '{}?secret={}'.format(parsed.path, fds['2'])
        stderr.connect()

        manager.start()

        while True:  # pragma: no cover
            for websocket in manager.websockets.values():
                if not websocket.terminated:
                    break
            else:
                break
            time.sleep(1)

        return stdout.data, stderr.data

class _CommandWebsocketClient(WebSocketBaseClient):  # pragma: no cover
    def __init__(self, manager, *args, **kwargs):
        self.manager = manager
        super(_CommandWebsocketClient, self).__init__(*args, **kwargs)

    def handshake_ok(self):
        self.manager.add(self)
        self.buffer = []

    def received_message(self, message):
        if message.encoding:
            self.buffer.append(message.data.decode(message.encoding))
        else:
            self.buffer.append(message.data.decode('utf-8'))

    @property
    def data(self):
        return ''.join(self.buffer)

class _StdinWebsocket(WebSocketBaseClient):  # pragma: no cover
    """A websocket client for handling stdin."""
    def __init__(self, manager, *args, **kwargs):
        self.manager = manager
        super(_StdinWebsocket, self).__init__(*args, **kwargs)

    def handshake_ok(self):
        self.manager.add(self)
        self.close()

class Snapshot(mixin.Waitable, mixin.Marshallable):
    """A container snapshot."""

    @classmethod
    def get(cls, client, container, name):
        try:
            response = client.api.containers[
                container.name].snapshots[name].get()
        except exceptions.LXDAPIException as e:
            if e.response.status_code == 404:
                raise exceptions.NotFound()
            raise

        snapshot = cls(
            _client=client, _container=container,
            **response.json()['metadata'])
        # Snapshot names are namespaced in LXD, as
        # container-name/snapshot-name. We hide that implementation
        # detail.
        snapshot.name = snapshot.name.split('/')[-1]
        return snapshot

    @classmethod
    def all(cls, client, container):
        response = client.api.containers[container.name].snapshots.get()

        return [cls(
                name=snapshot.split('/')[-1], _client=client,
                _container=container)
                for snapshot in response.json()['metadata']]

    @classmethod
    def create(cls, client, container, name, stateful=False, wait=False):
        response = client.api.containers[container.name].snapshots.post(json={
            'name': name, 'stateful': stateful})

        snapshot = cls(_client=client, _container=container, name=name)
        if wait:
            snapshot.wait_for_operation(response.json()['operation'])
        return snapshot

    def __init__(self, **kwargs):
        super(Snapshot, self).__init__()
        for key, value in six.iteritems(kwargs):
            setattr(self, key, value)

    def rename(self, new_name, wait=False):
        """Rename a snapshot."""
        response = self._client.api.containers[
            self._container.name].snapshots[self.name].post(
            json={'name': new_name})
        if wait:
            self.wait_for_operation(response.json()['operation'])
        self.name = new_name

    def delete(self, wait=False):
        """Delete a snapshot."""
        response = self._client.api.containers[
            self._container.name].snapshots[self.name].delete()

        if wait:
            self.wait_for_operation(response.json()['operation'])
<commit_msg>Fix lint<commit_after># Copyright (c) 2016 Canonical Ltd
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
import time

import six
from six.moves.urllib import parse
from ws4py.manager import WebSocketManager
from ws4py.client import WebSocketBaseClient

from pylxd import exceptions, managers, mixin
from pylxd.deprecation import deprecated
from pylxd.operation import Operation

class ContainerState():
    def __init__(self, **kwargs):
        for key, value in six.iteritems(kwargs):
            setattr(self, key, value)

class Container(mixin.Waitable, mixin.Marshallable):
    """An LXD Container.

    This class is not intended to be used directly, but rather to be used
    via `Client.containers.create`.
    """
    class FilesManager(object):
        """A pseudo-manager for namespacing file operations."""

        def __init__(self, client, container):
            self._client = client
            self._container = container

        def put(self, filepath, data):
            response = self._client.api.containers[
                self._container.name].files.post(
                params={'path': filepath}, data=data)
            return response.status_code == 200

        def get(self, filepath):
            try:
                response = self._client.api.containers[
                    self._container.name].files.get(
                    params={'path': filepath})
            except exceptions.LXDAPIException as e:
                # LXD 2.0.3+ return 404, not 500,
                if e.response.status_code in (500, 404):
                    raise exceptions.NotFound()
                raise
            return response.content

    __slots__ = [
        '_client',
        'architecture', 'config', 'created_at', 'devices', 'ephemeral',
        'expanded_config', 'expanded_devices', 'name', 'profiles', 'status'
    ]

    @classmethod
    def get(cls, client, name):
        """Get a container by name."""
        try:
            response = client.api.containers[name].get()
        except exceptions.LXDAPIException as e:
            if e.response.status_code == 404:
                raise exceptions.NotFound()
            raise

        container = cls(_client=client, **response.json()['metadata'])
        return container

    @classmethod
    def all(cls, client):
        """Get all containers.

        Containers returned from this method will only have the name
        set, as that is the only property returned from LXD. If more
        information is needed, `Container.fetch` is the method call
        that should be used.
        """
        response = client.api.containers.get()

        containers = []
        for url in response.json()['metadata']:
            name = url.split('/')[-1]
            containers.append(cls(_client=client, name=name))
        return containers

    @classmethod
    def create(cls, client, config, wait=False):
        """Create a new container config."""
        try:
            response = client.api.containers.post(json=config)
        except exceptions.LXDAPIException as e:
            raise exceptions.CreateFailed(e.response)

        if wait:
            Operation.wait_for_operation(client, response.json()['operation'])
        return cls(name=config['name'], _client=client)

    def __init__(self, **kwargs):
        super(Container, self).__init__()
        for key, value in six.iteritems(kwargs):
            setattr(self, key, value)

        self.snapshots = managers.SnapshotManager(self._client, self)
        self.files = self.FilesManager(self._client, self)

    def fetch(self):
        """Reload the container information."""
        try:
            response = self._client.api.containers[self.name].get()
        except exceptions.LXDAPIException as e:
            if e.response.status_code == 404:
                raise exceptions.NotFound()
            raise
        for key, value in six.iteritems(response.json()['metadata']):
            setattr(self, key, value)
    # XXX: rockstar (28 Mar 2016) - This method was named improperly
    # originally. It's being kept here for backwards compatibility.
    reload = deprecated(
        "Container.reload is deprecated. Please use Container.fetch")(
        fetch)

    def update(self, wait=False):
        """Update the container in lxd from local changes."""
        try:
            marshalled = self.marshall()
        except AttributeError:
            raise exceptions.ObjectIncomplete()
        # These two properties are explicitly not allowed.
        del marshalled['name']
        del marshalled['status']

        response = self._client.api.containers[self.name].put(
            json=marshalled)

        if wait:
            self.wait_for_operation(response.json()['operation'])

    def rename(self, name, wait=False):
        """Rename a container."""
        response = self._client.api.containers[
            self.name].post(json={'name': name})

        if wait:
            self.wait_for_operation(response.json()['operation'])
        self.name = name

    def delete(self, wait=False):
        """Delete the container."""
        response = self._client.api.containers[self.name].delete()

        if wait:
            self.wait_for_operation(response.json()['operation'])

    def _set_state(self, state, timeout=30, force=True, wait=False):
        response = self._client.api.containers[self.name].state.put(json={
            'action': state,
            'timeout': timeout,
            'force': force
        })
        if wait:
            self.wait_for_operation(response.json()['operation'])
            self.fetch()

    def state(self):
        response = self._client.api.containers[self.name].state.get()
        state = ContainerState(**response.json()['metadata'])
        return state

    def start(self, timeout=30, force=True, wait=False):
        """Start the container."""
        return self._set_state('start',
                               timeout=timeout,
                               force=force,
                               wait=wait)

    def stop(self, timeout=30, force=True, wait=False):
        """Stop the container."""
        return self._set_state('stop',
                               timeout=timeout,
                               force=force,
                               wait=wait)

    def restart(self, timeout=30, force=True, wait=False):
        """Restart the container."""
        return self._set_state('restart',
                               timeout=timeout,
                               force=force,
                               wait=wait)

    def freeze(self, timeout=30, force=True, wait=False):
        """Freeze the container."""
        return self._set_state('freeze',
                               timeout=timeout,
                               force=force,
                               wait=wait)

    def unfreeze(self, timeout=30, force=True, wait=False):
        """Unfreeze the container."""
        return self._set_state('unfreeze',
                               timeout=timeout,
                               force=force,
                               wait=wait)

    @deprecated('Container.snapshot is deprecated. Please use Container.snapshots.create')  # NOQA
    def snapshot(self, name, stateful=False, wait=False):  # pragma: no cover
        """Take a snapshot of the container."""
        self.snapshots.create(name, stateful, wait)

    @deprecated('Container.list_snapshots is deprecated. Please use Container.snapshots.all')  # NOQA
    def list_snapshots(self):  # pragma: no cover
        """List all container snapshots."""
        return [s.name for s in self.snapshots.all()]

    @deprecated('Container.rename_snapshot is deprecated. Please use Snapshot.rename')  # NOQA
    def rename_snapshot(self, old, new, wait=False):  # pragma: no cover
        """Rename a snapshot."""
        snapshot = self.snapshots.get(old)
        snapshot.rename(new, wait=wait)

    @deprecated('Container.delete_snapshot is deprecated. Please use Snapshot.delete')  # NOQA
    def delete_snapshot(self, name, wait=False):  # pragma: no cover
        """Delete a snapshot."""
        snapshot = self.snapshots.get(name)
        snapshot.delete(wait=wait)

    @deprecated('Container.get_file is deprecated. Please use Container.files.get')  # NOQA
    def get_file(self, filepath):  # pragma: no cover
        """Get a file from the container."""
        return self.files.get(filepath)

    @deprecated('Container.put_file is deprecated. Please use Container.files.put')  # NOQA
    def put_file(self, filepath, data):  # pragma: no cover
        """Put a file on the container."""
        return self.files.put(filepath, data)

    def execute(self, commands, environment={}):
        """Execute a command on the container."""
        if isinstance(commands, six.string_types):
            raise TypeError("First argument must be a list.")
        response = self._client.api.containers[self.name]['exec'].post(json={
            'command': commands,
            'environment': environment,
            'wait-for-websocket': True,
            'interactive': False,
        })

        fds = response.json()['metadata']['metadata']['fds']
        operation_id = response.json()['operation'].split('/')[-1]
        parsed = parse.urlparse(
            self._client.api.operations[operation_id].websocket._api_endpoint)

        manager = WebSocketManager()

        stdin = _StdinWebsocket(manager, self._client.websocket_url)
        stdin.resource = '{}?secret={}'.format(parsed.path, fds['0'])
        stdin.connect()
        stdout = _CommandWebsocketClient(manager, self._client.websocket_url)
        stdout.resource = '{}?secret={}'.format(parsed.path, fds['1'])
        stdout.connect()
        stderr = _CommandWebsocketClient(manager, self._client.websocket_url)
        stderr.resource = '{}?secret={}'.format(parsed.path, fds['2'])
        stderr.connect()

        manager.start()

        while True:  # pragma: no cover
            for websocket in manager.websockets.values():
                if not websocket.terminated:
                    break
            else:
                break
            time.sleep(1)

        return stdout.data, stderr.data

class _CommandWebsocketClient(WebSocketBaseClient):  # pragma: no cover
    def __init__(self, manager, *args, **kwargs):
        self.manager = manager
        super(_CommandWebsocketClient, self).__init__(*args, **kwargs)

    def handshake_ok(self):
        self.manager.add(self)
        self.buffer = []

    def received_message(self, message):
        if message.encoding:
            self.buffer.append(message.data.decode(message.encoding))
        else:
            self.buffer.append(message.data.decode('utf-8'))

    @property
    def data(self):
        return ''.join(self.buffer)

class _StdinWebsocket(WebSocketBaseClient):  # pragma: no cover
    """A websocket client for handling stdin."""
    def __init__(self, manager, *args, **kwargs):
        self.manager = manager
        super(_StdinWebsocket, self).__init__(*args, **kwargs)

    def handshake_ok(self):
        self.manager.add(self)
        self.close()

class Snapshot(mixin.Waitable, mixin.Marshallable):
    """A container snapshot."""

    @classmethod
    def get(cls, client, container, name):
        try:
            response = client.api.containers[
                container.name].snapshots[name].get()
        except exceptions.LXDAPIException as e:
            if e.response.status_code == 404:
                raise exceptions.NotFound()
            raise

        snapshot = cls(
            _client=client, _container=container,
            **response.json()['metadata'])
        # Snapshot names are namespaced in LXD, as
        # container-name/snapshot-name. We hide that implementation
        # detail.
        snapshot.name = snapshot.name.split('/')[-1]
        return snapshot

    @classmethod
    def all(cls, client, container):
        response = client.api.containers[container.name].snapshots.get()

        return [cls(
                name=snapshot.split('/')[-1], _client=client,
                _container=container)
                for snapshot in response.json()['metadata']]

    @classmethod
    def create(cls, client, container, name, stateful=False, wait=False):
        response = client.api.containers[container.name].snapshots.post(json={
            'name': name, 'stateful': stateful})

        snapshot = cls(_client=client, _container=container, name=name)
        if wait:
            snapshot.wait_for_operation(response.json()['operation'])
        return snapshot

    def __init__(self, **kwargs):
        super(Snapshot, self).__init__()
        for key, value in six.iteritems(kwargs):
            setattr(self, key, value)

    def rename(self, new_name, wait=False):
        """Rename a snapshot."""
        response = self._client.api.containers[
            self._container.name].snapshots[self.name].post(
            json={'name': new_name})
        if wait:
            self.wait_for_operation(response.json()['operation'])
        self.name = new_name

    def delete(self, wait=False):
        """Delete a snapshot."""
        response = self._client.api.containers[
            self._container.name].snapshots[self.name].delete()

        if wait:
            self.wait_for_operation(response.json()['operation'])
<|endoftext|>


<commit_before>"""
WSGI config for cs122 project.

It exposes the WSGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/
"""

import os
from whitenoise.django import DjangoWhiteNoise
from django.core.wsgi import get_wsgi_application

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cs122.settings")
application = get_wsgi_application()
application = DjangoWhiteNoise(application)
<commit_msg>init settings before importing whitenoise mod in wsgi<commit_after>"""
WSGI config for cs122 project.

It exposes the WSGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/
"""

import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cs122.settings")
from whitenoise.django import DjangoWhiteNoise

application = get_wsgi_application()
application = DjangoWhiteNoise(application)
<|endoftext|>


<commit_before>import os
from collections import OrderedDict
import torch.nn as nn
from torchvision.datasets import MNIST
import torchvision.transforms as transforms
import torch
import torch.nn.functional as F
from test_tube import HyperOptArgumentParser
from torch import optim
from torch.utils.data import DataLoader
from torch.utils.data.distributed import DistributedSampler

from pytorch_lightning.root_module.root_module import LightningModule

class LightningTemplateModel(LightningModule):
    """
    Sample model to show how to define a template
    """

    def __init__(self, hparams):
        """
        Pass in parsed HyperOptArgumentParser to the model
        :param hparams:
        """
        # init superclass
        super(LightningTemplateModel, self).__init__(hparams)

        self.batch_size = hparams.batch_size

        # build model
        self.__build_model()

    # ---------------------
    # MODEL SETUP
    # ---------------------
    def __build_model(self):
        """
        Layout model
        :return:
        """
        self.c_d1 = nn.Linear(in_features=self.hparams.in_features, out_features=self.hparams.hidden_dim)
        self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
        self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)

        self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim, out_features=self.hparams.out_features)

    # ---------------------
    # TRAINING
    # ---------------------
    def forward(self, x):
        """
        No special modification required for lightning, define as you normally would
        :param x:
        :return:
        """

        x = self.c_d1(x)
        x = torch.tanh(x)
        x = self.c_d1_bn(x)
        x = self.c_d1_drop(x)

        x = self.c_d2(x)
        logits = F.log_softmax(x, dim=1)

        return logits

    def loss(self, labels, logits):
        nll = F.nll_loss(logits, labels)
        return nll

    def training_step(self, data_batch, batch_i):
        """
        Lightning calls this inside the training loop
        :param data_batch:
        :return:
        """
        # forward pass
        x, y = data_batch
        x = x.view(x.size(0), -1)
        y_hat = self.forward(x)

        # calculate loss
        loss_val = self.loss(y, y_hat)

        output = OrderedDict({
            'loss': loss_val,
            'tqdm_metrics': {}
        })
        return output

    def validation_step(self, data_batch, batch_i):
        """
        Lightning calls this inside the validation loop
        :param data_batch:
        :return:
        """
        x, y = data_batch
        x = x.view(x.size(0), -1)
        y_hat = self.forward(x)

        loss_val = self.loss(y, y_hat)

        # acc
        labels_hat = torch.argmax(y_hat, dim=1)
        val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)

        output = OrderedDict({
            'val_loss': loss_val,
            'val_acc': torch.tensor(val_acc),
        })
        return output

    def validation_end(self, outputs):
        """
        Called at the end of validation to aggregate outputs
        :param outputs: list of individual outputs of each validation step
        :return:
        """
        val_loss_mean = 0
        val_acc_mean = 0
        for output in outputs:
            val_loss_mean += output['val_loss']
            val_acc_mean += output['val_acc']

        val_loss_mean /= len(outputs)
        val_acc_mean /= len(outputs)
        tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
        return tqdm_dic

    def update_tng_log_metrics(self, logs):
        return logs

    # ---------------------
    # MODEL SAVING
    # ---------------------
    def get_save_dict(self):
        checkpoint = {'state_dict': self.state_dict()}
        return checkpoint

    def load_model_specific(self, checkpoint):
        self.load_state_dict(checkpoint['state_dict'])
        pass

    # ---------------------
    # TRAINING SETUP
    # ---------------------
    def configure_optimizers(self):
        """
        return whatever optimizers we want here
        :return: list of optimizers
        """
        optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
        return [optimizer]

    def __dataloader(self, train):
        # init data generators
        transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
        dataset = MNIST(root=self.hparams.data_root, train=train, transform=transform, download=True)

        # when using multi-node we need to add the datasampler
        train_sampler = None
        batch_size = self.hparams.batch_size

        try:
            if self.on_gpu:
                train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank)
                batch_size = batch_size // self.trainer.world_size  # scale batch size
        except Exception as e:
            pass

        should_shuffle = train_sampler is None
        loader = DataLoader(
            dataset=dataset,
            batch_size=batch_size,
            shuffle=should_shuffle,
            sampler=train_sampler
        )

        return loader

    @property
    def tng_dataloader(self):
        if self._tng_dataloader is None:
            try:
                self._tng_dataloader = self.__dataloader(train=True)
            except Exception as e:
                print(e)
                raise e
        return self._tng_dataloader

    @property
    def val_dataloader(self):
        if self._val_dataloader is None:
            try:
                self._val_dataloader = self.__dataloader(train=False)
            except Exception as e:
                print(e)
                raise e
        return self._val_dataloader

    @property
    def test_dataloader(self):
        if self._test_dataloader is None:
            try:
                self._test_dataloader = self.__dataloader(train=False)
            except Exception as e:
                print(e)
                raise e
        return self._test_dataloader

    @staticmethod
    def add_model_specific_args(parent_parser, root_dir):
        """
        Parameters you define here will be available to your model through self.hparams
        :param parent_parser:
        :param root_dir:
        :return:
        """
        parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])

        # param overwrites
        # parser.set_defaults(gradient_clip=5.0)

        # network params
        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
        parser.add_argument('--in_features', default=28*28, type=int)
        parser.add_argument('--out_features', default=10, type=int)
        parser.add_argument('--hidden_dim', default=50000, type=int) # use 500 for CPU, 50000 for GPU to see speed difference

        # data
        parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)

        # training params (opt)
        parser.opt_list('--learning_rate', default=0.001*8, type=float, options=[0.0001, 0.0005, 0.001, 0.005],
                        tunable=False)
        parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False)

        # if using 2 nodes with 4 gpus each the batch size here (256) will be 256 / (2*8) = 16 per gpu
        parser.opt_list('--batch_size', default=256*8, type=int, options=[32, 64, 128, 256], tunable=False,
                        help='batch size will be divided over all the gpus being used across all nodes')
        return parser
<commit_msg>set dp as default backend<commit_after>import os
from collections import OrderedDict
import torch.nn as nn
from torchvision.datasets import MNIST
import torchvision.transforms as transforms
import torch
import torch.nn.functional as F
from test_tube import HyperOptArgumentParser
from torch import optim
from torch.utils.data import DataLoader
from torch.utils.data.distributed import DistributedSampler

from pytorch_lightning.root_module.root_module import LightningModule

class LightningTemplateModel(LightningModule):
    """
    Sample model to show how to define a template
    """

    def __init__(self, hparams):
        """
        Pass in parsed HyperOptArgumentParser to the model
        :param hparams:
        """
        # init superclass
        super(LightningTemplateModel, self).__init__(hparams)

        self.batch_size = hparams.batch_size

        # build model
        self.__build_model()

    # ---------------------
    # MODEL SETUP
    # ---------------------
    def __build_model(self):
        """
        Layout model
        :return:
        """
        self.c_d1 = nn.Linear(in_features=self.hparams.in_features, out_features=self.hparams.hidden_dim)
        self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
        self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)

        self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim, out_features=self.hparams.out_features)

    # ---------------------
    # TRAINING
    # ---------------------
    def forward(self, x):
        """
        No special modification required for lightning, define as you normally would
        :param x:
        :return:
        """

        x = self.c_d1(x)
        x = torch.tanh(x)
        x = self.c_d1_bn(x)
        x = self.c_d1_drop(x)

        x = self.c_d2(x)
        logits = F.log_softmax(x, dim=1)

        return logits

    def loss(self, labels, logits):
        nll = F.nll_loss(logits, labels)
        return nll

    def training_step(self, data_batch, batch_i):
        """
        Lightning calls this inside the training loop
        :param data_batch:
        :return:
        """
        # forward pass
        x, y = data_batch
        x = x.view(x.size(0), -1)
        y_hat = self.forward(x)

        # calculate loss
        loss_val = self.loss(y, y_hat)

        output = OrderedDict({
            'loss': loss_val
        })
        return output

    def validation_step(self, data_batch, batch_i):
        """
        Lightning calls this inside the validation loop
        :param data_batch:
        :return:
        """
        x, y = data_batch
        x = x.view(x.size(0), -1)
        y_hat = self.forward(x)

        loss_val = self.loss(y, y_hat)

        # acc
        labels_hat = torch.argmax(y_hat, dim=1)
        val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)

        output = OrderedDict({
            'val_loss': loss_val,
            'val_acc': torch.tensor(val_acc),
        })
        return output

    def validation_end(self, outputs):
        """
        Called at the end of validation to aggregate outputs
        :param outputs: list of individual outputs of each validation step
        :return:
        """
        val_loss_mean = 0
        val_acc_mean = 0
        for output in outputs:
            val_loss_mean += output['val_loss']
            val_acc_mean += output['val_acc']

        val_loss_mean /= len(outputs)
        val_acc_mean /= len(outputs)
        tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
        return tqdm_dic

    def update_tng_log_metrics(self, logs):
        return logs

    # ---------------------
    # MODEL SAVING
    # ---------------------
    def get_save_dict(self):
        checkpoint = {'state_dict': self.state_dict()}
        return checkpoint

    def load_model_specific(self, checkpoint):
        self.load_state_dict(checkpoint['state_dict'])
        pass

    # ---------------------
    # TRAINING SETUP
    # ---------------------
    def configure_optimizers(self):
        """
        return whatever optimizers we want here
        :return: list of optimizers
        """
        optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
        return [optimizer]

    def __dataloader(self, train):
        # init data generators
        transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
        dataset = MNIST(root=self.hparams.data_root, train=train, transform=transform, download=True)

        # when using multi-node we need to add the datasampler
        train_sampler = None
        batch_size = self.hparams.batch_size

        try:
            if self.on_gpu:
                train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank)
                batch_size = batch_size // self.trainer.world_size  # scale batch size
        except Exception as e:
            pass

        should_shuffle = train_sampler is None
        loader = DataLoader(
            dataset=dataset,
            batch_size=batch_size,
            shuffle=should_shuffle,
            sampler=train_sampler
        )

        return loader

    @property
    def tng_dataloader(self):
        if self._tng_dataloader is None:
            try:
                self._tng_dataloader = self.__dataloader(train=True)
            except Exception as e:
                print(e)
                raise e
        return self._tng_dataloader

    @property
    def val_dataloader(self):
        if self._val_dataloader is None:
            try:
                self._val_dataloader = self.__dataloader(train=False)
            except Exception as e:
                print(e)
                raise e
        return self._val_dataloader

    @property
    def test_dataloader(self):
        if self._test_dataloader is None:
            try:
                self._test_dataloader = self.__dataloader(train=False)
            except Exception as e:
                print(e)
                raise e
        return self._test_dataloader

    @staticmethod
    def add_model_specific_args(parent_parser, root_dir):
        """
        Parameters you define here will be available to your model through self.hparams
        :param parent_parser:
        :param root_dir:
        :return:
        """
        parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])

        # param overwrites
        # parser.set_defaults(gradient_clip=5.0)

        # network params
        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
        parser.add_argument('--in_features', default=28*28, type=int)
        parser.add_argument('--out_features', default=10, type=int)
        parser.add_argument('--hidden_dim', default=50000, type=int) # use 500 for CPU, 50000 for GPU to see speed difference

        # data
        parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)

        # training params (opt)
        parser.opt_list('--learning_rate', default=0.001*8, type=float, options=[0.0001, 0.0005, 0.001, 0.005],
                        tunable=False)
        parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False)

        # if using 2 nodes with 4 gpus each the batch size here (256) will be 256 / (2*8) = 16 per gpu
        parser.opt_list('--batch_size', default=256*8, type=int, options=[32, 64, 128, 256], tunable=False,
                        help='batch size will be divided over all the gpus being used across all nodes')
        return parser
<|endoftext|>


<commit_before>"""Setup CLI configuration."""
# :license: MIT, see LICENSE for more details.
import os.path

import click

import SoftLayer
from SoftLayer import auth
from SoftLayer.CLI import config
from SoftLayer.CLI import environment
from SoftLayer.CLI import exceptions
from SoftLayer.CLI import formatting
from SoftLayer import transports
from SoftLayer import utils

def get_api_key(client, username, secret):
    """Attempts API-Key and password auth to get an API key.

    This will also generate an API key if one doesn't exist
    """

    # Try to use a client with username/api key
    if len(secret) == 64:
        try:
            # real_transport = getattr(client.transport, 'transport', transport)
            if isinstance(client.transport.transport, transports.RestTransport):
                client.auth = auth.BasicHTTPAuthentication(username, secret)
            else:
                client.auth = auth.BasicAuthentication(username, secret)
            client['Account'].getCurrentUser()
            return secret
        except SoftLayer.SoftLayerAPIError as ex:
            if 'invalid api token' not in ex.faultString.lower():
                raise
    else:
        # Try to use a client with username/password
        client.authenticate_with_password(username, secret)

        user_record = client['Account'].getCurrentUser(
            mask='id, apiAuthenticationKeys')
        api_keys = user_record['apiAuthenticationKeys']
        if len(api_keys) == 0:
            return client['User_Customer'].addApiAuthenticationKey(
                id=user_record['id'])
        return api_keys[0]['authenticationKey']

@click.command()
@environment.pass_env
def cli(env):
    """Edit configuration."""

    username, secret, endpoint_url, timeout = get_user_input(env)

    env.client.transport.transport.endpoint_url = endpoint_url
    api_key = get_api_key(env.client, username, secret)

    path = '~/.softlayer'
    if env.config_file:
        path = env.config_file
    config_path = os.path.expanduser(path)

    env.out(env.fmt(config.config_table({'username': username,
                                         'api_key': api_key,
                                         'endpoint_url': endpoint_url,
                                         'timeout': timeout})))

    if not formatting.confirm('Are you sure you want to write settings '
                              'to "%s"?' % config_path, default=True):
        raise exceptions.CLIAbort('Aborted.')

    # Persist the config file. Read the target config file in before
    # setting the values to avoid clobbering settings
    parsed_config = utils.configparser.RawConfigParser()
    parsed_config.read(config_path)
    try:
        parsed_config.add_section('softlayer')
    except utils.configparser.DuplicateSectionError:
        pass

    parsed_config.set('softlayer', 'username', username)
    parsed_config.set('softlayer', 'api_key', api_key)
    parsed_config.set('softlayer', 'endpoint_url', endpoint_url)
    parsed_config.set('softlayer', 'timeout', timeout)

    config_fd = os.fdopen(os.open(config_path,
                                  (os.O_WRONLY | os.O_CREAT | os.O_TRUNC),
                                  0o600),
                          'w')
    try:
        parsed_config.write(config_fd)
    finally:
        config_fd.close()

    env.fout("Configuration Updated Successfully")

def get_user_input(env):
    """Ask for username, secret (api_key or password) and endpoint_url."""

    defaults = config.get_settings_from_client(env.client)

    # Ask for username
    username = env.input('Username', default=defaults['username'])

    # Ask for 'secret' which can be api_key or their password
    secret = env.getpass('API Key or Password', default=defaults['api_key'])

    # Ask for which endpoint they want to use
    endpoint = defaults.get('endpoint_url', 'public')
    endpoint_type = env.input(
        'Endpoint (public|private|custom)', default=endpoint)
    endpoint_type = endpoint_type.lower()

    if endpoint_type == 'public':
        endpoint_url = SoftLayer.API_PUBLIC_ENDPOINT
    elif endpoint_type == 'private':
        endpoint_url = SoftLayer.API_PRIVATE_ENDPOINT
    else:
        if endpoint_type == 'custom':
            endpoint_url = env.input('Endpoint URL', default=endpoint)
        else:
            endpoint_url = endpoint

    print("SETTING enpoint to %s "% endpoint_url)

    # Ask for timeout
    timeout = env.input('Timeout', default=defaults['timeout'] or 0)

    return username, secret, endpoint_url, timeout
<commit_msg>fixing a bug where the input endpoint differes from the config endpoint<commit_after>"""Setup CLI configuration."""
# :license: MIT, see LICENSE for more details.
import os.path

import click

import SoftLayer
from SoftLayer import auth
from SoftLayer.CLI import config
from SoftLayer.CLI import environment
from SoftLayer.CLI import exceptions
from SoftLayer.CLI import formatting
from SoftLayer import transports
from SoftLayer import utils

def get_api_key(client, username, secret):
    """Attempts API-Key and password auth to get an API key.

    This will also generate an API key if one doesn't exist
    """

    # Try to use a client with username/api key
    if len(secret) == 64:
        try:
            client['Account'].getCurrentUser()
            return secret
        except SoftLayer.SoftLayerAPIError as ex:
            if 'invalid api token' not in ex.faultString.lower():
                raise
    else:
        # Try to use a client with username/password
        client.authenticate_with_password(username, secret)

        user_record = client['Account'].getCurrentUser(mask='id, apiAuthenticationKeys')
        api_keys = user_record['apiAuthenticationKeys']
        if len(api_keys) == 0:
            return client['User_Customer'].addApiAuthenticationKey(id=user_record['id'])
        return api_keys[0]['authenticationKey']

@click.command()
@environment.pass_env
def cli(env):
    """Edit configuration."""

    username, secret, endpoint_url, timeout = get_user_input(env)

    env.client.transport.transport.endpoint_url = endpoint_url
    new_client = SoftLayer.client(username=username, api_key=secret, endpoint_url=endpoint_url, timeout=timeout)
    api_key = get_api_key(new_client, username, secret)

    path = '~/.softlayer'
    if env.config_file:
        path = env.config_file
    config_path = os.path.expanduser(path)

    env.out(env.fmt(config.config_table({'username': username,
                                         'api_key': api_key,
                                         'endpoint_url': endpoint_url,
                                         'timeout': timeout})))

    if not formatting.confirm('Are you sure you want to write settings '
                              'to "%s"?' % config_path, default=True):
        raise exceptions.CLIAbort('Aborted.')

    # Persist the config file. Read the target config file in before
    # setting the values to avoid clobbering settings
    parsed_config = utils.configparser.RawConfigParser()
    parsed_config.read(config_path)
    try:
        parsed_config.add_section('softlayer')
    except utils.configparser.DuplicateSectionError:
        pass

    parsed_config.set('softlayer', 'username', username)
    parsed_config.set('softlayer', 'api_key', api_key)
    parsed_config.set('softlayer', 'endpoint_url', endpoint_url)
    parsed_config.set('softlayer', 'timeout', timeout)

    config_fd = os.fdopen(os.open(config_path,
                                  (os.O_WRONLY | os.O_CREAT | os.O_TRUNC),
                                  0o600),
                          'w')
    try:
        parsed_config.write(config_fd)
    finally:
        config_fd.close()

    env.fout("Configuration Updated Successfully")

def get_user_input(env):
    """Ask for username, secret (api_key or password) and endpoint_url."""

    defaults = config.get_settings_from_client(env.client)

    # Ask for username
    username = env.input('Username', default=defaults['username'])

    # Ask for 'secret' which can be api_key or their password
    secret = env.getpass('API Key or Password', default=defaults['api_key'])

    # Ask for which endpoint they want to use
    endpoint = defaults.get('endpoint_url', 'public')
    endpoint_type = env.input(
        'Endpoint (public|private|custom)', default=endpoint)
    endpoint_type = endpoint_type.lower()

    if endpoint_type == 'public':
        endpoint_url = SoftLayer.API_PUBLIC_ENDPOINT
    elif endpoint_type == 'private':
        endpoint_url = SoftLayer.API_PRIVATE_ENDPOINT
    else:
        if endpoint_type == 'custom':
            endpoint_url = env.input('Endpoint URL', default=endpoint)
        else:
            endpoint_url = endpoint

    print("SETTING enpoint to %s "% endpoint_url)

    # Ask for timeout
    timeout = env.input('Timeout', default=defaults['timeout'] or 0)

    return username, secret, endpoint_url, timeout
<|endoftext|>


<commit_before>"""
Functions for mapping from one 2D Cartesian coordinate system to another.

These are useful for defining projections with nonlinear magnifications,
reductions, or other transformations of the input coordinate system.

Generally, these function objects should work for an arbitrary (x,y) pair,
returning new, remapped (x,y), although some classes of mappers may
define their range and domain, with undesirable or undefined behavior
outside those regions.  It is best if such objects raise a suitable
exception in those circumstances.

$Id$
"""
__version__='$Revision$'

from numpy import exp,log,sqrt,sin,cos
from math import atan,pi,atan2

from topo.base.parameterizedobject import Parameter
from topo.base.parameterclasses import Number,Enumeration
from topo.base.functionfamilies import CoordinateMapperFn, IdentityMF
from topo.misc.utils import signabs

##########################################################################
# coordinate mapper functions

class ConstantMapper(CoordinateMapperFn):
    """
    Coordinate Mapper that uses constant mapping values.

    Outputs constant coordinate (x_cons, y_cons).
    """
    x_cons = Number(default=0.0,bounds=(0,None),doc="""
       The maximum range of the mapping input.""")
    y_cons = Number(default=0.0,bounds=(0,None), doc="""
       The y coordinate of the mapping output.""")

    def __call__(self, x, y):
        """
        Arguments x and y are used to follow the abstraction class
        CoordinateMapperFn only.
        """
        return self.x_cons, self.y_cons

class SingleDimensionMapper(CoordinateMapperFn):
    """
    Coordinate Mapper that uses an origin-centered 1-D mapping function.

    An abstract mapping function for coordinate mappers that remap
    based on the radius, x, or y individually. Subclasses should override
    _map_fn(self,z).
    """
    _abstract_class_name = "SingleDimensionMapper"

    in_range = Number(default=0.5*sqrt(2),bounds=(0,None),doc="""
       The maximum range of the mapping input.""")
    out_range = Number(default=0.5*sqrt(2),bounds=(0,None), doc="""
       The maximum range of the output.""")
    remap_dimension = Enumeration(default='radius',
                                  available=['radius','x','y','xy'],
                                  doc="""
        The dimension to remap. ('xy' remaps x and y independently)""")

    def __call__(self,x,y):

        if self.remap_dimension == 'radius':
            r = sqrt(x**2 + y**2)
            a = atan2(x,y)
            new_r = self._map_fn(r)
            xout = new_r * sin(a)
            yout = new_r * cos(a)
        else:
            if 'x' in self.remap_dimension:
                xout = self._map_fn(x)
            else:
                xout = x

            if 'y' in self.remap_dimension:
                yout = self._map_fn(y)
            else:
                yout = y

        return xout,yout

    def _map_fn(self,z):
        raise NotImplementedError

class MagnifyingMapper(SingleDimensionMapper):
    """
    Exponential (magnifying) mapping function.

    Provides a mapping that magnifies the center of the activity image.
    Parameter k indicates amount of magnification, k = 0 means no magnification,
    """

    k = Number(default=1.0,bounds=(0,None))

    def _map_fn(self,z):
        k = self.k
        if k == 0:
            return z
        else:
            sgn,z = signabs(z)
            return sgn * self.out_range * (exp(z/self.in_range*k)-1)/(exp(k)-1)

class ReducingMapper(SingleDimensionMapper):
    """
    Provides a mapping that reduces the center of the activity.

    Roughly the inverse of Magnifying Mapper.  k indicates amount of reduction.
    """
    k = Number(default=1.0,bounds=(0,None))

    def _map_fn(self,z):
        k = self.k
        sgn,z = signabs(z)
        return sgn * self.out_range * log(z/self.in_range*k+1)/log(k+1)

class OttesSCMotorMapper(CoordinateMapperFn):
    """
    Mapping function for mapping motor output from superior colliculus.

    This mapping is taken from Ottes et al. (1986) Vision Research 26:857-873.

    Default constant values are from Table 1, ibid.  The mapping allows the creation
    of a single sheet representing both colliculi, one in the x-positive hemisheet and
    the other in the x-negative hemisheet.
    """

    A = Number(default=5.3, doc="""
       Shape parameter A, in degrees""")
    Bu = Number(default=1.8, doc="""
       Rostral-caudal scale parameter, in mm""")
    Bv = Number(default=1.8, doc="""
       Orthogonal (medial-lateral?) scale paraemter in mm/deg""")

    mm_scale = Number(default=8.0,doc="""
       Scale factor to convert constants Bu and Bv from mm to sheet
       units.  Expressed in mm/unit. """)

    decoder_sheet = Parameter(default=None)

    def __call__(self,x,y):

        A = self.A
        Bu = (self.Bu / self.mm_scale)
        Bv = self.Bv / self.mm_scale

        R = x * self.decoder_sheet.amplitude_scale
        phi = y * self.decoder_sheet.direction_scale * pi/180

        Rsign,R = signabs(R)

        u = Rsign * Bu * (log(sqrt(R**2 + A**2 + 2*A*R*cos(phi))) - log(A))
        v = Bv * atan((R*sin(phi))/(R*cos(phi)+A))

        return u,v
<commit_msg>Factored Ottes SC mapping.<commit_after>"""
Functions for mapping from one 2D Cartesian coordinate system to another.

These are useful for defining projections with nonlinear magnifications,
reductions, or other transformations of the input coordinate system.

Generally, these function objects should work for an arbitrary (x,y) pair,
returning new, remapped (x,y), although some classes of mappers may
define their range and domain, with undesirable or undefined behavior
outside those regions.  It is best if such objects raise a suitable
exception in those circumstances.

$Id$
"""
__version__='$Revision$'

from numpy import exp,log,sqrt,sin,cos,zeros
from math import atan,pi,atan2

from topo.base.parameterizedobject import Parameter
from topo.base.parameterclasses import Number,Enumeration
from topo.base.functionfamilies import CoordinateMapperFn, IdentityMF
from topo.misc.utils import signabs

##########################################################################
# coordinate mapper functions

class ConstantMapper(CoordinateMapperFn):
    """
    Coordinate Mapper that uses constant mapping values.

    Outputs constant coordinate (x_cons, y_cons).
    """
    x_cons = Number(default=0.0,bounds=(0,None),doc="""
       The maximum range of the mapping input.""")
    y_cons = Number(default=0.0,bounds=(0,None), doc="""
       The y coordinate of the mapping output.""")

    def __call__(self, x, y):
        """
        Arguments x and y are used to follow the abstraction class
        CoordinateMapperFn only.
        """
        return self.x_cons, self.y_cons

class SingleDimensionMapper(CoordinateMapperFn):
    """
    Coordinate Mapper that uses an origin-centered 1-D mapping function.

    An abstract mapping function for coordinate mappers that remap
    based on the radius, x, or y individually. Subclasses should override
    _map_fn(self,z).
    """
    _abstract_class_name = "SingleDimensionMapper"

    in_range = Number(default=0.5*sqrt(2),bounds=(0,None),doc="""
       The maximum range of the mapping input.""")
    out_range = Number(default=0.5*sqrt(2),bounds=(0,None), doc="""
       The maximum range of the output.""")
    remap_dimension = Enumeration(default='radius',
                                  available=['radius','x','y','xy'],
                                  doc="""
        The dimension to remap. ('xy' remaps x and y independently)""")

    def __call__(self,x,y):

        if self.remap_dimension == 'radius':
            r = sqrt(x**2 + y**2)
            a = atan2(x,y)
            new_r = self._map_fn(r)
            xout = new_r * sin(a)
            yout = new_r * cos(a)
        else:
            if 'x' in self.remap_dimension:
                xout = self._map_fn(x)
            else:
                xout = x

            if 'y' in self.remap_dimension:
                yout = self._map_fn(y)
            else:
                yout = y

        return xout,yout

    def _map_fn(self,z):
        raise NotImplementedError

class MagnifyingMapper(SingleDimensionMapper):
    """
    Exponential (magnifying) mapping function.

    Provides a mapping that magnifies the center of the activity image.
    Parameter k indicates amount of magnification, k = 0 means no magnification,
    """

    k = Number(default=1.0,bounds=(0,None))

    def _map_fn(self,z):
        k = self.k
        if k == 0:
            return z
        else:
            sgn,z = signabs(z)
            return sgn * self.out_range * (exp(z/self.in_range*k)-1)/(exp(k)-1)

class ReducingMapper(SingleDimensionMapper):
    """
    Provides a mapping that reduces the center of the activity.

    Roughly the inverse of Magnifying Mapper.  k indicates amount of reduction.
    """
    k = Number(default=1.0,bounds=(0,None))

    def _map_fn(self,z):
        k = self.k
        sgn,z = signabs(z)
        return sgn * self.out_range * log(z/self.in_range*k+1)/log(k+1)

class OttesSCMapper(CoordinateMapperFn):
    """
    Abstract class for superior colliculus mappings.

    Subclasses of this class implement afferent and efferent mappings
    from Ottes et al. (1986) Vision Research 26:857-873.

    Default constant values are from Table 1, ibid.
    """
    __abstract_class = "OttesSCMapper"

    A = Number(default=5.3, doc="""
       Shape parameter A, in degrees""")
    Bu = Number(default=1.8, doc="""
       Rostral-caudal scale parameter, in mm""")
    Bv = Number(default=1.8, doc="""
       Orthogonal (medial-lateral?) scale paraemter in mm/deg""")

    mm_scale = Number(default=8.0,doc="""
       Scale factor to convert constants Bu and Bv from mm to sheet
       units.  Expressed in mm/unit. """)

    amplitude_scale = Number(default=1,doc="""
        Scale factor for saccade command amplitude, expressed in
        degrees per unit of sheet.  Indicates how large a
        saccade is represented by the x-component of the command
        input.""")

    direction_scale = Number(default=1,doc="""
        Scale factor for saccade command direction, expressed in
        degrees per unit of sheet.  Indicates what direction of saccade
        is represented by the y-component of the command input.""")

    def __call__(self,x,y):
        raise NotImplementedError

class OttesSCMotorMapper(OttesSCMapper):
    """
    Efferent superior colliculus mapping.

    Provides the output/motor mapping from SC as defined by Ottes et al.
    (see superclass docs for reference)

    The mapping allows the creation of a single sheet representing
    both colliculi, one in the x-positive hemisheet and the other in
    the x-negative hemisheet.
    """
    def __call__(self,x,y):

        A = self.A
        Bu = self.Bu / self.mm_scale
        Bv = self.Bv / self.mm_scale

        R = x * self.amplitude_scale
        phi = y * self.direction_scale

        Rsign,R = signabs(R)

        u,v = ottes_mapping(R,phi,A,Bu,Bv)
        return Rsign*u,v

class OttesSCSenseMapper(OttesSCMapper):
    """
    Afferent superior colliculus mapping.

    Provides the input/retinal mapping to SC as defined by Ottes et al.
    (see superclass docs for reference)

    The mapping allows the creation of a single sheet representing
    both colliculi, one in the x-positive hemisheet and the other in
    the x-negative hemisheet.

    [NOTE: see warning in docs for ottes_inverse_mapping()]
    """

    def __call__(self,x,y):

        A = self.A
        Bu = self.Bu
        Bv = self.Bv

        u = x * self.mm_scale
        v = y * self.mm_scale

        usgn,u = signabs(u)

        R,phi = ottes_inverse_mapping(u,v,A,Bu,Bv)

        return (usgn*R/self.amplitude_scale,
                phi/self.direction_scale)

def ottes_mapping(R,phi,A,Bu,Bv):
    """
    The efferent mapping function from Ottes et al. (1986)
    Vision Research 26:857-873.

    Takes saccade with amplitude R (in degrees) and direction
    phi (in degrees), and returns a location u,v on the colliculus
    in mm, where the u axis is rostral/caudal, and the v axis  is
    medial/lateral.
    """
    phi *= pi/180
    u = Bu * (log(sqrt(R**2 + A**2 + 2*A*R*cos(phi))) - log(A))
    v = Bv * atan((R*sin(phi))/(R*cos(phi)+A))
    return u,v

def ottes_inverse_mapping(u,v,A,Bu,Bv):
    """
    The inverse funtion provided in the appendix of Ottes et al. (1986)
    Vision Research 26:857-873

    Supposedly takes a location u,v on the colliculus in mm and maps
    to a retinal eccentricity R and direction phi, in degrees.
    However, this function is not actually the mathematical inverse of
    the forward mapping, nor even close.
    """
    # JPALERT: As mentioned above, this is not actually the inverse of
    # the efferent mapping. I believe the formula in the paper is
    # in error, though it's possible that there's a bug below that
    # I've missed.

    rads = pi/180
    R   = A * sqrt(exp(2*u/Bu) - 2*exp(u/Bu)*cos(rads*v/Bv) + 1)
    phi = atan( (exp(u/Bu)*sin(rads*v/Bv)) / (exp(u/Bu)*cos(rads*v/Bv) -1) )
    return R,phi*180/pi

<|endoftext|>


<commit_before>"""
Unit test for Simulation
$Id$
"""
__version__='$Revision$'

import unittest
import copy
import pickle

from numpy.oldnumeric import array
from topo.base.simulation import Simulation,EPConnection,EPConnectionEvent,Event
from topo.ep.basic import *

from topo.base.cf import CFSheet, CFProjection

from utils import new_simulation

import topo

# CEBALERT: not a complete test of Simulation

class TestSimulation(unittest.TestCase):

    def test_register_is_true(self):
        sim1 = new_simulation(name="test_singleton")
        assert sim1 is topo.sim

        sid = id(topo.sim['S'])

        sim2 = copy.copy(topo.sim)
        assert sim2 is sim1

        sim3 = copy.deepcopy(topo.sim)
        assert sim3 is sim1

        self.assertEqual(id(sim3['S']),sid)

        topo.sim['S'].precedence=111
        p = pickle.dumps(topo.sim,2)
        topo.sim['S'].precedence=5

        Simulation.register=False # to ensure the object's own register is being used
        sim4 = pickle.loads(p)
        Simulation.register=True

        assert sim4 is sim1
        assert topo.sim['S'].precedence==111,"%s"%topo.sim['S'].precedence

    def test_register_is_false(self):
        sim1 = new_simulation(register=False)
        assert sim1 is not topo.sim

        sid = id(sim1['S'])

        sim2 = copy.copy(sim1)
        assert sim2 is not sim1
        assert sim2 is not topo.sim

        sim3 = copy.deepcopy(sim1)
        assert sim3 is not sim1
        assert sim3 is not topo.sim

        self.assertNotEqual(id(sim3['S']),sid)

        new_simulation(register=True)
        sim1['S'].precedence=111
        p = pickle.dumps(sim1,2)
        topo.sim['S'].precedence=5
        sim4 = pickle.loads(p)

        assert sim4 is not sim1
        assert sim4 is not topo.sim
        assert topo.sim['S'].precedence==5

    def test_event_copy(self):
        """
        Test to make sure that EPConnectionEvent copies the underlying data
        on construction.
        """
        s = Simulation()
        data = array([4,3])
        epc = EPConnection()
        se = EPConnectionEvent(1,epc,data)
        se.data[0] = 5
        assert data[0] != se.data[0], 'Matrices should be different'
        se2 = copy.copy(se)
        assert se is not se2, 'Objects are the same'

    def test_state_stack(self):
        s = Simulation()
        s['pulse1'] = PulseGenerator(period = 1)
        s['pulse2'] = PulseGenerator(period = 3)
        s['sum_unit'] = SumUnit()
        s.connect('pulse1','sum_unit',delay=1)
        s.connect('pulse2','sum_unit',delay=1)
        s.run(1.0)
        s.state_push()
        self.assertEqual(len(s._events_stack),1)
        s.state_pop()
        self.assertEqual(len(s._events_stack),0)

    def test_event_cmp(self):

        e1 = Event(1)
        e1a = Event(1)
        e2 = Event(2)

        assert e1 is not e1a
        assert e1 ==  e1a
        assert e1 < e2
        assert e1 == e1
        assert e2 > e1
        assert e2 == e2
        assert e1 != e2
        assert e2 != e1

    def test_event_insert(self):
        s = Simulation()

        e1 = Event(1)
        e1a = Event(1)
        e2 = Event(2)
        e2a = Event(2)

        s.enqueue_event(e1)
        s.enqueue_event(e2)
        s.enqueue_event(e2a)
        s.enqueue_event(e1a)

        s.enqueue_event(Event(0))

        assert len(s.events) == 5, 'Event queue has %d events, should have 5.' % len(s.events)

        assert s.events[1] == e1
        assert s.events[2] == e1a
        assert s.events[3] == e2
        assert s.events[4] == e2a

    def test_get_objects(self):
        s = Simulation()

        s['pulse1'] = PulseGenerator(period = 1)
        s['pulse2'] = PulseGenerator(period = 3)
        s['sum_unit'] = SumUnit()
        n1 = s['pulse1'].name
        n2 = s['pulse2'].name

        s.connect('pulse1','sum_unit',delay=1)
        s.connect('pulse2','sum_unit',delay=1)
        t1 = s.objects()
        e1 = [ep for ep in t1.values() if isinstance(ep,PulseGenerator) and ep.name == n1]
        t2 = s.objects()
        e2 = [ep for ep in t2.values() if isinstance(ep,PulseGenerator) and ep.name == n2]
        assert e1.pop().name == n1, 'Object names do not match'
        assert e2.pop().name == n2, 'Object names do not match'

##     def test_garbage_collection(self):
##         """something"""
##         s = Simulation(register=False)
##         s['v1']=CFSheet(nominal_density=24)
##         s['v2']=CFSheet(nominal_density=24)
##         s.connect('v1','v2',name='Afferent',delay=0.05,connection_type=CFProjection)
##         from weakref import WeakValueDictionary as W
##         w=W()
##         w['v1'] = s['v1']
##         w['v2'] = s['v2']

##         # need to cause s to be deleted: s in this namespace should
##         # be only reference to s (apart from circular ones...)
##         del s

##         try:
##             w['v1']
##         except KeyError:
##             pass
##         else:
##             raise ValueError("v1 was not deleted")

##         try:
##             w['v2']
##         except KeyError:
##             pass
##         else:
##             raise ValueError("v2 was not deleted")

suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestSimulation))
<commit_msg>Fix rerefences to topo.ep.basic<commit_after>"""
Unit test for Simulation
$Id$
"""
__version__='$Revision$'

import unittest
import copy
import pickle

from numpy.oldnumeric import array
from topo.base.simulation import Simulation,EPConnection,EPConnectionEvent,Event
from topo.ep import *

from topo.base.cf import CFSheet, CFProjection

from utils import new_simulation

import topo

# CEBALERT: not a complete test of Simulation

class TestSimulation(unittest.TestCase):

    def test_register_is_true(self):
        sim1 = new_simulation(name="test_singleton")
        assert sim1 is topo.sim

        sid = id(topo.sim['S'])

        sim2 = copy.copy(topo.sim)
        assert sim2 is sim1

        sim3 = copy.deepcopy(topo.sim)
        assert sim3 is sim1

        self.assertEqual(id(sim3['S']),sid)

        topo.sim['S'].precedence=111
        p = pickle.dumps(topo.sim,2)
        topo.sim['S'].precedence=5

        Simulation.register=False # to ensure the object's own register is being used
        sim4 = pickle.loads(p)
        Simulation.register=True

        assert sim4 is sim1
        assert topo.sim['S'].precedence==111,"%s"%topo.sim['S'].precedence

    def test_register_is_false(self):
        sim1 = new_simulation(register=False)
        assert sim1 is not topo.sim

        sid = id(sim1['S'])

        sim2 = copy.copy(sim1)
        assert sim2 is not sim1
        assert sim2 is not topo.sim

        sim3 = copy.deepcopy(sim1)
        assert sim3 is not sim1
        assert sim3 is not topo.sim

        self.assertNotEqual(id(sim3['S']),sid)

        new_simulation(register=True)
        sim1['S'].precedence=111
        p = pickle.dumps(sim1,2)
        topo.sim['S'].precedence=5
        sim4 = pickle.loads(p)

        assert sim4 is not sim1
        assert sim4 is not topo.sim
        assert topo.sim['S'].precedence==5

    def test_event_copy(self):
        """
        Test to make sure that EPConnectionEvent copies the underlying data
        on construction.
        """
        s = Simulation()
        data = array([4,3])
        epc = EPConnection()
        se = EPConnectionEvent(1,epc,data)
        se.data[0] = 5
        assert data[0] != se.data[0], 'Matrices should be different'
        se2 = copy.copy(se)
        assert se is not se2, 'Objects are the same'

    def test_state_stack(self):
        s = Simulation()
        s['pulse1'] = PulseGenerator(period = 1)
        s['pulse2'] = PulseGenerator(period = 3)
        s['sum_unit'] = SumUnit()
        s.connect('pulse1','sum_unit',delay=1)
        s.connect('pulse2','sum_unit',delay=1)
        s.run(1.0)
        s.state_push()
        self.assertEqual(len(s._events_stack),1)
        s.state_pop()
        self.assertEqual(len(s._events_stack),0)

    def test_event_cmp(self):

        e1 = Event(1)
        e1a = Event(1)
        e2 = Event(2)

        assert e1 is not e1a
        assert e1 ==  e1a
        assert e1 < e2
        assert e1 == e1
        assert e2 > e1
        assert e2 == e2
        assert e1 != e2
        assert e2 != e1

    def test_event_insert(self):
        s = Simulation()

        e1 = Event(1)
        e1a = Event(1)
        e2 = Event(2)
        e2a = Event(2)

        s.enqueue_event(e1)
        s.enqueue_event(e2)
        s.enqueue_event(e2a)
        s.enqueue_event(e1a)

        s.enqueue_event(Event(0))

        assert len(s.events) == 5, 'Event queue has %d events, should have 5.' % len(s.events)

        assert s.events[1] == e1
        assert s.events[2] == e1a
        assert s.events[3] == e2
        assert s.events[4] == e2a

    def test_get_objects(self):
        s = Simulation()

        s['pulse1'] = PulseGenerator(period = 1)
        s['pulse2'] = PulseGenerator(period = 3)
        s['sum_unit'] = SumUnit()
        n1 = s['pulse1'].name
        n2 = s['pulse2'].name

        s.connect('pulse1','sum_unit',delay=1)
        s.connect('pulse2','sum_unit',delay=1)
        t1 = s.objects()
        e1 = [ep for ep in t1.values() if isinstance(ep,PulseGenerator) and ep.name == n1]
        t2 = s.objects()
        e2 = [ep for ep in t2.values() if isinstance(ep,PulseGenerator) and ep.name == n2]
        assert e1.pop().name == n1, 'Object names do not match'
        assert e2.pop().name == n2, 'Object names do not match'

##     def test_garbage_collection(self):
##         """something"""
##         s = Simulation(register=False)
##         s['v1']=CFSheet(nominal_density=24)
##         s['v2']=CFSheet(nominal_density=24)
##         s.connect('v1','v2',name='Afferent',delay=0.05,connection_type=CFProjection)
##         from weakref import WeakValueDictionary as W
##         w=W()
##         w['v1'] = s['v1']
##         w['v2'] = s['v2']

##         # need to cause s to be deleted: s in this namespace should
##         # be only reference to s (apart from circular ones...)
##         del s

##         try:
##             w['v1']
##         except KeyError:
##             pass
##         else:
##             raise ValueError("v1 was not deleted")

##         try:
##             w['v2']
##         except KeyError:
##             pass
##         else:
##             raise ValueError("v2 was not deleted")

suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestSimulation))
<|endoftext|>


<commit_before>"""
This module queries process memory allocation metrics from the operating
system. It provides a platform independent layer to get the amount of virtual
and physical memory allocated to the Python process.

Different mechanisms are implemented: The resource module is tried first.  If
the resource module does not report the required information (Linux/Solaris),
try to read the stat file and then to execute the ps command. If the resource
module does not exist, try a fallback for Windows using the win32 module. If all
fails, return 0 for each attribute.

Windows without the win32 module is not supported.

    >>> from pympler.process import ProcessMemoryInfo
    >>> pmi = ProcessMemoryInfo()
    >>> print "Virtual size [Byte]:", pmi.vsz # doctest: +ELLIPSIS
    Virtual size [Byte]: ...
"""

from os import getpid
from subprocess  import Popen, PIPE
from threading import enumerate

try:
    from resource import getpagesize as _getpagesize
except ImportError:
    _getpagesize = lambda : 4096

class _ProcessMemoryInfo(object):
    """Stores information about various process-level memory metrics. The
    virtual size is stored in attribute `vsz`, the physical memory allocated to
    the process in `rss`, and the number of (major) pagefaults in `pagefaults`.
    This is an abstract base class which needs to be overridden by operating
    system specific implementations. This is done when importing the module.
    """

    pagesize = _getpagesize()

    def __init__(self):
        self.pid = getpid()
        self.rss = 0
        self.vsz = 0
        self.pagefaults = 0
        self.os_specific = []
        self.update()

    def update(self):
        """
        Refresh the information using platform instruments. Returns true if this
        operation yields useful values on the current platform.
        """
        return False

ProcessMemoryInfo = _ProcessMemoryInfo

def is_available():
    """
    Convenience function to check if the current platform is supported by this
    module.
    """
    return ProcessMemoryInfo().update()

class _ProcessMemoryInfoPS(_ProcessMemoryInfo):
    def update(self):
        """
        Get virtual and resident size of current process via 'ps'.
        This should work for MacOS X, Solaris, Linux. Returns true if it was
        successful.
        """
        try:
            p = Popen(['/bin/ps', '-p%s' % self.pid, '-o', 'rss,vsz'],
                      stdout=PIPE, stderr=PIPE)
        except OSError:
            pass
        else:
            s = p.communicate()[0].split()
            if p.returncode == 0 and len(s) >= 2:
                self.vsz = int(s[-1]) * 1024
                self.rss = int(s[-2]) * 1024
                return True
        return False

class _ProcessMemoryInfoProc(_ProcessMemoryInfo):

    key_map = {
        'VmPeak'       : 'Peak virtual memory size',
        'VmSize'       : 'Virtual memory size',
        'VmLck'        : 'Locked memory size',
        'VmHWM'        : 'Peak resident set size',
        'VmRSS'        : 'Resident set size',
        'VmStk'        : 'Size of stack segment',
        'VmData'       : 'Size of data segment',
        'VmExe'        : 'Size of code segment',
        'VmLib'        : 'Shared library code size',
        'VmPTE'        : 'Page table entries size',
    }

    def update(self):
        """
        Get virtual size of current process by reading the process' stat file.
        This should work for Linux.
        """
        try:
            stat = open('/proc/self/stat')
            status = open('/proc/self/status')
        except IOError:
            pass
        else:
            stats = stat.read().split()
            self.vsz = int( stats[22] )
            self.rss = int( stats[23] ) * self.pagesize
            self.pagefaults = int( stats[11] )

            for entry in status.readlines():
                key, value = entry.split(':')
                key = self.key_map.get(key)
                if key:
                    self.os_specific.append((key, value.strip()))

            stat.close()
            status.close()
            return True
        return False

try:
    from resource import getrusage, RUSAGE_SELF

    class _ProcessMemoryInfoResource(_ProcessMemoryInfo):
        def update(self):
            """
            Get resident set size of current process through resource module. Only
            available on Unix. Does not work with any platform tested on.
            """
            usage = getrusage(RUSAGE_SELF)
            self.rss = usage.ru_maxrss * self.pagesize
            self.vsz = usage.ru_maxrss * self.pagesize # XXX rss is not vsz
            self.pagefault = usage.ru_majflt
            return self.rss != 0

    if _ProcessMemoryInfoResource().update():
        ProcessMemoryInfo = _ProcessMemoryInfoResource
    elif _ProcessMemoryInfoProc().update():
        ProcessMemoryInfo = _ProcessMemoryInfoProc
    elif _ProcessMemoryInfoPS().update():
        ProcessMemoryInfo = _ProcessMemoryInfoPS

except ImportError:
    try:
        # Requires pywin32
        from win32process import GetProcessMemoryInfo
        from win32api import GetCurrentProcess
    except ImportError:
        # TODO Emit Warning:
        #print "It is recommended to install pywin32 when running pympler on Microsoft Windows."
        pass
    else:
        class _ProcessMemoryInfoWin32(_ProcessMemoryInfo):
            def update(self):
                process_handle = GetCurrentProcess()
                memory_info = GetProcessMemoryInfo( process_handle )
                self.vsz       = memory_info['PagefileUsage']
                self.rss       = memory_info['WorkingSetSize']
                self.pagefault = memory_info['PageFaultCount']
                return True

        ProcessMemoryInfo = _ProcessMemoryInfoWin32

class ThreadInfo(object):
    """Collect information about an active thread."""
    pass

def get_current_threads():
    """Get a list of `ThreadInfo` objects."""
    threads = []
    for thread in enumerate():
        tinfo = ThreadInfo()
        tinfo.ident = thread.ident
        tinfo.name = thread.name
        tinfo.daemon = thread.daemon
        threads.append(tinfo)
    return threads

<commit_msg>little fixes in process module<commit_after>"""
This module queries process memory allocation metrics from the operating
system. It provides a platform independent layer to get the amount of virtual
and physical memory allocated to the Python process.

Different mechanisms are implemented: The resource module is tried first.  If
the resource module does not report the required information (Linux/Solaris),
try to read the stat file and then to execute the ps command. If the resource
module does not exist, try a fallback for Windows using the win32 module. If all
fails, return 0 for each attribute.

Windows without the win32 module is not supported.

    >>> from pympler.process import ProcessMemoryInfo
    >>> pmi = ProcessMemoryInfo()
    >>> print "Virtual size [Byte]:", pmi.vsz # doctest: +ELLIPSIS
    Virtual size [Byte]: ...
"""

from os import getpid
from subprocess  import Popen, PIPE
from threading import enumerate as enumerate_threads

try:
    from resource import getpagesize as _getpagesize
except ImportError:
    _getpagesize = lambda : 4096

class _ProcessMemoryInfo(object):
    """Stores information about various process-level memory metrics. The
    virtual size is stored in attribute `vsz`, the physical memory allocated to
    the process in `rss`, and the number of (major) pagefaults in `pagefaults`.
    This is an abstract base class which needs to be overridden by operating
    system specific implementations. This is done when importing the module.
    """

    pagesize = _getpagesize()

    def __init__(self):
        self.pid = getpid()
        self.rss = 0
        self.vsz = 0
        self.pagefaults = 0
        self.os_specific = []
        self.update()

    def update(self):
        """
        Refresh the information using platform instruments. Returns true if this
        operation yields useful values on the current platform.
        """
        return False

ProcessMemoryInfo = _ProcessMemoryInfo

def is_available():
    """
    Convenience function to check if the current platform is supported by this
    module.
    """
    return ProcessMemoryInfo().update()

class _ProcessMemoryInfoPS(_ProcessMemoryInfo):
    def update(self):
        """
        Get virtual and resident size of current process via 'ps'.
        This should work for MacOS X, Solaris, Linux. Returns true if it was
        successful.
        """
        try:
            p = Popen(['/bin/ps', '-p%s' % self.pid, '-o', 'rss,vsz'],
                      stdout=PIPE, stderr=PIPE)
        except OSError:
            pass
        else:
            s = p.communicate()[0].split()
            if p.returncode == 0 and len(s) >= 2:
                self.vsz = int(s[-1]) * 1024
                self.rss = int(s[-2]) * 1024
                return True
        return False

class _ProcessMemoryInfoProc(_ProcessMemoryInfo):

    key_map = {
        'VmPeak'       : 'Peak virtual memory size',
        'VmSize'       : 'Virtual memory size',
        'VmLck'        : 'Locked memory size',
        'VmHWM'        : 'Peak resident set size',
        'VmRSS'        : 'Resident set size',
        'VmStk'        : 'Size of stack segment',
        'VmData'       : 'Size of data segment',
        'VmExe'        : 'Size of code segment',
        'VmLib'        : 'Shared library code size',
        'VmPTE'        : 'Page table entries size',
    }

    def update(self):
        """
        Get virtual size of current process by reading the process' stat file.
        This should work for Linux.
        """
        try:
            stat = open('/proc/self/stat')
            status = open('/proc/self/status')
        except IOError:
            pass
        else:
            stats = stat.read().split()
            self.vsz = int( stats[22] )
            self.rss = int( stats[23] ) * self.pagesize
            self.pagefaults = int( stats[11] )

            for entry in status.readlines():
                key, value = entry.split(':')
                key = self.key_map.get(key)
                if key:
                    self.os_specific.append((key, value.strip()))

            stat.close()
            status.close()
            return True
        return False

try:
    from resource import getrusage, RUSAGE_SELF

    class _ProcessMemoryInfoResource(_ProcessMemoryInfo):
        def update(self):
            """
            Get resident set size of current process through resource module. Only
            available on Unix. Does not work with any platform tested on.
            """
            usage = getrusage(RUSAGE_SELF)
            self.rss = usage.ru_maxrss * self.pagesize
            self.vsz = usage.ru_maxrss * self.pagesize # XXX rss is not vsz
            self.pagefaults = usage.ru_majflt
            return self.rss != 0

    if _ProcessMemoryInfoResource().update():
        ProcessMemoryInfo = _ProcessMemoryInfoResource
    elif _ProcessMemoryInfoProc().update():
        ProcessMemoryInfo = _ProcessMemoryInfoProc
    elif _ProcessMemoryInfoPS().update():
        ProcessMemoryInfo = _ProcessMemoryInfoPS

except ImportError:
    try:
        # Requires pywin32
        from win32process import GetProcessMemoryInfo
        from win32api import GetCurrentProcess
    except ImportError:
        # TODO Emit Warning:
        #print "It is recommended to install pywin32 when running pympler on Microsoft Windows."
        pass
    else:
        class _ProcessMemoryInfoWin32(_ProcessMemoryInfo):
            def update(self):
                process_handle = GetCurrentProcess()
                memory_info = GetProcessMemoryInfo( process_handle )
                self.vsz        = memory_info['PagefileUsage']
                self.rss        = memory_info['WorkingSetSize']
                self.pagefaults = memory_info['PageFaultCount']
                return True

        ProcessMemoryInfo = _ProcessMemoryInfoWin32

class ThreadInfo(object):
    """Collect information about an active thread."""
    pass

def get_current_threads():
    """Get a list of `ThreadInfo` objects."""
    threads = []
    for thread in enumerate_threads():
        tinfo = ThreadInfo()
        try:
            tinfo.ident = thread.ident
        except AttributeError:
            tinfo.ident = None # TODO
        try:
            tinfo.name = thread.name
        except AttributeError:
            tinfo.name = thread.getName()
        try:
            tinfo.daemon = thread.daemon
        except AttributeError:
            tinfo.daemon = thread.isDaemon()
        threads.append(tinfo)
    return threads

<|endoftext|>


<commit_before>from functools import partial
from itertools import accumulate as accumulate_, chain, islice, tee

MYPY = False
if MYPY:
    from typing import Callable, Iterable, Iterator, List, Optional, Set, Tuple, TypeVar
    T = TypeVar('T')
    U = TypeVar('U')

filter_ = partial(filter, None)  # type: Callable[[Iterable[Optional[T]]], Iterator[T]]
flatten = chain.from_iterable

def accumulate(iterable, initial=None):
    # type: (Iterable[int], int) -> Iterable[int]
    if initial is None:
        return accumulate_(iterable)
    else:
        return accumulate_(chain([initial], iterable))

def pairwise(iterable):
    # type: (Iterable[T]) -> Iterable[Tuple[T, T]]
    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
    a, b = tee(iterable)
    next(b, None)
    return zip(a, b)

def unique(iterable):
    # type: (Iterable[T]) -> Iterator[T]
    seen = set()  # type: Set[T]
    for item in iterable:
        if item in seen:
            continue
        seen.add(item)
        yield item

def drop(n, iterable):
    # type: (int, Iterable[T]) -> Iterator[T]
    return islice(iterable, n, None)

def tail(iterable):
    # type: (Iterable[T]) -> Iterator[T]
    return drop(1, iterable)

def unzip(zipped):
    # type: (Iterable[Tuple[T, U]]) -> Tuple[Tuple[T, ...], Tuple[U, ...]]
    return tuple(zip(*zipped))  # type: ignore

# Below functions taken from https://github.com/erikrose/more-itertools
# Copyright (c) 2012 Erik Rose

def take(n, iterable):
    # type: (int, Iterable[T]) -> List[T]
    """Return first *n* items of the iterable as a list.
        >>> take(3, range(10))
        [0, 1, 2]
    If there are fewer than *n* items in the iterable, all of them are
    returned.
        >>> take(10, range(3))
        [0, 1, 2]
    """
    return list(islice(iterable, n))

def chunked(iterable, n):
    """Break *iterable* into lists of length *n*:

        >>> list(chunked([1, 2, 3, 4, 5, 6], 3))
        [[1, 2, 3], [4, 5, 6]]

    If the length of *iterable* is not evenly divisible by *n*, the last
    returned list will be shorter:

        >>> list(chunked([1, 2, 3, 4, 5, 6, 7, 8], 3))
        [[1, 2, 3], [4, 5, 6], [7, 8]]

    To use a fill-in value instead, see the :func:`grouper` recipe.

    :func:`chunked` is useful for splitting up a computation on a large number
    of keys into batches, to be pickled and sent off to worker processes. One
    example is operations on rows in MySQL, which does not implement
    server-side cursors properly and would otherwise load the entire dataset
    into RAM on the client.

    """
    return iter(partial(take, n, iter(iterable)), [])

def partition(pred, iterable):
    # type: (Optional[Callable[[T], bool]], Iterable[T]) -> Tuple[Iterator[T], Iterator[T]]
    """
    Returns a 2-tuple of iterables derived from the input iterable.
    The first yields the items that have ``pred(item) == False``.
    The second yields the items that have ``pred(item) == True``.

        >>> is_odd = lambda x: x % 2 != 0
        >>> iterable = range(10)
        >>> even_items, odd_items = partition(is_odd, iterable)
        >>> list(even_items), list(odd_items)
        ([0, 2, 4, 6, 8], [1, 3, 5, 7, 9])

    If *pred* is None, :func:`bool` is used.

        >>> iterable = [0, 1, False, True, '', ' ']
        >>> false_items, true_items = partition(None, iterable)
        >>> list(false_items), list(true_items)
        ([0, False, ''], [1, True, ' '])

    """
    if pred is None:
        pred = bool

    evaluations = ((pred(x), x) for x in iterable)
    t1, t2 = tee(evaluations)
    return (
        (x for (cond, x) in t1 if not cond),
        (x for (cond, x) in t2 if cond),
    )
<commit_msg>Implement `fns.peek`<commit_after>from functools import partial
from itertools import accumulate as accumulate_, chain, islice, tee

MYPY = False
if MYPY:
    from typing import Callable, Iterable, Iterator, List, Optional, Set, Tuple, TypeVar
    T = TypeVar('T')
    U = TypeVar('U')

filter_ = partial(filter, None)  # type: Callable[[Iterable[Optional[T]]], Iterator[T]]
flatten = chain.from_iterable

def accumulate(iterable, initial=None):
    # type: (Iterable[int], int) -> Iterable[int]
    if initial is None:
        return accumulate_(iterable)
    else:
        return accumulate_(chain([initial], iterable))

def pairwise(iterable):
    # type: (Iterable[T]) -> Iterable[Tuple[T, T]]
    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
    a, b = tee(iterable)
    next(b, None)
    return zip(a, b)

def unique(iterable):
    # type: (Iterable[T]) -> Iterator[T]
    seen = set()  # type: Set[T]
    for item in iterable:
        if item in seen:
            continue
        seen.add(item)
        yield item

def peek(iterable):
    # type: (Iterable[T]) -> Tuple[T, Iterable[T]]
    it = iter(iterable)
    head = next(it)
    return head, chain([head], it)

def drop(n, iterable):
    # type: (int, Iterable[T]) -> Iterator[T]
    return islice(iterable, n, None)

def tail(iterable):
    # type: (Iterable[T]) -> Iterator[T]
    return drop(1, iterable)

def unzip(zipped):
    # type: (Iterable[Tuple[T, U]]) -> Tuple[Tuple[T, ...], Tuple[U, ...]]
    return tuple(zip(*zipped))  # type: ignore

# Below functions taken from https://github.com/erikrose/more-itertools
# Copyright (c) 2012 Erik Rose

def take(n, iterable):
    # type: (int, Iterable[T]) -> List[T]
    """Return first *n* items of the iterable as a list.
        >>> take(3, range(10))
        [0, 1, 2]
    If there are fewer than *n* items in the iterable, all of them are
    returned.
        >>> take(10, range(3))
        [0, 1, 2]
    """
    return list(islice(iterable, n))

def chunked(iterable, n):
    """Break *iterable* into lists of length *n*:

        >>> list(chunked([1, 2, 3, 4, 5, 6], 3))
        [[1, 2, 3], [4, 5, 6]]

    If the length of *iterable* is not evenly divisible by *n*, the last
    returned list will be shorter:

        >>> list(chunked([1, 2, 3, 4, 5, 6, 7, 8], 3))
        [[1, 2, 3], [4, 5, 6], [7, 8]]

    To use a fill-in value instead, see the :func:`grouper` recipe.

    :func:`chunked` is useful for splitting up a computation on a large number
    of keys into batches, to be pickled and sent off to worker processes. One
    example is operations on rows in MySQL, which does not implement
    server-side cursors properly and would otherwise load the entire dataset
    into RAM on the client.

    """
    return iter(partial(take, n, iter(iterable)), [])

def partition(pred, iterable):
    # type: (Optional[Callable[[T], bool]], Iterable[T]) -> Tuple[Iterator[T], Iterator[T]]
    """
    Returns a 2-tuple of iterables derived from the input iterable.
    The first yields the items that have ``pred(item) == False``.
    The second yields the items that have ``pred(item) == True``.

        >>> is_odd = lambda x: x % 2 != 0
        >>> iterable = range(10)
        >>> even_items, odd_items = partition(is_odd, iterable)
        >>> list(even_items), list(odd_items)
        ([0, 2, 4, 6, 8], [1, 3, 5, 7, 9])

    If *pred* is None, :func:`bool` is used.

        >>> iterable = [0, 1, False, True, '', ' ']
        >>> false_items, true_items = partition(None, iterable)
        >>> list(false_items), list(true_items)
        ([0, False, ''], [1, True, ' '])

    """
    if pred is None:
        pred = bool

    evaluations = ((pred(x), x) for x in iterable)
    t1, t2 = tee(evaluations)
    return (
        (x for (cond, x) in t1 if not cond),
        (x for (cond, x) in t2 if cond),
    )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

def load_nametable(nt_file):
    f = open(nt_file)
    nt_content = f.read()
    nt_bin = []
    for nt in nt_content:
        nt_bin.append(ord(nt))
    return nt_bin

def get_nametable(index, nt):
    tile_index = index * 1024
    nametable = []
    for y in range(32):
        line = []
        for x in range(32):
            dx = tile_index / 32
            dy = tile_index % 32
            spr_index = nt[tile_index]
            line.append(spr_index)
            tile_index += 1
        nametable.append(line)
    return nametable

def length(nt):
    return len(nt) / 1024<commit_msg>PEP8 fixes on nametable.py<commit_after># -*- coding: utf-8 -*-

def load_nametable(nt_file):
    f = open(nt_file)
    nt_content = f.read()
    nt_bin = []
    for nt in nt_content:
        nt_bin.append(ord(nt))
    return nt_bin

def get_nametable(index, nt):
    tile_index = index * 1024
    nametable = []
    for y in range(32):
        line = []
        for x in range(32):
            # dx = tile_index / 32
            # dy = tile_index % 32
            spr_index = nt[tile_index]
            line.append(spr_index)
            tile_index += 1
        nametable.append(line)
    return nametable

def length(nt):
    return len(nt) / 1024
<|endoftext|>