<commit_before>import asyncio
import websockets
import random
from collections import OrderedDict
import ejson

@asyncio.coroutine
def noop(*a): return

class Subscription:
    def __init__(self, name, ready_cb, added_cb, changed_cb, removed_cb):
        self.id = str(random.randint(0, 1000000))
        self.name = name
        self.data = OrderedDict()
        self.ready_cb = ready_cb
        self.added_cb = added_cb
        self.changed_cb = changed_cb
        self.removed_cb = removed_cb

    def sorted(self, key):
        return OrderedDict(sorted(self.data.items(), key=lambda d: d[1][key]))

class MethodCall:
    def __init__(self, callback):
        self.id = str(random.randint(1, 1000000))
        self.callback = callback
        self.finished = False
        self.result, self.error = None, None

    @asyncio.coroutine
    def onfinished(self, result, error):
        self.finished = True
        self.result, self.error = result, error
        if self.callback:
            yield from self.callback(result, error)

class DDPClient:
    def __init__(self, address):
        self.connected = False
        self.address = address
        self.session = None

        self.subs = {}
        self.calls = {}

        self.callcache = []

    @asyncio.coroutine
    def connect(self):
        self.websocket = yield from websockets.connect(self.address)

        while not self.websocket.open: yield from asyncio.sleep(0.01)

        msg = {'msg': 'connect', 'version': '1', 'support': ['1']}
        if self.session: msg['session'] = self.session
        yield from self.websocket.send(ejson.dumps(msg))

        asyncio.get_event_loop().create_task(self.recvloop())
        while not self.connected:
            yield from asyncio.sleep(0.1)

    @asyncio.coroutine
    def subscribe(self, name, *params, ready_cb = noop,
                  added_cb = noop, changed_cb = noop, removed_cb = noop):
        sub = Subscription(name, ready_cb, added_cb, changed_cb, removed_cb)
        self.subs[name] = sub
        yield from self.websocket.send(ejson.dumps(
            {'msg': 'sub',
             'id': sub.id,
             'name': name,
             'params': params}
        ))

        return sub

    @asyncio.coroutine
    def call(self, method, *params, callback = None, wait = True):
        c = MethodCall(callback)
        self.calls[c.id] = c

        yield from self.websocket.send(ejson.dumps(
            {'msg': 'method',
             'method': method,
             'params': params,
             'id': c.id}
        ))

        if wait:
            while not c.finished:
                yield from asyncio.sleep(0.1)

            return c.result, c.error

    @asyncio.coroutine
    def call_cached(self, method, params = [], callback = lambda *a: None):
        if self.connected:
            yield from self.call(method, params, callback)
        else:
            self.callcache.append((method, params, callback))

    @asyncio.coroutine
    def recvloop(self):
        while self.websocket.open:
            while len(self.callcache) > 0:
                self.call(*self.callcache.pop(0))

            msg = yield from self.websocket.recv()
            if not msg: continue
            msg = ejson.loads(msg)

            if msg.get('msg') == 'connected':
                self.connected = True
                self.session = msg['session']

            elif msg.get('msg') == 'ping':
                yield from self.websocket.send(ejson.dumps({'msg': 'pong', 'id': msg.get('id')}))

            elif msg.get('msg') == 'ready':
                for sub in self.subs.values():
                    if sub.id in msg['subs']:
                        yield from sub.ready_cb(sub)

            elif msg.get('msg') == 'added':
                sub = self.subs.get(msg['collection'])
                if sub:
                    sub.data[msg['id']] = msg['fields']
                    yield from sub.added_cb(sub, msg['id'], msg['fields'])

            elif msg.get('msg') == 'changed':
                sub = self.subs.get(msg['collection'])
                if sub:
                    if msg.get('fields'):
                        sub.data[msg['id']].update(msg['fields'])
                        yield from sub.changed_cb(sub, msg['id'], msg['fields'])
                    elif msg.get('cleared'):
                        for key in msg['cleared']:
                            del sub.data[key]
                        yield from sub.changed_cb(sub, msg['id'], msg['cleared'])

            elif msg.get('msg') == 'removed':
                sub = self.subs.get(msg['collection'])
                if sub:
                    del sub.data[msg['id']]
                    yield from sub.removed_cb(sub, msg['id'])

            elif msg.get('msg') == 'result':
                c = self.calls.get(msg['id'])
                if c:
                    yield from c.onfinished(msg.get('result'), msg.get('error'))

        self.connected = False
        while True:
            self.websocket = yield from websockets.connect(self.address)
            if self.websocket.open:
                yield from self.connect()
                return
            else:
                yield from asyncio.sleep(1)

<commit_msg>Fixed bug affecting older versions of asyncio<commit_after>import asyncio
import websockets
import random
from collections import OrderedDict
import ejson

@asyncio.coroutine
def noop(*a): return

class Subscription:
    def __init__(self, name, ready_cb, added_cb, changed_cb, removed_cb):
        self.id = str(random.randint(0, 1000000))
        self.name = name
        self.data = OrderedDict()
        self.ready_cb = ready_cb
        self.added_cb = added_cb
        self.changed_cb = changed_cb
        self.removed_cb = removed_cb

    def sorted(self, key):
        return OrderedDict(sorted(self.data.items(), key=lambda d: d[1][key]))

class MethodCall:
    def __init__(self, callback):
        self.id = str(random.randint(1, 1000000))
        self.callback = callback
        self.finished = False
        self.result, self.error = None, None

    @asyncio.coroutine
    def onfinished(self, result, error):
        self.finished = True
        self.result, self.error = result, error
        if self.callback:
            yield from self.callback(result, error)

class DDPClient:
    def __init__(self, address):
        self.connected = False
        self.address = address
        self.session = None

        self.subs = {}
        self.calls = {}

        self.callcache = []

    @asyncio.coroutine
    def connect(self):
        self.websocket = yield from websockets.connect(self.address)

        while not self.websocket.open: yield from asyncio.sleep(0.01)

        msg = {'msg': 'connect', 'version': '1', 'support': ['1']}
        if self.session: msg['session'] = self.session
        yield from self.websocket.send(ejson.dumps(msg))

        asyncio.async(self.recvloop())
        while not self.connected:
            yield from asyncio.sleep(0.1)

    @asyncio.coroutine
    def subscribe(self, name, *params, ready_cb = noop,
                  added_cb = noop, changed_cb = noop, removed_cb = noop):
        sub = Subscription(name, ready_cb, added_cb, changed_cb, removed_cb)
        self.subs[name] = sub
        yield from self.websocket.send(ejson.dumps(
            {'msg': 'sub',
             'id': sub.id,
             'name': name,
             'params': params}
        ))

        return sub

    @asyncio.coroutine
    def call(self, method, *params, callback = None, wait = True):
        c = MethodCall(callback)
        self.calls[c.id] = c

        yield from self.websocket.send(ejson.dumps(
            {'msg': 'method',
             'method': method,
             'params': params,
             'id': c.id}
        ))

        if wait:
            while not c.finished:
                yield from asyncio.sleep(0.1)

            return c.result, c.error

    @asyncio.coroutine
    def call_cached(self, method, params = [], callback = lambda *a: None):
        if self.connected:
            yield from self.call(method, params, callback)
        else:
            self.callcache.append((method, params, callback))

    @asyncio.coroutine
    def recvloop(self):
        while self.websocket.open:
            while len(self.callcache) > 0:
                self.call(*self.callcache.pop(0))

            msg = yield from self.websocket.recv()
            if not msg: continue
            msg = ejson.loads(msg)

            if msg.get('msg') == 'connected':
                self.connected = True
                self.session = msg['session']

            elif msg.get('msg') == 'ping':
                yield from self.websocket.send(ejson.dumps({'msg': 'pong', 'id': msg.get('id')}))

            elif msg.get('msg') == 'ready':
                for sub in self.subs.values():
                    if sub.id in msg['subs']:
                        yield from sub.ready_cb(sub)

            elif msg.get('msg') == 'added':
                sub = self.subs.get(msg['collection'])
                if sub:
                    sub.data[msg['id']] = msg['fields']
                    yield from sub.added_cb(sub, msg['id'], msg['fields'])

            elif msg.get('msg') == 'changed':
                sub = self.subs.get(msg['collection'])
                if sub:
                    if msg.get('fields'):
                        sub.data[msg['id']].update(msg['fields'])
                        yield from sub.changed_cb(sub, msg['id'], msg['fields'])
                    elif msg.get('cleared'):
                        for key in msg['cleared']:
                            del sub.data[key]
                        yield from sub.changed_cb(sub, msg['id'], msg['cleared'])

            elif msg.get('msg') == 'removed':
                sub = self.subs.get(msg['collection'])
                if sub:
                    del sub.data[msg['id']]
                    yield from sub.removed_cb(sub, msg['id'])

            elif msg.get('msg') == 'result':
                c = self.calls.get(msg['id'])
                if c:
                    yield from c.onfinished(msg.get('result'), msg.get('error'))

        self.connected = False
        while True:
            self.websocket = yield from websockets.connect(self.address)
            if self.websocket.open:
                yield from self.connect()
                return
            else:
                yield from asyncio.sleep(1)

<|endoftext|>


<commit_before><commit_msg>Licence block for code added in r11586<commit_after><|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Kernel-side code for an IPython based visualization tool."""

import base64

from google.colab import output

import IPython

import jax.numpy as jnp
from jax import tree_map

from jax_md import dataclasses
from jax_md import partition

import json

import numpy as onp

# INTERNAL_FILE_IMPORT

renderer_code = IPython.display.HTML(
  url=('https://raw.githubusercontent.com/google/jax-md/master/'
       'jax_md/colab_tools/visualization.html')
)

SIMULATION_IDX = 0

def to_np(*xs):
  return [onp.array(x) if isinstance(x, jnp.ndarray) else x for x in xs]

@dataclasses.dataclass
class Disk:
  """Disk geometry elements.

  Args:
    position: An array of shape `(steps, count, dim)` or `(count, dim)`
      specifying possibly time varying positions. Here `dim` is the spatial
      dimension.
    size: An array of shape (steps, count)`, `(count,)`, or `()` specifying
      possibly time-varying / per-disk diameters.
    color: An array of shape `(steps, count, 3)` or `(count,)` specifying
      possibly time-varying / per-disk RGB colors.
    count: The number of disks.
  """
  position: jnp.ndarray
  size: jnp.ndarray
  color: jnp.ndarray
  count: int = dataclasses.static_field()

  def __init__(self, position, diameter=1.0, color=None):
    if color is None:
      color = jnp.array([0.8, 0.8, 1.0])

    position, diameter, color = to_np(position, diameter, color)

    object.__setattr__(self, 'position', position)
    object.__setattr__(self, 'size', diameter)
    object.__setattr__(self, 'color', color)
    object.__setattr__(self, 'count', position.shape[-2])

  def __repr__(self):
    return 'Disk'

@dataclasses.dataclass
class Sphere:
  """Sphere geometry elements.

  Args:
    position: An array of shape `(steps, count, dim)` or `(count, dim)`
      specifying possibly time varying positions. Here `dim` is the spatial
      dimension.
    size: An array of shape (steps, count)`, `(count,)`, or `()` specifying
      possibly time-varying / per-sphere diameters.
    color: An array of shape `(steps, count, 3)` or `(count,)` specifying
      possibly time-varying / per-sphere RGB colors.
    count: The number of spheres.
  """
  position: jnp.ndarray
  size: jnp.ndarray
  color: jnp.ndarray
  count: int

  def __init__(self, position, diameter=1.0, color=None):
    if color is None:
      color = jnp.array([0.8, 0.8, 1.0])

    position, diameter, color = to_np(position, diameter, color)

    object.__setattr__(self, 'position', position)
    object.__setattr__(self, 'size', diameter)
    object.__setattr__(self, 'color', color)
    object.__setattr__(self, 'count', position.shape[-2])

  def __repr__(self):
    return 'Sphere'

@dataclasses.dataclass
class Bond:
  """Bonds are lines between geometric objects.

  Args:
    reference_geometry: The name of the geometry object to draw bonds between.
    neighbor_idx: An array of ids of objects that should have bonds drawn
      between them. This uses the same encoding as in `partition.neighbor_list`.
      Essentially, ids is a `(steps, count, max_neighbors)` or
      `(count, max_neighbors)` array of integers. `neighbor_idx[i, j] < count`
      denotes a bond between object `i` and `neighbor[i, j]`.
    diameter: The width of the line between the objects.
    color: An array of shape `(3,)` specifying the RGB color of the bonds.
    count: The number of objects.
    max_neighbors: The maximum number of bonds a central object can have.
  """
  reference_geometry: str
  neighbor_idx: jnp.ndarray
  diameter: jnp.ndarray
  color: jnp.ndarray
  count: int
  max_neighbors: int

  def __init__(self, reference_geometry, idx, diameter=None, color=None):
    if color is None:
      color = onp.array([0.8, 0.8, 1.0])
    if diameter is None:
      diameter = onp.array(0.2)

    if isinstance(idx, partition.NeighborList):
      idx = (idx.idx if idx.format is partition.Dense
             else partition.to_dense(idx))

    idx, diameter, color = to_np(idx, diameter, color)

    object.__setattr__(self, 'reference_geometry', reference_geometry)
    object.__setattr__(self, 'neighbor_idx', idx)
    object.__setattr__(self, 'diameter', diameter)
    object.__setattr__(self, 'color', color)
    object.__setattr__(self, 'count', idx.shape[-2])
    object.__setattr__(self, 'max_neighbors', idx.shape[-1])

  def __repr__(self):
    return 'Bond'

TYPE_DIMENSIONS = {
    'position': 2,
    'size': 1,
    'color': 2,
    'neighbor_idx': 2,
    'diameter': 1,
}

def _encode(R):
  dtype = R.dtype
  if dtype == jnp.float64:
    dtype = jnp.float32
  if dtype == jnp.int64:
    dtype = jnp.int32
  dtype = jnp.float32
  return base64.b64encode(onp.array(R, dtype).tobytes()).decode('utf-8')

def _to_json(data):
  try:
    return IPython.display.JSON(data=data)
  except:
    return IPython.display.JSON(data=json.dumps(data))

def render(box_size,
           geometry,
           buffer_size=None,
           background_color=None,
           resolution=None,
           frame_rate=None,
           verbose=False):
  """Creates a rendering front-end along with callbacks in the host program.

  Args:
    box_size: A float or an array of shape `(spatial_dimension,)`. Specifies
      the size of the simulation volume. Used to position the camera.
    geometry: A dictionary containing names paired with geometric objects such
      as Disk, Sphere, or Bond.
    buffer_size: The maximum number of timesteps to send to the font-end in a
      single call.
    background_color: An array of shape (3,) specifying the background color of
      the visualization.
    resolution: The resolution of the renderer.
    frame_rate: An optional integer specifying the target frames-per-second
      for the renderer.
    verbose: Specifies whether or not the client should emit information and
      error messages. Useful for debugging visualizations, but adds clutter.
  """
  global SIMULATION_IDX
  # INTERNAL_RENDERER_CODE_LOADING

  simulation_idx = SIMULATION_IDX

  frame_count = None
  dimension = None

  if not isinstance(geometry, dict):
    geometry = { 'all': geometry }

  for geom in geometry.values():
    if hasattr(geom, 'position'):
      assert dimension is None or geom.position.shape[-1] == dimension
      dimension = geom.position.shape[-1]

      if geom.position.ndim == 3:
        assert frame_count is None or frame_count == geom.position.shape[0]
        frame_count = geom.position.shape[0]

  assert dimension is not None

  if isinstance(box_size, (jnp.ndarray, np.ndarray)):
    if box_size.shape:
      assert box_size.shape == (dimension,)
      box_size = list(box_size)
    else:
      box_size = [float(box_size),] * dimension
  elif isinstance(box_size, float) or isinstance(box_size, int):
    box_size = [box_size,] * dimension

  def get_metadata():
    metadata = {
        'box_size': box_size,
        'dimension': dimension,
        'geometry': [k for k in geometry.keys()],
        'simulation_idx': simulation_idx
    }

    if frame_count is not None:
      metadata['frame_count'] = frame_count

    if buffer_size is not None:
      metadata['buffer_size'] = buffer_size

    if background_color is not None:
      metadata['background_color'] = background_color

    if resolution is not None:
      metadata['resolution'] = resolution

    if frame_rate is not None:
      metadata['frame_rate'] = frame_rate

    if verbose:
      metadata['verbose'] = True

    return _to_json(metadata)
  output.register_callback('GetSimulationMetadata', get_metadata)

  def get_dynamic_geometry_metadata(name):
    assert name in geometry

    geom = geometry[name]
    geom_dict = dataclasses.asdict(geom)

    geom_metadata = {
        'shape': str(geom),
        'fields': {},
    }

    for field in geom_dict:
      if isinstance(geom_dict[field], list):
        geom_dict[field] = onp.array(geom_dict[field])
      if not isinstance(geom_dict[field], onp.ndarray):
        geom_metadata[field] = geom_dict[field]
        continue
      if len(geom_dict[field].shape) == TYPE_DIMENSIONS[field] + 1:
        geom_metadata['fields'][field] = 'dynamic'
      elif len(geom_dict[field].shape) == TYPE_DIMENSIONS[field]:
        geom_metadata['fields'][field] = 'static'
      elif len(geom_dict[field].shape) == TYPE_DIMENSIONS[field] - 1:
        geom_metadata['fields'][field] = 'global'
    return _to_json(geom_metadata)
  output.register_callback(f'GetGeometryMetadata{SIMULATION_IDX}',
                           get_dynamic_geometry_metadata)

  def get_array_chunk(name, field, offset, size):
    assert name in geometry

    geom = dataclasses.asdict(geometry[name])
    assert field in geom
    array = geom[field]
    if isinstance(array, list):
      array = onp.array(array)

    return _to_json({
        'array_chunk': _encode(array[offset:(offset + size)])
    })
  output.register_callback(f'GetArrayChunk{SIMULATION_IDX}', get_array_chunk)

  def get_array(name, field):
    assert name in geometry

    geom = dataclasses.asdict(geometry[name])
    assert field in geom
    array = geom[field]
    if isinstance(array, list):
      array = onp.array(array)

    return _to_json({ 'array': _encode(array) })
  output.register_callback(f'GetArray{SIMULATION_IDX}', get_array)

  SIMULATION_IDX += 1

  IPython.display.display(renderer_code)
<commit_msg>Small fixes to renderer.<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Kernel-side code for an IPython based visualization tool."""

import base64

from google.colab import output

import IPython

import jax.numpy as jnp
from jax import tree_map

from jax_md import dataclasses
from jax_md import partition

import json

import numpy as np

# INTERNAL_FILE_IMPORT

renderer_code = IPython.display.HTML(
  url=('https://raw.githubusercontent.com/google/jax-md/master/'
       'jax_md/colab_tools/visualization.html')
)

SIMULATION_IDX = 0

def to_np(*xs):
  return [np.array(x) if isinstance(x, jnp.ndarray) else x for x in xs]

@dataclasses.dataclass
class Disk:
  """Disk geometry elements.

  Args:
    position: An array of shape `(steps, count, dim)` or `(count, dim)`
      specifying possibly time varying positions. Here `dim` is the spatial
      dimension.
    size: An array of shape (steps, count)`, `(count,)`, or `()` specifying
      possibly time-varying / per-disk diameters.
    color: An array of shape `(steps, count, 3)` or `(count,)` specifying
      possibly time-varying / per-disk RGB colors.
    count: The number of disks.
  """
  position: jnp.ndarray
  size: jnp.ndarray
  color: jnp.ndarray
  count: int = dataclasses.static_field()

  def __init__(self, position, diameter=1.0, color=None):
    if color is None:
      color = jnp.array([0.8, 0.8, 1.0])

    position, diameter, color = to_np(position, diameter, color)

    object.__setattr__(self, 'position', position)
    object.__setattr__(self, 'size', diameter)
    object.__setattr__(self, 'color', color)
    object.__setattr__(self, 'count', position.shape[-2])

  def __repr__(self):
    return 'Disk'

@dataclasses.dataclass
class Sphere:
  """Sphere geometry elements.

  Args:
    position: An array of shape `(steps, count, dim)` or `(count, dim)`
      specifying possibly time varying positions. Here `dim` is the spatial
      dimension.
    size: An array of shape (steps, count)`, `(count,)`, or `()` specifying
      possibly time-varying / per-sphere diameters.
    color: An array of shape `(steps, count, 3)` or `(count,)` specifying
      possibly time-varying / per-sphere RGB colors.
    count: The number of spheres.
  """
  position: jnp.ndarray
  size: jnp.ndarray
  color: jnp.ndarray
  count: int

  def __init__(self, position, diameter=1.0, color=None):
    if color is None:
      color = jnp.array([0.8, 0.8, 1.0])

    position, diameter, color = to_np(position, diameter, color)

    object.__setattr__(self, 'position', position)
    object.__setattr__(self, 'size', diameter)
    object.__setattr__(self, 'color', color)
    object.__setattr__(self, 'count', position.shape[-2])

  def __repr__(self):
    return 'Sphere'

@dataclasses.dataclass
class Bond:
  """Bonds are lines between geometric objects.

  Args:
    reference_geometry: The name of the geometry object to draw bonds between.
    neighbor_idx: An array of ids of objects that should have bonds drawn
      between them. This uses the same encoding as in `partition.neighbor_list`.
      Essentially, ids is a `(steps, count, max_neighbors)` or
      `(count, max_neighbors)` array of integers. `neighbor_idx[i, j] < count`
      denotes a bond between object `i` and `neighbor[i, j]`.
    diameter: The width of the line between the objects.
    color: An array of shape `(3,)` specifying the RGB color of the bonds.
    count: The number of objects.
    max_neighbors: The maximum number of bonds a central object can have.
  """
  reference_geometry: str
  neighbor_idx: jnp.ndarray
  diameter: jnp.ndarray
  color: jnp.ndarray
  count: int
  max_neighbors: int

  def __init__(self, reference_geometry, idx, diameter=None, color=None):
    if color is None:
      color = np.array([0.8, 0.8, 1.0])
    if diameter is None:
      diameter = np.array(0.2)

    if isinstance(idx, partition.NeighborList):
      idx = (idx.idx if idx.format is partition.Dense
             else partition.to_dense(idx))

    idx, diameter, color = to_np(idx, diameter, color)

    object.__setattr__(self, 'reference_geometry', reference_geometry)
    object.__setattr__(self, 'neighbor_idx', idx)
    object.__setattr__(self, 'diameter', diameter)
    object.__setattr__(self, 'color', color)
    object.__setattr__(self, 'count', idx.shape[-2])
    object.__setattr__(self, 'max_neighbors', idx.shape[-1])

  def __repr__(self):
    return 'Bond'

TYPE_DIMENSIONS = {
    'position': 2,
    'size': 1,
    'color': 2,
    'neighbor_idx': 2,
    'diameter': 1,
}

def _encode(R):
  dtype = R.dtype
  if dtype == jnp.float64:
    dtype = jnp.float32
  if dtype == jnp.int64:
    dtype = jnp.int32
  dtype = jnp.float32
  return base64.b64encode(np.array(R, dtype).tobytes()).decode('utf-8')

def _to_json(data):
  try:
    return IPython.display.JSON(data=data)
  except:
    return IPython.display.JSON(data=json.dumps(data))

def render(box_size,
           geometry,
           buffer_size=None,
           background_color=None,
           resolution=None,
           frame_rate=None,
           verbose=False):
  """Creates a rendering front-end along with callbacks in the host program.

  Args:
    box_size: A float or an array of shape `(spatial_dimension,)`. Specifies
      the size of the simulation volume. Used to position the camera.
    geometry: A dictionary containing names paired with geometric objects such
      as Disk, Sphere, or Bond.
    buffer_size: The maximum number of timesteps to send to the font-end in a
      single call.
    background_color: An array of shape (3,) specifying the background color of
      the visualization.
    resolution: The resolution of the renderer.
    frame_rate: An optional integer specifying the target frames-per-second
      for the renderer.
    verbose: Specifies whether or not the client should emit information and
      error messages. Useful for debugging visualizations, but adds clutter.
  """
  global SIMULATION_IDX
  # INTERNAL_RENDERER_CODE_LOADING

  simulation_idx = SIMULATION_IDX

  frame_count = None
  dimension = None

  if not isinstance(geometry, dict):
    geometry = { 'all': geometry }

  for geom in geometry.values():
    if hasattr(geom, 'position'):
      assert dimension is None or geom.position.shape[-1] == dimension
      dimension = geom.position.shape[-1]

      if geom.position.ndim == 3:
        assert frame_count is None or frame_count == geom.position.shape[0]
        frame_count = geom.position.shape[0]

  assert dimension is not None

  if isinstance(box_size, (jnp.ndarray, np.ndarray)):
    if box_size.shape:
      assert box_size.shape == (dimension,)
      box_size = list(box_size)
    else:
      box_size = [float(box_size),] * dimension
  elif isinstance(box_size, float) or isinstance(box_size, int):
    box_size = [box_size,] * dimension

  def get_metadata():
    metadata = {
        'box_size': box_size,
        'dimension': dimension,
        'geometry': [k for k in geometry.keys()],
        'simulation_idx': simulation_idx
    }

    if frame_count is not None:
      metadata['frame_count'] = frame_count

    if buffer_size is not None:
      metadata['buffer_size'] = buffer_size

    if background_color is not None:
      metadata['background_color'] = background_color

    if resolution is not None:
      metadata['resolution'] = resolution

    if frame_rate is not None:
      metadata['frame_rate'] = frame_rate

    if verbose:
      metadata['verbose'] = True

    return _to_json(metadata)
  output.register_callback('GetSimulationMetadata', get_metadata)

  def get_dynamic_geometry_metadata(name):
    assert name in geometry

    geom = geometry[name]
    geom_dict = dataclasses.asdict(geom)

    geom_metadata = {
        'shape': str(geom),
        'fields': {},
    }

    for field in geom_dict:
      if isinstance(geom_dict[field], list):
        geom_dict[field] = np.array(geom_dict[field])
      if not isinstance(geom_dict[field], np.ndarray):
        geom_metadata[field] = geom_dict[field]
        continue
      if len(geom_dict[field].shape) == TYPE_DIMENSIONS[field] + 1:
        geom_metadata['fields'][field] = 'dynamic'
      elif len(geom_dict[field].shape) == TYPE_DIMENSIONS[field]:
        geom_metadata['fields'][field] = 'static'
      elif len(geom_dict[field].shape) == TYPE_DIMENSIONS[field] - 1:
        geom_metadata['fields'][field] = 'global'
    return _to_json(geom_metadata)
  output.register_callback(f'GetGeometryMetadata{SIMULATION_IDX}',
                           get_dynamic_geometry_metadata)

  def get_array_chunk(name, field, offset, size):
    assert name in geometry

    geom = dataclasses.asdict(geometry[name])
    assert field in geom
    array = geom[field]
    if isinstance(array, list):
      array = np.array(array)

    return _to_json({
        'array_chunk': _encode(array[offset:(offset + size)])
    })
  output.register_callback(f'GetArrayChunk{SIMULATION_IDX}', get_array_chunk)

  def get_array(name, field):
    assert name in geometry

    geom = dataclasses.asdict(geometry[name])
    assert field in geom
    array = geom[field]
    if isinstance(array, list):
      array = np.array(array)

    return _to_json({ 'array': _encode(array) })
  output.register_callback(f'GetArray{SIMULATION_IDX}', get_array)

  SIMULATION_IDX += 1

  IPython.display.display(renderer_code)
<|endoftext|>


<commit_before>"""sjourney URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.9/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url, patterns, include
from django.contrib import admin
from sjourney.app.views import HomeView, AppView, logout
from sjourney.app.api import UserListAPIView, SocialAuthUserListAPIView
from sjourney.app.api import SocialAuthUserDetailAPIView
from sjourney.app.api import CategoryListAPIView, CategoryDetailAPIView
from sjourney.app.api import PictureListAPIView, PictureDetailAPIView

urlpatterns = [
    url(r'^admin/', include(admin.site.urls)),
    url('', include('social.apps.django_app.urls', namespace='social')),
    url(r'^$', HomeView.as_view()),
    url(r'^home/$', AppView.as_view()),
    url(r'^logout/$', logout),

    url(r'^api/v1/users/$', UserListAPIView.as_view()),
    url(r'^api/v1/susers/$', SocialAuthUserListAPIView.as_view()),
    url(r'^api/v1/susers/(?P<pk>[0-9]+)',SocialAuthUserDetailAPIView.as_view()),
    url(r'^api/v1/pics/category/$', CategoryListAPIView.as_view()),
    url(r'^api/v1/pics/category/(?P<pk>[0-9]+)', CategoryDetailAPIView.as_view()),
    url(r'^api/v1/pics/$', PictureListAPIView.as_view()),
    url(r'^api/v1/pics/(?P<pk>[0-9]+)', PictureDetailAPIView.as_view()),
    url(r'^docs/', include('rest_framework_docs.urls')),
]
<commit_msg>[Feature #120481309] Have the image accessible via url<commit_after>"""sjourney URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.9/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url, patterns, include
from django.contrib import admin
from sjourney.app.views import HomeView, AppView, logout
from sjourney.app.api import UserListAPIView, SocialAuthUserListAPIView
from sjourney.app.api import SocialAuthUserDetailAPIView
from sjourney.app.api import CategoryListAPIView, CategoryDetailAPIView
from sjourney.app.api import PictureListAPIView, PictureDetailAPIView
import settings

urlpatterns = [
    url(r'^admin/', include(admin.site.urls)),
    url('', include('social.apps.django_app.urls', namespace='social')),
    url(r'^$', HomeView.as_view()),
    url(r'^home/$', AppView.as_view()),
    url(r'^logout/$', logout),

    url(r'^api/v1/users/$', UserListAPIView.as_view()),
    url(r'^api/v1/susers/$', SocialAuthUserListAPIView.as_view()),
    url(r'^api/v1/susers/(?P<pk>[0-9]+)',SocialAuthUserDetailAPIView.as_view()),
    url(r'^api/v1/pics/category/$', CategoryListAPIView.as_view()),
    url(r'^api/v1/pics/category/(?P<pk>[0-9]+)', CategoryDetailAPIView.as_view()),
    url(r'^api/v1/pics/$', PictureListAPIView.as_view()),
    url(r'^api/v1/pics/(?P<pk>[0-9]+)', PictureDetailAPIView.as_view()),

    url(r'^media/(?P<path>.*)$', 'django.views.static.serve', {
        'document_root': settings.MEDIA_ROOT,
    }),
    url(r'^docs/', include('rest_framework_docs.urls')),
]
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
"""
@file
@brief Magic command to handle files
"""
import sys, os, pandas

from IPython.core.magic import Magics, magics_class, line_magic, cell_magic
from IPython.core.magic import line_cell_magic
from IPython.core.display import HTML

from pyquickhelper.sync.synchelper import explore_folder_iterfile
from pyquickhelper import MagicCommandParser, run_cmd
from .format_helper import format_file_size, format_file_mtime
from .content_helper import file_head, file_tail

@magics_class
class MagicFile(Magics):
    """
    Defines magic commands to list the content of a folder

    .. versionadded:: 1.1
    """

    @staticmethod
    def head_parser():
        """
        defines the way to parse the magic command ``%head``
        """
        parser = MagicCommandParser(description='display the first lines of a text file')
        parser.add_argument('f', type=str, help='filename')
        parser.add_argument('-n', '--n', type=int, default=10, help='number of lines to display')
        parser.add_argument('-e', '--encoding', default="utf8", help='file encoding')
        return parser
    _parser_head = None

    @line_magic
    def head(self, line):
        """
        defines ``%head``
        which displays the first lines of a file
        """
        if MagicFile._parser_head is None:
            MagicFile._parser_head = MagicFile.head_parser()
        parser = MagicFile._parser_head

        try:
            args = parser.parse_cmd(line)
        except SystemExit:
            print( parser.print_help() )
            args = None

        if args is not None:
            rows = file_head(args.f, args.n, args.encoding)
            return HTML("
\n{0}\n
".format("".join(rows)))

    @staticmethod
    def tail_parser():
        """
        defines the way to parse the magic command ``%tail``
        """
        parser = MagicCommandParser(description='display the last lines of a text file')
        parser.add_argument('f', type=str, help='filename')
        parser.add_argument('-n', '--n', type=int, default=10, help='number of lines to display')
        parser.add_argument('-e', '--encoding', default="utf8", help='file encoding')
        return parser
    _parser_tail = None

    @line_magic
    def tail(self, line):
        """
        defines ``%tail``
        which displays the last lines of a file
        """
        if MagicFile._parser_tail is None:
            MagicFile._parser_tail = MagicFile.tail_parser()
        parser = MagicFile._parser_tail

        try:
            args = parser.parse_cmd(line)
        except SystemExit:
            print( parser.print_help() )
            args = None

        if args is not None:
            rows = file_tail(args.f, args.n, args.encoding)
            return HTML("
\n{0}\n
".format("".join(rows)))

    @staticmethod
    def lsr_parser():
        """
        defines the way to parse the magic command ``%lsr``
        """
        parser = MagicCommandParser(description='display the content of a folder as a dataframe')
        parser.add_argument('path', type=str, nargs="?", help='path', default=".")
        parser.add_argument('-f', '--filter', type=str, default=".*", help='filter, same syntax as a regular expression')
        return parser
    _parser_lsr = None

    @line_magic
    def lsr(self, line):
        """
        define ``%lsr``, the method stops after around 10000 files --> you should precise the filter.
        """
        if MagicFile._parser_lsr is None:
            MagicFile._parser_lsr = MagicFile.lsr_parser()
        parser = MagicFile._parser_lsr

        try:
            args = parser.parse_cmd(line)
        except SystemExit:
            print( parser.print_help() )
            args = None

        if args is not None:
            if args.path is None or len(args.path) == 0 :
                filename = "."
            else:
                filename = args.path
            pattern = args.filter

            if "*" in filename:
                pattern = filename
                filename = "."

            iter = explore_folder_iterfile(filename, pattern)
            rows = [ ]
            for r in iter :
                d = os.path.isfile(r)
                if d :
                    st = os.stat(r)
                    r = {   "name":r,
                            "size":format_file_size(st.st_size),
                            "last_modified":format_file_mtime(st.st_mtime),
                            "directory":False }
                else:
                    r = { "name":r, "directory":True }
                rows.append(r)
            return pandas.DataFrame(rows)

    @cell_magic
    def PYTHON(self, line, cell = None):
        """
        defines command ``%%PIG``
        """
        if line in [None, ""] :
            print("Usage:")
            print("     %%PYTHON <filename>")
            print("")
            print("The command store the content of the cell as a local file.")
        else:
            filename = line.strip()
            with open(filename, "w", encoding="utf8") as f :
                f.write("# -*- coding: utf8 -*-\n")
                f.write(cell.replace("\r",""))

    @cell_magic
    def runpy(self, line, cell = None):
        """
        defines command ``%%runpy``

        run a python script which accepts standards input and produces standard outputs,
        a timeout is set up at 10s

        .. versionadded:: 1.1
        """
        if line in [None, ""] :
            print("Usage:")
            print("     %%runpy <pythonfile.py> <args>")
            print("     first row")
            print("     second row")
            print("     ...")
        else:
            filename = line.strip().split()
            if len(filename) == 0:
                self.runpy("")
            else:
                args = " ".join(filename[1:])
                filename = filename[0]
                cmd = sys.executable.replace("pythonw", "python") + " " + filename + " " + args
                tosend = cell
                out,err = run_cmd(cmd, wait=True, sin=tosend, communicate=True, timeout=10, shell=False)
                if len(err) > 0 :
                    return HTML ('Error

\n%s\n
' % err)
                else:
                    return HTML ('
\n%s\n
' % out)

def register_file_magics():
    """
    register magics function, can be called from a notebook
    """
    from IPython import get_ipython
    ip = get_ipython()
    ip.register_magics(MagicFile)<commit_msg>propagate changes introduced in pyquickhelper<commit_after>#-*- coding: utf-8 -*-
"""
@file
@brief Magic command to handle files
"""
import sys, os, pandas

from IPython.core.magic import Magics, magics_class, line_magic, cell_magic
from IPython.core.magic import line_cell_magic
from IPython.core.display import HTML

from pyquickhelper.filehelper.synchelper import explore_folder_iterfile
from pyquickhelper import MagicCommandParser, run_cmd
from .format_helper import format_file_size, format_file_mtime
from .content_helper import file_head, file_tail

@magics_class
class MagicFile(Magics):
    """
    Defines magic commands to list the content of a folder

    .. versionadded:: 1.1
    """

    @staticmethod
    def head_parser():
        """
        defines the way to parse the magic command ``%head``
        """
        parser = MagicCommandParser(description='display the first lines of a text file')
        parser.add_argument('f', type=str, help='filename')
        parser.add_argument('-n', '--n', type=int, default=10, help='number of lines to display')
        parser.add_argument('-e', '--encoding', default="utf8", help='file encoding')
        return parser
    _parser_head = None

    @line_magic
    def head(self, line):
        """
        defines ``%head``
        which displays the first lines of a file
        """
        if MagicFile._parser_head is None:
            MagicFile._parser_head = MagicFile.head_parser()
        parser = MagicFile._parser_head

        try:
            args = parser.parse_cmd(line)
        except SystemExit:
            print( parser.print_help() )
            args = None

        if args is not None:
            rows = file_head(args.f, args.n, args.encoding)
            return HTML("
\n{0}\n
".format("".join(rows)))

    @staticmethod
    def tail_parser():
        """
        defines the way to parse the magic command ``%tail``
        """
        parser = MagicCommandParser(description='display the last lines of a text file')
        parser.add_argument('f', type=str, help='filename')
        parser.add_argument('-n', '--n', type=int, default=10, help='number of lines to display')
        parser.add_argument('-e', '--encoding', default="utf8", help='file encoding')
        return parser
    _parser_tail = None

    @line_magic
    def tail(self, line):
        """
        defines ``%tail``
        which displays the last lines of a file
        """
        if MagicFile._parser_tail is None:
            MagicFile._parser_tail = MagicFile.tail_parser()
        parser = MagicFile._parser_tail

        try:
            args = parser.parse_cmd(line)
        except SystemExit:
            print( parser.print_help() )
            args = None

        if args is not None:
            rows = file_tail(args.f, args.n, args.encoding)
            return HTML("
\n{0}\n
".format("".join(rows)))

    @staticmethod
    def lsr_parser():
        """
        defines the way to parse the magic command ``%lsr``
        """
        parser = MagicCommandParser(description='display the content of a folder as a dataframe')
        parser.add_argument('path', type=str, nargs="?", help='path', default=".")
        parser.add_argument('-f', '--filter', type=str, default=".*", help='filter, same syntax as a regular expression')
        return parser
    _parser_lsr = None

    @line_magic
    def lsr(self, line):
        """
        define ``%lsr``, the method stops after around 10000 files --> you should precise the filter.
        """
        if MagicFile._parser_lsr is None:
            MagicFile._parser_lsr = MagicFile.lsr_parser()
        parser = MagicFile._parser_lsr

        try:
            args = parser.parse_cmd(line)
        except SystemExit:
            print( parser.print_help() )
            args = None

        if args is not None:
            if args.path is None or len(args.path) == 0 :
                filename = "."
            else:
                filename = args.path
            pattern = args.filter

            if "*" in filename:
                pattern = filename
                filename = "."

            iter = explore_folder_iterfile(filename, pattern)
            rows = [ ]
            for r in iter :
                d = os.path.isfile(r)
                if d :
                    st = os.stat(r)
                    r = {   "name":r,
                            "size":format_file_size(st.st_size),
                            "last_modified":format_file_mtime(st.st_mtime),
                            "directory":False }
                else:
                    r = { "name":r, "directory":True }
                rows.append(r)
            return pandas.DataFrame(rows)

    @cell_magic
    def PYTHON(self, line, cell = None):
        """
        defines command ``%%PIG``
        """
        if line in [None, ""] :
            print("Usage:")
            print("     %%PYTHON <filename>")
            print("")
            print("The command store the content of the cell as a local file.")
        else:
            filename = line.strip()
            with open(filename, "w", encoding="utf8") as f :
                f.write("# -*- coding: utf8 -*-\n")
                f.write(cell.replace("\r",""))

    @cell_magic
    def runpy(self, line, cell = None):
        """
        defines command ``%%runpy``

        run a python script which accepts standards input and produces standard outputs,
        a timeout is set up at 10s

        .. versionadded:: 1.1
        """
        if line in [None, ""] :
            print("Usage:")
            print("     %%runpy <pythonfile.py> <args>")
            print("     first row")
            print("     second row")
            print("     ...")
        else:
            filename = line.strip().split()
            if len(filename) == 0:
                self.runpy("")
            else:
                args = " ".join(filename[1:])
                filename = filename[0]
                cmd = sys.executable.replace("pythonw", "python") + " " + filename + " " + args
                tosend = cell
                out,err = run_cmd(cmd, wait=True, sin=tosend, communicate=True, timeout=10, shell=False)
                if len(err) > 0 :
                    return HTML ('Error

\n%s\n
' % err)
                else:
                    return HTML ('
\n%s\n
' % out)

def register_file_magics():
    """
    register magics function, can be called from a notebook
    """
    from IPython import get_ipython
    ip = get_ipython()
    ip.register_magics(MagicFile)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from image import Image, TileBuilder, TileTopologyIterator
from merger import Merger
from locator import Locator
from information import WorkflowInformation
from timing import WorkflowTiming

__author__ = "Romain Mormont <r.mormont@student.ulg.ac.be>"

class SLDCWorkflow(object):
    """
    A workflow for finding objects on large images and computing a class for these objects.
    """

    def __init__(self, segmenter, dispatcher_classifier, tile_builder,
                 tile_max_width=1024, tile_max_height=1024, tile_overlap=5,
                 boundary_thickness=7):
        """Constructor for SLDCWorkflow objects

        Parameters
        ----------
        segmenter: Segmenter
            The segmenter to use for the "Segment" step
        dispatcher_classifier: DispatcherClassifier
            The dispatcher classifier to use for the "Dispatch" and "Classify" steps
        tile_builder: TileBuilder
            An object for building tiles
        tile_max_width: int, optional (default: 1024)
            The maximum width of the tiles when iterating over the image
        tile_max_height: int, optional (default: 1024)
            The maximum height of the tiles when iterating over the image
        tile_overlap: int, optional (default: 5)
            The number of pixels of overlap between tiles when iterating over the image
        boundary_thickness:
            The thickness between of the boundaries between the tiles for merging
        """
        self._tile_max_width = tile_max_width
        self._tile_max_height = tile_max_height
        self._tile_overlap = tile_overlap
        self._tile_builder = tile_builder
        self._segmenter = segmenter
        self._locator = Locator()
        self._merger = Merger(boundary_thickness)
        self._dispatch_classifier = dispatcher_classifier

    def process(self, image, logger):
        """Process the image using the SLDCWorkflow
        Parameters
        ----------
        image: Image
            The image to process
        Returns
        -------
        polygons_classes: array of 2-tuples
            An array containing the found polygons as well as the predicted class. These data are
            structured in an array of tuple where a tuple contains as its first element the polygon
            object (shapely.geometry.Polygon) and as second element the predicted class (integer code).
        logger: Logger
            A logger
        Notes
        -----
        This method doesn't modify the image passed as parameter.
        This method doesn't modify the object's attributes.
        """
        timing = WorkflowTiming()
        tile_topology = image.tile_topology(max_width=self._tile_max_width, max_height=self._tile_max_height,
                                            overlap=self._tile_overlap)
        tile_iterator = TileTopologyIterator(self._tile_builder, tile_topology, silent_fail=True)

        # segment locate
        polygons_tiles = list()
        logger.info("SLDCWorkflow : start segment/locate.")
        for i, tile in enumerate(tile_iterator):
            # log percentage of progress if there are enough tiles
            if tile_topology.tile_count > 25 and (i + 1) % (tile_topology.tile_count // 10) == 0:
                percentage = 100.0 * i / tile_topology.tile_count
                logger.info("SLDCWorkflow : {}% of the tiles processed (segment/locate).\n".format(percentage) +
                            "SLDCWorkflow : segment/locate duration is {} s until now.".format(timing.sl_total_duration()))
            polygons_tiles.append((tile, self._segment_locate(tile, timing)))

        # log end of segment locate
        logger.info("SLDCWorkflow : end segment/locate.\n" +
                    "SLDCWorkflow : {} tile(s) processed in {} s.\n".format(len(polygons_tiles), timing.sl_total_duration()) +
                    "SLDCWorkflow : {} polygon(s) found.".format(sum([len(polygons) for _, polygons in polygons_tiles])))

        # merge
        logger.info("SLDCWorkflow : start merging")
        timing.start_merging()
        polygons = self._merger.merge(polygons_tiles, tile_topology)
        timing.end_merging()
        logger.info("SLDCWorkflow : end merging.\n" +
                    "SLDCWorkflow : {} polygon(s) found.\n".format(len(polygons)) +
                    "SLDCWorkflow : executed in {} s.".format(timing.duration_of(WorkflowTiming.MERGING)))

        # dispatch classify
        logger.info("SLDCWorkflow : start dispatch/classify.")
        timing.start_dispatch_classify()
        predictions, dispatch_indexes = self._dispatch_classifier.dispatch_classify_batch(image, polygons, logger)
        timing.end_dispatch_classify()
        logger.info("SLDCWorkflow : end dispatch/classify.\n" +
                    "SLDCWorkflow : executed in {} s.".format(timing.duration_of(WorkflowTiming.DISPATCH_CLASSIFY)))

        return WorkflowInformation(polygons, dispatch_indexes, predictions, timing, metadata=self.get_metadata())

    def _segment_locate(self, tile, timing):
        timing.start_fetching()
        np_image = tile.np_image
        timing.end_fetching()
        timing.start_segment()
        segmented = self._segmenter.segment(np_image)
        timing.end_segment()
        timing.start_location()
        located = self._locator.locate(segmented, offset=tile.offset)
        timing.end_location()
        return located

    def get_metadata(self):
        """Return the metadata associated with this workflow
        The metadata should be a way for the implementor to document the results of the workflow execution.
        Returns
        -------
        metadata: string
            The workflow metadata
        """
        return "Workflow class: {}\n".format(self.__class__.__name__)
<commit_msg>handle tile extraction exception in segment locate<commit_after># -*- coding: utf-8 -*-

from image import Image, TileBuilder, TileTopologyIterator
from merger import Merger
from locator import Locator
from information import WorkflowInformation
from sldc import TileExtractionException
from timing import WorkflowTiming

__author__ = "Romain Mormont <r.mormont@student.ulg.ac.be>"

class SLDCWorkflow(object):
    """
    A workflow for finding objects on large images and computing a class for these objects.
    """

    def __init__(self, segmenter, dispatcher_classifier, tile_builder,
                 tile_max_width=1024, tile_max_height=1024, tile_overlap=5,
                 boundary_thickness=7):
        """Constructor for SLDCWorkflow objects

        Parameters
        ----------
        segmenter: Segmenter
            The segmenter to use for the "Segment" step
        dispatcher_classifier: DispatcherClassifier
            The dispatcher classifier to use for the "Dispatch" and "Classify" steps
        tile_builder: TileBuilder
            An object for building tiles
        tile_max_width: int, optional (default: 1024)
            The maximum width of the tiles when iterating over the image
        tile_max_height: int, optional (default: 1024)
            The maximum height of the tiles when iterating over the image
        tile_overlap: int, optional (default: 5)
            The number of pixels of overlap between tiles when iterating over the image
        boundary_thickness:
            The thickness between of the boundaries between the tiles for merging
        """
        self._tile_max_width = tile_max_width
        self._tile_max_height = tile_max_height
        self._tile_overlap = tile_overlap
        self._tile_builder = tile_builder
        self._segmenter = segmenter
        self._locator = Locator()
        self._merger = Merger(boundary_thickness)
        self._dispatch_classifier = dispatcher_classifier

    def process(self, image, logger):
        """Process the image using the SLDCWorkflow
        Parameters
        ----------
        image: Image
            The image to process
        Returns
        -------
        polygons_classes: array of 2-tuples
            An array containing the found polygons as well as the predicted class. These data are
            structured in an array of tuple where a tuple contains as its first element the polygon
            object (shapely.geometry.Polygon) and as second element the predicted class (integer code).
        logger: Logger
            A logger
        Notes
        -----
        This method doesn't modify the image passed as parameter.
        This method doesn't modify the object's attributes.
        """
        timing = WorkflowTiming()
        tile_topology = image.tile_topology(max_width=self._tile_max_width, max_height=self._tile_max_height,
                                            overlap=self._tile_overlap)
        tile_iterator = TileTopologyIterator(self._tile_builder, tile_topology, silent_fail=True)

        # segment locate
        polygons_tiles = list()
        logger.info("SLDCWorkflow : start segment/locate.")
        for i, tile in enumerate(tile_iterator):
            # log percentage of progress if there are enough tiles
            if tile_topology.tile_count > 25 and (i + 1) % (tile_topology.tile_count // 10) == 0:
                percentage = 100.0 * i / tile_topology.tile_count
                logger.info("SLDCWorkflow : {}% of the tiles processed (segment/locate).\n".format(percentage) +
                            "SLDCWorkflow : segment/locate duration is {} s until now.".format(timing.sl_total_duration()))
            polygons_tiles.append((tile, self._segment_locate(tile, timing, logger)))

        # log end of segment locate
        logger.info("SLDCWorkflow : end segment/locate.\n" +
                    "SLDCWorkflow : {} tile(s) processed in {} s.\n".format(len(polygons_tiles), timing.sl_total_duration()) +
                    "SLDCWorkflow : {} polygon(s) found on those tiles.".format(sum([len(polygons) for _, polygons in polygons_tiles])))

        # merge
        logger.info("SLDCWorkflow : start merging")
        timing.start_merging()
        polygons = self._merger.merge(polygons_tiles, tile_topology)
        timing.end_merging()
        logger.info("SLDCWorkflow : end merging.\n" +
                    "SLDCWorkflow : {} polygon(s) found.\n".format(len(polygons)) +
                    "SLDCWorkflow : executed in {} s.".format(timing.duration_of(WorkflowTiming.MERGING)))

        # dispatch classify
        logger.info("SLDCWorkflow : start dispatch/classify.")
        timing.start_dispatch_classify()
        predictions, dispatch_indexes = self._dispatch_classifier.dispatch_classify_batch(image, polygons, logger)
        timing.end_dispatch_classify()
        logger.info("SLDCWorkflow : end dispatch/classify.\n" +
                    "SLDCWorkflow : executed in {} s.".format(timing.duration_of(WorkflowTiming.DISPATCH_CLASSIFY)))

        return WorkflowInformation(polygons, dispatch_indexes, predictions, timing, metadata=self.get_metadata())

    def _segment_locate(self, tile, timing, logger):
        """Fetch the tile and then perform the segment and locate steps
        Parameters
        ----------
        tile: Tile
            The tile to process for the segment locate
        timing: WorkflowTiming
            The workflow timing object for measuring the execution times of the various steps
        logger: Logger
            A logger
        Returns
        -------
        polygons: list of Polygon
            List containing the polygons found by the locate step
        """
        try:
            timing.start_fetching()
            np_image = tile.np_image
            timing.end_fetching()
            timing.start_segment()
            segmented = self._segmenter.segment(np_image)
            timing.end_segment()
            timing.start_location()
            located = self._locator.locate(segmented, offset=tile.offset)
            timing.end_location()
            return located
        except TileExtractionException as e:
            logger.warning("SLDCWorkflow : skip tile {} because it couldn't be extracted.\n".format(tile.identifier) +
                           "SLDCWorkflow : fetch error : {}".format(e.message))
            return []

    def get_metadata(self):
        """Return the metadata associated with this workflow
        The metadata should be a way for the implementor to document the results of the workflow execution.
        Returns
        -------
        metadata: string
            The workflow metadata
        """
        return "Workflow class: {}\n".format(self.__class__.__name__)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import

from . import views

__version__ = '0.9.1'
<commit_msg>Bump version number<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import

from . import views

__version__ = '0.9.2'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import codecs
import sys
import urllib

from wekeypedia.wikipedia_page import WikipediaPage as Page, url2title, url2lang
from wekeypedia.wikipedia_network import WikipediaNetwork
from wekeypedia.exporter.nx_json import NetworkxJson

source = "List_of_geometry_topics"

hl = WikipediaNetwork("")
hl.keywords = hl.get_page_links(source)

hl.build()

exporter = NetworkxJson(hl.graph)
exporter.nx_export("adjacency", "_data/network.json")

# print hl.keywords
print ""

def fetch_page(source):
  print "📄  fetching: %s" % source.encode('utf-8')

  p = Page()
  r = p.fetch_from_api_title(source.strip(), { "redirects":"true", "prop": "info|revisions", "inprop": "url", "rvprop": "content" })

  with codecs.open("path_points/%s.md" % (source), "w", "utf-8-sig") as file:
    file.write("---\n")

    # REWRITE ME
    result = r["query"]["pages"][r["query"]["pages"].keys()[0]]
    for k in result.keys():
      if k != "revisions":
        # print k
        # print result[k]

        content = result[k]

        if k == "fullurl":
          content = urllib.unquote(content)

        file.write("%s: %s\n" % (k, unicode(content)))

    file.write("---\n\n")

    if "revisions" in result.keys():
      file.write(result["revisions"][0]["*"])

for k in hl.keywords:
  fetch_page(k)<commit_msg>retrieve an html version of the content and store it directly into a json format<commit_after># -*- coding: utf-8 -*-
import os
import codecs
import sys
import urllib
import json

from wekeypedia.wikipedia_page import WikipediaPage as Page, url2title, url2lang
from wekeypedia.wikipedia_network import WikipediaNetwork
from wekeypedia.exporter.nx_json import NetworkxJson

source = "List_of_geometry_topics"

hl = WikipediaNetwork("")
hl.keywords = hl.get_page_links(source)

hl.build()

exporter = NetworkxJson(hl.graph)
exporter.nx_export("adjacency", "_data/network.json")

# print hl.keywords
print ""

def fetch_page(source):
  print "📄  fetching: %s" % source.encode('utf-8')

  p = Page()
  r = p.fetch_from_api_title(source.strip(), { "redirects":"true", "rvparse" : "true", "prop": "info|revisions", "inprop": "url", "rvprop": "content" })

  with open("path_points/%s.json" % (source), "w") as f:
    json.dump(r, f)

  with codecs.open("_path_points/%s.md" % (source), "w", "utf-8-sig") as file:
    file.write("---\n")

    # REWRITE ME
    result = r["query"]["pages"][r["query"]["pages"].keys()[0]]
    for k in result.keys():
      if k != "revisions":
        # print k
        # print result[k]

        content = result[k]

        if k == "fullurl":
          content = urllib.unquote(content)

        file.write("%s: %s\n" % (k, unicode(content)))

    file.write("---\n\n")

    if "revisions" in result.keys():
      file.write(result["revisions"][0]["*"])

for k in hl.keywords:
  fetch_page(k)<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Abstract base classes for the scalable modelling of infrastructure systems

"""
from __future__ import absolute_import, division, print_function

import logging
from abc import ABC, abstractmethod, abstractproperty

__author__ = "Will Usher"
__copyright__ = "Will Usher"
__license__ = "mit"

_logger = logging.getLogger(__name__)

class Commodity(ABC):
    """
    """
    def __init__(self, name):
        self._name = name

    def name(self):
        pass

class SectorModel(ABC):
    """An abstract representation of the sector model with inputs and outputs.

    Attributes
    ==========
    input_hooks :
    output_hooks :

    """
    @abstractmethod
    def optimise(self, method, decision_vars, objective_function):
        """Performs an optimisation of the sector model assets

        Arguments
        =========
        method : function
            Provides the
        decision_vars : list
            Defines the decision variables
        objective_function : function
            Defines the objective function
        """
        pass

    @abstractmethod
    def simulate(self):
        """Performs an operational simulation of the sector model

        Note
        ====
        The term simulation may refer to operational optimisation, rather than
        simulation-only. This process is described as simulation to distinguish
        from the definition of investments in capacity, versus operation using
        the given capacity

        """
        pass

    @abstractproperty
    def model_executable(self):
        """The path to the model executable
        """
        pass

class Interface(ABC):
    """Provides the interface between a sector model and other interfaces

    Attributes
    ==========
    inputs : :class:`abstract.Input`
    region_id : int
    timestep : int

    Returns
    =======

    Methods
    =======

    """
    def __init__(self, inputs):
        self._inputs = inputs

    @property
    def inputs(self):
        return self._inputs

    @inputs.setter
    def inputs(self, list_of_inputs):
        for single_input in list_of_inputs:
            assert isinstance(single_input, Input)
        self._inputs = list_of_inputs

class Input(ABC):
    """An input is a dependency which is used in the sector model
    """

    def __init__(self):
        """
        """
        pass

class Asset(ABC):
    """
    """
    def __init__(self):
        """
        """
        pass

if __name__ == "__main__":
    pass
<commit_msg>Added more abstract classes to help outline the infrastructure model wrapper<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Abstract base classes for the scalable modelling of infrastructure systems

"""
from __future__ import absolute_import, division, print_function

import logging
from abc import ABC, abstractmethod

__author__ = "Will Usher"
__copyright__ = "Will Usher"
__license__ = "mit"

_logger = logging.getLogger(__name__)

class Commodity(ABC):
    """
    """
    names = []

    def __init__(self, name, emission_factor):
        self._name = name
        self.names.append(name)
        self._emissions_factor = emission_factor

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, value):
        self._name = value

    @classmethod
    def print_commodities(self):
        for commodity in self.names:
            print("{}".format(commodity))

class SectorModel(ABC):
    """An abstract representation of the sector model with inputs and outputs.

    Attributes
    ==========
    input_hooks : dict
        A mapping between :class:`Dependency` and sector model inputs
    output_hooks : dict
        A mapping between :class:`Output` and sector model outputs
    run_successful

    """
    def __init__(self):
        self._run_successful = None

    @property
    def run_successful(self):
        """Indicates whether the simulation or optimisation run was successful

        """
        return self._run_successful

    @run_successful.setter
    def run_successful(self, outcome):
        self._run_successful = outcome

    @abstractmethod
    def optimise(self, method, decision_vars, objective_function):
        """Performs an optimisation of the sector model assets

        Arguments
        =========
        method : function
            Provides the
        decision_vars : list
            Defines the decision variables
        objective_function : function
            Defines the objective function
        """
        pass

    @abstractmethod
    def simulate(self):
        """Performs an operational simulation of the sector model

        Note
        ====
        The term simulation may refer to operational optimisation, rather than
        simulation-only. This process is described as simulation to distinguish
        from the definition of investments in capacity, versus operation using
        the given capacity

        """
        pass

    @property
    @abstractmethod
    def model_executable(self):
        """The path to the model executable
        """
        return self._model_executable

    @model_executable.setter
    @abstractmethod
    def model_executable(self, value):
        """
        """
        self._model_executable = value

class Interface(ABC):
    """Provides the interface between a sector model and other interfaces

    Attributes
    ==========
    inputs : :class:`Input`
    region : str
        The unique identifier for the region
    timestep : int
        The current timestep
    sector_model : :class:`SectorModel`
        The sector model wrapped by the Interface

    """
    def __init__(self, inputs, region, timestep, sector_model):
        self._inputs = inputs
        self._region = region
        self._timestep = timestep
        self._sector_model = sector_model

    @property
    def inputs(self):
        return self._inputs

    @inputs.setter
    def inputs(self, list_of_inputs):
        for single_input in list_of_inputs:
            assert isinstance(single_input, Input)
        self._inputs = list_of_inputs

class Input(ABC):
    """An input is a sector model input exposed to the :class:`Interface`
    """

    inputs = []

    def __init__(self, name, region, timestep):
        self._name = name
        self._region = region
        self._timestep = timestep
        input_tuple = (name, region, timestep)
        self.inputs.append(input_tuple)

    @classmethod
    def list_inputs(self):
        for input_tuple in self.inputs:
            print('{}'.format(input_tuple))

class Dependency(Input):
    """A dependency is a type of input which links interfaces
    """

class Decision(ABC):
    """A decision denotes an Asset target, and bounds for the current Interface

    Normally, an instance of a decision corresponds to an investment decision
    or policy to increase capacity, or decrease demand

    """
    def __init__(self, target_asset, lower_bound, upper_bound):
        """The asset which is targetted by the decision
        """
        self._target_asset = target_asset
        """The upper bound on the value of the investment in the target asset
        """
        self._upper_bound = upper_bound
        """The lower bound on the value of the investment in the target asset
        """
        self._lower_bound = lower_bound

class Asset(ABC):
    """An asset is a decision targetted capacity that persists across timesteps

    Examples of assets include power stations, water treatment plants, roads,
    railway tracks, airports, ports, centres of demand such as houses or
    factories, waste processing plants etc.

    An Asset is targetted by and influenced by a :class:`Decision` but only
    need to be defined in the :class:`Interface` if targetted
    by a :class:`Decision`

    The inter-timestep state of a model is a function of the Asset-base.

    The Asset-state is also persisted (written to the datastore)

    """
    def __init__(self, name, capacity):
        """
        """
        self._name = name
        self._capacity = capacity
        self._new_capacity = None

    @classmethod
    @abstractmethod
    def write_assets_to_datastore(self):
        """Writes the current :class:`State` of the Asset to the datastore
        """
        pass

    @abstractmethod
    def get_decisions(self):
        """Returns a container of :class:`Decision`
        """
        pass

    @property
    def new_capacity(self):
        return self._new_capacity

    @classmethod
    def get_state(self):
        return self

class AbstractState(ABC):
    """
    """
    @abstractmethod
    def update_state(self):
        pass

class State(AbstractState):
    """A static representation of an Interface's assets

    The state is used to record (and persist) the inter-temporal transition
    from one time-step to the next of the :class:`Interface`
    """
    def __init__(self, region, timestep, sector_model):
        self._region = region
        self._timestep = timestep
        self._sector_model = sector_model
        self.assets = Asset.get_state()

    def update_state(self):
        self.increment_timestep()
        self.update_asset_capacities()

    def increment_timestep(self, increment=1):
        """Increments the timestep by step-length
        """
        self._timestep += increment

    def update_asset_capacities(self):
        """
        """
        for asset in self.assets:
            asset_decisions = asset.get_decisions()
            asset.capacity += asset_decisions.new_capacity
            asset.capacity -= asset.retiring_capacity(self._timestep)

class AbstractModel(ABC):
    """An instance of a model contains Interfaces wrapped around sector models
    """
    def __init__(self):
        self._commodities = []
        self._regions = []
        self.timesteps = []

    @property
    def commodities(self):
        return self._commodities

    @commodities.setter
    def commodities(self, commodities):
        self._commodities = commodities

    @property
    def regions(self):
        return self._regions

    @regions.setter
    def regions(self, value):
        self._regions = value

    @property
    def timesteps(self):
        return self.timesteps

    @timesteps.setter
    def timesteps(self, value):
        self.timesteps = value

    @abstractmethod
    def attach_interface(self):
        """
        """
        pass

class Model(AbstractModel):
    def attach_interface(self):
        pass

class AbstractModelBuilder(ABC):
    """Instantiates and validates a model before releasing it for operation
    """
    def __init__(self):
        self.model = None

    def initialise_model(self):
        self.model = Model()

    @abstractmethod
    def add_commodities(self):
        pass

    @abstractmethod
    def add_regions(self):
        pass

    @abstractmethod
    def add_timeseteps(self):
        pass

    def validate(self):
        self.validate_commodities()
        self.validate_regions()
        self.validate_timesteps()

    @abstractmethod
    def validate_commodities(self):
        pass

    @abstractmethod
    def validate_regions(self):
        pass

    @abstractmethod
    def validate_timesteps(self):
        pass

if __name__ == "__main__":
    pass
<|endoftext|>


<commit_before>
from .util import geocode
from .auxil import getAuxdata
<commit_msg>included function auxil.gpt<commit_after>
from .util import geocode
from .auxil import getAuxdata, gpt
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
Tests the iso-age library

History
-------
2011-10-02 - Created by Jonathan Sick

"""

__all__ = ['']
import os
import tables
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab

import snapshotlib
from fspsq import ParameterSet

from mclib import MagTableDef
from mclib import get_metallicity_LUT

from griddata import griddata

def main():
    #tauzGrid = TauZGrid("tauz", dbname='fsps')
    #tauzGrid.reset()
    #tauzGrid.generate_grid()
    #tauzGrid.compute_models(nThreads=2, maxN=10, clean=True)
    #tauzGrid.plot_color("tauzgrid", 'MegaCam_g', 'MegaCam_i')
    mclib = MonteCarloLibrary('mc.4')
    mclib.reset()
    mclib.define_samples(n=100)
    mclib.compute_models(nThreads=1, maxN=10, clean=True)

class TauZGrid(snapshotlib.SnapshotLibrary):
    """Build a grid of Zs and Taus. Observe each at the age of the universe."""
    def __init__(self, libname, dbname="fsps", age=13.7):
        super(TauZGrid, self).__init__(libname, dbname=dbname, age=age)

    def generate_grid(self):
        zmets = range(1,23,4)
        #zmets = [3, 5, 10, 15, 20]
        #taus = np.arange(50.,100.,0.5)
        #taus = [0.5,1., 5., 10.]
        taus = np.linspace(0.1,100,10.)
        for zmet in zmets:
            for tau in taus:
                pset = ParameterSet(None, sfh=1,
                        zmet=int(zmet), tau=float(tau), tage=13.7,
                        const=0.5)
                self.register_pset(pset)

    def plot_color(self, plotPath, c1Name, c2Name):
        """Plots colour vs tau for each metallicity."""
        fig = plt.figure(figsize=(6,6))
        ax = fig.add_subplot(111)
        zmets = self.collection.distinct('pset.zmet')
        zmets.sort()
        print zmets
        for zmet in zmets:
            docs = self.collection.find({"pset.zmet":zmet})
            taus = []
            c1 = []
            c2 = []
            for doc in docs:
                taus.append(doc['pset']['tau'])
                c1.append(doc['obs'][c1Name])
                c2.append(doc['obs'][c2Name])
            taus = np.array(taus)
            sort = np.argsort(taus)
            taus = taus[sort]
            c1 = np.array(c1)
            c2 = np.array(c2)
            c1 = c1[sort]
            c2 = c2[sort]
            c = c1 - c2
            ax.plot(taus, c, '-')

        ax.set_xlabel(r"$\tau$")
        ax.set_ylabel("%s - %s" % (c1Name,c2Name))
        fig.savefig(plotPath+".pdf", format='pdf')

class MonteCarloLibrary(snapshotlib.SnapshotLibrary):
    """docstring for MonteCarloLibrary"""
    def __init__(self, libname, dbname='fsps', age=13.7):
        super(MonteCarloLibrary, self).__init__(libname,dbname=dbname,age=age)

    def define_samples(self, n=50000):
        """Define the set of models."""
        for i in xrange(n):
            pset = ParameterSet(None, # automatically create a name
                sfh=1,
                imf_type=1, # Chabrier 2003
                dust_type=2, # Calzetti 2000 attenuation curve
                zmet=int(self._sample_zmet()),
                tau=float(self._sample_tau()),
                const=float(self._sample_const()),
                sf_start=float(self._sample_sf_start()),
                fburst=float(self._sample_fburst()),
                tburst=float(self._sample_tburst()),
                dust2=float(self._sample_dust2()),
                tage=13.7
                )
            self.register_pset(pset)

    def _sample_zmet(self):
        """Returns a random metallicity"""
        return np.random.randint(1,23)

    def _sample_tau(self):
        """Returns a random e-folding of SFR"""
        return np.random.uniform(0.1, 10,)

    def _sample_const(self):
        """Returns fraction of mass formed as a constant mode of SF"""
        return np.random.uniform(0.1,0.5)

    def _sample_sf_start(self):
        """Start time of SFH in Gyr"""
        return np.random.uniform(0.5,3.)

    def _sample_fburst(self):
        """Fraction of mass formed in an instantaneous burst of SF."""
        return np.random.uniform(0.,0.5)

    def _sample_tburst(self):
        """Time of the burst after the BB."""
        return np.random.uniform(1.5,12.5)

    def _sample_dust2(self):
        """Optical depth of ambient ISM."""
        return np.random.uniform(0.1,1.)

    def create_mag_table(self, outputPath, t=13.7,
            isocType="pdva", specType="basel"):
        """Create an HDF5 table of that describes a set of magnitudes for
        stellar population realizations at a defined age (gyr from BigBang)"""
        if os.path.exists(outputPath): os.remove(outputPath)
        title = os.path.splitext(os.path.basename(outputPath))[0]
        h5file = tables.openFile(outputPath, mode="w", title=title)
        table = h5file.createTable("/", 'mags', MagTableDef, "Mag Model Table")
        print h5file
        docs = self.collection.find({"compute_complete":True,
            "np_data": {"$exists": 1}}) # , limit=2
        print "working on %i docs to read" % docs.count()
        lut = get_metallicity_LUT(isocType, specType)
        rows = []
        cols = ['Z','tau','age','mass','lbol','sfr','TMASS_J','TMASS_H',
                'TMASS_Ks','MegaCam_u','MegaCam_g','MegaCam_r','MegaCam_i',
                'MegaCam_z','GALEX_NUV','GALEX_FUV']
        for doc in docs:
            print "reading", doc['_id']
            # print doc.keys()
            # print doc['np_data']
            npData = doc['np_data']
            # print npData.dtype
            # binData = Binary(doc['np_data']['data'])
            # print type(binData)
            # npData = pickle.load(binData)
            nRows = len(npData)
            # Append model information (about SFH, dust, etc)
            zmet = doc['pset']['zmet']
            Z = lut[zmet-1]
            zmets = np.ones(nRows, dtype=np.float) * Z
            tau = doc['pset']['tau']
            taus = np.ones(nRows, dtype=np.float) * tau
            npDataAll = mlab.rec_append_fields(npData, ['Z','tau'],[zmets,taus])
            # Trim the recarray to just the desired fields
            npDataTrim = mlab.rec_keep_fields(npDataAll,
                ['Z','tau','age','mass','lbol','sfr','TMASS_J','TMASS_H',
                'TMASS_Ks','MegaCam_u','MegaCam_g','MegaCam_r','MegaCam_i',
                'MegaCam_z','GALEX_NUV','GALEX_FUV'])
            # select row closest to the target age
            ageGyr = 10.**npDataTrim['age'] / 10.**9
            i = np.argmin((ageGyr - t)**2)
            row = np.array(npDataTrim[i], copy=True)
            print i, row.shape, row.dtype
            print row['Z'], row['tau'],row['TMASS_J'],row['TMASS_Ks']
            rows.append(row)
            # Append to HDF5
            for col in cols:
                table.row[col] = row[col]
            table.row.append()
            #table.append(npDataTrim)
        #mlab.recs_join(key, name, rows, jointype='outer', missing=0.0, postfixes=None)
        h5file.flush()
        h5file.close()

    def bin_cc_index(self, c1I, c2I, h5Path):
        """Bin the mag table into a colour colour diagram.

        Produces an color-color table with indexes into models in
        the mags HDF5 table.

        Parameters
        ----------

        c1I : tuple of two str
            Names of the two bands that make the first colour
        c2I : tuple of two str
            Names of the two bands that make the second colour
        """
        h5file = tables.openFile(h5Path, mode='a')
        magTable = h5file.root.mags
        c1 = np.array([x[c1I[0]]-x[c1I[1]] for x in magTable])
        c2 = np.array([x[c2I[0]]-x[c2I[1]] for x in magTable])
        mass = np.array([x['mass'] for x in magTable])
        logL = np.array([x['lbol'] for x in magTable])
        logML = mass - logL
        grid, gridN, wherebin = griddata(c1, c2, logML, binsize=0.05,
                retbin=True, retloc=True)
        print "grid", grid.shape
        # Set up the cc table
        if 'cc' in h5file.root:
            print "cc already exists"
            h5file.root.cc._f_remove()
        ccDtype = np.dtype([('c1',np.int),('c2',np.int),('xi',np.int),
            ('yi',np.int),('ml',np.float)])
        ccTable = h5file.createTable("/", 'cc', ccDtype,
                "CC Table %s-%s %s-%s" % (c1I[0],c1I[1],c2I[0],c2I[1]))
        ny, nx = grid.shape
        c1colors = np.arange(c1.min(), c1.max()+0.05, 0.05)
        c2colors = np.arange(c2.min(), c2.max()+0.05, 0.05)
        print "g-i", c1colors
        print "i-Ks", c2colors
        for i in xrange(ny):
            for j in xrange(nx):
                ccTable.row['c1'] = c1colors[j]
                ccTable.row['c2'] = c2colors[i]
                ccTable.row['xi'] = i
                ccTable.row['yi'] = j
                ccTable.row['ml'] = grid[i,j]
                ccTable.row.append()
        h5file.flush()
        h5file.close()

if __name__ == '__main__':
    main()

<commit_msg>Implemented cc M/L look up table<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
Tests the iso-age library

History
-------
2011-10-02 - Created by Jonathan Sick

"""

__all__ = ['']
import os
import tables
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab

import snapshotlib
from fspsq import ParameterSet

from mclib import MagTableDef
from mclib import get_metallicity_LUT

from griddata import griddata

def main():
    #tauzGrid = TauZGrid("tauz", dbname='fsps')
    #tauzGrid.reset()
    #tauzGrid.generate_grid()
    #tauzGrid.compute_models(nThreads=2, maxN=10, clean=True)
    #tauzGrid.plot_color("tauzgrid", 'MegaCam_g', 'MegaCam_i')

    mclib = MonteCarloLibrary('mc.4')
    #mclib.reset()
    #mclib.define_samples(n=10000)
    #mclib.compute_models(nThreads=8, maxN=100, clean=True)
    mclib.create_mag_table("mc4.h5")
    mclib.bin_cc_index(("MegaCam_i","TMASS_Ks"),("MegaCam_g","MegaCam_i"),
            "mc4.h5")
    mclib.plot_cc_lut("mc4.h5", r"$i^\prime-K_s$", r"$g^\prime-i^\prime$")

class TauZGrid(snapshotlib.SnapshotLibrary):
    """Build a grid of Zs and Taus. Observe each at the age of the universe."""
    def __init__(self, libname, dbname="fsps", age=13.7):
        super(TauZGrid, self).__init__(libname, dbname=dbname, age=age)

    def generate_grid(self):
        zmets = range(1,23,4)
        #zmets = [3, 5, 10, 15, 20]
        #taus = np.arange(50.,100.,0.5)
        #taus = [0.5,1., 5., 10.]
        taus = np.linspace(0.1,100,10.)
        for zmet in zmets:
            for tau in taus:
                pset = ParameterSet(None, sfh=1,
                        zmet=int(zmet), tau=float(tau), tage=13.7,
                        const=0.5)
                self.register_pset(pset)

    def plot_color(self, plotPath, c1Name, c2Name):
        """Plots colour vs tau for each metallicity."""
        fig = plt.figure(figsize=(6,6))
        ax = fig.add_subplot(111)
        zmets = self.collection.distinct('pset.zmet')
        zmets.sort()
        print zmets
        for zmet in zmets:
            docs = self.collection.find({"pset.zmet":zmet})
            taus = []
            c1 = []
            c2 = []
            for doc in docs:
                taus.append(doc['pset']['tau'])
                c1.append(doc['obs'][c1Name])
                c2.append(doc['obs'][c2Name])
            taus = np.array(taus)
            sort = np.argsort(taus)
            taus = taus[sort]
            c1 = np.array(c1)
            c2 = np.array(c2)
            c1 = c1[sort]
            c2 = c2[sort]
            c = c1 - c2
            ax.plot(taus, c, '-')

        ax.set_xlabel(r"$\tau$")
        ax.set_ylabel("%s - %s" % (c1Name,c2Name))
        fig.savefig(plotPath+".pdf", format='pdf')

class MonteCarloLibrary(snapshotlib.SnapshotLibrary):
    """docstring for MonteCarloLibrary"""
    def __init__(self, libname, dbname='fsps', age=13.7):
        super(MonteCarloLibrary, self).__init__(libname,dbname=dbname,age=age)

    def define_samples(self, n=50000):
        """Define the set of models."""
        for i in xrange(n):
            pset = ParameterSet(None, # automatically create a name
                sfh=1,
                imf_type=1, # Chabrier 2003
                dust_type=2, # Calzetti 2000 attenuation curve
                zmet=int(self._sample_zmet()),
                tau=float(self._sample_tau()),
                const=float(self._sample_const()),
                sf_start=float(self._sample_sf_start()),
                fburst=float(self._sample_fburst()),
                tburst=float(self._sample_tburst()),
                dust2=float(self._sample_dust2()),
                tage=13.7
                )
            self.register_pset(pset)

    def _sample_zmet(self):
        """Returns a random metallicity"""
        return np.random.randint(1,23)

    def _sample_tau(self):
        """Returns a random e-folding of SFR"""
        return np.random.uniform(0.1, 10,)

    def _sample_const(self):
        """Returns fraction of mass formed as a constant mode of SF"""
        return np.random.uniform(0.1,0.5)

    def _sample_sf_start(self):
        """Start time of SFH in Gyr"""
        return np.random.uniform(0.5,3.)

    def _sample_fburst(self):
        """Fraction of mass formed in an instantaneous burst of SF."""
        return np.random.uniform(0.,0.5)

    def _sample_tburst(self):
        """Time of the burst after the BB."""
        return np.random.uniform(1.5,12.5)

    def _sample_dust2(self):
        """Optical depth of ambient ISM."""
        return np.random.uniform(0.1,1.)

    def create_mag_table(self, outputPath,
            isocType="pdva", specType="basel"):
        """Create an HDF5 table of that describes a set of magnitudes for
        stellar population realizations at a defined age. Assumes a
        snapshot data set for that age.
        """
        if os.path.exists(outputPath): os.remove(outputPath)
        title = os.path.splitext(os.path.basename(outputPath))[0]
        h5file = tables.openFile(outputPath, mode="w", title=title)
        table = h5file.createTable("/", 'mags', MagTableDef, "Mag Model Table")
        print h5file
        docs = self.collection.find({"compute_complete":True,
            "obs": {"$exists": 1}}) # , limit=2
        print "working on %i docs to read" % docs.count()
        lut = get_metallicity_LUT(isocType, specType)
        obsCols = ['mass','lbol','sfr','TMASS_J','TMASS_H',
                'TMASS_Ks','MegaCam_u','MegaCam_g','MegaCam_r','MegaCam_i',
                'MegaCam_z','GALEX_NUV','GALEX_FUV']
        for doc in docs:
            print "reading", doc['_id']
            obs = doc['obs']
            # Append model information (about SFH, dust, etc)
            zmet = doc['pset']['zmet']
            Z = lut[zmet-1]
            # Append to HDF5
            table.row['Z'] = Z
            table.row['tau'] = doc['pset']['tau']
            table.row['age'] = doc['pset']['tage']
            for col in obsCols:
                table.row[col] = obs[col]
            table.row.append()
        h5file.flush()
        h5file.close()

    def bin_cc_index(self, c1I, c2I, h5Path):
        """Bin the mag table into a colour colour diagram.

        Produces an color-color table with indexes into models in
        the mags HDF5 table.

        Parameters
        ----------

        c1I : tuple of two str
            Names of the two bands that make the first colour
        c2I : tuple of two str
            Names of the two bands that make the second colour
        """
        h5file = tables.openFile(h5Path, mode='a')
        magTable = h5file.root.mags
        c1 = np.array([x[c1I[0]]-x[c1I[1]] for x in magTable])
        c2 = np.array([x[c2I[0]]-x[c2I[1]] for x in magTable])
        mass = np.array([x['mass'] for x in magTable])
        logL = np.array([x['lbol'] for x in magTable])
        logML = mass - logL
        grid, gridN, wherebin = griddata(c1, c2, logML, binsize=0.05,
                retbin=True, retloc=True)
        print "grid", grid.shape
        # Set up the cc table
        if 'cc' in h5file.root:
            print "cc already exists"
            h5file.root.cc._f_remove()
        ccDtype = np.dtype([('c1',np.int),('c2',np.int),('xi',np.int),
            ('yi',np.int),('ml',np.float)])
        ccTable = h5file.createTable("/", 'cc', ccDtype,
                "CC Table %s-%s %s-%s" % (c1I[0],c1I[1],c2I[0],c2I[1]))
        ny, nx = grid.shape
        c1colors = np.arange(c1.min(), c1.max()+0.05, 0.05)
        c2colors = np.arange(c2.min(), c2.max()+0.05, 0.05)
        for i in xrange(ny):
            for j in xrange(nx):
                ccTable.row['c1'] = float(c1colors[j])
                ccTable.row['c2'] = float(c2colors[i])
                ccTable.row['yi'] = i
                ccTable.row['xi'] = j
                ccTable.row['ml'] = grid[i,j]
                ccTable.row.append()
        h5file.flush()
        h5file.close()

    def plot_cc_lut(self, h5Path, xlabel, ylabel):
        """Create the g-i,i-Ks M/L look up table plot."""
        h5file = tables.openFile(h5Path, mode='a')
        ccTable = h5file.root.cc
        c1, c2, xi, yi, ml = [],[],[],[],[]
        for row in ccTable:
            c1.append(row['c1'])
            c2.append(row['c2'])
            xi.append(row['xi'])
            yi.append(row['yi'])
            ml.append(row['ml'])
        c1 = np.array(c1)
        c2 = np.array(c2)
        xi = np.array(xi, dtype=np.int)
        yi = np.array(yi, dtype=np.int)
        print xi
        print yi
        nx = max(xi)+1
        ny = max(yi)+1

        # Extent is the physical limits (left, right, bottom, top)
        extent = [min(c1), max(c1), min(c2), max(c2)]
        lut = np.empty((ny,nx), dtype=np.float)
        for i in xrange(len(c1)):
            lut[ny-yi[i]-1,xi[i]] = ml[i]
        print lut.shape

        fig = plt.figure(figsize=(4.,3.5)) # set width,height in inches
        fig.subplots_adjust(left=0.15, bottom=0.1, right=0.85, top=0.99)
        ax = fig.add_subplot(111)
        im = ax.imshow(lut, cmap=mpl.cm.jet, aspect='equal', extent=extent,
            interpolation='nearest')
        # Create the colorbar
        # see http://matplotlib.sourceforge.net/api/pyplot_api.html#matplotlib.pyplot.colorbar
        cbar = fig.colorbar(mappable=im, cax=None, ax=ax, orientation='vertical',
            fraction=0.1, pad=0.0, shrink=0.8,)
        cbar.set_label(r'$\langle\log M/L_\mathrm{bol}\rangle$')
        ax.set_xlabel(xlabel)
        ax.set_ylabel(ylabel)
        fig.savefig(h5Path+".pdf", format="pdf") # can also do "eps", etc.

if __name__ == '__main__':
    main()

<|endoftext|>