<commit_before># Copyright (C) 2013-2014  Stefano Zacchiroli <zack@upsilon.cc>
#
# This file is part of Debsources.
#
# Debsources is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import logging
import os
import subprocess

from sqlalchemy import sql

import dbutils

from models import Ctag, File, MAX_KEY_LENGTH

conf = None

CTAGS_FLAGS = [ '--recurse',
                '--excmd=number',
                '--fields=+lnz',
                # '--extra=+q',
                '--sort=no',
                '--links=no',
]

MY_NAME = 'ctags'
MY_EXT = '.' + MY_NAME
ctags_path = lambda pkgdir: pkgdir + MY_EXT

# maximum number of ctags after which a (bulk) insert is sent to the DB
BULK_FLUSH_THRESHOLD = 20000

# maximum number of detailed warnings for malformed tags that will be emitted.
# used to avoid flooding logs
BAD_TAGS_THRESHOLD = 5

def parse_ctags(path):
    """parse exuberant ctags tags file

    for each tag yield a tag dictionary::

      { 'tag':  'TAG_NAME',
        'path': 'PATH/WITH/IN/PACKAGE',
        'line': LINE_NUMBER, # int
        'kind': 'TAG_KIND', # 1 letter
        'language': 'TAG_LANGUAGE',
      }
    """
    def parse_tag(line):
        tag = { 'kind': None, 'line': None, 'language': None }
        # initialize with extension fields which are not guaranteed to exist

        fields = line.rstrip().split('\t')
        tag['tag'] = fields[0].decode()	# will fail when encountering encoding issues;
					# that is intended
        tag['path'] = fields[1]
        # note: ignore fields[2], ex_cmd

        for ext in fields[3:]:	# parse extension fields
            k, v = ext.split(':', 1) # caution: "typeref:struct:__RAW_READ_INFO"
            if k == 'kind':
                tag['kind'] = v
            elif k == 'line':
                tag['line'] = int(v)
            elif k == 'language':
                tag['language'] = v.lower()
            else:
                pass	# ignore other fields

        assert tag['line'] is not None
        assert len(tag['tag']) <= MAX_KEY_LENGTH
        return tag

    with open(path) as ctags:
        bad_tags = 0
        for line in ctags:
            # e.g. 'music\tsound.c\t13;"\tkind:v\tline:13\tlanguage:C\tfile:\n'
            # see CTAGS(1), section "TAG FILE FORMAT"
            if line.startswith('!_TAG'):	# skip ctags metadata
                continue
            try:
                yield parse_tag(line)
            except:
                bad_tags += 1
                if bad_tags <= BAD_TAGS_THRESHOLD:
                    logging.warn('ignore malformed tag "%s"' % line.rstrip())
        if bad_tags > BAD_TAGS_THRESHOLD:
            logging.warn('%d extra malformed tag(s) ignored' %
                         (bad_tags - BAD_TAGS_THRESHOLD))

def add_package(session, pkg, pkgdir, file_table):
    global conf
    logging.debug('add-package %s' % pkg)

    ctagsfile = ctags_path(pkgdir)
    ctagsfile_tmp = ctagsfile + '.new'

    if 'hooks.fs' in conf['backends']:
        if not os.path.exists(ctagsfile): # extract tags only if needed
            workdir = os.getcwd()
            try:
                cmd = [ 'ctags' ] + CTAGS_FLAGS + [ '-f', '-' ]
                os.chdir(pkgdir) # execute in pkgdir to get relative paths right
                with open(ctagsfile_tmp, 'w') as out,\
                     open(os.devnull, 'w') as null:
                    subprocess.check_call(cmd, stdout=out, stderr=null)
                os.rename(ctagsfile_tmp, ctagsfile)
            finally:
                os.chdir(workdir)

    if 'hooks.db' in conf['backends']:
        db_package = dbutils.lookup_package(session, pkg['package'], pkg['version'])
        curfile = {None: None}	# poor man's cache for last <relpath, file_id>;
                             # rely on the fact that ctags file are path-sorted
        insert_q = sql.insert(Ctag.__table__)
        insert_params = []
        if not session.query(Ctag).filter_by(package_id=db_package.id).first():
            # ASSUMPTION: if *a* ctag of this package has already been added to
            # the db in the past, then *all* of them have, as additions are
            # part of the same transaction
            for tag in parse_ctags(ctagsfile):
                params = ({'package_id': db_package.id,
                           'tag': tag['tag'],
                           # 'file_id': 	# will be filled below
                           'line': tag['line'],
                           'kind': tag['kind'],
                           'language': tag['language'],
                       })
                relpath = tag['path']
                if file_table:
                    try:
                        params['file_id'] = file_table[relpath]
                    except KeyError:
                        continue
                else:
                    try:
                        params['file_id'] = curfile[relpath]
                    except KeyError:
                        file_ = session.query(File).filter_by(package_id=db_package.id,
                                                              path=relpath).first()
                        if not file_:
                            continue
                        curfile = { relpath: file_.id }
                        params['file_id'] = file_.id
                insert_params.append(params)
                if len(insert_params) >= BULK_FLUSH_THRESHOLD:
                    session.execute(insert_q, insert_params)
                    session.flush()
                    insert_params = []
            if insert_params:	# might be empty if there are no ctags at all!
                session.execute(insert_q, insert_params)
                session.flush()

def rm_package(session, pkg, pkgdir, file_table):
    global conf
    logging.debug('rm-package %s' % pkg)

    if 'hooks.fs' in conf['backends']:
        ctagsfile = ctags_path(pkgdir)
        if os.path.exists(ctagsfile):
            os.unlink(ctagsfile)

    if 'hooks.db' in conf['backends']:
        db_package = dbutils.lookup_package(session, pkg['package'], pkg['version'])
        session.query(Ctag) \
               .filter_by(package_id=db_package.id) \
               .delete()

def init_plugin(debsources):
    global conf
    conf = debsources['config']
    debsources['subscribe']('add-package', add_package, title=MY_NAME)
    debsources['subscribe']('rm-package',  rm_package,  title=MY_NAME)
    debsources['declare_ext'](MY_EXT, MY_NAME)
<commit_msg>ctags plugin: write tags file incrementally, avoding tempfile<commit_after># Copyright (C) 2013-2014  Stefano Zacchiroli <zack@upsilon.cc>
#
# This file is part of Debsources.
#
# Debsources is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import logging
import os
import subprocess

from sqlalchemy import sql

import dbutils

from models import Ctag, File, MAX_KEY_LENGTH

conf = None

CTAGS_FLAGS = [ '--recurse',
                '--excmd=number',
                '--fields=+lnz',
                # '--extra=+q',
                '--sort=no',
                '--links=no',
]

MY_NAME = 'ctags'
MY_EXT = '.' + MY_NAME
ctags_path = lambda pkgdir: pkgdir + MY_EXT

# maximum number of ctags after which a (bulk) insert is sent to the DB
BULK_FLUSH_THRESHOLD = 20000

# maximum number of detailed warnings for malformed tags that will be emitted.
# used to avoid flooding logs
BAD_TAGS_THRESHOLD = 5

def parse_ctags(path):
    """parse exuberant ctags tags file

    for each tag yield a tag dictionary::

      { 'tag':  'TAG_NAME',
        'path': 'PATH/WITH/IN/PACKAGE',
        'line': LINE_NUMBER, # int
        'kind': 'TAG_KIND', # 1 letter
        'language': 'TAG_LANGUAGE',
      }
    """
    def parse_tag(line):
        tag = { 'kind': None, 'line': None, 'language': None }
        # initialize with extension fields which are not guaranteed to exist

        fields = line.rstrip().split('\t')
        tag['tag'] = fields[0].decode()	# will fail when encountering encoding issues;
					# that is intended
        tag['path'] = fields[1]
        # note: ignore fields[2], ex_cmd

        for ext in fields[3:]:	# parse extension fields
            k, v = ext.split(':', 1) # caution: "typeref:struct:__RAW_READ_INFO"
            if k == 'kind':
                tag['kind'] = v
            elif k == 'line':
                tag['line'] = int(v)
            elif k == 'language':
                tag['language'] = v.lower()
            else:
                pass	# ignore other fields

        assert tag['line'] is not None
        assert len(tag['tag']) <= MAX_KEY_LENGTH
        return tag

    with open(path) as ctags:
        bad_tags = 0
        for line in ctags:
            # e.g. 'music\tsound.c\t13;"\tkind:v\tline:13\tlanguage:C\tfile:\n'
            # see CTAGS(1), section "TAG FILE FORMAT"
            if line.startswith('!_TAG'):	# skip ctags metadata
                continue
            try:
                yield parse_tag(line)
            except:
                bad_tags += 1
                if bad_tags <= BAD_TAGS_THRESHOLD:
                    logging.warn('ignore malformed tag "%s"' % line.rstrip())
        if bad_tags > BAD_TAGS_THRESHOLD:
            logging.warn('%d extra malformed tag(s) ignored' %
                         (bad_tags - BAD_TAGS_THRESHOLD))

def add_package(session, pkg, pkgdir, file_table):
    global conf
    logging.debug('add-package %s' % pkg)

    ctagsfile = ctags_path(pkgdir)
    ctagsfile_tmp = ctagsfile + '.new'

    if 'hooks.fs' in conf['backends']:
        if not os.path.exists(ctagsfile): # extract tags only if needed
            workdir = os.getcwd()
            try:
                cmd = [ 'ctags' ] + CTAGS_FLAGS + [ '-o', ctagsfile_tmp ]
                os.chdir(pkgdir) # execute in pkgdir to get relative paths right
                with open(os.devnull, 'w') as null:
                    subprocess.check_call(cmd, stderr=null)
                os.rename(ctagsfile_tmp, ctagsfile)
            finally:
                os.chdir(workdir)

    if 'hooks.db' in conf['backends']:
        db_package = dbutils.lookup_package(session, pkg['package'], pkg['version'])
        curfile = {None: None}	# poor man's cache for last <relpath, file_id>;
                             # rely on the fact that ctags file are path-sorted
        insert_q = sql.insert(Ctag.__table__)
        insert_params = []
        if not session.query(Ctag).filter_by(package_id=db_package.id).first():
            # ASSUMPTION: if *a* ctag of this package has already been added to
            # the db in the past, then *all* of them have, as additions are
            # part of the same transaction
            for tag in parse_ctags(ctagsfile):
                params = ({'package_id': db_package.id,
                           'tag': tag['tag'],
                           # 'file_id': 	# will be filled below
                           'line': tag['line'],
                           'kind': tag['kind'],
                           'language': tag['language'],
                       })
                relpath = tag['path']
                if file_table:
                    try:
                        params['file_id'] = file_table[relpath]
                    except KeyError:
                        continue
                else:
                    try:
                        params['file_id'] = curfile[relpath]
                    except KeyError:
                        file_ = session.query(File).filter_by(package_id=db_package.id,
                                                              path=relpath).first()
                        if not file_:
                            continue
                        curfile = { relpath: file_.id }
                        params['file_id'] = file_.id
                insert_params.append(params)
                if len(insert_params) >= BULK_FLUSH_THRESHOLD:
                    session.execute(insert_q, insert_params)
                    session.flush()
                    insert_params = []
            if insert_params:	# might be empty if there are no ctags at all!
                session.execute(insert_q, insert_params)
                session.flush()

def rm_package(session, pkg, pkgdir, file_table):
    global conf
    logging.debug('rm-package %s' % pkg)

    if 'hooks.fs' in conf['backends']:
        ctagsfile = ctags_path(pkgdir)
        if os.path.exists(ctagsfile):
            os.unlink(ctagsfile)

    if 'hooks.db' in conf['backends']:
        db_package = dbutils.lookup_package(session, pkg['package'], pkg['version'])
        session.query(Ctag) \
               .filter_by(package_id=db_package.id) \
               .delete()

def init_plugin(debsources):
    global conf
    conf = debsources['config']
    debsources['subscribe']('add-package', add_package, title=MY_NAME)
    debsources['subscribe']('rm-package',  rm_package,  title=MY_NAME)
    debsources['declare_ext'](MY_EXT, MY_NAME)
<|endoftext|>


<commit_before># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""The TensorBoard Custom Scalars plugin.

This plugin lets the user create scalars plots with custom run-tag combinations
by specifying regular expressions.

See `http_api.md` in this directory for specifications of the routes for
this plugin.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import re

from google.protobuf import json_format
import numpy as np
import tensorflow as tf
from werkzeug import wrappers

from tensorboard.backend import http_util
from tensorboard.plugins import base_plugin
from tensorboard.plugins.custom_scalar import layout_pb2
from tensorboard.plugins.custom_scalar import metadata
from tensorboard.plugins.scalar import metadata as scalars_metadata
from tensorboard.plugins.scalar import scalars_plugin

# The name of the property in the response for whether the regex is valid.
_REGEX_VALID_PROPERTY = 'regex_valid'

# The name of the property in the response for the payload (tag to ScalarEvents
# mapping).
_TAG_TO_EVENTS_PROPERTY = 'tag_to_events'

# The number of seconds to wait in between checks for the config file specifying
# layout.
_CONFIG_FILE_CHECK_THROTTLE = 60

class CustomScalarsPlugin(base_plugin.TBPlugin):
  """CustomScalars Plugin for TensorBoard."""

  plugin_name = metadata.PLUGIN_NAME

  def __init__(self, context):
    """Instantiates ScalarsPlugin via TensorBoard core.

    Args:
      context: A base_plugin.TBContext instance.
    """
    self._logdir = context.logdir
    self._multiplexer = context.multiplexer
    self._plugin_name_to_instance = context.plugin_name_to_instance

  def _get_scalars_plugin(self):
    """Tries to get the scalars plugin.

    Returns:
      The scalars plugin. Or None if it is not yet registered.
    """
    if scalars_metadata.PLUGIN_NAME in self._plugin_name_to_instance:
      # The plugin is registered.
      return self._plugin_name_to_instance[scalars_metadata.PLUGIN_NAME]
    # The plugin is not yet registered.
    return None

  def get_plugin_apps(self):
    return {
        '/download_data': self.download_data_route,
        '/layout': self.layout_route,
        '/scalars': self.scalars_route,
    }

  def is_active(self):
    """This plugin is active if 2 conditions hold.

    1. The scalars plugin is registered and active.
    2. There is a custom layout for the dashboard.
    """
    scalars_plugin_instance = self._get_scalars_plugin()
    if not (scalars_plugin_instance and
            scalars_plugin_instance.is_active()):
      return False

    for run in self._multiplexer.Runs():
      try:
        self._multiplexer.Tensors(run, metadata.CONFIG_SUMMARY_TAG)
      except KeyError:
        continue
      return True

    return False

  @wrappers.Request.application
  def download_data_route(self, request):
    run = request.args.get('run')
    tag = request.args.get('tag')
    response_format = request.args.get('format')
    try:
      body, mime_type = self.download_data_impl(run, tag, response_format)
    except ValueError as e:
      return http_util.Respond(
          request=request,
          content=str(e),
          content_type='text/plain',
          code=500)
    return http_util.Respond(request, body, mime_type)

  def download_data_impl(self, run, tag, response_format):
    """Provides a response for downloading scalars data for a data series.

    Args:
      run: The run.
      tag: The specific tag.
      response_format: A string. One of the values of the OutputFormat enum of
        the scalar plugin.

    Raises:
      ValueError: If the scalars plugin is not registered.

    Returns:
      2 entities:
        - A JSON object response body.
        - A mime type (string) for the response.
    """
    scalars_plugin_instance = self._get_scalars_plugin()
    if not scalars_plugin_instance:
      raise ValueError(('Failed to respond to request for /download_data. '
                        'The scalars plugin is oddly not registered.'))

    body, mime_type = scalars_plugin_instance.scalars_impl(
        tag, run, response_format)
    return body, mime_type

  @wrappers.Request.application
  def scalars_route(self, request):
    """Given a tag regex and single run, return ScalarEvents.

    This route takes 2 GET params:
    run: A run string to find tags for.
    tag: A string that is a regex used to find matching tags.
    The response is a JSON object:
    {
      // Whether the regular expression is valid. Also false if empty.
      regexValid: boolean,

      // An object mapping tag name to a list of ScalarEvents.
      payload: Object<string, ScalarEvent[]>,
    }
    """
    # TODO: return HTTP status code for malformed requests
    tag_regex_string = request.args.get('tag')
    run = request.args.get('run')
    mime_type = 'application/json'

    try:
      body = self.scalars_impl(run, tag_regex_string)
    except ValueError as e:
      return http_util.Respond(
          request=request,
          content=str(e),
          content_type='text/plain',
          code=500)

    # Produce the response.
    return http_util.Respond(request, body, mime_type)

  def scalars_impl(self, run, tag_regex_string):
    """Given a tag regex and single run, return ScalarEvents.

    Args:
      run: A run string.
      tag_regex_string: A regular expression that captures portions of tags.

    Raises:
      ValueError: if the scalars plugin is not registered.

    Returns:
      A dictionary that is the JSON-able response.
    """
    if not tag_regex_string:
      # The user provided no regex.
      return {
          _REGEX_VALID_PROPERTY: False,
          _TAG_TO_EVENTS_PROPERTY: {},
      }

    # Construct the regex.
    try:
      regex = re.compile(tag_regex_string)
    except re.error:
      return {
          _REGEX_VALID_PROPERTY: False,
          _TAG_TO_EVENTS_PROPERTY: {},
      }

    # Fetch the tags for the run. Filter for tags that match the regex.
    run_to_data = self._multiplexer.PluginRunToTagToContent(
        scalars_metadata.PLUGIN_NAME)

    tag_to_data = None
    try:
      tag_to_data = run_to_data[run]
    except KeyError:
      # The run could not be found. Perhaps a configuration specified a run that
      # TensorBoard has not read from disk yet.
      payload = {}

    if tag_to_data:
      scalars_plugin_instance = self._get_scalars_plugin()
      if not scalars_plugin_instance:
        raise ValueError(('Failed to respond to request for /scalars. '
                          'The scalars plugin is oddly not registered.'))

      form = scalars_plugin.OutputFormat.JSON
      payload = {tag: scalars_plugin_instance.scalars_impl(tag, run, form)[0]
                 for tag in tag_to_data.keys()
                 if regex.match(tag)}

    return {
        _REGEX_VALID_PROPERTY: True,
        _TAG_TO_EVENTS_PROPERTY: payload,
    }

  @wrappers.Request.application
  def layout_route(self, request):
    r"""Fetches the custom layout specified by the config file in the logdir.

    If more than 1 run contains a layout, this method merges the layouts by
    merging charts within individual categories. If 2 categories with the same
    name are found, the charts within are merged. The merging is based on the
    order of the runs to which the layouts are written.

    The response is a JSON object mirroring properties of the Layout proto if a
    layout for any run is found:

    {
      categories: [
        {
          title: 'Biases',
          chart: [
            {
              title: 'biases',
              tag: 'foo/layer\d+/biases'
            },
            {
              title: 'biases',
              tag: 'foo/logits/biases'
            },
          ],
        },
        {
          title: 'Activations',
          chart: [
            {
              title: 'activations',
              tag: 'foo/layer0/activations',
            },
          ],
        },
      ],
    }

    The response is an empty object if no layout could be found.
    """
    body = self.layout_impl()
    return http_util.Respond(request, body, 'application/json')

  def layout_impl(self):
    # Keep a mapping between and category so we do not create duplicate
    # categories.
    title_to_category = {}

    merged_layout = None
    runs = list(self._multiplexer.Runs())
    runs.sort()
    for run in runs:
      try:
        tensor_events = self._multiplexer.Tensors(
            run, metadata.CONFIG_SUMMARY_TAG)
      except KeyError:
        # This run lacks a layout.
        continue

      # This run has a layout. Merge it with the ones currently found.
      string_array = tf.make_ndarray(tensor_events[0].tensor_proto)
      content = np.asscalar(string_array)
      layout_proto = layout_pb2.Layout()
      layout_proto.ParseFromString(tf.compat.as_bytes(content))

      if merged_layout:
        # Append the categories within this layout to the merged layout.
        for category in layout_proto.category:
          if category.title in title_to_category:
            # A category with this name has been seen before. Do not create a
            # new one. Merge their charts.
            title_to_category[category.title].chart.extend(category.chart)
          else:
            # This category has not been seen before.
            merged_layout.category.add().MergeFrom(category)
            title_to_category[category.title] = category
      else:
        # This is the first layout encountered.
        merged_layout = layout_proto
        for category in layout_proto.category:
          title_to_category[category.title] = category

    if merged_layout:
      return json_format.MessageToJson(
          merged_layout, including_default_value_fields=True)
    else:
      # No layout was found.
      return {}
<commit_msg>Avoid catching KeyError within custom-scalars (#738)<commit_after># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""The TensorBoard Custom Scalars plugin.

This plugin lets the user create scalars plots with custom run-tag combinations
by specifying regular expressions.

See `http_api.md` in this directory for specifications of the routes for
this plugin.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import re

from google.protobuf import json_format
import numpy as np
import tensorflow as tf
from werkzeug import wrappers

from tensorboard.backend import http_util
from tensorboard.plugins import base_plugin
from tensorboard.plugins.custom_scalar import layout_pb2
from tensorboard.plugins.custom_scalar import metadata
from tensorboard.plugins.scalar import metadata as scalars_metadata
from tensorboard.plugins.scalar import scalars_plugin

# The name of the property in the response for whether the regex is valid.
_REGEX_VALID_PROPERTY = 'regex_valid'

# The name of the property in the response for the payload (tag to ScalarEvents
# mapping).
_TAG_TO_EVENTS_PROPERTY = 'tag_to_events'

# The number of seconds to wait in between checks for the config file specifying
# layout.
_CONFIG_FILE_CHECK_THROTTLE = 60

class CustomScalarsPlugin(base_plugin.TBPlugin):
  """CustomScalars Plugin for TensorBoard."""

  plugin_name = metadata.PLUGIN_NAME

  def __init__(self, context):
    """Instantiates ScalarsPlugin via TensorBoard core.

    Args:
      context: A base_plugin.TBContext instance.
    """
    self._logdir = context.logdir
    self._multiplexer = context.multiplexer
    self._plugin_name_to_instance = context.plugin_name_to_instance

  def _get_scalars_plugin(self):
    """Tries to get the scalars plugin.

    Returns:
      The scalars plugin. Or None if it is not yet registered.
    """
    if scalars_metadata.PLUGIN_NAME in self._plugin_name_to_instance:
      # The plugin is registered.
      return self._plugin_name_to_instance[scalars_metadata.PLUGIN_NAME]
    # The plugin is not yet registered.
    return None

  def get_plugin_apps(self):
    return {
        '/download_data': self.download_data_route,
        '/layout': self.layout_route,
        '/scalars': self.scalars_route,
    }

  def is_active(self):
    """This plugin is active if 2 conditions hold.

    1. The scalars plugin is registered and active.
    2. There is a custom layout for the dashboard.

    Returns: A boolean. Whether the plugin is active.
    """
    if not self._multiplexer:
      return False

    scalars_plugin_instance = self._get_scalars_plugin()
    if not (scalars_plugin_instance and
            scalars_plugin_instance.is_active()):
      return False

    # This plugin is active if any run has a layout.
    return bool(self._multiplexer.PluginRunToTagToContent(metadata.PLUGIN_NAME))

  @wrappers.Request.application
  def download_data_route(self, request):
    run = request.args.get('run')
    tag = request.args.get('tag')
    response_format = request.args.get('format')
    try:
      body, mime_type = self.download_data_impl(run, tag, response_format)
    except ValueError as e:
      return http_util.Respond(
          request=request,
          content=str(e),
          content_type='text/plain',
          code=500)
    return http_util.Respond(request, body, mime_type)

  def download_data_impl(self, run, tag, response_format):
    """Provides a response for downloading scalars data for a data series.

    Args:
      run: The run.
      tag: The specific tag.
      response_format: A string. One of the values of the OutputFormat enum of
        the scalar plugin.

    Raises:
      ValueError: If the scalars plugin is not registered.

    Returns:
      2 entities:
        - A JSON object response body.
        - A mime type (string) for the response.
    """
    scalars_plugin_instance = self._get_scalars_plugin()
    if not scalars_plugin_instance:
      raise ValueError(('Failed to respond to request for /download_data. '
                        'The scalars plugin is oddly not registered.'))

    body, mime_type = scalars_plugin_instance.scalars_impl(
        tag, run, response_format)
    return body, mime_type

  @wrappers.Request.application
  def scalars_route(self, request):
    """Given a tag regex and single run, return ScalarEvents.

    This route takes 2 GET params:
    run: A run string to find tags for.
    tag: A string that is a regex used to find matching tags.
    The response is a JSON object:
    {
      // Whether the regular expression is valid. Also false if empty.
      regexValid: boolean,

      // An object mapping tag name to a list of ScalarEvents.
      payload: Object<string, ScalarEvent[]>,
    }
    """
    # TODO: return HTTP status code for malformed requests
    tag_regex_string = request.args.get('tag')
    run = request.args.get('run')
    mime_type = 'application/json'

    try:
      body = self.scalars_impl(run, tag_regex_string)
    except ValueError as e:
      return http_util.Respond(
          request=request,
          content=str(e),
          content_type='text/plain',
          code=500)

    # Produce the response.
    return http_util.Respond(request, body, mime_type)

  def scalars_impl(self, run, tag_regex_string):
    """Given a tag regex and single run, return ScalarEvents.

    Args:
      run: A run string.
      tag_regex_string: A regular expression that captures portions of tags.

    Raises:
      ValueError: if the scalars plugin is not registered.

    Returns:
      A dictionary that is the JSON-able response.
    """
    if not tag_regex_string:
      # The user provided no regex.
      return {
          _REGEX_VALID_PROPERTY: False,
          _TAG_TO_EVENTS_PROPERTY: {},
      }

    # Construct the regex.
    try:
      regex = re.compile(tag_regex_string)
    except re.error:
      return {
          _REGEX_VALID_PROPERTY: False,
          _TAG_TO_EVENTS_PROPERTY: {},
      }

    # Fetch the tags for the run. Filter for tags that match the regex.
    run_to_data = self._multiplexer.PluginRunToTagToContent(
        scalars_metadata.PLUGIN_NAME)

    tag_to_data = None
    try:
      tag_to_data = run_to_data[run]
    except KeyError:
      # The run could not be found. Perhaps a configuration specified a run that
      # TensorBoard has not read from disk yet.
      payload = {}

    if tag_to_data:
      scalars_plugin_instance = self._get_scalars_plugin()
      if not scalars_plugin_instance:
        raise ValueError(('Failed to respond to request for /scalars. '
                          'The scalars plugin is oddly not registered.'))

      form = scalars_plugin.OutputFormat.JSON
      payload = {tag: scalars_plugin_instance.scalars_impl(tag, run, form)[0]
                 for tag in tag_to_data.keys()
                 if regex.match(tag)}

    return {
        _REGEX_VALID_PROPERTY: True,
        _TAG_TO_EVENTS_PROPERTY: payload,
    }

  @wrappers.Request.application
  def layout_route(self, request):
    r"""Fetches the custom layout specified by the config file in the logdir.

    If more than 1 run contains a layout, this method merges the layouts by
    merging charts within individual categories. If 2 categories with the same
    name are found, the charts within are merged. The merging is based on the
    order of the runs to which the layouts are written.

    The response is a JSON object mirroring properties of the Layout proto if a
    layout for any run is found:

    {
      categories: [
        {
          title: 'Biases',
          chart: [
            {
              title: 'biases',
              tag: 'foo/layer\d+/biases'
            },
            {
              title: 'biases',
              tag: 'foo/logits/biases'
            },
          ],
        },
        {
          title: 'Activations',
          chart: [
            {
              title: 'activations',
              tag: 'foo/layer0/activations',
            },
          ],
        },
      ],
    }

    The response is an empty object if no layout could be found.
    """
    body = self.layout_impl()
    return http_util.Respond(request, body, 'application/json')

  def layout_impl(self):
    # Keep a mapping between and category so we do not create duplicate
    # categories.
    title_to_category = {}

    merged_layout = None
    runs = list(self._multiplexer.PluginRunToTagToContent(metadata.PLUGIN_NAME))
    runs.sort()
    for run in runs:
      tensor_events = self._multiplexer.Tensors(
          run, metadata.CONFIG_SUMMARY_TAG)

      # This run has a layout. Merge it with the ones currently found.
      string_array = tf.make_ndarray(tensor_events[0].tensor_proto)
      content = np.asscalar(string_array)
      layout_proto = layout_pb2.Layout()
      layout_proto.ParseFromString(tf.compat.as_bytes(content))

      if merged_layout:
        # Append the categories within this layout to the merged layout.
        for category in layout_proto.category:
          if category.title in title_to_category:
            # A category with this name has been seen before. Do not create a
            # new one. Merge their charts.
            title_to_category[category.title].chart.extend(category.chart)
          else:
            # This category has not been seen before.
            merged_layout.category.add().MergeFrom(category)
            title_to_category[category.title] = category
      else:
        # This is the first layout encountered.
        merged_layout = layout_proto
        for category in layout_proto.category:
          title_to_category[category.title] = category

    if merged_layout:
      return json_format.MessageToJson(
          merged_layout, including_default_value_fields=True)
    else:
      # No layout was found.
      return {}
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from runner.koan import *
import re
class AboutRegex(Koan):
    """
        These koans are based on the Ben's book: Regular Expressions in 10 minutes.
        I found this books very useful so I decided to write a koans in order to practice everything I had learned from it.
        http://www.forta.com/books/0672325667/
    """

    def test_matching_literal_text(self):
        """
            Lesson 1 Matching Literal String
        """
        string = "Hello, my name is Felix and this koans are based on the Ben's book: Regular Expressions in 10 minutes."
        m = re.search(__, string)
        self.assertTrue(m and m.group(0) and m.group(0)== 'Felix', "I want my name")

    def test_matching_literal_text_how_many(self):
        """
            Lesson 1 How many matches?

            The default behaviour of most regular expression engines is to return just the first match.
            In python you have the next options:

                match()    -->  Determine if the RE matches at the beginning of the string.
                search()   -->  Scan through a string, looking for any location where this RE matches.
                findall()  -->  Find all substrings where the RE matches, and returns them as a list.
                finditer() -->  Find all substrings where the RE matches, and returns them as an iterator.

        """
        string = "Hello, my name is Felix and this koans are based on the Ben's book: Regular Expressions in 10 minutes. Repeat My name is Felix"
        m = re.match('Felix', string) #TIP: Maybe match it's not the best option

        # I want to know how many times appears my name
        self.assertEqual(m, __)

    def test_matching_literal_text_not_case_sensitivity(self):
        """
            Lesson 1 Matching Literal String non case sensitivity.
            Most regex implementations also support matches that are not case sensitive. In python you can use re.IGNORECASE, in
            Javascript you can specify the optional i flag.
            In Ben's book you can see more languages.

        """
        string = "Hello, my name is Felix or felix and this koans is based on the Ben's book: Regular Expressions in 10 minutes."

        self.assertEqual(re.findall("felix", string), __)
        self.assertEqual(re.findall("felix", string, re.IGNORECASE), __)

    def test_matching_any_character(self):
        """
            Lesson 1 Matching any character

            . matches any character, alphabetic characters, digits and .
        """
        string = "pecks.xlx\n"    \
                + "orders1.xls\n" \
                + "apec1.xls\n"   \
                + "na1.xls\n"     \
                + "na2.xls\n"     \
                + "sa1.xls"

        # TIP: remember the name of this lesson

        change_this_search_string = 'a..xlx' # <-- I want to find all uses of myArray
        self.assertEquals(len(re.findall(change_this_search_string, string)),3)

    def test_matching_set_character(self):
        """
            Lesson 2 Matching sets of characters

            A set of characters is defined using the metacharacters [ and ]. Everything between them is part of the set and
            any one of the set members must match (but not all).
        """
        string = "sales.xlx\n"    \
                + "sales1.xls\n"  \
                + "orders3.xls\n" \
                + "apac1.xls\n" \
                + "sales2.xls\n"  \
                + "na1.xls\n"  \
                + "na2.xls\n"  \
                + "sa1.xls\n"  \
                + "ca1.xls"
        # I want to find all files for North America(na) or South America(sa), but not (ca)
        # TIP you can use the pattern .a. which matches in above test but in this case matches more than you want
        change_this_search_string = '[nsc]a[2-9].xls'
        self.assertEquals(len(re.findall(change_this_search_string, string)),3)

    def test_anything_but_matching(self):
        """
            Lesson 2 Using character set ranges
            Occasionally, you'll want a list of characters that you don't want to match.
            Character sets can be negated using the ^ metacharacter.

        """
        string = "sales.xlx\n"    \
                + "sales1.xls\n"  \
                + "orders3.xls\n" \
                + "apac1.xls\n" \
                + "sales2.xls\n"  \
                + "sales3.xls\n"  \
                + "europe2.xls\n"  \
                + "sam.xls\n"  \
                + "na1.xls\n"  \
                + "na2.xls\n"  \
                + "sa1.xls\n"  \
                + "ca1.xls"

        # I want to find the name sam
        change_this_search_string = '[^nc]am'
        self.assertEquals(re.findall(change_this_search_string, string), ['sam.xls'])

<commit_msg>Koan done: regex<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from runner.koan import *
import re
class AboutRegex(Koan):
    """
        These koans are based on the Ben's book: Regular Expressions in 10 minutes.
        I found this books very useful so I decided to write a koans in order to practice everything I had learned from it.
        http://www.forta.com/books/0672325667/
    """

    def test_matching_literal_text(self):
        """
            Lesson 1 Matching Literal String
        """
        string = "Hello, my name is Felix and this koans are based on the Ben's book: Regular Expressions in 10 minutes."
        m = re.search(r'Felix', string)
        self.assertTrue(m and m.group(0) and m.group(0)== 'Felix', "I want my name")

    def test_matching_literal_text_how_many(self):
        """
            Lesson 1 How many matches?

            The default behaviour of most regular expression engines is to return just the first match.
            In python you have the next options:

                match()    -->  Determine if the RE matches at the beginning of the string.
                search()   -->  Scan through a string, looking for any location where this RE matches.
                findall()  -->  Find all substrings where the RE matches, and returns them as a list.
                finditer() -->  Find all substrings where the RE matches, and returns them as an iterator.

        """
        string = "Hello, my name is Felix and this koans are based on the Ben's book: Regular Expressions in 10 minutes. Repeat My name is Felix"
        m = len(re.findall('Felix', string)) #TIP: Maybe match it's not the best option

        # I want to know how many times appears my name
        self.assertEqual(m, 2)

    def test_matching_literal_text_not_case_sensitivity(self):
        """
            Lesson 1 Matching Literal String non case sensitivity.
            Most regex implementations also support matches that are not case sensitive. In python you can use re.IGNORECASE, in
            Javascript you can specify the optional i flag.
            In Ben's book you can see more languages.

        """
        string = "Hello, my name is Felix or felix and this koans is based on the Ben's book: Regular Expressions in 10 minutes."

        self.assertEqual(re.findall("felix", string), ['felix'])
        self.assertEqual(re.findall("felix", string, re.IGNORECASE), ['Felix', 'felix'])

    def test_matching_any_character(self):
        """
            Lesson 1 Matching any character

            . matches any character, alphabetic characters, digits and .
        """
        string = "pecks.xlx\n"    \
                + "orders1.xls\n" \
                + "apec1.xls\n"   \
                + "na1.xls\n"     \
                + "na2.xls\n"     \
                + "sa1.xls"

        # TIP: remember the name of this lesson

        change_this_search_string = r'a.\.xl.' # <-- I want to find all uses of myArray
        self.assertEquals(len(re.findall(change_this_search_string, string)),3)

    def test_matching_set_character(self):
        """
            Lesson 2 Matching sets of characters

            A set of characters is defined using the metacharacters [ and ]. Everything between them is part of the set and
            any one of the set members must match (but not all).
        """
        string = "sales.xlx\n"    \
                + "sales1.xls\n"  \
                + "orders3.xls\n" \
                + "apac1.xls\n" \
                + "sales2.xls\n"  \
                + "na1.xls\n"  \
                + "na2.xls\n"  \
                + "sa1.xls\n"  \
                + "ca1.xls"
        # I want to find all files for North America(na) or South America(sa), but not (ca)
        # TIP you can use the pattern .a. which matches in above test but in this case matches more than you want
        change_this_search_string = '[ns]a[0-9].xls'
        self.assertEquals(len(re.findall(change_this_search_string, string)),3)

    def test_anything_but_matching(self):
        """
            Lesson 2 Using character set ranges
            Occasionally, you'll want a list of characters that you don't want to match.
            Character sets can be negated using the ^ metacharacter.

        """
        string = "sales.xlx\n"    \
                + "sales1.xls\n"  \
                + "orders3.xls\n" \
                + "apac1.xls\n" \
                + "sales2.xls\n"  \
                + "sales3.xls\n"  \
                + "europe2.xls\n"  \
                + "sam.xls\n"  \
                + "na1.xls\n"  \
                + "na2.xls\n"  \
                + "sa1.xls\n"  \
                + "ca1.xls"

        # I want to find the name sam
        change_this_search_string = '[^nc]am.xls'
        self.assertEquals(re.findall(change_this_search_string, string), ['sam.xls'])

<|endoftext|>


<commit_before>import sys
#
import numpy
#
import tabular_predDB.cython.ContinuousComponentModel as CCM
import tabular_predDB.cython.MultinomialComponentModel as MCM

class Bunch(dict):
    def __getattr__(self, key):
        if self.has_key(key):
            return self.get(key, None)
        else:
            raise AttributeError(key)
    def __setattr__(self, key, value):
        self[key] = value

Constraints = Bunch

def simple_predictive_sample(M_c, X_L, X_D, Y, Q, get_next_seed):
    num_rows = len(X_D[0])
    num_cols = len(M_c['column_metadata'])
    which_row = Q[0][0]
    assert(all([which_tuple[0]==which_row for which_tuple in Q]))
    # FIXME: handle unobserved rows
    is_observed_row = which_row < num_rows
    assert(is_observed_row)
    x = []
    if not is_observed_row:
        SEED = get_next_seed()
        x = simple_predictive_sample_unobserved(
            M_c, X_L, X_D, Y, Q, SEED)
    else:
        for query in Q:
            which_column = query[1]
            is_observed_col = which_column < num_cols
            # FIXME: not handling unobserved columns for now
            assert(is_observed_col)
            SEED = get_next_seed()
            sample = simple_predictive_sample_observed(
                M_c, X_L, X_D, which_row, which_column, SEED)
            x.append(sample)
    return x

def simple_predictive_sample_observed_old(M_c, X_L, X_D, which_row, which_column, SEED):
    modeltype = M_c['column_metadata'][which_column]['modeltype']
    column_hypers = X_L['column_hypers'][which_column]
    which_view = X_L['column_partition']['assignments'][which_column]
    view_state_i = X_L['view_state'][which_view]
    #
    column_name = M_c['idx_to_name'][str(which_column)]
    column_names = X_L['view_state'][which_view]['column_names']
    which_column_name = column_names.index(column_name)
    column_component_suffstats = \
        X_L['view_state'][which_view]['column_component_suffstats']
    column_component_suffstats_i = \
        column_component_suffstats[which_column_name]
    #
    which_cluster = X_D[which_view][which_row]
    cluster_count = sum(numpy.array(X_D[which_view])==which_cluster)
    component_suffstats = column_component_suffstats_i[which_cluster]
    component_model_constructor = None
    if modeltype == 'normal_inverse_gamma':
        component_model_constructor = CCM.p_ContinuousComponentModel
    elif modeltype == 'symmetric_dirichlet_discrete':
        component_model_constructor = MCM.p_MultinomialComponentModel
        # FIXME: this is a hack
        component_suffstats = dict(counts=component_suffstats)
    else:
        assert False, "simple_predictive_sample_observed: unknown modeltype: %s" % modeltype
    component_model = component_model_constructor(column_hypers,
                                                  count=cluster_count,
                                                  **component_suffstats)
    print component_model
    draw = component_model.get_draw(SEED)
    return draw

def simple_predictive_sample_observed(M_c, X_L, X_D, which_row, which_column, SEED):
    which_view = X_L['column_partition']['assignments'][which_column]
    which_cluster = X_D[which_view][which_row]
    cluster_model = create_cluster_model_from_X_L(M_c, X_L,
                                                  which_view, which_cluster)
    component_model = cluster_model[which_column]
    print component_model
    draw = component_model.get_draw(SEED)
    return draw

# determine views involved
# for each view:
#   determine probability of cluster in a view
#     crp probability
#     data probability
#   sample a cluster
#   generate all data from cluster
# join data

# def get_column_reordering(M_c, column_names):
#     name_to_idx = M_c['name_to_idx']
#     global_column_indices = [
#         name_to_idx[column_name]
#         for column_name in column_names
#         ]
#     column_reordering = numpy.argsort(global_column_indices)
#     return column_reordering

# def determine_constraints_in_view(view_state_i, M_c, Y):
#     Y_subset = []
#     name_to_idx = M_c['name_to_idx']
#     these_column_names = set(view_state_i['column_names'])
#     for y in Y:
#         constraint_column = y.index
#         constraint_column_name = name_to_idx[constraint_column]
#         if constraint_column_name in these_column_names:
#             Y_subset.append(y)
#     return Y_subset

# def determine_constraint_view_tuples(X_L, M_c, Y):
#     constraint_view_tuples = []
#     for view_state_i in X_L['view_state']:
#         constraints_in_view = determine_constraints_in_view(
#             view_state_i, M_c, Y)
#         constraint_view_tuple = (constraints_in_view, view_state_i)
#         constraint_view_tuples.append(constraint_view_tuple)
#     return constraint_view_tuples

def determine_cluster_crp_logps(view_state_i):
    counts = view_state_i['row_paritition_model']['counts']
    alpha = numpy.exp(view_state_i['row_paritition_model']['log_alpha'])
    counts_appended = numpy.append(counts, alpha)
    sum_counts_appended = sum(counts_appended)
    logps = numpy.log(counts_appended / float(sum_counts_appended))
    return logps

def extract_view_info(M_c, X_L, view_idx):
    view_state_i = X_L['view_state'][view_idx]
    column_names = view_state_i['column_names']
    column_component_suffstats = view_state_i['column_component_suffstats']
    # ensure view_state_i ordering matches global ordering
    global_column_indices = names_to_global_indices(column_names, M_c)
    column_metadata = numpy.array([
        M_c['column_metadata'][col_idx]
        for col_idx in global_column_indices
        ])
    column_hypers = numpy.array([
            X_L['column_hypers'][col_idx]
            for col_idx in global_column_indices
            ])
    zipped_column_info = zip(column_metadata, column_hypers, column_component_suffstats)
    zipped_column_info = dict(zip(global_column_indices, zipped_column_info))
    row_partition_model = view_state_i['row_partition_model']
    return zipped_column_info, row_partition_model

def get_column_info_subset(zipped_column_info, column_indices):
    column_info_subset = dict()
    for column_index in column_indices:
        if column_index in zipped_column_info:
            column_info_subset[column_index] = \
                zipped_column_info[column_index]
    return column_info_subset

def names_to_global_indices(column_names, M_c):
    name_to_idx = M_c['name_to_idx']
    # FIXME: str(column_name) is hack
    return [name_to_idx[str(column_name)] for column_name in column_names]

def get_component_model_constructor(modeltype):
    if modeltype == 'normal_inverse_gamma':
        component_model_constructor = CCM.p_ContinuousComponentModel
    elif modeltype == 'symmetric_dirichlet_discrete':
        component_model_constructor = MCM.p_MultinomialComponentModel
    else:
        assert False, "get_model_constructor: unknown modeltype: %s" % modeltype
    return component_model_constructor

def create_component_model(column_metadata, column_hypers, count, suffstats):
    modeltype = column_metadata['modeltype']
    component_model_constructor = get_component_model_constructor(modeltype)
    # FIXME: this is a hack
    if modeltype == 'symmetric_dirichlet_discrete':
        suffstats = dict(counts=suffstats)
    component_model = component_model_constructor(column_hypers, count,
                                                  **suffstats)
    return component_model

def create_cluster_model(zipped_column_info, row_partition_model,
                         cluster_idx):
    count = row_partition_model['counts'][cluster_idx]
    cluster_component_models = dict()
    for global_column_idx in zipped_column_info:
        column_metadata, column_hypers, column_component_suffstats = \
            zipped_column_info[global_column_idx]
        cluster_component_suffstats = column_component_suffstats[cluster_idx]
        component_model = create_component_model(
            column_metadata, column_hypers, count,
            cluster_component_suffstats)
        cluster_component_models[global_column_idx] = component_model
    return cluster_component_models

def create_empty_cluster_model(zipped_column_info):
    cluster_component_models = dict()
    for global_column_idx in zipped_column_info:
        column_metadata, column_hypers, column_component_suffstats = \
            zipped_column_info[global_column_idx]
        component_model = create_component_model(column_metadata,
                                                 column_hypers, None, None)
        cluster_component_models[global_column_idx] = component_model
    return cluster_component_models

def create_cluster_models(M_c, X_L, view_idx, constraints=None):
    zipped_column_info, row_partition_model = extract_view_info(
        M_c, X_L, view_idx)
    if constraints is not None:
        constraint_columns = [constraint.index for constraint in constraints]
        zipped_column_info = get_column_info_subset(
            zipped_column_info, constraint_columns)
    cluster_models = []
    for cluster_idx in range():
        cluster_model = create_cluster_model(
            zipped_column_info, row_partition_model, cluster_idx
            )
        cluster_models.append(cluster_model)
    empty_cluster_model = create_empty_cluster_model(zipped_column_info)
    cluster_models.append(empty_cluster_model)
    return cluster_models

def determine_cluster_data_logp(cluster_model, column_constraints):
    logp = 0
    for column_constraint in column_constraints:
        constraint_index = column_constraint.index
        constraint_value = column_constraint.value
        component_model = cluster_model[constraint_index]
        logp += component_model.cacl_element_predictive_logp(constraint_value)
    return logp

def determine_cluster_data_logps(M_c, X_L, X_D, Y, view_idx):
    cluster_models = create_cluster_models(M_c, X_L, view_idx, constraints=Y)
    for cluster_model in cluster_models:
        logp = determine_cluster_data_logp(cluster_model)
        logps.append(logp)
    return logps

def determine_cluster_logps(M_c, X_L, X_D, Y, view_idx):
    view_state_i = X_L['view_state'][view_idx]
    cluster_crp_logps = determine_cluster_crp_logps(view_state_i)
    cluster_crp_logps = numpy.array(cluster_crp_logps)
    cluster_data_logps = determine_cluster_data_logps(M_c, X_L, X_D, Y,
                                                      view_idx)
    cluster_data_logps = numpy.array(cluster_data_logps)
    #
    cluster_logps = cluster_crp_logps + cluster_data_logps
    return cluster_logps

def sample_from_cluster(cluster_model, SEED):
    random_state = numpy.random.RandomState(SEED)
    sample = []
    for component_model in cluster_model:
        seed_i = random_state.randint(sys.MAXINT)
        sample_i = component_model.get_draw(seed_i)
        sample.append(sample_i)
    return sample

def create_cluster_model_from_X_L(M_c, X_L, view_idx, cluster_idx):
    zipped_column_info, row_partition_model = extract_view_info(
        M_c, X_L, view_idx)
    cluster_model = create_cluster_model(
        zipped_column_info, row_partition_model, cluster_idx
        )
    return cluster_model

def sample_from_view(M_c, X_L, X_D, Y, view_idx, SEED):
    random_state = numpy.random.RandomState(SEED)
    cluster_logps = determine_cluster_logps(M_c, X_L, X_D, Y, view_idx)
    draw = numpy.nonzero(numpy.random.multinomial(1, cluster_logps))[0]
    cluster_model = create_cluster_model_from_X_L(M_c, X_L, view_idx, draw)
    sample = sample_from_cluster(cluster_model)
    return sample

def simple_predictive_sample_unobserved(M_c, X_L, X_D, Y, which_column,
                                        SEED):
    num_views = len(X_D)
    cluster_logps_list = []
    for view_idx in range(num_views):
        cluster_logps = determine_cluster_logps(M_c, X_L, X_D, Y, view_idx)
        cluster_logps_list.append(cluster_logp)
    # sample a cluster index from each view
    random_state = numpy.random.RandomState(SEED)
    sample_list = []
    for view_idx, cluster_logps in enumerate(cluster_lopgs_list):
        draw = numpy.nonzero(numpy.random.multinomial(1, cluster_logps))[0]
        cluster_model = create_cluster_model_from_X_L(M_c, X_L, view_idx, draw)
        sample = sample_from_cluster(cluster_model)
        sample_list.append(sample)
    # FIXME: clean up and make a lookup from individual sample dictionaries
    return sample_list

def sample_from_view_cluster(M_c, X_L, X_D, Q, SEED):
    pass

<commit_msg>remove debugging prints<commit_after>import sys
#
import numpy
#
import tabular_predDB.cython.ContinuousComponentModel as CCM
import tabular_predDB.cython.MultinomialComponentModel as MCM

class Bunch(dict):
    def __getattr__(self, key):
        if self.has_key(key):
            return self.get(key, None)
        else:
            raise AttributeError(key)
    def __setattr__(self, key, value):
        self[key] = value

Constraints = Bunch

def simple_predictive_sample(M_c, X_L, X_D, Y, Q, get_next_seed):
    num_rows = len(X_D[0])
    num_cols = len(M_c['column_metadata'])
    which_row = Q[0][0]
    assert(all([which_tuple[0]==which_row for which_tuple in Q]))
    # FIXME: handle unobserved rows
    is_observed_row = which_row < num_rows
    assert(is_observed_row)
    x = []
    if not is_observed_row:
        SEED = get_next_seed()
        x = simple_predictive_sample_unobserved(
            M_c, X_L, X_D, Y, Q, SEED)
    else:
        for query in Q:
            which_column = query[1]
            is_observed_col = which_column < num_cols
            # FIXME: not handling unobserved columns for now
            assert(is_observed_col)
            SEED = get_next_seed()
            sample = simple_predictive_sample_observed(
                M_c, X_L, X_D, which_row, which_column, SEED)
            x.append(sample)
    return x

def simple_predictive_sample_observed_old(M_c, X_L, X_D, which_row, which_column, SEED):
    modeltype = M_c['column_metadata'][which_column]['modeltype']
    column_hypers = X_L['column_hypers'][which_column]
    which_view = X_L['column_partition']['assignments'][which_column]
    view_state_i = X_L['view_state'][which_view]
    #
    column_name = M_c['idx_to_name'][str(which_column)]
    column_names = X_L['view_state'][which_view]['column_names']
    which_column_name = column_names.index(column_name)
    column_component_suffstats = \
        X_L['view_state'][which_view]['column_component_suffstats']
    column_component_suffstats_i = \
        column_component_suffstats[which_column_name]
    #
    which_cluster = X_D[which_view][which_row]
    cluster_count = sum(numpy.array(X_D[which_view])==which_cluster)
    component_suffstats = column_component_suffstats_i[which_cluster]
    component_model_constructor = None
    if modeltype == 'normal_inverse_gamma':
        component_model_constructor = CCM.p_ContinuousComponentModel
    elif modeltype == 'symmetric_dirichlet_discrete':
        component_model_constructor = MCM.p_MultinomialComponentModel
        # FIXME: this is a hack
        component_suffstats = dict(counts=component_suffstats)
    else:
        assert False, "simple_predictive_sample_observed: unknown modeltype: %s" % modeltype
    component_model = component_model_constructor(column_hypers,
                                                  count=cluster_count,
                                                  **component_suffstats)
    draw = component_model.get_draw(SEED)
    return draw

def simple_predictive_sample_observed(M_c, X_L, X_D, which_row, which_column, SEED):
    which_view = X_L['column_partition']['assignments'][which_column]
    which_cluster = X_D[which_view][which_row]
    cluster_model = create_cluster_model_from_X_L(M_c, X_L,
                                                  which_view, which_cluster)
    component_model = cluster_model[which_column]
    draw = component_model.get_draw(SEED)
    return draw

# determine views involved
# for each view:
#   determine probability of cluster in a view
#     crp probability
#     data probability
#   sample a cluster
#   generate all data from cluster
# join data

# def get_column_reordering(M_c, column_names):
#     name_to_idx = M_c['name_to_idx']
#     global_column_indices = [
#         name_to_idx[column_name]
#         for column_name in column_names
#         ]
#     column_reordering = numpy.argsort(global_column_indices)
#     return column_reordering

# def determine_constraints_in_view(view_state_i, M_c, Y):
#     Y_subset = []
#     name_to_idx = M_c['name_to_idx']
#     these_column_names = set(view_state_i['column_names'])
#     for y in Y:
#         constraint_column = y.index
#         constraint_column_name = name_to_idx[constraint_column]
#         if constraint_column_name in these_column_names:
#             Y_subset.append(y)
#     return Y_subset

# def determine_constraint_view_tuples(X_L, M_c, Y):
#     constraint_view_tuples = []
#     for view_state_i in X_L['view_state']:
#         constraints_in_view = determine_constraints_in_view(
#             view_state_i, M_c, Y)
#         constraint_view_tuple = (constraints_in_view, view_state_i)
#         constraint_view_tuples.append(constraint_view_tuple)
#     return constraint_view_tuples

def determine_cluster_crp_logps(view_state_i):
    counts = view_state_i['row_paritition_model']['counts']
    alpha = numpy.exp(view_state_i['row_paritition_model']['log_alpha'])
    counts_appended = numpy.append(counts, alpha)
    sum_counts_appended = sum(counts_appended)
    logps = numpy.log(counts_appended / float(sum_counts_appended))
    return logps

def extract_view_info(M_c, X_L, view_idx):
    view_state_i = X_L['view_state'][view_idx]
    column_names = view_state_i['column_names']
    column_component_suffstats = view_state_i['column_component_suffstats']
    # ensure view_state_i ordering matches global ordering
    global_column_indices = names_to_global_indices(column_names, M_c)
    column_metadata = numpy.array([
        M_c['column_metadata'][col_idx]
        for col_idx in global_column_indices
        ])
    column_hypers = numpy.array([
            X_L['column_hypers'][col_idx]
            for col_idx in global_column_indices
            ])
    zipped_column_info = zip(column_metadata, column_hypers, column_component_suffstats)
    zipped_column_info = dict(zip(global_column_indices, zipped_column_info))
    row_partition_model = view_state_i['row_partition_model']
    return zipped_column_info, row_partition_model

def get_column_info_subset(zipped_column_info, column_indices):
    column_info_subset = dict()
    for column_index in column_indices:
        if column_index in zipped_column_info:
            column_info_subset[column_index] = \
                zipped_column_info[column_index]
    return column_info_subset

def names_to_global_indices(column_names, M_c):
    name_to_idx = M_c['name_to_idx']
    # FIXME: str(column_name) is hack
    return [name_to_idx[str(column_name)] for column_name in column_names]

def get_component_model_constructor(modeltype):
    if modeltype == 'normal_inverse_gamma':
        component_model_constructor = CCM.p_ContinuousComponentModel
    elif modeltype == 'symmetric_dirichlet_discrete':
        component_model_constructor = MCM.p_MultinomialComponentModel
    else:
        assert False, "get_model_constructor: unknown modeltype: %s" % modeltype
    return component_model_constructor

def create_component_model(column_metadata, column_hypers, count, suffstats):
    modeltype = column_metadata['modeltype']
    component_model_constructor = get_component_model_constructor(modeltype)
    # FIXME: this is a hack
    if modeltype == 'symmetric_dirichlet_discrete':
        suffstats = dict(counts=suffstats)
    component_model = component_model_constructor(column_hypers, count,
                                                  **suffstats)
    return component_model

def create_cluster_model(zipped_column_info, row_partition_model,
                         cluster_idx):
    count = row_partition_model['counts'][cluster_idx]
    cluster_component_models = dict()
    for global_column_idx in zipped_column_info:
        column_metadata, column_hypers, column_component_suffstats = \
            zipped_column_info[global_column_idx]
        cluster_component_suffstats = column_component_suffstats[cluster_idx]
        component_model = create_component_model(
            column_metadata, column_hypers, count,
            cluster_component_suffstats)
        cluster_component_models[global_column_idx] = component_model
    return cluster_component_models

def create_empty_cluster_model(zipped_column_info):
    cluster_component_models = dict()
    for global_column_idx in zipped_column_info:
        column_metadata, column_hypers, column_component_suffstats = \
            zipped_column_info[global_column_idx]
        component_model = create_component_model(column_metadata,
                                                 column_hypers, None, None)
        cluster_component_models[global_column_idx] = component_model
    return cluster_component_models

def create_cluster_models(M_c, X_L, view_idx, constraints=None):
    zipped_column_info, row_partition_model = extract_view_info(
        M_c, X_L, view_idx)
    if constraints is not None:
        constraint_columns = [constraint.index for constraint in constraints]
        zipped_column_info = get_column_info_subset(
            zipped_column_info, constraint_columns)
    cluster_models = []
    for cluster_idx in range():
        cluster_model = create_cluster_model(
            zipped_column_info, row_partition_model, cluster_idx
            )
        cluster_models.append(cluster_model)
    empty_cluster_model = create_empty_cluster_model(zipped_column_info)
    cluster_models.append(empty_cluster_model)
    return cluster_models

def determine_cluster_data_logp(cluster_model, column_constraints):
    logp = 0
    for column_constraint in column_constraints:
        constraint_index = column_constraint.index
        constraint_value = column_constraint.value
        component_model = cluster_model[constraint_index]
        logp += component_model.cacl_element_predictive_logp(constraint_value)
    return logp

def determine_cluster_data_logps(M_c, X_L, X_D, Y, view_idx):
    cluster_models = create_cluster_models(M_c, X_L, view_idx, constraints=Y)
    for cluster_model in cluster_models:
        logp = determine_cluster_data_logp(cluster_model)
        logps.append(logp)
    return logps

def determine_cluster_logps(M_c, X_L, X_D, Y, view_idx):
    view_state_i = X_L['view_state'][view_idx]
    cluster_crp_logps = determine_cluster_crp_logps(view_state_i)
    cluster_crp_logps = numpy.array(cluster_crp_logps)
    cluster_data_logps = determine_cluster_data_logps(M_c, X_L, X_D, Y,
                                                      view_idx)
    cluster_data_logps = numpy.array(cluster_data_logps)
    #
    cluster_logps = cluster_crp_logps + cluster_data_logps
    return cluster_logps

def sample_from_cluster(cluster_model, SEED):
    random_state = numpy.random.RandomState(SEED)
    sample = []
    for component_model in cluster_model:
        seed_i = random_state.randint(sys.MAXINT)
        sample_i = component_model.get_draw(seed_i)
        sample.append(sample_i)
    return sample

def create_cluster_model_from_X_L(M_c, X_L, view_idx, cluster_idx):
    zipped_column_info, row_partition_model = extract_view_info(
        M_c, X_L, view_idx)
    cluster_model = create_cluster_model(
        zipped_column_info, row_partition_model, cluster_idx
        )
    return cluster_model

def sample_from_view(M_c, X_L, X_D, Y, view_idx, SEED):
    random_state = numpy.random.RandomState(SEED)
    cluster_logps = determine_cluster_logps(M_c, X_L, X_D, Y, view_idx)
    draw = numpy.nonzero(numpy.random.multinomial(1, cluster_logps))[0]
    cluster_model = create_cluster_model_from_X_L(M_c, X_L, view_idx, draw)
    sample = sample_from_cluster(cluster_model)
    return sample

def simple_predictive_sample_unobserved(M_c, X_L, X_D, Y, which_column,
                                        SEED):
    num_views = len(X_D)
    cluster_logps_list = []
    for view_idx in range(num_views):
        cluster_logps = determine_cluster_logps(M_c, X_L, X_D, Y, view_idx)
        cluster_logps_list.append(cluster_logp)
    # sample a cluster index from each view
    random_state = numpy.random.RandomState(SEED)
    sample_list = []
    for view_idx, cluster_logps in enumerate(cluster_lopgs_list):
        draw = numpy.nonzero(numpy.random.multinomial(1, cluster_logps))[0]
        cluster_model = create_cluster_model_from_X_L(M_c, X_L, view_idx, draw)
        sample = sample_from_cluster(cluster_model)
        sample_list.append(sample)
    # FIXME: clean up and make a lookup from individual sample dictionaries
    return sample_list

def sample_from_view_cluster(M_c, X_L, X_D, Q, SEED):
    pass

<|endoftext|>


<commit_before># ----------------------------------------------------------------------------
# PyWavefront
# Copyright (c) 2018 Kurt Yoder
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#  * Neither the name of PyWavefront nor the names of its
#    contributors may be used to endorse or promote products
#    derived from this software without specific prior written
#    permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# ----------------------------------------------------------------------------
import pyglet
from pyglet.gl import *

from pywavefront import Wavefront
from pywavefront.mesh import Mesh
from pywavefront.material import Material

VERTEX_FORMATS = {
    'V3F': GL_V3F,
    'C3F_V3F': GL_C3F_V3F,
    'N3F_V3F': GL_N3F_V3F,
    'T2F_V3F': GL_T2F_V3F,
    # 'C3F_N3F_V3F': GL_C3F_N3F_V3F,  # Unsupported
    'T2F_C3F_V3F': GL_T2F_C3F_V3F,
    'T2F_N3F_V3F': GL_T2F_N3F_V3F,
    # 'T2F_C3F_N3F_V3F': GL_T2F_C3F_N3F_V3F,  # Unsupported
}

def draw(instance):
    """Generic draw function"""
    # Draw Wavefront instance
    if isinstance(instance, Wavefront):
        draw_materials(instance.materials)
    # Draw single material
    elif isinstance(instance, Material):
        draw_material(instance)
    # Draw dict of materials
    elif isinstance(instance, dict):
        draw_materials(instance)
    else:
        raise ValueError("Cannot figure out how to draw: {}".format(instance))

def draw_materials(materials):
    """Draw a dict of meshes"""
    for name, material in materials.items():
        draw_material(material)

def draw_material(material, face=GL_FRONT_AND_BACK):
    """Draw a single material"""
    if material.gl_floats is None:
        material.gl_floats = (GLfloat * len(material.vertices))(*material.vertices)
        material.triangle_count = len(material.vertices) / material.vertex_size

    vertex_format = VERTEX_FORMATS.get(material.vertex_format)
    if not vertex_format:
        raise ValueError("Vertex format {} not supported by pyglet".format(vertex_format))

    glPushClientAttrib(GL_CLIENT_VERTEX_ARRAY_BIT)
    glPushAttrib(GL_CURRENT_BIT | GL_ENABLE_BIT | GL_LIGHTING_BIT)
    glEnable(GL_CULL_FACE)
    glCullFace(GL_BACK)

    if material.texture and material.has_uvs:
        bind_texture(material.texture)
    else:
        glDisable(GL_TEXTURE_2D)

    glMaterialfv(face, GL_DIFFUSE, gl_light(material.diffuse))
    glMaterialfv(face, GL_AMBIENT, gl_light(material.ambient))
    glMaterialfv(face, GL_SPECULAR, gl_light(material.specular))
    glMaterialfv(face, GL_EMISSION, gl_light(material.emissive))
    glMaterialf(face, GL_SHININESS, material.shininess)
    glEnable(GL_LIGHT0)

    if material.has_normals:
        glEnable(GL_LIGHTING)
    else:
        glDisable(GL_LIGHTING)

    glInterleavedArrays(vertex_format, 0, material.gl_floats)
    glDrawArrays(GL_TRIANGLES, 0, int(material.triangle_count))

    glPopAttrib()
    glPopClientAttrib()

def gl_light(lighting):
    """Return a GLfloat with length 4, containing the 4 lighting values."""
    return (GLfloat * 4)(*(lighting))

def bind_texture(texture):
    """Draw a single texture"""
    if not getattr(texture, 'image', None):
        texture.image = load_image(texture.image_name)

    glEnable(texture.image.target)
    glBindTexture(texture.image.target, texture.image.id)
    gl.glTexParameterf(texture.image.target,
                       gl.GL_TEXTURE_WRAP_S, gl.GL_CLAMP_TO_EDGE)
    gl.glTexParameterf(texture.image.target,
                       gl.GL_TEXTURE_WRAP_T, gl.GL_CLAMP_TO_EDGE)

def load_image(name):
    """Load an image"""
    image = pyglet.image.load(name).texture
    verify_dimensions(image)
    return image

def verify_dimensions(image):
    verify(image, 'width')
    verify(image, 'height')

def verify(image, dimension):
    value = image.__getattribute__(dimension)

    while value > 1:
        div_float = float(value) / 2.0
        div_int = int(div_float)

        if not (div_float == div_int):
            raise Exception('image %s is %d, which is not a power of 2' % (
                dimension, image.__getattribute__(dimension)))

        value = div_int
<commit_msg>Visualization should cap shinyness<commit_after># ----------------------------------------------------------------------------
# PyWavefront
# Copyright (c) 2018 Kurt Yoder
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#  * Neither the name of PyWavefront nor the names of its
#    contributors may be used to endorse or promote products
#    derived from this software without specific prior written
#    permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# ----------------------------------------------------------------------------
import pyglet
from pyglet.gl import *

from pywavefront import Wavefront
from pywavefront.mesh import Mesh
from pywavefront.material import Material

VERTEX_FORMATS = {
    'V3F': GL_V3F,
    'C3F_V3F': GL_C3F_V3F,
    'N3F_V3F': GL_N3F_V3F,
    'T2F_V3F': GL_T2F_V3F,
    # 'C3F_N3F_V3F': GL_C3F_N3F_V3F,  # Unsupported
    'T2F_C3F_V3F': GL_T2F_C3F_V3F,
    'T2F_N3F_V3F': GL_T2F_N3F_V3F,
    # 'T2F_C3F_N3F_V3F': GL_T2F_C3F_N3F_V3F,  # Unsupported
}

def draw(instance):
    """Generic draw function"""
    # Draw Wavefront instance
    if isinstance(instance, Wavefront):
        draw_materials(instance.materials)
    # Draw single material
    elif isinstance(instance, Material):
        draw_material(instance)
    # Draw dict of materials
    elif isinstance(instance, dict):
        draw_materials(instance)
    else:
        raise ValueError("Cannot figure out how to draw: {}".format(instance))

def draw_materials(materials):
    """Draw a dict of meshes"""
    for name, material in materials.items():
        draw_material(material)

def draw_material(material, face=GL_FRONT_AND_BACK):
    """Draw a single material"""
    if material.gl_floats is None:
        material.gl_floats = (GLfloat * len(material.vertices))(*material.vertices)
        material.triangle_count = len(material.vertices) / material.vertex_size

    vertex_format = VERTEX_FORMATS.get(material.vertex_format)
    if not vertex_format:
        raise ValueError("Vertex format {} not supported by pyglet".format(vertex_format))

    glPushClientAttrib(GL_CLIENT_VERTEX_ARRAY_BIT)
    glPushAttrib(GL_CURRENT_BIT | GL_ENABLE_BIT | GL_LIGHTING_BIT)
    glEnable(GL_CULL_FACE)
    glCullFace(GL_BACK)

    if material.texture and material.has_uvs:
        bind_texture(material.texture)
    else:
        glDisable(GL_TEXTURE_2D)

    glMaterialfv(face, GL_DIFFUSE, gl_light(material.diffuse))
    glMaterialfv(face, GL_AMBIENT, gl_light(material.ambient))
    glMaterialfv(face, GL_SPECULAR, gl_light(material.specular))
    glMaterialfv(face, GL_EMISSION, gl_light(material.emissive))
    glMaterialf(face, GL_SHININESS, min(128.0, material.shininess))
    glEnable(GL_LIGHT0)

    if material.has_normals:
        glEnable(GL_LIGHTING)
    else:
        glDisable(GL_LIGHTING)

    glInterleavedArrays(vertex_format, 0, material.gl_floats)
    glDrawArrays(GL_TRIANGLES, 0, int(material.triangle_count))

    glPopAttrib()
    glPopClientAttrib()

def gl_light(lighting):
    """Return a GLfloat with length 4, containing the 4 lighting values."""
    return (GLfloat * 4)(*(lighting))

def bind_texture(texture):
    """Draw a single texture"""
    if not getattr(texture, 'image', None):
        texture.image = load_image(texture.image_name)

    glEnable(texture.image.target)
    glBindTexture(texture.image.target, texture.image.id)
    gl.glTexParameterf(texture.image.target,
                       gl.GL_TEXTURE_WRAP_S, gl.GL_CLAMP_TO_EDGE)
    gl.glTexParameterf(texture.image.target,
                       gl.GL_TEXTURE_WRAP_T, gl.GL_CLAMP_TO_EDGE)

def load_image(name):
    """Load an image"""
    image = pyglet.image.load(name).texture
    verify_dimensions(image)
    return image

def verify_dimensions(image):
    verify(image, 'width')
    verify(image, 'height')

def verify(image, dimension):
    value = image.__getattribute__(dimension)

    while value > 1:
        div_float = float(value) / 2.0
        div_int = int(div_float)

        if not (div_float == div_int):
            raise Exception('image %s is %d, which is not a power of 2' % (
                dimension, image.__getattribute__(dimension)))

        value = div_int
<|endoftext|>