<commit_before>from . import nn
from . import util

import sys
import os
import logging

import chainer.optimizers as O
import chainer.links as L
from chainer import Variable
from chainer import cuda

def _log_str(lst):
    s = []
    for k, v in lst:
        s.append(k + ':')
        if isinstance(v, float):
            v_str = '{:.6f}'.format(v)
        else:
            v_str = str(v)
        s.append(v_str)
    return '\t'.join(s)

def train(args):
    if args.gpu is not None:
        cuda.get_device(args.gpu).use()

    os.makedirs(args.model)

    # set up logger
    logger = logging.getLogger()
    logging.basicConfig(level=logging.INFO)
    log_path = os.path.join(args.model, 'log')
    file_handler = logging.FileHandler(log_path)
    fmt = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    file_handler.setFormatter(fmt)
    logger.addHandler(file_handler)

    # set up optimizer
    optim_name = args.optim[0]
    assert not args.decay_lr or optim_name == 'SGD', 'learning-rate decay is only supported for SGD'
    optim_args = map(float, args.optim[1:])
    optimizer = getattr(O, optim_name)(*optim_args)

    # load data
    logger.info('Loading data...')
    corpus, vocab_word, vocab_char, vocab_tag = util.load_conll(args.data, args.vocab)

    # load pre-trained embeddings
    init_emb = None
    if args.init_emb:
        logger.info('Loading word embeddings...')
        assert args.init_emb_words
        init_emb = util.load_init_emb(args.init_emb, args.init_emb_words, vocab_word)
        emb_dim = init_emb.shape[1]
    else:
        emb_dim = args.word_emb

    # create batches
    logger.info('Creating batches...')
    batches = util.create_batches(corpus, vocab_word, vocab_char, vocab_tag, args.batch,
                                  args.word_window, args.char_window, gpu=args.gpu, shuffle=not args.no_shuffle)

    # set up tagger
    tagger = nn.NnTagger(
            word_vocab_size=vocab_word.size(), word_emb_dim=emb_dim, word_window_size=args.word_window, word_init_emb=init_emb, word_hidden_dim=args.word_hidden,
            use_char=args.use_char, char_vocab_size=vocab_char.size(), char_emb_dim=args.char_emb, char_window_size=args.char_window, char_hidden_dim=args.char_hidden,
            tag_num=vocab_tag.size())
    classifier = L.Classifier(tagger)

    initial_lr = None
    if args.decay_lr:
        initial_lr = optimizer.lr

    # set up GPU
    if args.gpu is not None:
        classifier.to_gpu()

    optimizer.setup(classifier)

    # create directory
    vocab_word.save(os.path.join(args.model, 'vocab_word'))
    vocab_char.save(os.path.join(args.model, 'vocab_char'))
    vocab_tag.save(os.path.join(args.model, 'vocab_tag'))

    # training loop
    for n in range(args.epoch):
        # decay learning rate
        if args.decay_lr:
            optimizer.lr = initial_lr / (n + 1)
            logger.info('Learning rate set to: {}'.format(optimizer.lr))

        for i, ((word_ids_data, (char_ids_data, char_boundaries)), t_data) in enumerate(batches):
            batch_size, batch_length = word_ids_data.shape

            word_ids = Variable(word_ids_data)
            char_ids = Variable(char_ids_data)
            t = Variable(t_data)
            batch = word_ids, (char_ids, char_boundaries)
            optimizer.update(classifier, batch, t)

            logger.info(_log_str([
                ('epoch', n),
                ('batch', i),
                ('loss', float(classifier.loss.data)),
                ('acc', float(classifier.accuracy.data)),
                ('size', batch_size),
                ('len', batch_length),
            ]))

        # save current model
        dest_path = os.path.join(args.model, 'epoch' + str(n))
        tagger.save(dest_path)
<commit_msg>Print time per minibatch<commit_after>from . import nn
from . import util

import sys
import os
import logging
import time

import chainer.optimizers as O
import chainer.links as L
from chainer import Variable
from chainer import cuda

def _log_str(lst):
    s = []
    for k, v in lst:
        s.append(k + ':')
        if isinstance(v, float):
            v_str = '{:.6f}'.format(v)
        else:
            v_str = str(v)
        s.append(v_str)
    return '\t'.join(s)

def train(args):
    if args.gpu is not None:
        cuda.get_device(args.gpu).use()

    os.makedirs(args.model)

    # set up logger
    logger = logging.getLogger()
    logging.basicConfig(level=logging.INFO)
    log_path = os.path.join(args.model, 'log')
    file_handler = logging.FileHandler(log_path)
    fmt = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    file_handler.setFormatter(fmt)
    logger.addHandler(file_handler)

    # set up optimizer
    optim_name = args.optim[0]
    assert not args.decay_lr or optim_name == 'SGD', 'learning-rate decay is only supported for SGD'
    optim_args = map(float, args.optim[1:])
    optimizer = getattr(O, optim_name)(*optim_args)

    # load data
    logger.info('Loading data...')
    corpus, vocab_word, vocab_char, vocab_tag = util.load_conll(args.data, args.vocab)

    # load pre-trained embeddings
    init_emb = None
    if args.init_emb:
        logger.info('Loading word embeddings...')
        assert args.init_emb_words
        init_emb = util.load_init_emb(args.init_emb, args.init_emb_words, vocab_word)
        emb_dim = init_emb.shape[1]
    else:
        emb_dim = args.word_emb

    # create batches
    logger.info('Creating batches...')
    batches = util.create_batches(corpus, vocab_word, vocab_char, vocab_tag, args.batch,
                                  args.word_window, args.char_window, gpu=args.gpu, shuffle=not args.no_shuffle)

    # set up tagger
    tagger = nn.NnTagger(
            word_vocab_size=vocab_word.size(), word_emb_dim=emb_dim, word_window_size=args.word_window, word_init_emb=init_emb, word_hidden_dim=args.word_hidden,
            use_char=args.use_char, char_vocab_size=vocab_char.size(), char_emb_dim=args.char_emb, char_window_size=args.char_window, char_hidden_dim=args.char_hidden,
            tag_num=vocab_tag.size())
    classifier = L.Classifier(tagger)

    initial_lr = None
    if args.decay_lr:
        initial_lr = optimizer.lr

    # set up GPU
    if args.gpu is not None:
        classifier.to_gpu()

    optimizer.setup(classifier)

    # create directory
    vocab_word.save(os.path.join(args.model, 'vocab_word'))
    vocab_char.save(os.path.join(args.model, 'vocab_char'))
    vocab_tag.save(os.path.join(args.model, 'vocab_tag'))

    # training loop
    for n in range(args.epoch):
        # decay learning rate
        if args.decay_lr:
            optimizer.lr = initial_lr / (n + 1)
            logger.info('Learning rate set to: {}'.format(optimizer.lr))

        for i, ((word_ids_data, (char_ids_data, char_boundaries)), t_data) in enumerate(batches):
            batch_size, batch_length = word_ids_data.shape

            time_start = time.time()
            word_ids = Variable(word_ids_data)
            char_ids = Variable(char_ids_data)
            t = Variable(t_data)
            batch = word_ids, (char_ids, char_boundaries)
            optimizer.update(classifier, batch, t)
            time_end = time.time()
            time_delta = time_end - time_start

            logger.info(_log_str([
                ('epoch', n),
                ('batch', i),
                ('loss', float(classifier.loss.data)),
                ('acc', float(classifier.accuracy.data)),
                ('size', batch_size),
                ('len', batch_length),
                ('time', int(time_delta * 1000)),
            ]))

        # save current model
        dest_path = os.path.join(args.model, 'epoch' + str(n))
        tagger.save(dest_path)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
chanjo.cli
~~~~~~~~~~~

Command line interface (console entry points). Based on Click_.

.. _Click: http://click.pocoo.org/
"""
from __future__ import absolute_import, unicode_literals
import json
from pkg_resources import iter_entry_points

import click
from path import path
import toml as markup
from toolz import pipe
from toolz.curried import map

from . import (
  __title__, __version__, __banner__,
  annotator, builder, exporter, importer, sex_checker
)
from ._compat import text_type
from .config import Config, init_pipeline
from .store import Store
from .utils import (
  serialize_interval_plus, serialize_interval, id_generator
)

CONFIG_NAME="%(program)s.%(extension)s" % dict(
  program=__title__,
  extension=markup.__name__  # Works for JSON, TOML, YAML ...
)

# shared options/arguments that don't belong to any specific command
# path to a BAM-file (without opening as a stream)
bam_path_argument = click.argument('bam_path', type=click.Path(exists=True))
# open an input stream from a file (usually a BED-file)
in_argument = click.argument(
  'in_stream', type=click.File(), default='-', required=False)
# open an output stream (default is stdout) to write to
out_option = click.option('--out', type=click.File('w'), default='-',
                          help="define an output file other than 'stdout'")
# string to prepend to the contig ids to match e.g. the BAM-file
prepend_option = click.option(
  '--prepend', default='', help='prepend a string to each contig')

@click.group()
@click.option(
  '--config',
  default=CONFIG_NAME,
  type=click.File('w'),
  help='path to Chanjo config file'
)
@click.option('--db', type=text_type, help='path/URI of the SQL database')
@click.option(
  '--dialect',
  type=click.Choice(['sqlite', 'mysql']),
  help='type of SQL database'
)
@click.option(
  '--force', is_flag=True, help='overwrite existing assets without warning')
@click.option('--verbose', is_flag=True)
@click.version_option(__version__)
@click.pass_context
def cli(context, config, db, dialect, force, verbose):
  """Clinical sequencing coverage analysis tool."""
  # avoid setting global defaults in Click options, do it below when
  # updating the config object
  context.obj = Config(config, markup=markup)

  # global defaults
  db_path = db or context.obj.get('db', 'coverage.sqlite')
  db_dialect = dialect or context.obj.get('dialect', 'sqlite')

  context.obj['db'] = Store(db_path, dialect=db_dialect)
  context.obj['force'] = force

  # update the context with new defaults from the config file
  context.default_map = context.obj

@cli.command()
@click.option('--remove', is_flag=True, help='remove config option')
@click.argument('key')
@click.argument('value', required=False)
@click.pass_context
def config(context, key, value, remove):
  """Only handles string values."""
  # update or delete the config key-value pair
  # the key can be provide as a path to nested commands: "user.name"
  if remove:
    context.obj.unset(key, scope=context.obj.user_data)

  else:
    if value is None:
      raise ValueError("Unless remove a setting, you must submit a 'value'")

    context.obj.set(key, value, scope=context.obj.user_data)

  # persist updates to the config file
  context.obj.save()

@cli.command()
@click.pass_context
def init(context):
  """Walk user through setting up a new config file."""
  # print a nice welcome message
  click.echo(__banner__)

  # shortcut
  config = context.obj

  questions = [
    ('annotate.cutoff', 'sufficient coverage',
      config.get('annotate', {}).get('cutoff', 10)),
    ('dialect', 'preferred SQL-dialect', config.get('dialect', 'sqlite')),
    ('db', 'central database path/URI', config.get('db', 'coverage.sqlite'))
  ]

  # launch init pipeline
  init_pipeline(__title__, config, questions)

@cli.command()
@click.option('--adapter', default='ccds', help='plugin to use for conversion')
@click.option(
  'list_all', '--list', is_flag=True, help='show all installed adapters')
@out_option
@in_argument
@click.pass_context
def convert(context, in_stream, out, adapter, list_all):
  """Convert a reference database file to a Chanjo BED interval file

  \b
  IN_STREAM: interval reference file (e.g. CCDS database dump)
  """
  entry_points = iter_entry_points('chanjo.converters')

  if list_all:
    # list the installed converter options
    for entry_point in entry_points:
      # compose and print the message
      segments = dict(
        program=__title__,
        note=click.style('converter', fg='cyan'),
        plugin=entry_point.name
      )
      click.echo("%(program)s %(note)s %(plugin)s" % segments)

  else:
    # loop though each of the installed plugins
    for entry_point in entry_points:
      # break out when finding the matching adapter
      if entry_point.name == adapter:
        break

    # load entry point function
    converter_pipeline = entry_point.load()

    # execute converter pipeline
    bed_lines = pipe(
      converter_pipeline(in_stream),
      map(serialize_interval)         # stringify/bedify
    )

    # reduce/write the BED lines
    for bed_line in bed_lines:
      click.echo(bed_line, file=out)

@cli.command()
@in_argument
@click.pass_context
def build(context, in_stream):
  """Construct a new skeleton SQL interval store.

  \b
  IN_STREAM: Chanjo-style BED-file with interval definitions
  """
  # build a new skeleton SQL interval store
  builder.pipeline(
    chanjo_db=context.obj['db'],
    bed_stream=in_stream,
    overwrite=context.obj['force']
  )

@cli.command()
@out_option
@click.option(
  '--header/--no-header', is_flag=True, default=True, help="include headers")
@click.pass_context
def export(context, out, header):
  """Export an interval BED stream from an existing Chanjo store."""
  # build a new skeleton SQL interval store
  bed_lines = exporter.pipeline(
    chanjo_db=context.obj['db'],
    include_header=header
  )

  # reduce/write the BED lines
  for bed_line in bed_lines:
    click.echo(bed_line, file=out)

@cli.command()
@click.option('--sample', help='unique sample id (otherwise auto-generated)')
@click.option('--group', help='group id to associate samples e.g. in trios')
@click.option('--cutoff', default=10, help='cutoff for completeness')
@click.option(
  '--extend-by', default=0, help='dynamically extend intervals symetrically')
@prepend_option
@click.option(
  '--threshold',
  default=17000,
  help='base pair threshold for optimizing BAM-file reading'
)
@out_option
@bam_path_argument
@in_argument
@click.pass_context
def annotate(context, bam_path, in_stream, out, sample, group, cutoff,
             extend_by, prepend, threshold):
  """Annotate intervals in a BED-file/stream.

  \b
  BAM_PATH: Path to BAM-file
  IN_STREAM: Chanjo-style BED-file with interval definitions
  """
  # user defined sample id or randomly generated
  sample = (sample or id_generator())

  # step 1: metadata header
  metadata = dict(
    sample_id=sample,
    group_id=group,
    cutoff=cutoff,
    coverage_source=path(bam_path).abspath(),
    extension=extend_by
  )
  click.echo("#%s" % json.dumps(metadata), file=out)

  # step 2: annotate list of intervals with coverage and completeness
  bed_lines = pipe(
    annotator.pipeline(
      bed_stream=in_stream,
      bam_path=bam_path,
      cutoff=cutoff,
      extension=extend_by,
      contig_prepend=prepend,
      bp_threshold=threshold
    ),
    map(serialize_interval_plus)    # stringify/bedify
  )

  # reduce/write the BED lines
  for bed_line in bed_lines:
    click.echo(bed_line, file=out)

@cli.command(name='import')
@click.option('--json', is_flag=True, help="use legacy JSON 'annotate' output")
@in_argument
@click.pass_context
def import_(context, in_stream, json):
  """Import coverage annotations to an existing database.

  \b
  IN_STREAM: Chanjo-style BED-file with interval definitions
  """
  args = (context.obj['db'], in_stream)

  if json:
    # FYI: the ``bed_stream`` really is a JSON-file
    importer.json_pipeline(*args)
  else:
    importer.pipeline(*args)

@click.command('sex-check')
@prepend_option
@click.version_option(__version__)
@bam_path_argument
def sex_check(bam_path, prepend):
  """Sex Check - predict gender from a BAM-alignment.

  \b
  BAM_PATH: path to BAM-file
  """
  # run the sex checker pipeline
  gender = sex_checker.pipeline(bam_path, prepend=prepend)

  # print the results to the console for pipeability (csv)
  click.echo("#%(prepend)sX_coverage\t%(prepend)sY_coverage\tsex"
             % dict(prepend=prepend))
  click.echo('\t'.join(map(str, gender)))
<commit_msg>Avoid setting non-serialisable objects in context.obj in Click<commit_after># -*- coding: utf-8 -*-
"""
chanjo.cli
~~~~~~~~~~~

Command line interface (console entry points). Based on Click_.

.. _Click: http://click.pocoo.org/
"""
from __future__ import absolute_import, unicode_literals
import json
from pkg_resources import iter_entry_points

import click
from path import path
import toml as markup
from toolz import pipe
from toolz.curried import map

from . import (
  __title__, __version__, __banner__,
  annotator, builder, exporter, importer, sex_checker
)
from ._compat import text_type
from .config import Config, init_pipeline
from .store import Store
from .utils import (
  serialize_interval_plus, serialize_interval, id_generator
)

CONFIG_NAME="%(program)s.%(extension)s" % dict(
  program=__title__,
  extension=markup.__name__  # Works for JSON, TOML, YAML ...
)

# shared options/arguments that don't belong to any specific command
# path to a BAM-file (without opening as a stream)
bam_path_argument = click.argument('bam_path', type=click.Path(exists=True))
# open an input stream from a file (usually a BED-file)
in_argument = click.argument(
  'in_stream', type=click.File(), default='-', required=False)
# open an output stream (default is stdout) to write to
out_option = click.option('--out', type=click.File('w'), default='-',
                          help="define an output file other than 'stdout'")
# string to prepend to the contig ids to match e.g. the BAM-file
prepend_option = click.option(
  '--prepend', default='', help='prepend a string to each contig')

@click.group()
@click.option(
  '--config',
  default=CONFIG_NAME,
  type=click.File('w'),
  help='path to Chanjo config file'
)
@click.option('--db', type=text_type, help='path/URI of the SQL database')
@click.option(
  '--dialect',
  type=click.Choice(['sqlite', 'mysql']),
  help='type of SQL database'
)
@click.option(
  '--force', is_flag=True, help='overwrite existing assets without warning')
@click.option('--verbose', is_flag=True)
@click.version_option(__version__)
@click.pass_context
def cli(context, config, db, dialect, force, verbose):
  """Clinical sequencing coverage analysis tool."""
  # avoid setting global defaults in Click options, do it below when
  # updating the config object
  context.obj = Config(config, markup=markup)

  # global defaults
  db_path = db or context.obj.get('db', 'coverage.sqlite')
  db_dialect = dialect or context.obj.get('dialect', 'sqlite')

  context.db = Store(db_path, dialect=db_dialect)
  context.obj['force'] = force

  # update the context with new defaults from the config file
  context.default_map = context.obj

@cli.command()
@click.option('--remove', is_flag=True, help='remove config option')
@click.argument('key')
@click.argument('value', required=False)
@click.pass_context
def config(context, key, value, remove):
  """Only handles string values."""
  # update or delete the config key-value pair
  # the key can be provide as a path to nested commands: "user.name"
  if remove:
    context.obj.unset(key, scope=context.obj.user_data)

  else:
    if value is None:
      raise ValueError("Unless remove a setting, you must submit a 'value'")

    if value.isnumeric():
      value = int(value)

    context.obj.set(key, value, scope=context.obj.user_data)

  # persist updates to the config file
  context.obj.save()

@cli.command()
@click.pass_context
def init(context):
  """Walk user through setting up a new config file."""
  # print a nice welcome message
  click.echo(__banner__)

  questions = [
    ('annotate.cutoff', 'sufficient coverage',
      context.obj.get('annotate', {}).get('cutoff', 10)),
    ('dialect', 'preferred SQL-dialect', context.parent.db.dialect),
    ('db', 'central database path/URI', context.parent.db.uri)
  ]

  # launch init pipeline
  init_pipeline(__title__, context.obj, questions)

@cli.command()
@click.option('--adapter', default='ccds', help='plugin to use for conversion')
@click.option(
  'list_all', '--list', is_flag=True, help='show all installed adapters')
@out_option
@in_argument
@click.pass_context
def convert(context, in_stream, out, adapter, list_all):
  """Convert a reference database file to a Chanjo BED interval file

  \b
  IN_STREAM: interval reference file (e.g. CCDS database dump)
  """
  entry_points = iter_entry_points('chanjo.converters')

  if list_all:
    # list the installed converter options
    for entry_point in entry_points:
      # compose and print the message
      segments = dict(
        program=__title__,
        note=click.style('converter', fg='cyan'),
        plugin=entry_point.name
      )
      click.echo("%(program)s %(note)s %(plugin)s" % segments)

  else:
    # loop though each of the installed plugins
    for entry_point in entry_points:
      # break out when finding the matching adapter
      if entry_point.name == adapter:
        break

    # load entry point function
    converter_pipeline = entry_point.load()

    # execute converter pipeline
    bed_lines = pipe(
      converter_pipeline(in_stream),
      map(serialize_interval(bed=True))     # stringify/bedify
    )

    # reduce/write the BED lines
    for bed_line in bed_lines:
      click.echo(bed_line, file=out)

@cli.command()
@in_argument
@click.pass_context
def build(context, in_stream):
  """Construct a new skeleton SQL interval store.

  \b
  IN_STREAM: Chanjo-style BED-file with interval definitions
  """
  # build a new skeleton SQL interval store
  builder.pipeline(
    chanjo_db=context.parent.db,
    bed_stream=in_stream,
    overwrite=context.obj['force']
  )

@cli.command()
@out_option
@click.option(
  '--header/--no-header', is_flag=True, default=True, help="include headers")
@click.pass_context
def export(context, out, header):
  """Export an interval BED stream from an existing Chanjo store."""
  # build a new skeleton SQL interval store
  bed_lines = exporter.pipeline(
    chanjo_db=context.parent.db,
    include_header=header
  )

  # reduce/write the BED lines
  for bed_line in bed_lines:
    click.echo(bed_line, file=out)

@cli.command()
@click.option('--sample', help='unique sample id (otherwise auto-generated)')
@click.option('--group', help='group id to associate samples e.g. in trios')
@click.option('--cutoff', default=10, help='cutoff for completeness')
@click.option(
  '--extend-by', default=0, help='dynamically extend intervals symetrically')
@prepend_option
@click.option(
  '--threshold',
  default=17000,
  help='base pair threshold for optimizing BAM-file reading'
)
@out_option
@bam_path_argument
@in_argument
@click.pass_context
def annotate(context, bam_path, in_stream, out, sample, group, cutoff,
             extend_by, prepend, threshold):
  """Annotate intervals in a BED-file/stream.

  \b
  BAM_PATH: Path to BAM-file
  IN_STREAM: Chanjo-style BED-file with interval definitions
  """
  # user defined sample id or randomly generated
  sample = (sample or id_generator())

  # step 1: metadata header
  metadata = dict(
    sample_id=sample,
    group_id=group,
    cutoff=cutoff,
    coverage_source=path(bam_path).abspath(),
    extension=extend_by
  )
  click.echo("#%s" % json.dumps(metadata), file=out)

  # step 2: annotate list of intervals with coverage and completeness
  bed_lines = pipe(
    annotator.pipeline(
      bed_stream=in_stream,
      bam_path=bam_path,
      cutoff=cutoff,
      extension=extend_by,
      contig_prepend=prepend,
      bp_threshold=threshold
    ),
    map(serialize_interval_plus)    # stringify/bedify
  )

  # reduce/write the BED lines
  for bed_line in bed_lines:
    click.echo(bed_line, file=out)

@cli.command(name='import')
@click.option('--json', is_flag=True, help="use legacy JSON 'annotate' output")
@in_argument
@click.pass_context
def import_(context, in_stream, json):
  """Import coverage annotations to an existing database.

  \b
  IN_STREAM: Chanjo-style BED-file with interval definitions
  """
  args = (context.parent.db, in_stream)

  if json:
    # FYI: the ``bed_stream`` really is a JSON-file
    importer.json_pipeline(*args)
  else:
    importer.pipeline(*args)

@click.command('sex-check')
@prepend_option
@click.version_option(__version__)
@bam_path_argument
def sex_check(bam_path, prepend):
  """Sex Check - predict gender from a BAM-alignment.

  \b
  BAM_PATH: path to BAM-file
  """
  # run the sex checker pipeline
  gender = sex_checker.pipeline(bam_path, prepend=prepend)

  # print the results to the console for pipeability (csv)
  click.echo("#%(prepend)sX_coverage\t%(prepend)sY_coverage\tsex"
             % dict(prepend=prepend))
  click.echo('\t'.join(map(str, gender)))
<|endoftext|>


<commit_before>from decouple import config
from flask import (Flask, request, g, session, redirect, url_for, abort,
                   render_template)
from flask.ext.github import GitHub, GitHubError
from flask.ext.session import Session
from hookserver import HookServer
import redis

# Set these values in the .env file or env vars
GITHUB_CLIENT_ID = config('GITHUB_CLIENT_ID', '')
GITHUB_CLIENT_SECRET = config('GITHUB_CLIENT_SECRET', '')
GITHUB_ORG_ID = config('GITHUB_ORG_ID', 'jazzband')
GITHUB_SCOPE = config('GITHUB_SCOPE', 'read:org,user:email')
GITHUB_TEAM_ID = config('GITHUB_TEAM_ID', 0, cast=int)
GITHUB_ADMIN_TOKEN = config('GITHUB_ADMIN_TOKEN', '')

SECRET_KEY = config('SECRET_KEY', 'dev key')
DEBUG = config('DEBUG', False, cast=bool)
HOOKSERVER_SECRET = config('HOOKSERVER_SECRET', 'hook secret')
VALIDATE_IP = False

SESSION_TYPE = 'redis'
SESSION_COOKIE_NAME = 'jazzhands'
SESSION_COOKIE_SECURE = not DEBUG
SESSION_USE_SIGNER = config('SESSION_USE_SIGNER', True, cast=bool)
SESSION_REDIS = redis.from_url(config('REDIS_URL', 'redis://127.0.0.1:6379/0'))
PERMANENT_SESSION_LIFETIME = 60 * 60

# setup flask
app = HookServer(__name__, key=HOOKSERVER_SECRET)

# load decoupled config variables
app.config.from_object(__name__)

# setup github-flask
github = GitHub(app)

# setup session store
Session(app)

@app.before_request
def before_request():
    g.user_access_token = session.get('user_access_token', None)

@github.access_token_getter
def token_getter():
    return g.user_access_token

def add_to_org(user_login):
    """
    Adds the GitHub user with the given login to the org.
    """
    github.put(
        'teams/%s/memberships/%s' % (GITHUB_TEAM_ID, user_login),
        access_token=GITHUB_ADMIN_TOKEN
    )

def publicize_membership(user_login):
    """
    Publicizes the membership of the GitHub user with the given login.
    """
    github.put(
        'orgs/%s/public_members/%s' % (GITHUB_ORG_ID, user_login),
        access_token=GITHUB_ADMIN_TOKEN
    )

def is_member(user_login):
    """
    Checks if the GitHub user with the given login is member of the org.
    """
    try:
        github.get(
            'orgs/%s/members/%s' % (GITHUB_ORG_ID, user_login),
            access_token=GITHUB_ADMIN_TOKEN,
        )
        return True
    except GitHubError:
        return False

def verified_emails():
    """
    Checks if the authenticated GitHub user has any verified email addresses.
    """
    return any(
        [email for email in github.get('user/emails')
        if email.get('verified', False)],
    )

@app.errorhandler(403)
def forbidden(error):
    return render_template('forbidden.html')

@app.errorhandler(500)
def error(error):
    return render_template('error.html')

@app.route('/')
def start():
    if g.user_access_token:
        user_login = github.get('user').get('login', None)

        # fail if something went wrong
        if user_login is None:
            abort(500)

        # deny permission if there are no verified emails
        if not verified_emails():
            abort(403)

        membership = None
        user_is_member = is_member(user_login)

        if not user_is_member:
            try:
                membership = add_to_org(user_login)
            except GitHubError:
                pass

        return render_template(
            'index.html',
            next_url='https://github.com/orgs/jazzband/dashboard',
            membership=membership,
            org_id=GITHUB_ORG_ID,
            is_member=user_is_member,
        )
    else:
        return github.authorize(scope=app.config['GITHUB_SCOPE'])

@app.route('/callback')
@github.authorized_handler
def callback(access_token):
    next_url = url_for('start')

    if access_token is None:
        session.clear()
        return redirect(next_url)

    session['user_access_token'] = access_token
    return redirect(next_url)

@app.hook('ping')
def ping(data, guid):
    return 'pong'

@app.hook('membership')
def finish(data, guid):
    action = data.get('action')
    scope = data.get('scope')
    if action == 'added' and scope == 'team':
        team = data.get('team')
        if team and team['id'] == GITHUB_TEAM_ID:
            member = data.get('member')
            if member:
                try:
                    publicize_membership(member['user_login'])
                except GitHubError:
                    pass
    return 'ok'

if __name__ == '__main__':
    app.run(debug=True)
<commit_msg>Use corect field.<commit_after>from decouple import config
from flask import (Flask, request, g, session, redirect, url_for, abort,
                   render_template)
from flask.ext.github import GitHub, GitHubError
from flask.ext.session import Session
from hookserver import HookServer
import redis

# Set these values in the .env file or env vars
GITHUB_CLIENT_ID = config('GITHUB_CLIENT_ID', '')
GITHUB_CLIENT_SECRET = config('GITHUB_CLIENT_SECRET', '')
GITHUB_ORG_ID = config('GITHUB_ORG_ID', 'jazzband')
GITHUB_SCOPE = config('GITHUB_SCOPE', 'read:org,user:email')
GITHUB_TEAM_ID = config('GITHUB_TEAM_ID', 0, cast=int)
GITHUB_ADMIN_TOKEN = config('GITHUB_ADMIN_TOKEN', '')

SECRET_KEY = config('SECRET_KEY', 'dev key')
DEBUG = config('DEBUG', False, cast=bool)
HOOKSERVER_SECRET = config('HOOKSERVER_SECRET', 'hook secret')
VALIDATE_IP = False

SESSION_TYPE = 'redis'
SESSION_COOKIE_NAME = 'jazzhands'
SESSION_COOKIE_SECURE = not DEBUG
SESSION_USE_SIGNER = config('SESSION_USE_SIGNER', True, cast=bool)
SESSION_REDIS = redis.from_url(config('REDIS_URL', 'redis://127.0.0.1:6379/0'))
PERMANENT_SESSION_LIFETIME = 60 * 60

# setup flask
app = HookServer(__name__, key=HOOKSERVER_SECRET)

# load decoupled config variables
app.config.from_object(__name__)

# setup github-flask
github = GitHub(app)

# setup session store
Session(app)

@app.before_request
def before_request():
    g.user_access_token = session.get('user_access_token', None)

@github.access_token_getter
def token_getter():
    return g.user_access_token

def add_to_org(user_login):
    """
    Adds the GitHub user with the given login to the org.
    """
    github.put(
        'teams/%s/memberships/%s' % (GITHUB_TEAM_ID, user_login),
        access_token=GITHUB_ADMIN_TOKEN
    )

def publicize_membership(user_login):
    """
    Publicizes the membership of the GitHub user with the given login.
    """
    github.put(
        'orgs/%s/public_members/%s' % (GITHUB_ORG_ID, user_login),
        access_token=GITHUB_ADMIN_TOKEN
    )

def is_member(user_login):
    """
    Checks if the GitHub user with the given login is member of the org.
    """
    try:
        github.get(
            'orgs/%s/members/%s' % (GITHUB_ORG_ID, user_login),
            access_token=GITHUB_ADMIN_TOKEN,
        )
        return True
    except GitHubError:
        return False

def verified_emails():
    """
    Checks if the authenticated GitHub user has any verified email addresses.
    """
    return any(
        [email for email in github.get('user/emails')
        if email.get('verified', False)],
    )

@app.errorhandler(403)
def forbidden(error):
    return render_template('forbidden.html')

@app.errorhandler(500)
def error(error):
    return render_template('error.html')

@app.route('/')
def start():
    if g.user_access_token:
        user_login = github.get('user').get('login', None)

        # fail if something went wrong
        if user_login is None:
            abort(500)

        # deny permission if there are no verified emails
        if not verified_emails():
            abort(403)

        membership = None
        user_is_member = is_member(user_login)

        if not user_is_member:
            try:
                membership = add_to_org(user_login)
            except GitHubError:
                pass

        return render_template(
            'index.html',
            next_url='https://github.com/orgs/jazzband/dashboard',
            membership=membership,
            org_id=GITHUB_ORG_ID,
            is_member=user_is_member,
        )
    else:
        return github.authorize(scope=app.config['GITHUB_SCOPE'])

@app.route('/callback')
@github.authorized_handler
def callback(access_token):
    next_url = url_for('start')

    if access_token is None:
        session.clear()
        return redirect(next_url)

    session['user_access_token'] = access_token
    return redirect(next_url)

@app.hook('ping')
def ping(data, guid):
    return 'pong'

@app.hook('membership')
def finish(data, guid):
    action = data.get('action')
    scope = data.get('scope')
    if action == 'added' and scope == 'team':
        team = data.get('team')
        if team and team['id'] == GITHUB_TEAM_ID:
            member = data.get('member')
            if member:
                try:
                    publicize_membership(member['login'])
                except GitHubError:
                    pass
    return 'ok'

if __name__ == '__main__':
    app.run(debug=True)
<|endoftext|>


<commit_before># Copyright Least Authority Enterprises.
# See LICENSE for details.

"""
Tests for ``txkube.memory_kubernetes``.
"""

from zope.interface.verify import verifyClass

from hypothesis import given

from testtools.matchers import Equals, Is, IsInstance, AfterPreprocessing

from ..testing.integration import kubernetes_client_tests
from ..testing.strategies import iobjects
from ..testing import TestCase

from .. import memory_kubernetes
from .._memory import (
    _KubernetesState, IAgency, NullAgency, _incrementResourceVersion,
)

def get_kubernetes(case):
    """
    Create an in-memory test double provider of ``IKubernetes``.
    """
    return memory_kubernetes()

class KubernetesClientIntegrationTests(kubernetes_client_tests(get_kubernetes)):
    """
    Integration tests which interact with an in-memory-only Kubernetes
    deployment via ``txkube.memory_kubernetes``.
    """

class NullAgencyTests(TestCase):
    """
    Tests for ``NullAgency``.
    """
    def test_interface(self):
        """
        ``NullAgency`` implements ``IAgency``.
        """
        verifyClass(IAgency, NullAgency)

    @given(iobjects())
    def test_before_create(self, obj):
        """
        ``NullAgency.before_create`` returns the ``IObject`` passed to it with no
        modifications.
        """
        state = _KubernetesState()
        actual = NullAgency().before_create(state, obj)
        self.assertThat(actual, Equals(obj))

    @given(iobjects())
    def test_after_create(self, obj):
        """
        ``NullAgency.after_create`` returns the ``IObject`` passed to it with no
        modifications.
        """
        state = _KubernetesState()
        actual = NullAgency().after_create(state, obj)
        self.assertThat(actual, Equals(obj))

    @given(iobjects(), iobjects())
    def test_before_replace(self, old, new):
        """
        ``NullAgency.before_replace`` returns ``None``.
        """
        state = _KubernetesState()
        actual = NullAgency().before_replace(state, old, new)
        self.assertThat(actual, Is(None))

class IncrementResourceVersionTests(TestCase):
    """
    Tests for ``_incrementResourceVersion``.
    """
    def test_missing(self):
        """
        The next version after ``None`` is ``u"1"``.
        """
        version = _incrementResourceVersion(None)
        self.expectThat(version, IsInstance(unicode))
        self.expectThat(version, Equals(u"1"))

    def test_incremented(self):
        """
        Any version other than ``None`` is interpreted as a string representation
        of an integer and incremented.
        """
        version = _incrementResourceVersion(_incrementResourceVersion(None))
        updated = _incrementResourceVersion(version)
        self.expectThat(updated, IsInstance(unicode))
        self.expectThat(updated, AfterPreprocessing(int, Equals(int(version) + 1)))
<commit_msg>Import twisted.python.compat.unicode to fix Python 3.<commit_after># Copyright Least Authority Enterprises.
# See LICENSE for details.

"""
Tests for ``txkube.memory_kubernetes``.
"""

from zope.interface.verify import verifyClass

from hypothesis import given

from testtools.matchers import Equals, Is, IsInstance, AfterPreprocessing

from twisted.python.compat import unicode

from ..testing.integration import kubernetes_client_tests
from ..testing.strategies import iobjects
from ..testing import TestCase

from .. import memory_kubernetes
from .._memory import (
    _KubernetesState, IAgency, NullAgency, _incrementResourceVersion,
)

def get_kubernetes(case):
    """
    Create an in-memory test double provider of ``IKubernetes``.
    """
    return memory_kubernetes()

class KubernetesClientIntegrationTests(kubernetes_client_tests(get_kubernetes)):
    """
    Integration tests which interact with an in-memory-only Kubernetes
    deployment via ``txkube.memory_kubernetes``.
    """

class NullAgencyTests(TestCase):
    """
    Tests for ``NullAgency``.
    """
    def test_interface(self):
        """
        ``NullAgency`` implements ``IAgency``.
        """
        verifyClass(IAgency, NullAgency)

    @given(iobjects())
    def test_before_create(self, obj):
        """
        ``NullAgency.before_create`` returns the ``IObject`` passed to it with no
        modifications.
        """
        state = _KubernetesState()
        actual = NullAgency().before_create(state, obj)
        self.assertThat(actual, Equals(obj))

    @given(iobjects())
    def test_after_create(self, obj):
        """
        ``NullAgency.after_create`` returns the ``IObject`` passed to it with no
        modifications.
        """
        state = _KubernetesState()
        actual = NullAgency().after_create(state, obj)
        self.assertThat(actual, Equals(obj))

    @given(iobjects(), iobjects())
    def test_before_replace(self, old, new):
        """
        ``NullAgency.before_replace`` returns ``None``.
        """
        state = _KubernetesState()
        actual = NullAgency().before_replace(state, old, new)
        self.assertThat(actual, Is(None))

class IncrementResourceVersionTests(TestCase):
    """
    Tests for ``_incrementResourceVersion``.
    """
    def test_missing(self):
        """
        The next version after ``None`` is ``u"1"``.
        """
        version = _incrementResourceVersion(None)
        self.expectThat(version, IsInstance(unicode))
        self.expectThat(version, Equals(u"1"))

    def test_incremented(self):
        """
        Any version other than ``None`` is interpreted as a string representation
        of an integer and incremented.
        """
        version = _incrementResourceVersion(_incrementResourceVersion(None))
        updated = _incrementResourceVersion(version)
        self.expectThat(updated, IsInstance(unicode))
        self.expectThat(updated, AfterPreprocessing(int, Equals(int(version) + 1)))
<|endoftext|>


<commit_before>from ggame import App, Color, LineStyle, Sprite
from ggame import RectangleAsset, CircleAsset
import math

SCREEN_WIDTH = 1536
SCREEN_HEIGHT = 1024

red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
purple = Color(0xff00ff, 1.0)
thinline = LineStyle(0, black)

xcord = range(0, 1500, 1)

mode = input("Which type of function do you have? \nl for linear, t for Trig, p for Parabola ")
if mode == "l":
    k = float(input("f(x) = kx + b, set the value of k: "))
    b = float(input("f(x) = kx + b, set the value of b: "))
elif mode == "t":
    a = int(input("f(x) = a sin h (x + k) , set the value of a: "))

class Dot(Sprite):

    asset = RectangleAsset(2,2, thinline, red)

    def __init__(self, position):
        super().__init__(Dot.asset, position)
        self.x = 0

        #sprite = Sprite(Dot.asset, (self.x, 450-k*(self.x-750)-100*b))

    def step(self):
        self.x += 1
        sprite0 = Sprite(Dot.asset, (self.x, 450-k*(self.x-750)-100*b))

class Grapher(App):
    def __init__(self, width, height):
        super().__init__(width, height)

        black = Color(0x000000, 1.0)
        thinline = LineStyle(0, black)
        xaxis = RectangleAsset(1500,2, thinline, black)
        yaxis = RectangleAsset(2,1050, thinline, black)
        sprite1 = Sprite(xaxis, (0, 450))
        sprite2 = Sprite(yaxis, (750, 0))

        Dot((100,100))

        #sprite0 = [Sprite(Dot.asset, (x, 450-k*(x-750)-100*b)) for x in xcord]

    def step():
        for dot in self.getSpritesbyClass(Dot):
            dot.step()

myapp = Grapher(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()
<commit_msg>Updated from Brython Server: November 18, 2015 1:54:50 PM EST<commit_after>from ggame import App, Color, LineStyle, Sprite
from ggame import RectangleAsset, CircleAsset
import math

SCREEN_WIDTH = 1536
SCREEN_HEIGHT = 1024

red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
purple = Color(0xff00ff, 1.0)
thinline = LineStyle(0, black)

xcord = range(0, 1500, 1)

mode = input("Which type of function do you have? \nl for linear, t for Trig, p for Parabola ")
if mode == "l":
    k = float(input("f(x) = kx + b, set the value of k: "))
    b = float(input("f(x) = kx + b, set the value of b: "))
elif mode == "t":
    a = int(input("f(x) = a sin h (x + k) , set the value of a: "))

class Dot(Sprite):

    asset = RectangleAsset(2,2, thinline, red)

    def __init__(self, position):
        super().__init__(Dot.asset, position)
        self.x = 0

        self.dotmade = False

        #sprite = Sprite(Dot.asset, (self.x, 450-k*(self.x-750)-100*b))

    def step(self):
        self.x += 1
        if not self.dotmade and self.x < 1500:
            sprite0 = Sprite(Dot.asset, (self.x, 450-k*(self.x-750)-100*b))
            self.dotmade = True

class Grapher(App):
    def __init__(self, width, height):
        super().__init__(width, height)

        black = Color(0x000000, 1.0)
        thinline = LineStyle(0, black)
        xaxis = RectangleAsset(1500,2, thinline, black)
        yaxis = RectangleAsset(2,1050, thinline, black)
        sprite1 = Sprite(xaxis, (0, 450))
        sprite2 = Sprite(yaxis, (750, 0))

        Dot((self.x, 450-k*(self.x-750)-100*b))

        #sprite0 = [Sprite(Dot.asset, (x, 450-k*(x-750)-100*b)) for x in xcord]

    def step():
        for dot in self.getSpritesbyClass(Dot):
            dot.step()

myapp = Grapher(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()
<|endoftext|>


<commit_before>from __future__ import print_function, division
import time, threading, functools
import warnings

import numpy as np
import scipy.optimize, scipy.ndimage
from pyqtgraph import ptime
from ...Manager import getManager
from acq4.util import Qt
from acq4.util.Thread import Thread
from acq4.util.Mutex import Mutex
from acq4.util.debug import printExc
from six.moves import range

class TestPulseThread(Thread):
    """Background thread that runs periodic test pulses on a single patch clamp channel.
    """

    sigTestPulseFinished = Qt.Signal(object, object)  # device, result

    class StopRequested(Exception):
        pass

    def __init__(self, dev, params):
        Thread.__init__(self, name="TestPulseThread(%s)"%dev.name())
        self.dev = dev
        self._stop = False
        self.params = {
            'testPulseClass': TestPulse,
            'clampMode': None,
            'interval': None,
            'autoBiasEnabled': True,
            'autoBiasTarget': -70e-3,  # if None, use VC holding
            'autoBiasFollowRate': 0.5,
            'autoBiasMinCurrent': -1.5e-9,
            'autoBiasMaxCurrent': 1.5e9,
            'autoBiasVCCarryover': 0.7,
            'sampleRate': 500000,
            'downsample': 20,
            'vcPreDuration': 5e-3,
            'vcPulseDuration': 10e-3,
            'vcPostDuration': 5e-3,
            'vcHolding': None,
            'vcAmplitude': -10e-3,
            'vcAverage': 4,
            'icPreDuration': 10e-3,
            'icPulseDuration': 80e-3,
            'icPostDuration': 80e-3,
            'icHolding': None,
            'icAmplitude': -10e-12,
            'icAverage': 4,
            '_index': 0,
        }
        self._lastTask = None

        self._clampDev = self.dev.clampDevice
        self._daqName = self._clampDev.getDAQName()
        self._clampName = self._clampDev.name()
        self._manager = getManager()

        self.setParameters(**params)

    def setParameters(self, **kwds):
        newParams = self.params.copy()
        for k,v in kwds.items():
            if k not in self.params:
                raise KeyError("Unknown parameter %s" % k)
            newParams[k] = v
        newParams['_index'] += 1
        self.params = newParams

    def getParameter(self, param):
        return self.params[param]

    def start(self):
        self._stop = False
        Thread.start(self)

    def stop(self, block=False):
        self._stop = True
        if block:
            if not self.wait(10000):
                raise RuntimeError("Timed out waiting for test pulse thread exit.")

    def run(self):
        while True:
            try:
                self._checkStop()
                start = ptime.time()
                self.runOnce(_checkStop=True)

                interval = self.params['interval']
                if interval is None:
                    # start again immediately
                    continue

                # otherwise, wait until interval is over
                while True:
                    nextRun = start + self.params['interval']
                    now = ptime.time()
                    if now >= nextRun:
                        break
                    time.sleep(min(0.03, nextRun-now))
                    self._checkStop()
            except self.StopRequested:
                break
            except Exception:
                printExc("Error in test pulse thread:")
                time.sleep(2.0)

    def runOnce(self, _checkStop=False):
        currentMode = self._clampDev.getMode()
        params = self.params
        runMode = currentMode if params['clampMode'] is None else params['clampMode']
        if runMode == 'I=0':
            runMode = 'IC'

        # Can't reuse tasks yet; remove this when we can.
        self._lastTask = None

        if self._lastTask is None or self._lastTask._paramIndex != params['_index'] or self._lastTask._clampMode != runMode:
            taskParams = params.copy()

            # select parameters to use based on clamp mode
            for k in params:
                # rename like icPulseDuration => pulseDuration
                if k[:2] == runMode.lower():
                    taskParams[k[2].lower() + k[3:]] = taskParams[k]
                # remove all ic__ and vc__ params
                if k[:2] in ('ic', 'vc'):
                    taskParams.pop(k)
                taskParams['clampMode'] = runMode

            task = self.createTask(taskParams)
            task._paramIndex = params['_index']
            task._clampMode = runMode
            self._lastTask = task
            self._lastTaskParams = taskParams
        else:
            task = self._lastTask
            taskParams = self._lastTaskParams

        # if clamp mode changed while we were fiddling around, then abort.
        task.reserveDevices()
        try:
            if self._clampDev.getMode() != currentMode:
                task.releaseDevices()
                return

            task.execute()

            while not task.isDone():
                if _checkStop:
                    self._checkStop()
                time.sleep(0.01)

            tp = None
            if params['autoBiasEnabled']:
                # update bias before unlocking
                result = task.getResult()
                tp = self._makeTpResult(taskParams, result)
                self.updateAutoBias(tp)
        finally:
            task.releaseDevices()

        if tp is None:
            # no auto bias, release before doing analysis
            result = task.getResult()
            tp = self._makeTpResult(taskParams, result)

        self.sigTestPulseFinished.emit(self.dev, tp)

    def _makeTpResult(self, taskParams, result):
        tpClass = taskParams['testPulseClass']
        tp = tpClass(self._clampDev, taskParams, result)
        tp.analysis()
        return tp

    def createTask(self, params):
        duration = params['preDuration'] + params['pulseDuration'] + params['postDuration']
        numPts = int(float(duration * params['sampleRate']) * params['downsample']) // params['downsample']
        params['numPts'] = numPts  # send this back for analysis
        mode = params['clampMode']

        cmdData = np.empty(numPts * params['average'])
        holding = params['holding'] or self._clampDev.getHolding(mode)
        cmdData[:] = holding

        for i in range(params['average']):
            start = (numPts * i) + int(params['preDuration'] * params['sampleRate'])
            stop = start + int(params['pulseDuration'] * params['sampleRate'])
            cmdData[start:stop] += params['amplitude']

        cmd = {
            'protocol': {'duration': duration * params['average']},
            self._daqName: {'rate': params['sampleRate'], 'numPts': numPts * params['average'], 'downsample': params['downsample']},
            self._clampName: {
                'mode': mode,
                'command': cmdData,
                'recordState': ['BridgeBalResist', 'BridgeBalEnable'],
            }
        }
        if params['holding'] is not None:
            cmd[self._clampName]['holding'] = params['holding']

        return self._manager.createTask(cmd)

    def _checkStop(self):
        if self._stop:
            raise self.StopRequested()

    def updateAutoBias(self, tp):
        analysis = tp.analysis()
        mode = tp.clampMode()
        if mode.lower() == 'vc':
            # set ic holding from baseline current, multiplied by some factor for a little more added safety.
            self.dev.clampDevice.setHolding('IC', analysis['baselineCurrent'] * self.params['autoBiasVCCarryover'])
        else:
            target = self.params['autoBiasTarget']
            if target is None:
                target = self.params['vcHolding']

            rm = np.clip(analysis['steadyStateResistance'], 1e6, 10e9)
            vm = analysis['baselinePotential']

            dv = target - vm
            di = dv / rm

            holding = self.dev.clampDevice.getHolding(mode)
            newHolding = holding + di * self.params['autoBiasFollowRate']
            newHolding = np.clip(newHolding, self.params['autoBiasMinCurrent'], self.params['autoBiasMaxCurrent'])

            self.dev.clampDevice.setHolding(mode, newHolding)

class TestPulse(object):
    """Represents a single test pulse run, used to analyze and extract features.
    """
    def __init__(self, dev, taskParams, result):
        self.dev = dev
        self.devName = dev.name()
        self.taskParams = taskParams
        self.result = result
        self._analysis = None
        self._analysisLock = Mutex()
        self._average = None

    @property
    def data(self):
        if self._average is None:
            params = self.taskParams
            result = self.result[self.devName]
            if params['average'] == 1:
                self._average = result
            else:
                numPts = params['numPts'] // params['downsample']
                pri = result['Channel': 'primary']
                avg = np.zeros(numPts)
                for i in range(params['average']):
                    avg += pri[i*numPts:(i+1)*numPts]
                avg /= params['average']
                self._average = result['Time':0:numPts].copy()
                self._average['Channel': 'primary']._data[:] = avg

        return self._average

    def startTime(self):
        return self.result[self.devName]._info[-1]['startTime']

    def clampMode(self):
        return self.taskParams['clampMode']

    def analysis(self):
        with self._analysisLock:
            if self._analysis is not None:
                return self._analysis
            analysis = {}
            params = self.taskParams
            pri = self.data['Channel': 'primary']

            base = pri['Time': 0:params['preDuration']]
            peakStart = params['preDuration']
            peakStop = peakStart + 2e-3
            peak = pri['Time': peakStart:peakStop]
            ssStop = params['preDuration'] + params['pulseDuration']
            ssStart = ssStop - 2e-3
            steady  = pri['Time': ssStart:ssStop]

            if params['amplitude'] > 0:
                peakValue = peak.max()
            else:
                peakValue = peak.min()
            steadyValue = np.median(steady)
            baseValue = np.median(base)

            if params['clampMode'] == 'VC':
                analysis['baselinePotential'] = params['holding'] or 0
                analysis['baselineCurrent'] = baseValue
                analysis['peakResistance'] = params['amplitude'] / (peakValue - baseValue)
                analysis['steadyStateResistance'] = np.abs(params['amplitude'] / (steadyValue - baseValue))
                tauGuess = 3e-3
                yOffsetBounds = [-10e-9, 10e-9]
            else:
                bridge = self.data._info[-1]['ClampState']['ClampParams']['BridgeBalResist']
                bridgeOn = self.data._info[-1]['ClampState']['ClampParams']['BridgeBalEnable']
                if not bridgeOn:
                    bridge = 0.0
                analysis['baselineCurrent'] = params['holding'] or 0
                analysis['baselinePotential'] = baseValue
                analysis['peakResistance'] = bridge + (peakValue - baseValue) / params['amplitude']
                analysis['steadyStateResistance'] = np.abs(bridge + (steadyValue - baseValue) / params['amplitude'])
                tauGuess = 15e-3
                yOffsetBounds = [-150e-3, 100e-3]

            analysis['peakResistance'] = np.clip(analysis['peakResistance'], 0, 20e9)
            analysis['steadyStateResistance'] = np.clip(analysis['steadyStateResistance'], 0, 20e9)

            # do curve fitting
            pulseStart = params['preDuration'] #+ 150e-6
            pulseStop = params['preDuration'] + params['pulseDuration']
            pulse = pri['Time': pulseStart:pulseStop]
            t = pulse.xvals('Time')
            guess = (
                peakValue - steadyValue,  # amp
                tauGuess,  # tau
                steadyValue,  # yoffset
            )
            bounds = (
                np.array([yOffsetBounds[0], 50e-6, yOffsetBounds[0]]),
                np.array([yOffsetBounds[1], 500e-3, yOffsetBounds[1]])
            )
            xoffset = params['preDuration']
            pulseData = pulse.asarray()
            try:
                with warnings.catch_warnings():
                    warnings.simplefilter("ignore")
                    fit = scipy.optimize.curve_fit(exp, t-xoffset, pulseData, guess, maxfev=1000)  # uses leastsq
                # fit = scipy.optimize.curve_fit(exp, t-xoffset, pulse.asarray(), guess, bounds=bounds, max_nfev=1000)  # uses least_squares
                amp, tau, yoffset = fit[0]
            except RuntimeError:
                amp = tau = yoffset = np.nan

            analysis['fitExpAmp'] = amp
            analysis['fitExpTau'] = tau
            analysis['fitExpYOffset'] = yoffset
            analysis['fitExpXOffset'] = xoffset

            if params['clampMode'] == 'VC':
                # VC capacitance calculation adapted from Santos-Sacchi 1993
                # (not very accurate, probably because Q is calculated incorrectly)
                dt = t[1] - t[0]
                Q = (pulseData - yoffset).sum() * dt
                Rin = analysis['steadyStateResistance']
                Vc = params['amplitude']
                Rs_denom = (Q * Rin + tau * Vc)
                if Rs_denom != 0.0:
                    # Rs = (Rin * tau * Vc) / Rs_denom
                    Rs = analysis['peakResistance']
                    Rm = Rin - Rs
                    Cm = (Rin**2 * Q) / (Rm**2 * abs(Vc))
                else:
                    Rs = 0
                    Rm = 0
                    Cm = 0
                analysis['capacitance'] = Cm
            else:
                if analysis['steadyStateResistance'] > 0:
                    analysis['capacitance'] = tau / analysis['steadyStateResistance']
                else:
                    analysis['capacitance'] = np.nan

            # # detect bad fits
            # noise = (pulseData - scipy.ndimage.gaussian_filter(pulseData, 3)).std()
            # fitOk = tau < 1e-3 or tau > 0.5 or abs(amp) < noise * 2

            self._analysis = analysis
            return analysis

    def getFitData(self):
        params = self.taskParams
        analysis = self.analysis()
        pri = self.data['Channel': 'primary']
        pulseStart = params['preDuration']# + 150e-6
        pulseStop = params['preDuration'] + params['pulseDuration']
        pulse = pri['Time': pulseStart:pulseStop]
        t = pulse.xvals('Time')
        y = exp(t - analysis['fitExpXOffset'], analysis['fitExpAmp'], analysis['fitExpTau'], analysis['fitExpYOffset'])
        return t,y

def exp(t, amp, tau, yoffset):
    return yoffset + amp * np.exp(-t / tau)<commit_msg>just warn is okay<commit_after>from __future__ import print_function, division
import time, threading, functools
import warnings

import numpy as np
import scipy.optimize, scipy.ndimage
from pyqtgraph import ptime
from ...Manager import getManager
from acq4.util import Qt
from acq4.util.Thread import Thread
from acq4.util.Mutex import Mutex
from acq4.util.debug import printExc
from six.moves import range

class TestPulseThread(Thread):
    """Background thread that runs periodic test pulses on a single patch clamp channel.
    """

    sigTestPulseFinished = Qt.Signal(object, object)  # device, result

    class StopRequested(Exception):
        pass

    def __init__(self, dev, params):
        Thread.__init__(self, name="TestPulseThread(%s)"%dev.name())
        self.dev = dev
        self._stop = False
        self.params = {
            'testPulseClass': TestPulse,
            'clampMode': None,
            'interval': None,
            'autoBiasEnabled': True,
            'autoBiasTarget': -70e-3,  # if None, use VC holding
            'autoBiasFollowRate': 0.5,
            'autoBiasMinCurrent': -1.5e-9,
            'autoBiasMaxCurrent': 1.5e9,
            'autoBiasVCCarryover': 0.7,
            'sampleRate': 500000,
            'downsample': 20,
            'vcPreDuration': 5e-3,
            'vcPulseDuration': 10e-3,
            'vcPostDuration': 5e-3,
            'vcHolding': None,
            'vcAmplitude': -10e-3,
            'vcAverage': 4,
            'icPreDuration': 10e-3,
            'icPulseDuration': 80e-3,
            'icPostDuration': 80e-3,
            'icHolding': None,
            'icAmplitude': -10e-12,
            'icAverage': 4,
            '_index': 0,
        }
        self._lastTask = None

        self._clampDev = self.dev.clampDevice
        self._daqName = self._clampDev.getDAQName()
        self._clampName = self._clampDev.name()
        self._manager = getManager()

        self.setParameters(**params)

    def setParameters(self, **kwds):
        newParams = self.params.copy()
        for k,v in kwds.items():
            if k not in self.params:
                raise KeyError("Unknown parameter %s" % k)
            newParams[k] = v
        newParams['_index'] += 1
        self.params = newParams

    def getParameter(self, param):
        return self.params[param]

    def start(self):
        self._stop = False
        Thread.start(self)

    def stop(self, block=False):
        self._stop = True
        if block:
            if not self.wait(10000):
                raise RuntimeError("Timed out waiting for test pulse thread exit.")

    def run(self):
        while True:
            try:
                self._checkStop()
                start = ptime.time()
                self.runOnce(_checkStop=True)

                interval = self.params['interval']
                if interval is None:
                    # start again immediately
                    continue

                # otherwise, wait until interval is over
                while True:
                    nextRun = start + self.params['interval']
                    now = ptime.time()
                    if now >= nextRun:
                        break
                    time.sleep(min(0.03, nextRun-now))
                    self._checkStop()
            except self.StopRequested:
                break
            except Exception:
                printExc("Error in test pulse thread (will try again):", msgType='warning')
                time.sleep(2.0)

    def runOnce(self, _checkStop=False):
        currentMode = self._clampDev.getMode()
        params = self.params
        runMode = currentMode if params['clampMode'] is None else params['clampMode']
        if runMode == 'I=0':
            runMode = 'IC'

        # Can't reuse tasks yet; remove this when we can.
        self._lastTask = None

        if self._lastTask is None or self._lastTask._paramIndex != params['_index'] or self._lastTask._clampMode != runMode:
            taskParams = params.copy()

            # select parameters to use based on clamp mode
            for k in params:
                # rename like icPulseDuration => pulseDuration
                if k[:2] == runMode.lower():
                    taskParams[k[2].lower() + k[3:]] = taskParams[k]
                # remove all ic__ and vc__ params
                if k[:2] in ('ic', 'vc'):
                    taskParams.pop(k)
                taskParams['clampMode'] = runMode

            task = self.createTask(taskParams)
            task._paramIndex = params['_index']
            task._clampMode = runMode
            self._lastTask = task
            self._lastTaskParams = taskParams
        else:
            task = self._lastTask
            taskParams = self._lastTaskParams

        # if clamp mode changed while we were fiddling around, then abort.
        task.reserveDevices()
        try:
            if self._clampDev.getMode() != currentMode:
                task.releaseDevices()
                return

            task.execute()

            while not task.isDone():
                if _checkStop:
                    self._checkStop()
                time.sleep(0.01)

            tp = None
            if params['autoBiasEnabled']:
                # update bias before unlocking
                result = task.getResult()
                tp = self._makeTpResult(taskParams, result)
                self.updateAutoBias(tp)
        finally:
            task.releaseDevices()

        if tp is None:
            # no auto bias, release before doing analysis
            result = task.getResult()
            tp = self._makeTpResult(taskParams, result)

        self.sigTestPulseFinished.emit(self.dev, tp)

    def _makeTpResult(self, taskParams, result):
        tpClass = taskParams['testPulseClass']
        tp = tpClass(self._clampDev, taskParams, result)
        tp.analysis()
        return tp

    def createTask(self, params):
        duration = params['preDuration'] + params['pulseDuration'] + params['postDuration']
        numPts = int(float(duration * params['sampleRate']) * params['downsample']) // params['downsample']
        params['numPts'] = numPts  # send this back for analysis
        mode = params['clampMode']

        cmdData = np.empty(numPts * params['average'])
        holding = params['holding'] or self._clampDev.getHolding(mode)
        cmdData[:] = holding

        for i in range(params['average']):
            start = (numPts * i) + int(params['preDuration'] * params['sampleRate'])
            stop = start + int(params['pulseDuration'] * params['sampleRate'])
            cmdData[start:stop] += params['amplitude']

        cmd = {
            'protocol': {'duration': duration * params['average']},
            self._daqName: {'rate': params['sampleRate'], 'numPts': numPts * params['average'], 'downsample': params['downsample']},
            self._clampName: {
                'mode': mode,
                'command': cmdData,
                'recordState': ['BridgeBalResist', 'BridgeBalEnable'],
            }
        }
        if params['holding'] is not None:
            cmd[self._clampName]['holding'] = params['holding']

        return self._manager.createTask(cmd)

    def _checkStop(self):
        if self._stop:
            raise self.StopRequested()

    def updateAutoBias(self, tp):
        analysis = tp.analysis()
        mode = tp.clampMode()
        if mode.lower() == 'vc':
            # set ic holding from baseline current, multiplied by some factor for a little more added safety.
            self.dev.clampDevice.setHolding('IC', analysis['baselineCurrent'] * self.params['autoBiasVCCarryover'])
        else:
            target = self.params['autoBiasTarget']
            if target is None:
                target = self.params['vcHolding']

            rm = np.clip(analysis['steadyStateResistance'], 1e6, 10e9)
            vm = analysis['baselinePotential']

            dv = target - vm
            di = dv / rm

            holding = self.dev.clampDevice.getHolding(mode)
            newHolding = holding + di * self.params['autoBiasFollowRate']
            newHolding = np.clip(newHolding, self.params['autoBiasMinCurrent'], self.params['autoBiasMaxCurrent'])

            self.dev.clampDevice.setHolding(mode, newHolding)

class TestPulse(object):
    """Represents a single test pulse run, used to analyze and extract features.
    """
    def __init__(self, dev, taskParams, result):
        self.dev = dev
        self.devName = dev.name()
        self.taskParams = taskParams
        self.result = result
        self._analysis = None
        self._analysisLock = Mutex()
        self._average = None

    @property
    def data(self):
        if self._average is None:
            params = self.taskParams
            result = self.result[self.devName]
            if params['average'] == 1:
                self._average = result
            else:
                numPts = params['numPts'] // params['downsample']
                pri = result['Channel': 'primary']
                avg = np.zeros(numPts)
                for i in range(params['average']):
                    avg += pri[i*numPts:(i+1)*numPts]
                avg /= params['average']
                self._average = result['Time':0:numPts].copy()
                self._average['Channel': 'primary']._data[:] = avg

        return self._average

    def startTime(self):
        return self.result[self.devName]._info[-1]['startTime']

    def clampMode(self):
        return self.taskParams['clampMode']

    def analysis(self):
        with self._analysisLock:
            if self._analysis is not None:
                return self._analysis
            analysis = {}
            params = self.taskParams
            pri = self.data['Channel': 'primary']

            base = pri['Time': 0:params['preDuration']]
            peakStart = params['preDuration']
            peakStop = peakStart + 2e-3
            peak = pri['Time': peakStart:peakStop]
            ssStop = params['preDuration'] + params['pulseDuration']
            ssStart = ssStop - 2e-3
            steady  = pri['Time': ssStart:ssStop]

            if params['amplitude'] > 0:
                peakValue = peak.max()
            else:
                peakValue = peak.min()
            steadyValue = np.median(steady)
            baseValue = np.median(base)

            if params['clampMode'] == 'VC':
                analysis['baselinePotential'] = params['holding'] or 0
                analysis['baselineCurrent'] = baseValue
                analysis['peakResistance'] = params['amplitude'] / (peakValue - baseValue)
                analysis['steadyStateResistance'] = np.abs(params['amplitude'] / (steadyValue - baseValue))
                tauGuess = 3e-3
                yOffsetBounds = [-10e-9, 10e-9]
            else:
                bridge = self.data._info[-1]['ClampState']['ClampParams']['BridgeBalResist']
                bridgeOn = self.data._info[-1]['ClampState']['ClampParams']['BridgeBalEnable']
                if not bridgeOn:
                    bridge = 0.0
                analysis['baselineCurrent'] = params['holding'] or 0
                analysis['baselinePotential'] = baseValue
                analysis['peakResistance'] = bridge + (peakValue - baseValue) / params['amplitude']
                analysis['steadyStateResistance'] = np.abs(bridge + (steadyValue - baseValue) / params['amplitude'])
                tauGuess = 15e-3
                yOffsetBounds = [-150e-3, 100e-3]

            analysis['peakResistance'] = np.clip(analysis['peakResistance'], 0, 20e9)
            analysis['steadyStateResistance'] = np.clip(analysis['steadyStateResistance'], 0, 20e9)

            # do curve fitting
            pulseStart = params['preDuration'] #+ 150e-6
            pulseStop = params['preDuration'] + params['pulseDuration']
            pulse = pri['Time': pulseStart:pulseStop]
            t = pulse.xvals('Time')
            guess = (
                peakValue - steadyValue,  # amp
                tauGuess,  # tau
                steadyValue,  # yoffset
            )
            bounds = (
                np.array([yOffsetBounds[0], 50e-6, yOffsetBounds[0]]),
                np.array([yOffsetBounds[1], 500e-3, yOffsetBounds[1]])
            )
            xoffset = params['preDuration']
            pulseData = pulse.asarray()
            try:
                with warnings.catch_warnings():
                    warnings.simplefilter("ignore")
                    fit = scipy.optimize.curve_fit(exp, t-xoffset, pulseData, guess, maxfev=1000)  # uses leastsq
                # fit = scipy.optimize.curve_fit(exp, t-xoffset, pulse.asarray(), guess, bounds=bounds, max_nfev=1000)  # uses least_squares
                amp, tau, yoffset = fit[0]
            except RuntimeError:
                amp = tau = yoffset = np.nan

            analysis['fitExpAmp'] = amp
            analysis['fitExpTau'] = tau
            analysis['fitExpYOffset'] = yoffset
            analysis['fitExpXOffset'] = xoffset

            if params['clampMode'] == 'VC':
                # VC capacitance calculation adapted from Santos-Sacchi 1993
                # (not very accurate, probably because Q is calculated incorrectly)
                dt = t[1] - t[0]
                Q = (pulseData - yoffset).sum() * dt
                Rin = analysis['steadyStateResistance']
                Vc = params['amplitude']
                Rs_denom = (Q * Rin + tau * Vc)
                if Rs_denom != 0.0:
                    # Rs = (Rin * tau * Vc) / Rs_denom
                    Rs = analysis['peakResistance']
                    Rm = Rin - Rs
                    Cm = (Rin**2 * Q) / (Rm**2 * abs(Vc))
                else:
                    Rs = 0
                    Rm = 0
                    Cm = 0
                analysis['capacitance'] = Cm
            else:
                if analysis['steadyStateResistance'] > 0:
                    analysis['capacitance'] = tau / analysis['steadyStateResistance']
                else:
                    analysis['capacitance'] = np.nan

            # # detect bad fits
            # noise = (pulseData - scipy.ndimage.gaussian_filter(pulseData, 3)).std()
            # fitOk = tau < 1e-3 or tau > 0.5 or abs(amp) < noise * 2

            self._analysis = analysis
            return analysis

    def getFitData(self):
        params = self.taskParams
        analysis = self.analysis()
        pri = self.data['Channel': 'primary']
        pulseStart = params['preDuration']# + 150e-6
        pulseStop = params['preDuration'] + params['pulseDuration']
        pulse = pri['Time': pulseStart:pulseStop]
        t = pulse.xvals('Time')
        y = exp(t - analysis['fitExpXOffset'], analysis['fitExpAmp'], analysis['fitExpTau'], analysis['fitExpYOffset'])
        return t,y

def exp(t, amp, tau, yoffset):
    return yoffset + amp * np.exp(-t / tau)<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import os
import cPickle
import time
import sys
import math
import numpy as np
import shutil
from translate import utils
from translate.seq2seq_model import Seq2SeqModel
from collections import OrderedDict

class TranslationModel(object):
  def __init__(self, encoders, decoder, checkpoint_dir, learning_rate,
               learning_rate_decay_factor, keep_best=1, buckets=None, **kwargs):
    self.src_ext = [encoder.name for encoder in encoders]
    self.trg_ext = decoder.name
    self.extensions = self.src_ext + [self.trg_ext]

    # TODO: automatically find bucket sizes + handle multi-encoder setting
    # if binary_input:
    #   self.buckets = [(120, 10), (160, 10), (200, 15), (240, 15), (280, 15), (340, 20), (400, 20)]
    # elif character_level:
    #   self.buckets = [(20, 20), (40, 40), (60, 60), (100, 100)]  # quick fix
    # else:
    #   self.buckets = [(5, 10), (10, 15), (20, 25), (51, 51)]
    self.buckets = zip(*([encoder.buckets for encoder in encoders] + [decoder.buckets]))

    self.checkpoint_dir = checkpoint_dir
    self.keep_best = keep_best

    # list of extensions that use vector features instead of text features
    self.binary_input = [encoder.name for encoder in encoders if encoder.binary]
    self.character_level = [encoder.name for encoder in encoders if encoder.character_level]

    self.learning_rate = tf.Variable(learning_rate, trainable=False, name='learning_rate')
    self.learning_rate_decay_op = self.learning_rate.assign(self.learning_rate * learning_rate_decay_factor)

    with tf.device('/cpu:0'):
      self.global_step = tf.Variable(0, trainable=False, name='global_step')

    # main model
    self.model = Seq2SeqModel(encoders, decoder, self.buckets, self.learning_rate, self.global_step, **kwargs)
    self.vocabs = None

    self.saver = tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=5)

  def _read_data(self, filenames, max_train_size):
    utils.debug('reading vocabularies')
    self._read_vocab(filenames)

    utils.debug('reading training data')
    train_set = utils.read_dataset(filenames.train, self.extensions, self.vocabs, self.buckets,
                                   max_size=max_train_size, binary_input=self.binary_input,
                                   character_level=self.character_level)
    self.model.assign_data_set(train_set)

    utils.debug('reading development data')
    dev_set = utils.read_dataset(filenames.dev, self.extensions, self.vocabs, self.buckets,
                                 binary_input=self.binary_input, character_level=self.character_level)
    self.model.dev_set = dev_set

  def _read_vocab(self, filenames):
    if self.vocabs is not None:
      return

    # don't try reading vocabulary for encoders that take pre-computed features
    self.vocabs = [
      utils.initialize_vocabulary(vocab_path) if ext not in self.binary_input else None
      for ext, vocab_path in zip(self.extensions, filenames.vocab)
    ]
    self.src_vocab = self.vocabs[:-1]
    self.trg_vocab = self.vocabs[-1]
    self.ngrams = filenames.lm_path and utils.read_ngrams(filenames.lm_path, self.trg_vocab.vocab)

  def train(self, sess, filenames, beam_size, steps_per_checkpoint, steps_per_eval=None, bleu_script=None,
            max_train_size=None, eval_output=None, remove_unk=False, max_steps=0):
    utils.log('reading training and development data')
    self._read_data(filenames, max_train_size)
    previous_losses = []

    loss, time_, steps = 0, 0, 0

    utils.log('starting training')
    while True:
      model = self.model

      start_time = time.time()
      step_loss = self._train_step(sess, model)

      time_  += (time.time() - start_time)
      loss += step_loss
      steps += 1

      global_step = self.global_step.eval(sess)

      if steps_per_checkpoint and global_step % steps_per_checkpoint == 0:
        loss_ = loss / steps
        perplexity = math.exp(loss_) if loss_ < 300 else float('inf')

        utils.log('global step {} learning rate {:.4f} step-time {:.2f} perplexity {:.2f}'.format(
          global_step, self.model.learning_rate.eval(), time_ / steps, perplexity))

        # decay learning rate when loss is worse than last losses
        if len(previous_losses) > 2 and loss > max(previous_losses[-3:]):
          utils.debug('decreasing learning rate')
          sess.run(self.learning_rate_decay_op)

        previous_losses.append(loss)
        loss, time_, steps = 0, 0, 0

        self._eval_step(sess, self.model)
        self.save(sess)

      if steps_per_eval and bleu_script and global_step % steps_per_eval == 0:
        utils.log('starting BLEU eval')
        output = '{}.{}'.format(eval_output, global_step)
        score = self.evaluate(sess, filenames, beam_size, bleu_script, on_dev=True, output=output,
                              remove_unk=remove_unk)
        self._manage_best_checkpoints(global_step, score)

      if 0 < max_steps < global_step:
        utils.log('finished training')
        return

  def _train_step(self, sess, model):
    r = np.random.random_sample()
    bucket_id = min(i for i in xrange(len(model.train_bucket_scales)) if model.train_bucket_scales[i] > r)
    return model.step(sess, model.train_set, bucket_id)

  def _eval_step(self, sess, model):
    # compute perplexity on dev set
    for bucket_id in xrange(len(self.buckets)):
      if not model.dev_set[bucket_id]:
        utils.log("  eval: empty bucket {}".format(bucket_id))
        continue

      eval_loss = model.step(sess, model.dev_set, bucket_id, forward_only=True)
      perplexity = math.exp(eval_loss) if eval_loss < 300 else float('inf')
      utils.log("  eval: bucket {} perplexity {:.2f}".format(bucket_id, perplexity))

  def _decode_sentence(self, sess, src_sentences, beam_size=1, remove_unk=False):
    # TODO: this should be in utils
    token_ids = [utils.sentence_to_token_ids(sentence, vocab.vocab)
                 if vocab is not None else sentence   # when `sentence` is not a sentence but a vector...
                 for vocab, sentence in zip(self.vocabs, src_sentences)]
    max_len = self.buckets[-1][0] - 1  # TODO

    if any(len(ids_) > max_len for ids_ in token_ids):
      len_ = max(map(len, token_ids))
      utils.warn("line is too long ({} tokens), truncating".format(len_))
      token_ids = [ids_[:max_len] for ids_ in token_ids]

    if beam_size <= 1 and not isinstance(sess, list):
      trg_token_ids = self.model.greedy_decoding(sess, token_ids)
    else:
      hypotheses, scores = self.model.beam_search_decoding(sess, token_ids, beam_size, ngrams=self.ngrams,
                                                           reverse_vocab=self.trg_vocab.reverse)
      trg_token_ids = hypotheses[0]   # first hypothesis is the highest scoring one

    # remove EOS symbols from output
    if utils.EOS_ID in trg_token_ids:
      trg_token_ids = trg_token_ids[:trg_token_ids.index(utils.EOS_ID)]

    trg_tokens = [self.trg_vocab.reverse[i] if i < len(self.trg_vocab.reverse) else utils._UNK
                  for i in trg_token_ids]

    if remove_unk:
      trg_tokens = [token for token in trg_tokens if token != utils._UNK]

    if self.trg_ext in self.character_level:
      return ''.join(trg_tokens)
    else:
      return ' '.join(trg_tokens).replace('@@ ', '')  # merge subword units

  def decode(self, sess, filenames, beam_size, output=None, remove_unk=False):
    self._read_vocab(filenames)

    output_file = None
    try:
      output_file = sys.stdout if output is None else open(output, 'w')

      for lines in utils.read_lines(filenames.test[:-1], self.src_ext, self.binary_input):
        trg_sentence = self._decode_sentence(sess, lines, beam_size, remove_unk)
        output_file.write(trg_sentence + '\n')
        output_file.flush()
    finally:
      if output_file is not None:
        output_file.close()

  def evaluate(self, sess, filenames, beam_size, bleu_script, on_dev=False, output=None, remove_unk=False):
    self._read_vocab(filenames)
    if self.ngrams is not None:
      utils.debug('using external language model')

    filenames_ = filenames.dev if on_dev else filenames.test
    lines = list(utils.read_lines(filenames_, self.extensions, self.binary_input))

    hypotheses = [self._decode_sentence(sess, lines_[:-1], beam_size, remove_unk)
                  for lines_ in lines]
    references = [lines_[-1].strip().replace('@@ ', '') for lines_ in lines]

    bleu = utils.bleu_score(bleu_script, hypotheses, references)
    utils.log(bleu)
    if output is not None:
      with open(output, 'w') as f:
        f.writelines(line + '\n' for line in hypotheses)

    return bleu.score

  def _manage_best_checkpoints(self, step, score):
    score_filename = os.path.join(self.checkpoint_dir, 'bleu-scores.txt')
    # try loading previous scores
    try:
      with open(score_filename) as f:
        # list of pairs (score, step)
        scores = [(float(line.split()[0]), int(line.split()[1])) for line in f]
    except IOError:
      scores = []

    best_scores = sorted(scores, reverse=True)[:self.keep_best]

    if any(score_ < score for score_, _ in best_scores) or not best_scores:
      shutil.copy(os.path.join(self.checkpoint_dir, 'translate-{}'.format(step)),
                  os.path.join(self.checkpoint_dir, 'best-{}'.format(step)))

      if all(score_ < score for score_, _ in best_scores):
        path = os.path.abspath(os.path.join(self.checkpoint_dir, 'best'))
        try:  # remove old links
          os.remove(path)
        except OSError:
          pass
        # make symbolic links to best model
        os.symlink('{}-{}'.format(path, step), path)

      best_scores = sorted(best_scores + [(score, step)], reverse=True)

      for _, step_ in best_scores[self.keep_best:]:
        # remove checkpoints that are not in the top anymore
        try:
          os.remove(os.path.join(self.checkpoint_dir, 'best-{}'.format(step_)))
        except OSError:
          pass

    # save bleu scores
    scores.append((score, step))

    with open(score_filename, 'w') as f:
      for score_, step_ in scores:
        f.write('{} {}\n'.format(score_, step_))

  def initialize(self, sess, checkpoints=None, reset=False, reset_learning_rate=False):
    sess.run(tf.initialize_all_variables())
    if not reset:
      blacklist = ('learning_rate', 'dropout_keep_prob') if reset_learning_rate else ()
      load_checkpoint(sess, self.checkpoint_dir, blacklist=blacklist)

    if checkpoints is not None:  # load partial checkpoints
      for checkpoint in checkpoints:  # checkpoint files to load
        load_checkpoint(sess, None, checkpoint,
                        blacklist=('learning_rate', 'global_step', 'dropout_keep_prob'))

  def save(self, sess):
    save_checkpoint(sess, self.saver, self.checkpoint_dir, self.global_step)

def load_checkpoint(sess, checkpoint_dir, filename=None, blacklist=()):
  """ `checkpoint_dir` should be unique to this model
  if `filename` is None, we load last checkpoint, otherwise
    we ignore `checkpoint_dir` and load the given checkpoint file.
  """
  if filename is None:
    # load last checkpoint
    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
    if ckpt is not None:
      filename = ckpt.model_checkpoint_path
  else:
    checkpoint_dir = os.path.dirname(filename)

  var_file = os.path.join(checkpoint_dir, 'vars.pkl')

  if os.path.exists(var_file):
    with open(var_file, 'rb') as f:
      var_names = cPickle.load(f)
      variables = [var for var in tf.all_variables() if var.name in var_names]
  else:
    variables = tf.all_variables()

  # remove variables from blacklist
  variables = [var for var in variables if not any(var.name.startswith(prefix) for prefix in blacklist)]

  if filename is not None:
    utils.log('reading model parameters from {}'.format(filename))
    tf.train.Saver(variables).restore(sess, filename)

    utils.debug('retrieved parameters ({})'.format(len(variables)))
    for var in variables:
      utils.debug('  {} {}'.format(var.name, var.get_shape()))

def save_checkpoint(sess, saver, checkpoint_dir, step=None, name=None):
  """ `checkpoint_dir` should be unique to this model """
  var_file = os.path.join(checkpoint_dir, 'vars.pkl')
  name = name or 'translate'

  if not os.path.exists(checkpoint_dir):
    utils.log("creating directory {}".format(checkpoint_dir))
    os.makedirs(checkpoint_dir)

  with open(var_file, 'wb') as f:
    var_names = [var.name for var in tf.all_variables()]
    cPickle.dump(var_names, f)

  utils.log('saving model to {}'.format(checkpoint_dir))
  checkpoint_path = os.path.join(checkpoint_dir, name)
  saver.save(sess, checkpoint_path, step, write_meta_graph=False)
  utils.log('finished saving model')
<commit_msg>Fixed decoding bug<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import os
import cPickle
import time
import sys
import math
import numpy as np
import shutil
from translate import utils
from translate.seq2seq_model import Seq2SeqModel
from collections import OrderedDict

class TranslationModel(object):
  def __init__(self, encoders, decoder, checkpoint_dir, learning_rate,
               learning_rate_decay_factor, keep_best=1, buckets=None, **kwargs):
    self.src_ext = [encoder.name for encoder in encoders]
    self.trg_ext = decoder.name
    self.extensions = self.src_ext + [self.trg_ext]

    # TODO: automatically find bucket sizes + handle multi-encoder setting
    # if binary_input:
    #   self.buckets = [(120, 10), (160, 10), (200, 15), (240, 15), (280, 15), (340, 20), (400, 20)]
    # elif character_level:
    #   self.buckets = [(20, 20), (40, 40), (60, 60), (100, 100)]  # quick fix
    # else:
    #   self.buckets = [(5, 10), (10, 15), (20, 25), (51, 51)]
    self.buckets = zip(*([encoder.buckets for encoder in encoders] + [decoder.buckets]))

    self.checkpoint_dir = checkpoint_dir
    self.keep_best = keep_best

    # list of extensions that use vector features instead of text features
    self.binary_input = [encoder.name for encoder in encoders if encoder.binary]
    self.character_level = [encoder.name for encoder in encoders if encoder.character_level]

    self.learning_rate = tf.Variable(learning_rate, trainable=False, name='learning_rate')
    self.learning_rate_decay_op = self.learning_rate.assign(self.learning_rate * learning_rate_decay_factor)

    with tf.device('/cpu:0'):
      self.global_step = tf.Variable(0, trainable=False, name='global_step')

    # main model
    self.model = Seq2SeqModel(encoders, decoder, self.buckets, self.learning_rate, self.global_step, **kwargs)
    self.vocabs = None

    self.saver = tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=5)

  def _read_data(self, filenames, max_train_size):
    utils.debug('reading vocabularies')
    self._read_vocab(filenames)

    utils.debug('reading training data')
    train_set = utils.read_dataset(filenames.train, self.extensions, self.vocabs, self.buckets,
                                   max_size=max_train_size, binary_input=self.binary_input,
                                   character_level=self.character_level)
    self.model.assign_data_set(train_set)

    utils.debug('reading development data')
    dev_set = utils.read_dataset(filenames.dev, self.extensions, self.vocabs, self.buckets,
                                 binary_input=self.binary_input, character_level=self.character_level)
    self.model.dev_set = dev_set

  def _read_vocab(self, filenames):
    if self.vocabs is not None:
      return

    # don't try reading vocabulary for encoders that take pre-computed features
    self.vocabs = [
      utils.initialize_vocabulary(vocab_path) if ext not in self.binary_input else None
      for ext, vocab_path in zip(self.extensions, filenames.vocab)
    ]
    self.src_vocab = self.vocabs[:-1]
    self.trg_vocab = self.vocabs[-1]
    self.ngrams = filenames.lm_path and utils.read_ngrams(filenames.lm_path, self.trg_vocab.vocab)

  def train(self, sess, filenames, beam_size, steps_per_checkpoint, steps_per_eval=None, bleu_script=None,
            max_train_size=None, eval_output=None, remove_unk=False, max_steps=0):
    utils.log('reading training and development data')
    self._read_data(filenames, max_train_size)
    previous_losses = []

    loss, time_, steps = 0, 0, 0

    utils.log('starting training')
    while True:
      model = self.model

      start_time = time.time()
      step_loss = self._train_step(sess, model)

      time_  += (time.time() - start_time)
      loss += step_loss
      steps += 1

      global_step = self.global_step.eval(sess)

      if steps_per_checkpoint and global_step % steps_per_checkpoint == 0:
        loss_ = loss / steps
        perplexity = math.exp(loss_) if loss_ < 300 else float('inf')

        utils.log('global step {} learning rate {:.4f} step-time {:.2f} perplexity {:.2f}'.format(
          global_step, self.model.learning_rate.eval(), time_ / steps, perplexity))

        # decay learning rate when loss is worse than last losses
        if len(previous_losses) > 2 and loss > max(previous_losses[-3:]):
          utils.debug('decreasing learning rate')
          sess.run(self.learning_rate_decay_op)

        previous_losses.append(loss)
        loss, time_, steps = 0, 0, 0

        self._eval_step(sess, self.model)
        self.save(sess)

      if steps_per_eval and bleu_script and global_step % steps_per_eval == 0:
        utils.log('starting BLEU eval')
        output = '{}.{}'.format(eval_output, global_step)
        score = self.evaluate(sess, filenames, beam_size, bleu_script, on_dev=True, output=output,
                              remove_unk=remove_unk)
        self._manage_best_checkpoints(global_step, score)

      if 0 < max_steps < global_step:
        utils.log('finished training')
        return

  def _train_step(self, sess, model):
    r = np.random.random_sample()
    bucket_id = min(i for i in xrange(len(model.train_bucket_scales)) if model.train_bucket_scales[i] > r)
    return model.step(sess, model.train_set, bucket_id)

  def _eval_step(self, sess, model):
    # compute perplexity on dev set
    for bucket_id in xrange(len(self.buckets)):
      if not model.dev_set[bucket_id]:
        utils.log("  eval: empty bucket {}".format(bucket_id))
        continue

      eval_loss = model.step(sess, model.dev_set, bucket_id, forward_only=True)
      perplexity = math.exp(eval_loss) if eval_loss < 300 else float('inf')
      utils.log("  eval: bucket {} perplexity {:.2f}".format(bucket_id, perplexity))

  def _decode_sentence(self, sess, src_sentences, beam_size=1, remove_unk=False):
    token_ids = [utils.sentence_to_token_ids(sentence, vocab.vocab)
                 if vocab is not None else sentence   # when `sentence` is not a sentence but a vector...
                 for vocab, sentence in zip(self.vocabs, src_sentences)]

    for ids_, max_len in zip(token_ids, self.buckets[-1][:-1]):
      if len(ids_) > max_len:
        utils.warn("line is too long ({} tokens), truncating".format(len(ids_)))
        ids_[:] = ids_[:max_len]

    if beam_size <= 1 and not isinstance(sess, list):
      trg_token_ids = self.model.greedy_decoding(sess, token_ids)
    else:
      hypotheses, scores = self.model.beam_search_decoding(sess, token_ids, beam_size, ngrams=self.ngrams,
                                                           reverse_vocab=self.trg_vocab.reverse)
      trg_token_ids = hypotheses[0]   # first hypothesis is the highest scoring one

    # remove EOS symbols from output
    if utils.EOS_ID in trg_token_ids:
      trg_token_ids = trg_token_ids[:trg_token_ids.index(utils.EOS_ID)]

    trg_tokens = [self.trg_vocab.reverse[i] if i < len(self.trg_vocab.reverse) else utils._UNK
                  for i in trg_token_ids]

    if remove_unk:
      trg_tokens = [token for token in trg_tokens if token != utils._UNK]

    if self.trg_ext in self.character_level:
      return ''.join(trg_tokens)
    else:
      return ' '.join(trg_tokens).replace('@@ ', '')  # merge subword units

  def decode(self, sess, filenames, beam_size, output=None, remove_unk=False):
    self._read_vocab(filenames)

    output_file = None
    try:
      output_file = sys.stdout if output is None else open(output, 'w')

      for lines in utils.read_lines(filenames.test[:-1], self.src_ext, self.binary_input):
        trg_sentence = self._decode_sentence(sess, lines, beam_size, remove_unk)
        output_file.write(trg_sentence + '\n')
        output_file.flush()
    finally:
      if output_file is not None:
        output_file.close()

  def evaluate(self, sess, filenames, beam_size, bleu_script, on_dev=False, output=None, remove_unk=False):
    self._read_vocab(filenames)
    if self.ngrams is not None:
      utils.debug('using external language model')

    filenames_ = filenames.dev if on_dev else filenames.test
    lines = list(utils.read_lines(filenames_, self.extensions, self.binary_input))

    hypotheses = [self._decode_sentence(sess, lines_[:-1], beam_size, remove_unk)
                  for lines_ in lines]
    references = [lines_[-1].strip().replace('@@ ', '') for lines_ in lines]

    bleu = utils.bleu_score(bleu_script, hypotheses, references)
    utils.log(bleu)
    if output is not None:
      with open(output, 'w') as f:
        f.writelines(line + '\n' for line in hypotheses)

    return bleu.score

  def _manage_best_checkpoints(self, step, score):
    score_filename = os.path.join(self.checkpoint_dir, 'bleu-scores.txt')
    # try loading previous scores
    try:
      with open(score_filename) as f:
        # list of pairs (score, step)
        scores = [(float(line.split()[0]), int(line.split()[1])) for line in f]
    except IOError:
      scores = []

    best_scores = sorted(scores, reverse=True)[:self.keep_best]

    if any(score_ < score for score_, _ in best_scores) or not best_scores:
      shutil.copy(os.path.join(self.checkpoint_dir, 'translate-{}'.format(step)),
                  os.path.join(self.checkpoint_dir, 'best-{}'.format(step)))

      if all(score_ < score for score_, _ in best_scores):
        path = os.path.abspath(os.path.join(self.checkpoint_dir, 'best'))
        try:  # remove old links
          os.remove(path)
        except OSError:
          pass
        # make symbolic links to best model
        os.symlink('{}-{}'.format(path, step), path)

      best_scores = sorted(best_scores + [(score, step)], reverse=True)

      for _, step_ in best_scores[self.keep_best:]:
        # remove checkpoints that are not in the top anymore
        try:
          os.remove(os.path.join(self.checkpoint_dir, 'best-{}'.format(step_)))
        except OSError:
          pass

    # save bleu scores
    scores.append((score, step))

    with open(score_filename, 'w') as f:
      for score_, step_ in scores:
        f.write('{} {}\n'.format(score_, step_))

  def initialize(self, sess, checkpoints=None, reset=False, reset_learning_rate=False):
    sess.run(tf.initialize_all_variables())
    if not reset:
      blacklist = ('learning_rate', 'dropout_keep_prob') if reset_learning_rate else ()
      load_checkpoint(sess, self.checkpoint_dir, blacklist=blacklist)

    if checkpoints is not None:  # load partial checkpoints
      for checkpoint in checkpoints:  # checkpoint files to load
        load_checkpoint(sess, None, checkpoint,
                        blacklist=('learning_rate', 'global_step', 'dropout_keep_prob'))

  def save(self, sess):
    save_checkpoint(sess, self.saver, self.checkpoint_dir, self.global_step)

def load_checkpoint(sess, checkpoint_dir, filename=None, blacklist=()):
  """ `checkpoint_dir` should be unique to this model
  if `filename` is None, we load last checkpoint, otherwise
    we ignore `checkpoint_dir` and load the given checkpoint file.
  """
  if filename is None:
    # load last checkpoint
    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
    if ckpt is not None:
      filename = ckpt.model_checkpoint_path
  else:
    checkpoint_dir = os.path.dirname(filename)

  var_file = os.path.join(checkpoint_dir, 'vars.pkl')

  if os.path.exists(var_file):
    with open(var_file, 'rb') as f:
      var_names = cPickle.load(f)
      variables = [var for var in tf.all_variables() if var.name in var_names]
  else:
    variables = tf.all_variables()

  # remove variables from blacklist
  variables = [var for var in variables if not any(var.name.startswith(prefix) for prefix in blacklist)]

  if filename is not None:
    utils.log('reading model parameters from {}'.format(filename))
    tf.train.Saver(variables).restore(sess, filename)

    utils.debug('retrieved parameters ({})'.format(len(variables)))
    for var in variables:
      utils.debug('  {} {}'.format(var.name, var.get_shape()))

def save_checkpoint(sess, saver, checkpoint_dir, step=None, name=None):
  """ `checkpoint_dir` should be unique to this model """
  var_file = os.path.join(checkpoint_dir, 'vars.pkl')
  name = name or 'translate'

  if not os.path.exists(checkpoint_dir):
    utils.log("creating directory {}".format(checkpoint_dir))
    os.makedirs(checkpoint_dir)

  with open(var_file, 'wb') as f:
    var_names = [var.name for var in tf.all_variables()]
    cPickle.dump(var_names, f)

  utils.log('saving model to {}'.format(checkpoint_dir))
  checkpoint_path = os.path.join(checkpoint_dir, name)
  saver.save(sess, checkpoint_path, step, write_meta_graph=False)
  utils.log('finished saving model')
<|endoftext|>