<commit_before>#!/usr/bin/env python
import argparse
import os, sys, subprocess, json, re

def run(*args):
    ret = subprocess.call(args,  stdout=sys.stdout, stderr=sys.stderr)
    if ret != 0:
        exit(ret)

def target_binary_name(target):
    return target['name'].replace('-', '_') + target['metadata']['extra_filename']

class Manifest(object):
    def __init__(self, dir):
        try:
            output = subprocess.check_output(['cargo', 'read-manifest', '--manifest-path', dir],
                                             stderr=sys.stderr)
        except subprocess.CalledProcessError as e:
            print(e.output.decode())
            exit(e.returncode)
        self.manifest = json.loads(output.decode())
    def targets(self):
        return self.manifest['targets']
    def lib_name(self):
        for target in self.targets():
            if 'lib' in target['kind']:
                return target['name'].replace('-', '_')
        return None

def cargo(version, manifest, args):
    is_nightly = version == 'nightly'
    cargo_args = args.cargo_args
    nightly_feature = os.environ.get('TRAVIS_CARGO_NIGHTLY_FEATURE', 'unstable')
    if cargo_args[0] == 'bench' and not is_nightly:
        print('skipping `cargo bench` on non-nightly version')
        return

    if is_nightly:
        for i in range(0, len(cargo_args)):
            if cargo_args[i] == '--features':
                cargo_args[i + 1] += ' ' + nightly_feature
                break
        else:
            cargo_args += ['--features', nightly_feature]
    if not args.quiet and '--verbose' not in cargo_args and '-v' not in cargo_args:
        cargo_args.append('--verbose')

    run('cargo', *cargo_args)

def doc_upload(version, manifest, args):
    branch = os.environ['TRAVIS_BRANCH']
    pr = os.environ['TRAVIS_PULL_REQUEST']
    token = os.environ['GH_TOKEN']
    repo = os.environ['TRAVIS_REPO_SLUG']

    lib_name = manifest.lib_name()
    if lib_name is None:
        sys.stderr.write('error: uploading docs for package with no library')
        exit(1)

    if branch == 'master' and pr == 'false':
        print('uploading docs...')
        sys.stdout.flush()
        with open('target/doc/index.html', 'w') as f:
            f.write('<meta http-equiv=refresh content=0;url=%s/index.html>' % lib_name)

        run('git', 'clone', 'https://github.com/davisp/ghp-import')
        run('./ghp-import/ghp-import', '-n', 'target/doc')
        run('git', 'push', '-fq', 'https://%s@github.com/%s.git' % (token, repo), 'gh-pages')

def coveralls(version, manifest, args):
    job_id = os.environ['TRAVIS_JOB_ID']

    test_binaries = []

    # look through the output of `cargo test` to find the test
    # binaries.
    # FIXME: the information cargo feeds us is
    # inconsistent/inaccurate, so using the output of read-manifest is
    # far too much trouble.
    try:
        output = subprocess.check_output(['cargo', 'test'],
                                         stderr=sys.stderr)
    except subprocess.CalledProcessError as e:
        print(e.output.decode())
        exit(e.returncode)
    output = output.decode()
    running = re.compile('^     Running target/debug/(.*)$', re.M)
    for line in running.finditer(output):
        test_binaries.append(line.group(1))

    # build kcov:
    init = '''
    sudo apt-get install libcurl4-openssl-dev libelf-dev libdw-dev cmake
    git clone --depth 1 https://github.com/SimonKagstrom/kcov
    mkdir kcov/build
    '''
    for line in init.split('\n'):
        line = line.strip()
        if line:
            print('Running: %s' % line)
            run(*line.split())
    current = os.getcwd()
    os.chdir('kcov/build')
    build = '''
    cmake ..
    make
    sudo make install
    '''
    for line in build.split('\n'):
        line = line.strip()
        if line:
            print('Running: %s' % line)
            run(*line.split())
    os.chdir(current)

    # record coverage for each binary
    for binary in test_binaries:
        print('Recording %s' % binary)
        run('kcov', '--exclude-pattern=/.cargo', 'target/kcov-' + binary,
            'target/debug/' + binary)
    # merge all the coverages and upload in one go
    print('Uploading coverage')
    run('kcov', '--merge', '--coveralls-id=' + job_id, 'target/kcov',
        *('target/kcov-' + b for b in test_binaries))

def main():
    parser = argparse.ArgumentParser(description = 'manages interactions between travis '
                                     'and cargo/rust compilers')

    parser.add_argument('-q','--quiet', action='store_true', default=False,
                        help='don\'t pass --verbose to cargo')
    parser.add_argument('--only', metavar='VERSION',
                        help='only run the given command if the specified version matches '
                        '`TRAVIS_RUST_VERSION`')
    subparsers = parser.add_subparsers(title = 'subcommands')

    p_cargo = subparsers.add_parser('cargo',
                                    help = 'run cargo, passing '
                                    '`--features $TRAVIS_CARGO_NIGHTLY_FEATURE` (default '
                                    '\'unstable\') if the nightly branch is detected, and '
                                    'skipping `cargo bench` if it is not')
    p_cargo.add_argument('cargo_args', metavar='ARGS', nargs='+')
    p_cargo.set_defaults(func = cargo)

    p_doc_upload = subparsers.add_parser('doc-upload',
                                         help = 'uses ghp-import to upload cargo-rendered docs '
                                         'from the master branch')
    p_doc_upload.set_defaults(func = doc_upload)

    p_coveralls = subparsers.add_parser('coveralls',
                                        help = 'runs all targets that are have `test = true` \
                                        and `debug = true`')
    p_coveralls.set_defaults(func = coveralls)

    args = parser.parse_args()

    version = os.environ['TRAVIS_RUST_VERSION']
    if args.only and args.only != version:
        return

    manifest = Manifest(os.getcwd())
    args.func(version, manifest, args)

if __name__ == '__main__':
    main()
<commit_msg>Pass arguments/features to `cargo test` for coveralls.<commit_after>#!/usr/bin/env python
import argparse
import os, sys, subprocess, json, re

def run(*args):
    ret = subprocess.call(args,  stdout=sys.stdout, stderr=sys.stderr)
    if ret != 0:
        exit(ret)

def target_binary_name(target):
    return target['name'].replace('-', '_') + target['metadata']['extra_filename']

class Manifest(object):
    def __init__(self, dir):
        try:
            output = subprocess.check_output(['cargo', 'read-manifest', '--manifest-path', dir],
                                             stderr=sys.stderr)
        except subprocess.CalledProcessError as e:
            print(e.output.decode())
            exit(e.returncode)
        self.manifest = json.loads(output.decode())
    def targets(self):
        return self.manifest['targets']
    def lib_name(self):
        for target in self.targets():
            if 'lib' in target['kind']:
                return target['name'].replace('-', '_')
        return None

def add_features(cargo_args, version):
    nightly_feature = os.environ.get('TRAVIS_CARGO_NIGHTLY_FEATURE', 'unstable')
    if version == 'nightly':
        for i in range(0, len(cargo_args)):
            if cargo_args[i] == '--features':
                cargo_args[i + 1] += ' ' + nightly_feature
                break
        else:
            cargo_args += ['--features', nightly_feature]

def cargo(version, manifest, args):
    cargo_args = args.cargo_args
    if cargo_args[0] == 'bench' and not is_nightly:
        print('skipping `cargo bench` on non-nightly version')
        return

    add_features(cargo_args, version)

    if not args.quiet and '--verbose' not in cargo_args and '-v' not in cargo_args:
        cargo_args.append('--verbose')

    run('cargo', *cargo_args)

def doc_upload(version, manifest, args):
    branch = os.environ['TRAVIS_BRANCH']
    pr = os.environ['TRAVIS_PULL_REQUEST']
    token = os.environ['GH_TOKEN']
    repo = os.environ['TRAVIS_REPO_SLUG']

    lib_name = manifest.lib_name()
    if lib_name is None:
        sys.stderr.write('error: uploading docs for package with no library')
        exit(1)

    if branch == 'master' and pr == 'false':
        print('uploading docs...')
        sys.stdout.flush()
        with open('target/doc/index.html', 'w') as f:
            f.write('<meta http-equiv=refresh content=0;url=%s/index.html>' % lib_name)

        run('git', 'clone', 'https://github.com/davisp/ghp-import')
        run('./ghp-import/ghp-import', '-n', 'target/doc')
        run('git', 'push', '-fq', 'https://%s@github.com/%s.git' % (token, repo), 'gh-pages')

def coveralls(version, manifest, args):
    job_id = os.environ['TRAVIS_JOB_ID']

    cargo_args = args.cargo_args
    add_features(cargo_args, version)

    test_binaries = []

    # look through the output of `cargo test` to find the test
    # binaries.
    # FIXME: the information cargo feeds us is
    # inconsistent/inaccurate, so using the output of read-manifest is
    # far too much trouble.
    try:
        output = subprocess.check_output(['cargo', 'test'] + list(cargo_args),
                                         stderr=sys.stderr)
    except subprocess.CalledProcessError as e:
        print(e.output.decode())
        exit(e.returncode)
    output = output.decode()
    running = re.compile('^     Running target/debug/(.*)$', re.M)
    for line in running.finditer(output):
        test_binaries.append(line.group(1))

    # build kcov:
    init = '''
    sudo apt-get install libcurl4-openssl-dev libelf-dev libdw-dev cmake
    git clone --depth 1 https://github.com/SimonKagstrom/kcov
    mkdir kcov/build
    '''
    for line in init.split('\n'):
        line = line.strip()
        if line:
            print('Running: %s' % line)
            run(*line.split())
    current = os.getcwd()
    os.chdir('kcov/build')
    build = '''
    cmake ..
    make
    sudo make install
    '''
    for line in build.split('\n'):
        line = line.strip()
        if line:
            print('Running: %s' % line)
            run(*line.split())
    os.chdir(current)

    # record coverage for each binary
    for binary in test_binaries:
        print('Recording %s' % binary)
        run('kcov', '--exclude-pattern=/.cargo', 'target/kcov-' + binary,
            'target/debug/' + binary)
    # merge all the coverages and upload in one go
    print('Uploading coverage')
    run('kcov', '--merge', '--coveralls-id=' + job_id, 'target/kcov',
        *('target/kcov-' + b for b in test_binaries))

def main():
    parser = argparse.ArgumentParser(description = 'manages interactions between travis '
                                     'and cargo/rust compilers')

    parser.add_argument('-q','--quiet', action='store_true', default=False,
                        help='don\'t pass --verbose to cargo')
    parser.add_argument('--only', metavar='VERSION',
                        help='only run the given command if the specified version matches '
                        '`TRAVIS_RUST_VERSION`')
    subparsers = parser.add_subparsers(title = 'subcommands')

    p_cargo = subparsers.add_parser('cargo',
                                    help = 'run cargo, passing '
                                    '`--features $TRAVIS_CARGO_NIGHTLY_FEATURE` (default '
                                    '\'unstable\') if the nightly branch is detected, and '
                                    'skipping `cargo bench` if it is not')
    p_cargo.add_argument('cargo_args', metavar='ARGS', nargs='+',
                         help = 'arguments to pass to `cargo`, including the subcommand')
    p_cargo.set_defaults(func = cargo)

    p_doc_upload = subparsers.add_parser('doc-upload',
                                         help = 'uses ghp-import to upload cargo-rendered docs '
                                         'from the master branch')
    p_doc_upload.set_defaults(func = doc_upload)

    p_coveralls = subparsers.add_parser('coveralls',
                                        help = 'runs all targets that are have `test = true` \
                                        and `debug = true`')
    p_coveralls.add_argument('cargo_args', metavar='ARGS', nargs='*',
                             help = 'arguments to pass to `cargo test`, '
                             'not including the subcommand')
    p_coveralls.set_defaults(func = coveralls)

    args = parser.parse_args()

    version = os.environ['TRAVIS_RUST_VERSION']
    if args.only and args.only != version:
        return

    manifest = Manifest(os.getcwd())
    args.func(version, manifest, args)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2019 The Trax Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Trax trainer."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import datetime
import os

from absl import app
from absl import flags
from absl import logging

import gin
import jax
import tensorflow.compat.v2 as tf
from trax import backend
from trax.supervised import trainer_lib
from trax.tf_numpy import numpy as tf_np

FLAGS = flags.FLAGS

flags.DEFINE_string('dataset', None, 'Which dataset to use.')
flags.DEFINE_string('model', None, 'Which model to train.')
flags.DEFINE_string('data_dir', None, 'Path to the directory with data.')
flags.DEFINE_string('output_dir', None,
                    'Path to the directory to save logs and checkpoints.')
flags.DEFINE_multi_string('config_file', None,
                          'Configuration file with parameters (.gin).')
flags.DEFINE_multi_string('config', None,
                          'Configuration parameters (gin string).')
flags.DEFINE_integer('log_level', logging.INFO, 'Log level.')
# TPU Flags
flags.DEFINE_bool('use_tpu', False, "Whether we're running on TPU.")
flags.DEFINE_string(
    'jax_xla_backend', 'xla',
    'Either "xla" for the XLA service directly, or "tpu_driver"'
    'for a TPU Driver backend.')
flags.DEFINE_string('jax_backend_target', 'local',
                    'Either "local" or "rpc:address" to connect to a '
                    'remote service target.')
# TF Flags
flags.DEFINE_bool('enable_eager_execution', True,
                  "Whether we're running TF in eager mode.")
flags.DEFINE_bool('tf_xla', True, 'Whether to turn on XLA for TF.')
flags.DEFINE_bool('tf_opt_pin_to_host', False, 'Whether to turn on TF '
                  'pin-to-host optimization.')
flags.DEFINE_bool('tf_opt_layout', False, 'Whether to turn on TF layout '
                  'optimization.')
flags.DEFINE_bool('tf_xla_forced_compile', False, 'Use forced-compilation '
                  'instead of auto-clustering for XLA. This flag only has '
                  'effects when --tf_xla is on.')
flags.DEFINE_bool('tf_allow_float64', False, 'Whether to allow float64 for TF.')

def _default_output_dir():
  """Default output directory."""
  try:
    dataset_name = gin.query_parameter('inputs.dataset_name')
  except ValueError:
    dataset_name = 'random'
  dir_name = '{model_name}_{dataset_name}_{timestamp}'.format(
      model_name=gin.query_parameter('train.model').configurable.name,
      dataset_name=dataset_name,
      timestamp=datetime.datetime.now().strftime('%Y%m%d_%H%M'),
  )
  dir_path = os.path.join('~', 'trax', dir_name)
  print()
  trainer_lib.log('No --output_dir specified')
  return dir_path

def _setup_gin():
  """Setup gin configuration."""
  # Imports for configurables
  # pylint: disable=g-import-not-at-top,unused-import,g-bad-import-order,reimported,unused-variable
  from trax import models as _trax_models
  from trax import optimizers as _trax_opt
  # pylint: disable=g-import-not-at-top,unused-import,g-bad-import-order,reimported,unused-variable

  configs = FLAGS.config or []
  # Override with --dataset and --model
  if FLAGS.dataset:
    configs.append("inputs.dataset_name='%s'" % FLAGS.dataset)
    if FLAGS.data_dir:
      configs.append("inputs.data_dir='%s'" % FLAGS.data_dir)
  if FLAGS.model:
    configs.append('train.model=@trax.models.%s' % FLAGS.model)
  gin.parse_config_files_and_bindings(FLAGS.config_file, configs)

def set_tf_allow_float64(b):
  tf_np.set_allow_float64(b)

@gin.configurable
def tf_init_tpu(worker='', protocol=None):
  """Initializes TPU for TensorFlow.

  Args:
    worker: The BNS address of the remote TPU worker. If it's empty (the default
      value), TF will assume the TPU devices are connected to the local host.
    protocol: The network protocol used to connect to the TPU worker.
  Returns:
    The device name of the TPU worker's CPU.
  """
  protocol = protocol or 'grpc'
  is_local = (worker in ('', 'local'))
  resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=worker)
  if not is_local:
    tf.config.experimental_connect_to_cluster(resolver, protocol=protocol)
  tf.tpu.experimental.initialize_tpu_system(resolver)
  if is_local:
    return ''
  else:
    return '/job:worker'

def main(_):

  logging.set_verbosity(FLAGS.log_level)

  if FLAGS.enable_eager_execution:
    tf.compat.v1.enable_eager_execution()

  if FLAGS.tf_xla:
    tf.config.optimizer.set_jit(True)
    backend.set_tf_xla_forced_compile(FLAGS.tf_xla_forced_compile)

  tf.config.optimizer.set_experimental_options(
      {'pin_to_host_optimization': FLAGS.tf_opt_pin_to_host}
  )

  tf.config.optimizer.set_experimental_options(
      {'layout_optimizer': FLAGS.tf_opt_layout}
  )

  set_tf_allow_float64(FLAGS.tf_allow_float64)

  _setup_gin()

  if FLAGS.enable_eager_execution and backend.get_name() in ('numpy', 'jax'):
    # Numpy backend doesn't benefit from having the input pipeline run on GPU,
    # and jax backend has GPU memory contention if TF uses the GPU. Gin must be
    # set up first before determining the backend.
    tf.config.experimental.set_visible_devices([], 'GPU')

  # Setup output directory
  output_dir = FLAGS.output_dir or _default_output_dir()
  trainer_lib.log('Using --output_dir %s' % output_dir)
  output_dir = os.path.expanduser(output_dir)

  # If on TPU, let JAX know.
  if FLAGS.use_tpu:
    jax.config.update('jax_platform_name', 'tpu')
    jax.config.update('jax_xla_backend', FLAGS.jax_xla_backend)
    jax.config.update('jax_backend_target', FLAGS.jax_backend_target)

  if FLAGS.use_tpu and backend.get_name() == 'tf':
    worker_cpu = tf_init_tpu()
    with tf.device(worker_cpu):
      if trainer_lib.num_devices() == 1:
        # TF's device priority is GPU > CPU > TPU, so we need to explicitly make
        # the TPU core the default device here.
        with tf.device('/device:TPU:0'):
          trainer_lib.train(output_dir=output_dir)
      else:
        trainer_lib.train(output_dir=output_dir)
  else:
    trainer_lib.train(output_dir=output_dir)

  trainer_lib.log('Finished training.')

if __name__ == '__main__':
  app.run(main)
<commit_msg>Start tightening up trainer code.<commit_after># coding=utf-8
# Copyright 2019 The Trax Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Trax trainer."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import datetime
import os

from absl import app
from absl import flags
from absl import logging

import gin
import jax
import tensorflow.compat.v2 as tf
from trax import backend
from trax.supervised import trainer_lib
from trax.tf_numpy import numpy as tf_np

FLAGS = flags.FLAGS

flags.DEFINE_string('dataset', None, 'Which dataset to use.')
flags.DEFINE_string('model', None, 'Which model to train.')
flags.DEFINE_string('data_dir', None, 'Path to the directory with data.')
flags.DEFINE_string('output_dir', None,
                    'Path to the directory to save logs and checkpoints.')
flags.DEFINE_multi_string('config_file', None,
                          'Configuration file with parameters (.gin).')
flags.DEFINE_multi_string('config', None,
                          'Configuration parameters (gin string).')
flags.DEFINE_integer('log_level', logging.INFO, 'Log level.')
# TPU Flags
flags.DEFINE_bool('use_tpu', False, "Whether we're running on TPU.")
flags.DEFINE_string(
    'jax_xla_backend', 'xla',
    'Either "xla" for the XLA service directly, or "tpu_driver"'
    'for a TPU Driver backend.')
flags.DEFINE_string('jax_backend_target', 'local',
                    'Either "local" or "rpc:address" to connect to a '
                    'remote service target.')

# TensorFlow Flags
flags.DEFINE_bool('enable_eager_execution', True,
                  "Whether we're running TF in eager mode.")
flags.DEFINE_bool('tf_xla', True, 'Whether to turn on XLA for TF.')
flags.DEFINE_bool('tf_opt_pin_to_host', False, 'Whether to turn on TF '
                  'pin-to-host optimization.')
flags.DEFINE_bool('tf_opt_layout', False, 'Whether to turn on TF layout '
                  'optimization.')
flags.DEFINE_bool('tf_xla_forced_compile', False, 'Use forced-compilation '
                  'instead of auto-clustering for XLA. This flag only has '
                  'effects when --tf_xla is on.')
flags.DEFINE_bool('tf_allow_float64', False, 'Whether to allow float64 for TF.')

def _tf_setup_from_flags():
  """Processes TensorFlow-relevant flags."""
  if FLAGS.enable_eager_execution:
    tf.compat.v1.enable_eager_execution()
  if FLAGS.tf_xla:
    tf.config.optimizer.set_jit(True)
    backend.set_tf_xla_forced_compile(FLAGS.tf_xla_forced_compile)
  tf.config.optimizer.set_experimental_options({
      'pin_to_host_optimization': FLAGS.tf_opt_pin_to_host,
      'layout_optimizer': FLAGS.tf_opt_layout,
  })
  tf_np.set_allow_float64(FLAGS.tf_allow_float64)

def _gin_parse_configs():
  """Initializes gin-controlled bindings."""
  # Imports for configurables
  # pylint: disable=g-import-not-at-top,unused-import,g-bad-import-order,reimported,unused-variable
  from trax import models as _trax_models
  from trax import optimizers as _trax_opt
  # pylint: disable=g-import-not-at-top,unused-import,g-bad-import-order,reimported,unused-variable

  configs = FLAGS.config or []
  # Override with --dataset and --model
  if FLAGS.dataset:
    configs.append("inputs.dataset_name='%s'" % FLAGS.dataset)
    if FLAGS.data_dir:
      configs.append("inputs.data_dir='%s'" % FLAGS.data_dir)
  if FLAGS.model:
    configs.append('train.model=@trax.models.%s' % FLAGS.model)
  gin.parse_config_files_and_bindings(FLAGS.config_file, configs)

def _output_dir_or_default():
  """Returns a path to the output directory."""
  if FLAGS.output_dir:
    output_dir = FLAGS.output_dir
    trainer_lib.log('Using --output_dir {}'.format(output_dir))
    return os.path.expanduser(output_dir)

  # Else, generate a default output dir (under the user's home directory).
  try:
    dataset_name = gin.query_parameter('inputs.dataset_name')
  except ValueError:
    dataset_name = 'random'
  output_name = '{model_name}_{dataset_name}_{timestamp}'.format(
      model_name=gin.query_parameter('train.model').configurable.name,
      dataset_name=dataset_name,
      timestamp=datetime.datetime.now().strftime('%Y%m%d_%H%M'),
  )
  output_dir = os.path.join('~', 'trax', output_name)
  output_dir = os.path.expanduser(output_dir)
  print()
  trainer_lib.log('No --output_dir specified')
  trainer_lib.log('Using default output_dir: {}'.format(output_dir))
  return output_dir

def _jax_and_tf_configure_for_devices():
  if FLAGS.use_tpu:
    jax.config.update('jax_platform_name', 'tpu')
    jax.config.update('jax_xla_backend', FLAGS.jax_xla_backend)
    jax.config.update('jax_backend_target', FLAGS.jax_backend_target)
  if FLAGS.enable_eager_execution and backend.get_name() in ('numpy', 'jax'):
    # Numpy backend doesn't benefit from having the input pipeline run on GPU,
    # and jax backend has GPU memory contention if TF uses the GPU. Gin must be
    # set up first before determining the backend.
    tf.config.experimental.set_visible_devices([], 'GPU')

def _train_using_tf(output_dir):
  worker_cpu = tf_init_tpu()
  with tf.device(worker_cpu):
    if trainer_lib.num_devices() == 1:
      # TF's device priority is GPU > CPU > TPU, so we need to explicitly make
      # the TPU core the default device here.
      with tf.device('/device:TPU:0'):
        trainer_lib.train(output_dir=output_dir)
    else:
      trainer_lib.train(output_dir=output_dir)

@gin.configurable
def tf_init_tpu(worker='', protocol=None):
  """Initializes TPU for TensorFlow.

  Args:
    worker: The BNS address of the remote TPU worker. If it's empty (the default
      value), TF will assume the TPU devices are connected to the local host.
    protocol: The network protocol used to connect to the TPU worker.
  Returns:
    The device name of the TPU worker's CPU.
  """
  protocol = protocol or 'grpc'
  is_local = (worker in ('', 'local'))
  resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=worker)
  if not is_local:
    tf.config.experimental_connect_to_cluster(resolver, protocol=protocol)
  tf.tpu.experimental.initialize_tpu_system(resolver)
  if is_local:
    return ''
  else:
    return '/job:worker'

def main(_):
  logging.set_verbosity(FLAGS.log_level)

  _tf_setup_from_flags()
  _gin_parse_configs()
  _jax_and_tf_configure_for_devices()

  output_dir = _output_dir_or_default()
  if FLAGS.use_tpu and backend.get_name() == 'tf':
    _train_using_tf(output_dir)
  else:
    trainer_lib.train(output_dir=output_dir)

  trainer_lib.log('Finished training.')

if __name__ == '__main__':
  app.run(main)
<|endoftext|>


<commit_before>import os
import time
import joblib
import random
import logging
import numpy as np

from classifier import models, selectors, kernels, utils, tests

from triager import db, app, config
from models import Project, TrainStatus as TS

def train_project(id):
    try:
        logging.info("Started training project %s" % id)
        project = Project.query.get(id)
        project.train_status = TS.TRAINING
        db.session.add(project)
        db.session.commit()

        # Config
        config.reload()
        ticket_limit = int(config.general__ticket_limit)
        min_class_occur = int(config.general__min_class_occur)
        C = float(config.svm__coefficient)
        cache_size = int(config.svm__cache_size)

        # retrieve data
        # TODO: add to configuration
        data = project.datasource.get_data()[-ticket_limit:]
        data = utils.filter_docs(data, min_class_occur=min_class_occur)

        # create training model
        logging.debug("Training model for project %s" % id)
        selector = selectors.TFIDFDecorator(selectors.StopWordsDecorator(
            selectors.BasicSelector()))
        kernel = kernels.GaussianKernel()
        model = models.SVMModel(
            feature_selector=selector, kernel=kernel,
            C=C, cache_size=cache_size)

        # train model
        model.train(data)
        logging.debug("Model for project %s successfully trained" % id)

        # create testing model
        logging.debug("Training model for project %s for testing" % id)
        selector_test = selectors.TFIDFDecorator(selectors.StopWordsDecorator(
            selectors.BasicSelector()))
        kernel_test = kernels.GaussianKernel()
        model_test = models.SVMModel(
            feature_selector=selector_test, kernel=kernel_test,
            C=C, cache_size=cache_size)

        # split data for testing
        random.shuffle(data)
        n = len(data)
        split_pct = 8/10.0
        split_x = int(np.ceil(n*split_pct))
        data_train = data[:split_x]
        data_test = data[split_x:]

        # train testing model
        model_test.train(data_train)
        logging.debug("Testing model for project %s successfully trained" % id)

        # testing model
        project.accuracy = tests.accuracy(model_test, data_test)
        project.precision, project.recall = tests.precision_and_recall(
            model_test, data_test)

        # save
        dump_dir = os.path.join(app.config['MODEL_FOLDER'], str(id))
        if not os.path.exists(dump_dir):
            os.mkdir(dump_dir)
        joblib.dump(model, os.path.join(dump_dir, 'svm.pkl'))
        project.train_status = TS.TRAINED
        project.last_training = time.time()
        db.session.add(project)
        db.session.commit()
        logging.info("Project %s successfully trained and updated." % id)
    except Exception as ex:
        logging.error("Failed to train project %s" % id)
        logging.exception(ex)

        project.train_status = TS.FAILED
        project.training_message = "Reason: %s" % ex
        db.session.add(project)
        db.session.commit()
<commit_msg>Add exception name to training message<commit_after>import os
import time
import joblib
import random
import logging
import numpy as np

from classifier import models, selectors, kernels, utils, tests

from triager import db, app, config
from models import Project, TrainStatus as TS

def train_project(id):
    try:
        logging.info("Started training project %s" % id)
        project = Project.query.get(id)
        project.train_status = TS.TRAINING
        db.session.add(project)
        db.session.commit()

        # Config
        config.reload()
        ticket_limit = int(config.general__ticket_limit)
        min_class_occur = int(config.general__min_class_occur)
        C = float(config.svm__coefficient)
        cache_size = int(config.svm__cache_size)

        # retrieve data
        # TODO: add to configuration
        data = project.datasource.get_data()[-ticket_limit:]
        data = utils.filter_docs(data, min_class_occur=min_class_occur)

        # create training model
        logging.debug("Training model for project %s" % id)
        selector = selectors.TFIDFDecorator(selectors.StopWordsDecorator(
            selectors.BasicSelector()))
        kernel = kernels.GaussianKernel()
        model = models.SVMModel(
            feature_selector=selector, kernel=kernel,
            C=C, cache_size=cache_size)

        # train model
        model.train(data)
        logging.debug("Model for project %s successfully trained" % id)

        # create testing model
        logging.debug("Training model for project %s for testing" % id)
        selector_test = selectors.TFIDFDecorator(selectors.StopWordsDecorator(
            selectors.BasicSelector()))
        kernel_test = kernels.GaussianKernel()
        model_test = models.SVMModel(
            feature_selector=selector_test, kernel=kernel_test,
            C=C, cache_size=cache_size)

        # split data for testing
        random.shuffle(data)
        n = len(data)
        split_pct = 8/10.0
        split_x = int(np.ceil(n*split_pct))
        data_train = data[:split_x]
        data_test = data[split_x:]

        # train testing model
        model_test.train(data_train)
        logging.debug("Testing model for project %s successfully trained" % id)

        # testing model
        project.accuracy = tests.accuracy(model_test, data_test)
        project.precision, project.recall = tests.precision_and_recall(
            model_test, data_test)

        # save
        dump_dir = os.path.join(app.config['MODEL_FOLDER'], str(id))
        if not os.path.exists(dump_dir):
            os.mkdir(dump_dir)
        joblib.dump(model, os.path.join(dump_dir, 'svm.pkl'))
        project.train_status = TS.TRAINED
        project.last_training = time.time()
        db.session.add(project)
        db.session.commit()
        logging.info("Project %s successfully trained and updated." % id)
    except Exception as ex:
        logging.error("Failed to train project %s" % id)
        logging.exception(ex)

        project.train_status = TS.FAILED
        project.training_message = "Reason: %s: %s" % (
            ex.__class__.__name__, ex)
        db.session.add(project)
        db.session.commit()
<|endoftext|>


<commit_before>import logging
import os
import sys

if sys.version_info < (3,0):
    import imp
else:
    from importlib import import_module
    from importlib.machinery import SourceFileLoader

import yaml
from jinja2 import Template

log = logging.getLogger('triptan')

REV_FILE_NAME_FORMAT = u'revision-{0:03d}.py'

def load_module_from_path(name, path):
    """
        Load a module from the given path.

        :param str name: name of the module
        :param str path: the path to load the module from

        :returns: the module at the path
        :rtype: module
    """
    if sys.version_info < (3,0):
        return imp.load_source(name, path)
    else:
        return SourceFileLoader(name, path).load_module()

class MigrationLoadingException(Exception):
    pass

class Triptan(object):
    """
        The main triptan object.
    """

    def __init__(self, path, config_file):
        self.path = path
        self.config_file = config_file
        self.load_config()

        self.revisions = []
        self.load_revisions()
        self.storage = None
        self.init_storage()

    def get_join_path(self, path):
        return os.path.join(self.path, path)

    @property
    def config_path(self):
        return self.get_join_path(self.config_file)

    @property
    def current_revision(self):
        return self.storage.get_current_revision()

    def load_config(self):
        with open(self.config_path, 'r') as f:
            self.config = yaml.load(f)

    def _get_revision(self, num):
        revisions_dir = self.get_join_path(self.config['revisions_location'])
        file_name = REV_FILE_NAME_FORMAT.format(num)
        rev_path = os.path.join(revisions_dir, file_name)
        if not os.path.exists(rev_path):
            return False
        try:
            return load_module_from_path("revision-{}".format(num), rev_path)
        except:
            raise MigrationLoadingException(
                "Revision #{} failed to load.".format(num)
            )

    def load_revisions(self, reset=True):
        i = 0
        if reset:
            self.revisions = []
        while self._get_revision(i):
            self.revisions.append(self._get_revision(i))
            i += 1

    def init_storage(self):
        log.debug(
            "Using %s.%s as storage.",
            self.config['storage']['module'],
            self.config['storage']['class']
        )
        store_mod = import_module(self.config['storage']['module'])
        store_class = getattr(store_mod, self.config['storage']['class'])

        self.storage = store_class(self.path, self.config['storage'])

    def new_revision(self, name=""):
        num = len(self.revisions)
        log.info("Creating new revision #%i", num)
        template_path = os.path.join(
            os.path.dirname(__file__),
            'templates/migration.py.jinja2'
        )
        with open(template_path, 'r') as f:
            template = Template(f.read())
        revisions_dir = self.get_join_path(self.config['revisions_location'])
        file_name = REV_FILE_NAME_FORMAT.format(num)
        revision_path = os.path.join(revisions_dir, file_name)
        with open(revision_path, 'w') as f:
            f.write(template.render(num=num, name=name))

        # now load the revisions again to make sure we are caught up with
        # the changes
        self.load_revisions()

    def migrate(self, target):
        """
            Runs the given migration.
        """
        log.info("Running migration: %s.", target)
        if target.startswith("-"):
            clean_target = target[1:]
            target_revision = self.current_revision - int(clean_target)
        elif target.startswith("+"):
            clean_target = target[1:]
            target_revision = self.current_revision + int(clean_target)
        elif target == 'head':
            target_revision = len(self.revisions) - 1
        else:
            target_revision = int(target)

        log.info("Current revision #%i.", self.current_revision)
        log.info("Final revision #%i.", target_revision)

        if target_revision > len(self.revisions) - 1 or target_revision < -1:
            log.error("Final revision does not exist.")
            return False

        while self.current_revision != target_revision:
            # store the current revision so that we don't get confused
            # while we are in an iteration
            current = self.current_revision
            if target_revision > current:
                next_revision = self.revisions[current + 1]
                log.info("Running up migration #%i", current + 1)
                next_revision.up()
                log.info("Setting current revision #%i", current + 1)
                self.storage.set_current_revision(current + 1)
            if target_revision < current:
                current_revision = self.revisions[current]
                log.info("Running down migration #%i", current)
                current_revision.down()
                log.info("Setting current revision #%i", current - 1)
                self.storage.set_current_revision(current - 1)

        return True

    @classmethod
    def setup(cls, path, config_file, config):
        """
            Will initialize triptan at the given path.
        """
        log.info("Setting up triptan for '%s'..", path)
        template_path = os.path.join(
            os.path.dirname(__file__),
            'templates/config.yml.jinja2'
        )
        with open(template_path, 'r') as f:
            template = Template(f.read())
        with open(os.path.join(path, config_file), 'w') as f:
            f.write(template.render(**config))
        log.info("Created config file '%s'.", config_file)
        os.makedirs(os.path.join(path, config['revisions_location']))
        log.info("Created revisions directory '%s'.",
                 config['revisions_location'])
        log.info("Done.")

class BaseStorage(object):
    """
        Stores the current state of the migrations.
    """

    def __init__(self, path, config):
        """
        Initializes the storage with the current storage configuration.

        :param dict config: the configuration dictionary
        """
        self.path = path
        self.config = config

    def get_current_revision(self):
        """
            Get the current revision from the store.

            :returns: the current revision
            :rtype: int
        """
        raise NotImplementedError()

    def set_current_revision(self, revision):
        """
            Set the current revision in the store.

            :param int revision: the revision to store.
        """
        raise NotImplementedError()
<commit_msg>fix py2.7 bug with imports.<commit_after>import logging
import os
import sys
from importlib import import_module

if sys.version_info < (3,0):
    import imp
else:
    from importlib.machinery import SourceFileLoader

import yaml
from jinja2 import Template

log = logging.getLogger('triptan')

REV_FILE_NAME_FORMAT = u'revision-{0:03d}.py'

def load_module_from_path(name, path):
    """
        Load a module from the given path.

        :param str name: name of the module
        :param str path: the path to load the module from

        :returns: the module at the path
        :rtype: module
    """
    if sys.version_info < (3,0):
        return imp.load_source(name, path)
    else:
        return SourceFileLoader(name, path).load_module()

class MigrationLoadingException(Exception):
    pass

class Triptan(object):
    """
        The main triptan object.
    """

    def __init__(self, path, config_file):
        self.path = path
        self.config_file = config_file
        self.load_config()

        self.revisions = []
        self.load_revisions()
        self.storage = None
        self.init_storage()

    def get_join_path(self, path):
        return os.path.join(self.path, path)

    @property
    def config_path(self):
        return self.get_join_path(self.config_file)

    @property
    def current_revision(self):
        return self.storage.get_current_revision()

    def load_config(self):
        with open(self.config_path, 'r') as f:
            self.config = yaml.load(f)

    def _get_revision(self, num):
        revisions_dir = self.get_join_path(self.config['revisions_location'])
        file_name = REV_FILE_NAME_FORMAT.format(num)
        rev_path = os.path.join(revisions_dir, file_name)
        if not os.path.exists(rev_path):
            return False
        try:
            return load_module_from_path("revision-{}".format(num), rev_path)
        except:
            raise MigrationLoadingException(
                "Revision #{} failed to load.".format(num)
            )

    def load_revisions(self, reset=True):
        i = 0
        if reset:
            self.revisions = []
        while self._get_revision(i):
            self.revisions.append(self._get_revision(i))
            i += 1

    def init_storage(self):
        log.debug(
            "Using %s.%s as storage.",
            self.config['storage']['module'],
            self.config['storage']['class']
        )
        store_mod = import_module(self.config['storage']['module'])
        store_class = getattr(store_mod, self.config['storage']['class'])

        self.storage = store_class(self.path, self.config['storage'])

    def new_revision(self, name=""):
        num = len(self.revisions)
        log.info("Creating new revision #%i", num)
        template_path = os.path.join(
            os.path.dirname(__file__),
            'templates/migration.py.jinja2'
        )
        with open(template_path, 'r') as f:
            template = Template(f.read())
        revisions_dir = self.get_join_path(self.config['revisions_location'])
        file_name = REV_FILE_NAME_FORMAT.format(num)
        revision_path = os.path.join(revisions_dir, file_name)
        with open(revision_path, 'w') as f:
            f.write(template.render(num=num, name=name))

        # now load the revisions again to make sure we are caught up with
        # the changes
        self.load_revisions()

    def migrate(self, target):
        """
            Runs the given migration.
        """
        log.info("Running migration: %s.", target)
        if target.startswith("-"):
            clean_target = target[1:]
            target_revision = self.current_revision - int(clean_target)
        elif target.startswith("+"):
            clean_target = target[1:]
            target_revision = self.current_revision + int(clean_target)
        elif target == 'head':
            target_revision = len(self.revisions) - 1
        else:
            target_revision = int(target)

        log.info("Current revision #%i.", self.current_revision)
        log.info("Final revision #%i.", target_revision)

        if target_revision > len(self.revisions) - 1 or target_revision < -1:
            log.error("Final revision does not exist.")
            return False

        while self.current_revision != target_revision:
            # store the current revision so that we don't get confused
            # while we are in an iteration
            current = self.current_revision
            if target_revision > current:
                next_revision = self.revisions[current + 1]
                log.info("Running up migration #%i", current + 1)
                next_revision.up()
                log.info("Setting current revision #%i", current + 1)
                self.storage.set_current_revision(current + 1)
            if target_revision < current:
                current_revision = self.revisions[current]
                log.info("Running down migration #%i", current)
                current_revision.down()
                log.info("Setting current revision #%i", current - 1)
                self.storage.set_current_revision(current - 1)

        return True

    @classmethod
    def setup(cls, path, config_file, config):
        """
            Will initialize triptan at the given path.
        """
        log.info("Setting up triptan for '%s'..", path)
        template_path = os.path.join(
            os.path.dirname(__file__),
            'templates/config.yml.jinja2'
        )
        with open(template_path, 'r') as f:
            template = Template(f.read())
        with open(os.path.join(path, config_file), 'w') as f:
            f.write(template.render(**config))
        log.info("Created config file '%s'.", config_file)
        os.makedirs(os.path.join(path, config['revisions_location']))
        log.info("Created revisions directory '%s'.",
                 config['revisions_location'])
        log.info("Done.")

class BaseStorage(object):
    """
        Stores the current state of the migrations.
    """

    def __init__(self, path, config):
        """
        Initializes the storage with the current storage configuration.

        :param dict config: the configuration dictionary
        """
        self.path = path
        self.config = config

    def get_current_revision(self):
        """
            Get the current revision from the store.

            :returns: the current revision
            :rtype: int
        """
        raise NotImplementedError()

    def set_current_revision(self, revision):
        """
            Set the current revision in the store.

            :param int revision: the revision to store.
        """
        raise NotImplementedError()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# trml2pdf - An RML to PDF converter
# Copyright (C) 2003, Fabien Pinckaers, UCL, FSA
# Contributors
#     Richard Waid <richard@iopen.net>
#     Klaas Freitag <freitag@kde.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

# 1. system
import io
import xml.dom.minidom
# 2. 3rd parties
from reportlab import platypus
from reportlab.pdfgen import canvas
# 3. local
from . import canv, flowable, styles, utils

def docinit(els):
    from reportlab.lib.fonts import addMapping
    from reportlab.pdfbase import pdfmetrics
    from reportlab.pdfbase.ttfonts import TTFont

    for node in els:
        for font in node.getElementsByTagName('registerFont'):
            name = font.getAttribute('fontName')
            fname = font.getAttribute('fontFile')
            pdfmetrics.registerFont(TTFont(name, fname))
        for font in node.getElementsByTagName('registerTTFont'):
            name = font.getAttribute('faceName')
            fname = font.getAttribute('fileName')
            pdfmetrics.registerFont(TTFont(name, fname))  # , subfontIndex=subfontIndex
        for font_family in node.getElementsByTagName('registerFontFamily'):
            normal = font_family.getAttribute('normal')
            bold = font_family.getAttribute('bold')
            italic = font_family.getAttribute('italic')
            bold_italic = font_family.getAttribute('boldItalic')
            addMapping(normal, 0, 0, normal)  # normal
            addMapping(normal, 1, 0, bold)  # bold
            addMapping(normal, 0, 1, italic)  # italic
            addMapping(normal, 1, 1, bold_italic)  # italic and bold

class RmlDoc(object):

    def __init__(self, data: str):
        self.dom = xml.dom.minidom.parseString(data)
        self.filename = self.dom.documentElement.getAttribute('filename')
        self.canvas = None
        self.styles = None

    def render(self, out):
        el = self.dom.documentElement.getElementsByTagName('docinit')
        if el:
            docinit(el)
        el = self.dom.documentElement.getElementsByTagName('stylesheet')
        self.styles = styles.RmlStyles(el)
        el = self.dom.documentElement.getElementsByTagName('template')
        if len(el):
            pt_obj = RmlTemplate(out, el[0], self)
            pt_obj.render(
                self.dom.documentElement.getElementsByTagName('story')[0])
        else:
            self.canvas = canvas.Canvas(out)
            pd = self.dom.documentElement.getElementsByTagName(
                'pageDrawing')[0]
            pd_obj = canv.RmlCanvas(self.canvas, None, self)
            pd_obj.render(pd)
            self.canvas.showPage()
            self.canvas.save()

class RmlTemplate(object):

    def __init__(self, out, node, doc):
        if not node.hasAttribute('pageSize'):
            page_size = (utils.unit_get('21cm'), utils.unit_get('29.7cm'))
        else:
            ps = [x.strip() for x in node.getAttribute('pageSize').replace(')', '').replace(
                '(', '').split(',')]
            page_size = (utils.unit_get(ps[0]), utils.unit_get(ps[1]))
        # cm = reportlab.lib.units.cm
        self.doc_tmpl = platypus.BaseDocTemplate(out, pagesize=page_size, **utils.attr_get(
            node,
            ['leftMargin', 'rightMargin', 'topMargin', 'bottomMargin'],
            {'allowSplitting': 'int', 'showBoundary': 'bool', 'title': 'str', 'author': 'str', 'rotation': 'int'}))
        self.page_templates = []
        self.styles = doc.styles
        self.doc = doc
        pts = node.getElementsByTagName('pageTemplate')
        for pt in pts:
            frames = []
            for frame_el in pt.getElementsByTagName('frame'):
                frame = platypus.Frame(**(utils.attr_get(
                    frame_el,
                    ['x1', 'y1', 'width', 'height', 'leftPadding', 'rightPadding', 'bottomPadding', 'topPadding'],
                    {'id': 'text', 'showBoundary': 'bool'})))
                frames.append(frame)
            gr = pt.getElementsByTagName('pageGraphics')
            if len(gr):
                drw = canv.RmlDraw(gr[0], self.doc.styles)
                self.page_templates.append(platypus.PageTemplate(
                    frames=frames, onPage=drw.render, **utils.attr_get(pt, [], {'id': 'str'})))
            else:
                self.page_templates.append(
                    platypus.PageTemplate(frames=frames, **utils.attr_get(pt, [], {'id': 'str'})))
        self.doc_tmpl.addPageTemplates(self.page_templates)

    def render(self, node_story):
        r = flowable.RmlFlowable(self.doc)
        fis = r.render(node_story)
        self.doc_tmpl.build(fis)
<commit_msg>Support changing page size on a per-page basis<commit_after># -*- coding: utf-8 -*-

# trml2pdf - An RML to PDF converter
# Copyright (C) 2003, Fabien Pinckaers, UCL, FSA
# Contributors
#     Richard Waid <richard@iopen.net>
#     Klaas Freitag <freitag@kde.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

# 1. system
import io
import xml.dom.minidom
# 2. 3rd parties
from reportlab import platypus
from reportlab.pdfgen import canvas
# 3. local
from . import canv, flowable, styles, utils

def docinit(els):
    from reportlab.lib.fonts import addMapping
    from reportlab.pdfbase import pdfmetrics
    from reportlab.pdfbase.ttfonts import TTFont

    for node in els:
        for font in node.getElementsByTagName('registerFont'):
            name = font.getAttribute('fontName')
            fname = font.getAttribute('fontFile')
            pdfmetrics.registerFont(TTFont(name, fname))
        for font in node.getElementsByTagName('registerTTFont'):
            name = font.getAttribute('faceName')
            fname = font.getAttribute('fileName')
            pdfmetrics.registerFont(TTFont(name, fname))  # , subfontIndex=subfontIndex
        for font_family in node.getElementsByTagName('registerFontFamily'):
            normal = font_family.getAttribute('normal')
            bold = font_family.getAttribute('bold')
            italic = font_family.getAttribute('italic')
            bold_italic = font_family.getAttribute('boldItalic')
            addMapping(normal, 0, 0, normal)  # normal
            addMapping(normal, 1, 0, bold)  # bold
            addMapping(normal, 0, 1, italic)  # italic
            addMapping(normal, 1, 1, bold_italic)  # italic and bold

class RmlDoc(object):

    def __init__(self, data: str):
        self.dom = xml.dom.minidom.parseString(data)
        self.filename = self.dom.documentElement.getAttribute('filename')
        self.canvas = None
        self.styles = None

    def render(self, out):
        el = self.dom.documentElement.getElementsByTagName('docinit')
        if el:
            docinit(el)
        el = self.dom.documentElement.getElementsByTagName('stylesheet')
        self.styles = styles.RmlStyles(el)
        el = self.dom.documentElement.getElementsByTagName('template')
        if len(el):
            pt_obj = RmlTemplate(out, el[0], self)
            pt_obj.render(
                self.dom.documentElement.getElementsByTagName('story')[0])
        else:
            self.canvas = canvas.Canvas(out)
            pd = self.dom.documentElement.getElementsByTagName(
                'pageDrawing')[0]
            pd_obj = canv.RmlCanvas(self.canvas, None, self)
            pd_obj.render(pd)
            self.canvas.showPage()
            self.canvas.save()

class RmlTemplate(object):

    def __init__(self, out, node, doc):
        if not node.hasAttribute('pageSize'):
            page_size = (utils.unit_get('21cm'), utils.unit_get('29.7cm'))
        else:
            ps = [x.strip() for x in node.getAttribute('pageSize').replace(')', '').replace(
                '(', '').split(',')]
            page_size = (utils.unit_get(ps[0]), utils.unit_get(ps[1]))
        # cm = reportlab.lib.units.cm
        self.doc_tmpl = platypus.BaseDocTemplate(out, pagesize=page_size, **utils.attr_get(
            node,
            ['leftMargin', 'rightMargin', 'topMargin', 'bottomMargin'],
            {'allowSplitting': 'int', 'showBoundary': 'bool', 'title': 'str', 'author': 'str', 'rotation': 'int'}))
        self.page_templates = []
        self.styles = doc.styles
        self.doc = doc
        pts = node.getElementsByTagName('pageTemplate')
        for pt in pts:
            if not pt.hasAttribute('pageSize'):
                 pageSize = (utils.unit_get('21cm'), utils.unit_get('29.7cm'))
             else:
                 ps = [x.strip() for x in pt.getAttribute('pageSize').replace(')', '').replace(
                     '(', '').split(',')]
                 pageSize = (utils.unit_get(ps[0]), utils.unit_get(ps[1]))
            frames = []
            for frame_el in pt.getElementsByTagName('frame'):
                frame = platypus.Frame(**(utils.attr_get(
                    frame_el,
                    ['x1', 'y1', 'width', 'height', 'leftPadding', 'rightPadding', 'bottomPadding', 'topPadding'],
                    {'id': 'text', 'showBoundary': 'bool'})))
                frames.append(frame)
            gr = pt.getElementsByTagName('pageGraphics')
            if len(gr):
                drw = canv.RmlDraw(gr[0], self.doc.styles)
                self.page_templates.append(platypus.PageTemplate(
                    frames=frames, pageSize=pageSize, onPage=drw.render, **utils.attr_get(pt, [], {'id': 'str'})))
            else:
                self.page_templates.append(
                    platypus.PageTemplate(frames=frames, pageSize=pageSize, **utils.attr_get(pt, [], {'id': 'str'})))
        self.doc_tmpl.addPageTemplates(self.page_templates)

    def render(self, node_story):
        r = flowable.RmlFlowable(self.doc)
        fis = r.render(node_story)
        self.doc_tmpl.build(fis)
<|endoftext|>


<commit_before>'''

Copyright (C) 2017-2018 The Board of Trustees of the Leland Stanford Junior
University.
Copyright (C) 2017-2018 Vanessa Sochat.

This program is free software: you can redistribute it and/or modify it
under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or (at your
option) any later version.

This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <https://www.gnu.org/licenses/>.

'''

from django.contrib.auth.decorators import login_required
from django.contrib import messages
from django.shortcuts import render, redirect
from django.http import (
    HttpResponseRedirect,
    JsonResponse
)

from shub.settings import USER_COLLECTIONS
from shub.apps.users.forms import TeamForm
from shub.apps.users.models import ( User, Team, MembershipInvite )
from shub.apps.users.permissions import (
    has_create_permission,
    is_invite_valid
)
from shub.apps.users.utils import get_user

import uuid

def get_team(tid):
    ''' get a team based on its primary id

       Parameters
       ==========
       tid: the id of the team to look up

   '''
    keyargs = {'id': tid}
    try:
        team = Team.objects.get(**keyargs)
    except Team.DoesNotExist:
        raise Http404
    else:
        return team

################################################################################
# TEAM VIEWS ###################################################################
################################################################################

@login_required
def edit_team(request, tid=None):
    '''edit_team is the view to edit an existing team, or create a new team.
    :parma tid: the team id to edit or create. If none, indicates a new team
    '''

    if tid:
        team = get_team(tid)
        edit_permission = team.has_edit_permission(request)
        title = "Edit Team"
    else:
        team = Team()
        edit_permission = True
        title = "New Team"

    if edit_permission:

        form = TeamForm(request.POST or None,
                        request.FILES or None,
                        instance=team)

        if form.is_valid():
            form.save()

            # An editor is always a member
            if request.user not in team.members.all():
                team.members.add(request.user)
                team.save()

            # If a team id was provided, we add request user as owner
            if not tid:
                team.owners.add(request.user)
                team.save()

            messages.info(request, 'Team updated.')

        context = {"form": form,
                   "edit_permission": edit_permission,
                   "title":title,
                   "team":team}

        return render(request, "teams/edit_team.html", context)

    # If user makes it down here, does not have permission
    messages.info(request, "Only team owners can edit teams.")
    return redirect("teams")

def view_teams(request):
    '''view all teams (log in not required)
    :parma tid: the team id to edit or create. If none, indicates a new team
    '''
    teams = Team.objects.all()

    # Does the user have permission to create a team?
    create_permission = has_create_permission(request)

    context = {"teams": teams,
               "has_create_permission" : create_permission }

    return render(request, "teams/all_teams.html", context)

@login_required
def view_team(request, tid, code=None):
    '''view the details about a team
    :parma tid: the team id to edit or create. If none, indicates a new team
    '''
    team = get_team(tid)

    # Need to create annotation counts with "total" for all members
    edit_permission = team.has_edit_permission(request)
    members = team.get_members()
    count = len(members)

    context = {"team": team,
               "edit_permission":edit_permission,
               "count": count,
               "members": members,
               "code": code}

    return render(request, "teams/team_details.html", context)

################################################################################
# TEAM REQUESTS ################################################################
################################################################################

@login_required
def join_team(request, tid, code=None):
    '''add a user to a new team. If the team is open, he/she can join without
       needing a code. If is invite only, the code must be checked.

       Parameters
       ==========
       tid: the team id to edit or create. If none, indicates a new team
       code: if the user is accessing an invite link, the code is checked
             against the generated request.
    '''

    team = get_team(tid)
    user = request.user
    add_user = True

    if team.permission == "invite":
        if code is None:
            messages.info(request, "This is not a valid invitation.")
            add_user = False
        else:
            add_user = is_invite_valid(team, code)
            if add_user is False:
                messages.info(request, "This code is invalid to join this team.")

    if add_user:

        if user not in team.get_members():
            team.members.add(user)
            team.save()
            messages.info(request,"You have been added to team %s" %(team.name))
        else:
            messages.info(request,"You are already a member of %s" %(team.name))

    return HttpResponseRedirect(team.get_absolute_url())

################################################################################
# TEAM ACTIONS #################################################################
################################################################################

@login_required
def leave_team(request, tid):
    '''leave team is the view for a user to leave his or her team. A user
       doesn't need permission to remove him or herself.

       Parameters
       ==========
       team: the team to remove the individual from
    '''
    team = get_team(tid)

    team = _remove_member(team, request.user)
    team = _remove_owner(team, request.user)

    return redirect('teams')

def _remove_member(team, user):
    '''remove a member from a team if they are a part of it.

       Parameters
       ==========
       team: the team to remove the individual from
       user: the user to remove
    '''
    if user in team.members.all():
        team.members.remove(user)
        messages.info('%s is removed from %s' %(user.username, team.name))
        team.save()
    return team

def _remove_owner(team, user):
    '''remove an owner from a team, if they are part of it, and there
       is at least one other owner remaining.

       Parameters
       ==========
       team: the team to remove the individual from
       user: the user to remove
    '''
    if team.owners.count() > 1 and user in team.owners.all():
        team.owners.remove(user)
        messages.info('%s is no longer owner of %s' %(user.username, team.name))
        team.save()
    else:
        messages.info('At least one owner must be present for a team.')

    return team

@login_required
def remove_member(request, tid, uid):
    '''remove a member from a team.

       Parameters
       ==========
       team: the team to remove the individual from
       user: the user to remove
    '''

    team = get_team(tid)
    member = get_user(uid)

    if request.user in team.owners.all():
        team = _remove_member(team, user)
    else:
        message = "You are not allowed to perform this action."
    return JsonResponse({"message":message})

@login_required
def remove_owner(request, tid, uid):
    '''remove a member from a team.

       Parameters
       ==========
       tid: the team id to remove the individual from
       uid: the user id to remove
    '''

    team = get_team(tid)
    member = get_user(uid)

    if request.user in team.owners.all():
        team = _remove_owner(team, user)
    else:
        message = "You are not allowed to perform this action."
    return JsonResponse({"message":message})

@login_required
def add_owner(request, tid, uid):
    '''promote a user to be owner of a team

       Parameters
       ==========
       tid: the team id to remove the individual from
       uid: the user id to remove
    '''

    team = get_team(tid)
    member = get_user(uid)

    if request.user in team.owners.all():
        team.owners.add(member)
        team.save()
    else:
        message = "You are not allowed to perform this action."
    return JsonResponse({"message":message})

@login_required
def delete_team(request, tid):
    '''delete a team entirely, must be an owner

       Parameters
       ==========
       tid: the team id

    '''
    team = get_team(tid)

    if request.user in team.owners.all():
        messages.info(request,'%s has been deleted.' %team.name)
        team.delete()
    else:
        messages.info(request, "You are not allowed to perform this action.")

    return redirect('teams')

@login_required
def generate_team_invite(request, tid):
    '''generate an invitation for a user, return to view.
       The invitation is valid for a day.
    '''
    team = get_team(tid)
    if request.user in team.owners.all():
        code = uuid.uuid4()
        new_invite = MembershipInvite.objects.create(team=team,
                                                     code=code)
        new_invite.save()
        url = new_invite.get_url()
        messages.info(request, 'This invitation is valid until use: %s' %url)
        return view_team(request, team.id, code=code)

    messages.info(request, "You do not have permission to invite to this team.")
    return HttpResponseRedirect(team.get_absolute_url())
<commit_msg>fixing bug with user-->member<commit_after>'''

Copyright (C) 2017-2018 The Board of Trustees of the Leland Stanford Junior
University.
Copyright (C) 2017-2018 Vanessa Sochat.

This program is free software: you can redistribute it and/or modify it
under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or (at your
option) any later version.

This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <https://www.gnu.org/licenses/>.

'''

from django.contrib.auth.decorators import login_required
from django.contrib import messages
from django.shortcuts import render, redirect
from django.http import (
    HttpResponseRedirect,
    JsonResponse
)

from shub.settings import USER_COLLECTIONS
from shub.apps.users.forms import TeamForm
from shub.apps.users.models import ( User, Team, MembershipInvite )
from shub.apps.users.permissions import (
    has_create_permission,
    is_invite_valid
)
from shub.apps.users.utils import get_user

import uuid

def get_team(tid):
    ''' get a team based on its primary id

       Parameters
       ==========
       tid: the id of the team to look up

   '''
    keyargs = {'id': tid}
    try:
        team = Team.objects.get(**keyargs)
    except Team.DoesNotExist:
        raise Http404
    else:
        return team

################################################################################
# TEAM VIEWS ###################################################################
################################################################################

@login_required
def edit_team(request, tid=None):
    '''edit_team is the view to edit an existing team, or create a new team.
    :parma tid: the team id to edit or create. If none, indicates a new team
    '''

    if tid:
        team = get_team(tid)
        edit_permission = team.has_edit_permission(request)
        title = "Edit Team"
    else:
        team = Team()
        edit_permission = True
        title = "New Team"

    if edit_permission:

        form = TeamForm(request.POST or None,
                        request.FILES or None,
                        instance=team)

        if form.is_valid():
            form.save()

            # An editor is always a member
            if request.user not in team.members.all():
                team.members.add(request.user)
                team.save()

            # If a team id was provided, we add request user as owner
            if not tid:
                team.owners.add(request.user)
                team.save()

            messages.info(request, 'Team updated.')

        context = {"form": form,
                   "edit_permission": edit_permission,
                   "title":title,
                   "team":team}

        return render(request, "teams/edit_team.html", context)

    # If user makes it down here, does not have permission
    messages.info(request, "Only team owners can edit teams.")
    return redirect("teams")

def view_teams(request):
    '''view all teams (log in not required)
    :parma tid: the team id to edit or create. If none, indicates a new team
    '''
    teams = Team.objects.all()

    # Does the user have permission to create a team?
    create_permission = has_create_permission(request)

    context = {"teams": teams,
               "has_create_permission" : create_permission }

    return render(request, "teams/all_teams.html", context)

@login_required
def view_team(request, tid, code=None):
    '''view the details about a team
    :parma tid: the team id to edit or create. If none, indicates a new team
    '''
    team = get_team(tid)

    # Need to create annotation counts with "total" for all members
    edit_permission = team.has_edit_permission(request)
    members = team.get_members()
    count = len(members)

    context = {"team": team,
               "edit_permission":edit_permission,
               "count": count,
               "members": members,
               "code": code}

    return render(request, "teams/team_details.html", context)

################################################################################
# TEAM REQUESTS ################################################################
################################################################################

@login_required
def join_team(request, tid, code=None):
    '''add a user to a new team. If the team is open, he/she can join without
       needing a code. If is invite only, the code must be checked.

       Parameters
       ==========
       tid: the team id to edit or create. If none, indicates a new team
       code: if the user is accessing an invite link, the code is checked
             against the generated request.
    '''

    team = get_team(tid)
    user = request.user
    add_user = True

    if team.permission == "invite":
        if code is None:
            messages.info(request, "This is not a valid invitation.")
            add_user = False
        else:
            add_user = is_invite_valid(team, code)
            if add_user is False:
                messages.info(request, "This code is invalid to join this team.")

    if add_user:

        if user not in team.get_members():
            team.members.add(user)
            team.save()
            messages.info(request,"You have been added to team %s" %(team.name))
        else:
            messages.info(request,"You are already a member of %s" %(team.name))

    return HttpResponseRedirect(team.get_absolute_url())

################################################################################
# TEAM ACTIONS #################################################################
################################################################################

@login_required
def leave_team(request, tid):
    '''leave team is the view for a user to leave his or her team. A user
       doesn't need permission to remove him or herself.

       Parameters
       ==========
       team: the team to remove the individual from
    '''
    team = get_team(tid)

    team = _remove_member(team, request.user)
    team = _remove_owner(team, request.user)

    return redirect('teams')

def _remove_member(team, user):
    '''remove a member from a team if they are a part of it.

       Parameters
       ==========
       team: the team to remove the individual from
       user: the user to remove
    '''
    if user in team.members.all():
        team.members.remove(user)
        messages.info('%s is removed from %s' %(user.username, team.name))
        team.save()
    return team

def _remove_owner(team, user):
    '''remove an owner from a team, if they are part of it, and there
       is at least one other owner remaining.

       Parameters
       ==========
       team: the team to remove the individual from
       user: the user to remove
    '''
    if team.owners.count() > 1 and user in team.owners.all():
        team.owners.remove(user)
        messages.info('%s is no longer owner of %s' %(user.username, team.name))
        team.save()
    else:
        messages.info('At least one owner must be present for a team.')

    return team

@login_required
def remove_member(request, tid, uid):
    '''remove a member from a team.

       Parameters
       ==========
       team: the team to remove the individual from
       user: the user to remove
    '''

    team = get_team(tid)
    member = get_user(uid)

    if request.user in team.owners.all():
        team = _remove_member(team, member)
    else:
        message = "You are not allowed to perform this action."
    return JsonResponse({"message":message})

@login_required
def remove_owner(request, tid, uid):
    '''remove a member from a team.

       Parameters
       ==========
       tid: the team id to remove the individual from
       uid: the user id to remove
    '''

    team = get_team(tid)
    member = get_user(uid)

    if request.user in team.owners.all():
        team = _remove_owner(team, member)
    else:
        message = "You are not allowed to perform this action."
    return JsonResponse({"message":message})

@login_required
def add_owner(request, tid, uid):
    '''promote a user to be owner of a team

       Parameters
       ==========
       tid: the team id to remove the individual from
       uid: the user id to remove
    '''

    team = get_team(tid)
    member = get_user(uid)

    if request.user in team.owners.all():
        team.owners.add(member)
        team.save()
    else:
        message = "You are not allowed to perform this action."
    return JsonResponse({"message":message})

@login_required
def delete_team(request, tid):
    '''delete a team entirely, must be an owner

       Parameters
       ==========
       tid: the team id

    '''
    team = get_team(tid)

    if request.user in team.owners.all():
        messages.info(request,'%s has been deleted.' %team.name)
        team.delete()
    else:
        messages.info(request, "You are not allowed to perform this action.")

    return redirect('teams')

@login_required
def generate_team_invite(request, tid):
    '''generate an invitation for a user, return to view.
       The invitation is valid for a day.
    '''
    team = get_team(tid)
    if request.user in team.owners.all():
        code = uuid.uuid4()
        new_invite = MembershipInvite.objects.create(team=team,
                                                     code=code)
        new_invite.save()
        url = new_invite.get_url()
        messages.info(request, 'This invitation is valid until use: %s' %url)
        return view_team(request, team.id, code=code)

    messages.info(request, "You do not have permission to invite to this team.")
    return HttpResponseRedirect(team.get_absolute_url())
<|endoftext|>


<commit_before>import re
import os
import os.path
import urllib.request, urllib.parse, urllib.error
from xml.sax.saxutils import escape, unescape

re_terminal_path = re.compile(r'[/\\]|(\w\:)')
endpoint_regex = re.compile(r'(\w[\w\d]*)\:(.*)')
endpoint_prefix_regex = re.compile(r'^(\w[\w\d]+)\:')

class PageNotFoundError(Exception):
    """ An error raised when no physical file
       is found for a given URL.
    """
    def __init__(self, url, message=None, *args):
        Exception.__init__(self, url, message, *args)

    def __str__(self):
        url = self.args[0]
        message = self.args[1]
        res = "Can't find page '%s'." % url
        if message:
            res += ' ' + message
        return res

class NamespaceNotFoundError(Exception):
    """ An error raised when no physical directory is found
        for a given URL.
    """
    pass

def find_wiki_root(path=None):
    if not path:
        path = os.getcwd()
    while True:
        if os.path.isfile(os.path.join(path, '.wikirc')):
            return path
        if (os.path.isdir(os.path.join(path, '.git')) or
                os.path.isdir(os.path.join(path, '.hg'))):
            return path
        path = os.path.dirname(path)
        if not path or re_terminal_path.match(path):
            break
    return None

def get_absolute_url(base_url, url, quote=False):
    endpoint, base_url = split_page_url(base_url)
    if base_url[0] != '/':
        raise ValueError("The base URL must be absolute. Got: %s" % base_url)

    if url.startswith('/'):
        # Absolute page URL.
        abs_url = url
    elif url.startswith('./'):
        # URL wants to be relative to the base url's name, instead
        # of its directory.
        abs_url = base_url + url[1:]
    else:
        # Relative page URL. Let's normalize all `..` in it,
        # which could also replace forward slashes by backslashes
        # on Windows, so we need to convert that back.
        urldir = os.path.dirname(base_url)
        raw_abs_url = os.path.join(urldir, url)
        abs_url = os.path.normpath(raw_abs_url).replace('\\', '/')
    if quote:
        abs_url = urllib.parse.quote(abs_url.encode('utf-8'))
    if endpoint:
        return '%s:%s' % (endpoint, abs_url)
    return abs_url

def is_endpoint_url(url):
    return endpoint_prefix_regex.match(url) is not None

def split_page_url(url):
    m = endpoint_regex.match(url)
    if m is None:
        return (None, url)
    endpoint = m.group(1)
    path = m.group(2)
    return (endpoint, path)

def get_meta_name_and_modifiers(name):
    """ Strips a meta name from any leading modifiers like `__` or `+`
        and returns both as a tuple. If no modifier was found, the
        second tuple value is `None`.
    """
    clean_name = name
    modifiers = None
    if name[:2] == '__':
        modifiers = '__'
        clean_name = name[3:]
    elif name[0] == '+':
        modifiers = '+'
        clean_name = name[1:]
    return (clean_name, modifiers)

def flatten_single_metas(meta):
    items = list(meta.items())
    for k, v in items:
        if isinstance(v, list):
            l = len(v)
            if l == 0:
                del meta[k]
            elif l == 1:
                meta[k] = v[0]
    return meta

html_escape_table = {'"': """, "'": "'"}
html_unescape_table = {v: k for k, v in list(html_escape_table.items())}

def html_escape(text):
    return escape(text, html_escape_table)

def html_unescape(text):
    return unescape(text, html_unescape_table)

<commit_msg>core: Fix bug with determining absolute URLs.<commit_after>import re
import os
import os.path
import urllib.request, urllib.parse, urllib.error
from xml.sax.saxutils import escape, unescape

re_terminal_path = re.compile(r'[/\\]|(\w\:)')
endpoint_regex = re.compile(r'(\w[\w\d]*)\:(.*)')
endpoint_prefix_regex = re.compile(r'^(\w[\w\d]+)\:')

class PageNotFoundError(Exception):
    """ An error raised when no physical file
       is found for a given URL.
    """
    def __init__(self, url, message=None, *args):
        Exception.__init__(self, url, message, *args)

    def __str__(self):
        url = self.args[0]
        message = self.args[1]
        res = "Can't find page '%s'." % url
        if message:
            res += ' ' + message
        return res

class NamespaceNotFoundError(Exception):
    """ An error raised when no physical directory is found
        for a given URL.
    """
    pass

def find_wiki_root(path=None):
    if not path:
        path = os.getcwd()
    while True:
        if os.path.isfile(os.path.join(path, '.wikirc')):
            return path
        if (os.path.isdir(os.path.join(path, '.git')) or
                os.path.isdir(os.path.join(path, '.hg'))):
            return path
        path = os.path.dirname(path)
        if not path or re_terminal_path.match(path):
            break
    return None

def get_absolute_url(base_url, url, quote=False):
    base_endpoint, base_url = split_page_url(base_url)
    if base_url[0] != '/':
        raise ValueError("The base URL must be absolute. Got: %s" % base_url)

    endpoint, url = split_page_url(url)
    if not endpoint:
        endpoint = base_endpoint

    if url.startswith('/'):
        # Absolute page URL.
        abs_url = url
    elif url.startswith('./'):
        # URL wants to be relative to the base url's name, instead
        # of its directory.
        abs_url = base_url + url[1:]
    else:
        # Relative page URL. Let's normalize all `..` in it,
        # which could also replace forward slashes by backslashes
        # on Windows, so we need to convert that back.
        urldir = os.path.dirname(base_url)
        raw_abs_url = os.path.join(urldir, url)
        abs_url = os.path.normpath(raw_abs_url).replace('\\', '/')

    if quote:
        abs_url = urllib.parse.quote(abs_url.encode('utf-8'))
    if endpoint:
        return '%s:%s' % (endpoint, abs_url)
    return abs_url

def is_endpoint_url(url):
    return endpoint_prefix_regex.match(url) is not None

def split_page_url(url):
    m = endpoint_regex.match(url)
    if m is None:
        return (None, url)
    endpoint = m.group(1)
    path = m.group(2)
    return (endpoint, path)

def get_meta_name_and_modifiers(name):
    """ Strips a meta name from any leading modifiers like `__` or `+`
        and returns both as a tuple. If no modifier was found, the
        second tuple value is `None`.
    """
    clean_name = name
    modifiers = None
    if name[:2] == '__':
        modifiers = '__'
        clean_name = name[3:]
    elif name[0] == '+':
        modifiers = '+'
        clean_name = name[1:]
    return (clean_name, modifiers)

def flatten_single_metas(meta):
    items = list(meta.items())
    for k, v in items:
        if isinstance(v, list):
            l = len(v)
            if l == 0:
                del meta[k]
            elif l == 1:
                meta[k] = v[0]
    return meta

html_escape_table = {'"': """, "'": "'"}
html_unescape_table = {v: k for k, v in list(html_escape_table.items())}

def html_escape(text):
    return escape(text, html_escape_table)

def html_unescape(text):
    return unescape(text, html_unescape_table)

<|endoftext|>