<commit_before># coding=utf-8
# Copyright 2018 The Tensor2Tensor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Reinforcement learning models and parameters."""

import collections
import functools
import operator

# Dependency imports

import gym
from tensor2tensor.layers import common_hparams
from tensor2tensor.utils import registry

import tensorflow as tf

@registry.register_hparams
def ppo_base_v1():
  """Set of hyperparameters."""
  hparams = common_hparams.basic_params1()
  hparams.learning_rate = 1e-4
  hparams.add_hparam("init_mean_factor", 0.1)
  hparams.add_hparam("init_logstd", 0.1)
  hparams.add_hparam("policy_layers", (100, 100))
  hparams.add_hparam("value_layers", (100, 100))
  hparams.add_hparam("num_agents", 30)
  hparams.add_hparam("clipping_coef", 0.2)
  hparams.add_hparam("gae_gamma", 0.99)
  hparams.add_hparam("gae_lambda", 0.95)
  hparams.add_hparam("entropy_loss_coef", 0.01)
  hparams.add_hparam("value_loss_coef", 1)
  hparams.add_hparam("optimization_epochs", 15)
  hparams.add_hparam("epoch_length", 200)
  hparams.add_hparam("epochs_num", 2000)
  hparams.add_hparam("eval_every_epochs", 10)
  hparams.add_hparam("num_eval_agents", 3)
  hparams.add_hparam("video_during_eval", True)
  hparams.add_hparam("save_models_every_epochs", 30)
  hparams.add_hparam("optimization_batch_size", 50)
  hparams.add_hparam("max_gradients_norm", 0.5)
  hparams.add_hparam("simulated_environment", True)
  return hparams

@registry.register_hparams
def continuous_action_base():
  hparams = ppo_base_v1()
  hparams.add_hparam("network", feed_forward_gaussian_fun)
  return hparams

@registry.register_hparams
def discrete_action_base():
  hparams = ppo_base_v1()
  hparams.add_hparam("network", feed_forward_categorical_fun)
  return hparams

@registry.register_hparams
def atari_base():
  """Atari base parameters."""
  hparams = discrete_action_base()
  hparams.learning_rate = 16e-5
  hparams.num_agents = 5
  hparams.epoch_length = 200
  hparams.gae_gamma = 0.985
  hparams.gae_lambda = 0.985
  hparams.entropy_loss_coef = 0.002
  hparams.value_loss_coef = 0.025
  hparams.optimization_epochs = 10
  hparams.epochs_num = 10000
  hparams.num_eval_agents = 1
  hparams.network = feed_forward_cnn_small_categorical_fun
  return hparams

NetworkOutput = collections.namedtuple(
    "NetworkOutput", "policy, value, action_postprocessing")

def feed_forward_gaussian_fun(action_space, config, observations):
  """Feed-forward Gaussian."""
  if not isinstance(action_space, gym.spaces.box.Box):
    raise ValueError("Expecting continuous action space.")

  mean_weights_initializer = tf.contrib.layers.variance_scaling_initializer(
      factor=config.init_mean_factor)
  logstd_initializer = tf.random_normal_initializer(config.init_logstd, 1e-10)

  flat_observations = tf.reshape(observations, [
      tf.shape(observations)[0], tf.shape(observations)[1],
      functools.reduce(operator.mul, observations.shape.as_list()[2:], 1)])

  with tf.variable_scope("network_parameters"):
    with tf.variable_scope("policy"):
      x = flat_observations
      for size in config.policy_layers:
        x = tf.contrib.layers.fully_connected(x, size, tf.nn.relu)
      mean = tf.contrib.layers.fully_connected(
          x, action_space.shape[0], tf.tanh,
          weights_initializer=mean_weights_initializer)
      logstd = tf.get_variable(
          "logstd", mean.shape[2:], tf.float32, logstd_initializer)
      logstd = tf.tile(
          logstd[None, None],
          [tf.shape(mean)[0], tf.shape(mean)[1]] + [1] * (mean.shape.ndims - 2))
    with tf.variable_scope("value"):
      x = flat_observations
      for size in config.value_layers:
        x = tf.contrib.layers.fully_connected(x, size, tf.nn.relu)
      value = tf.contrib.layers.fully_connected(x, 1, None)[..., 0]
  mean = tf.check_numerics(mean, "mean")
  logstd = tf.check_numerics(logstd, "logstd")
  value = tf.check_numerics(value, "value")

  policy = tf.contrib.distributions.MultivariateNormalDiag(mean,
                                                           tf.exp(logstd))

  return NetworkOutput(policy, value, lambda a: tf.clip_by_value(a, -2., 2))

def feed_forward_categorical_fun(action_space, config, observations):
  """Feed-forward categorical."""
  if not isinstance(action_space, gym.spaces.Discrete):
    raise ValueError("Expecting discrete action space.")
  flat_observations = tf.reshape(observations, [
      tf.shape(observations)[0], tf.shape(observations)[1],
      functools.reduce(operator.mul, observations.shape.as_list()[2:], 1)])
  with tf.variable_scope("network_parameters"):
    with tf.variable_scope("policy"):
      x = flat_observations
      for size in config.policy_layers:
        x = tf.contrib.layers.fully_connected(x, size, tf.nn.relu)
      logits = tf.contrib.layers.fully_connected(x, action_space.n,
                                                 activation_fn=None)
    with tf.variable_scope("value"):
      x = flat_observations
      for size in config.value_layers:
        x = tf.contrib.layers.fully_connected(x, size, tf.nn.relu)
      value = tf.contrib.layers.fully_connected(x, 1, None)[..., 0]
  policy = tf.contrib.distributions.Categorical(logits=logits)
  return NetworkOutput(policy, value, lambda a: a)

def feed_forward_cnn_small_categorical_fun(action_space, config, observations):
  """Small cnn network with categorical output."""
  del config

  obs_shape = observations.shape.as_list()
  x = tf.reshape(observations, [-1] + obs_shape[2:])

  with tf.variable_scope("network_parameters"):
    x = tf.to_float(x) / 255.0
    x = tf.contrib.layers.conv2d(x, 32, [5, 5], [2, 2],
                                 activation_fn=tf.nn.relu, padding="SAME")
    x = tf.contrib.layers.conv2d(x, 32, [5, 5], [2, 2],
                                 activation_fn=tf.nn.relu, padding="SAME")

    flat_x = tf.reshape(
        x, [tf.shape(observations)[0], tf.shape(observations)[1],
            functools.reduce(operator.mul, x.shape.as_list()[1:], 1)])

    x = tf.contrib.layers.fully_connected(flat_x, 128, tf.nn.relu)
    logits = tf.contrib.layers.fully_connected(x, action_space.n,
                                               activation_fn=None)

    value = tf.contrib.layers.fully_connected(x, 1, activation_fn=None)[..., 0]
    policy = tf.contrib.distributions.Categorical(logits=logits)

  return NetworkOutput(policy, value, lambda a: a)
<commit_msg>Fixing RL test.<commit_after># coding=utf-8
# Copyright 2018 The Tensor2Tensor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Reinforcement learning models and parameters."""

import collections
import functools
import operator

# Dependency imports

import gym
from tensor2tensor.layers import common_hparams
from tensor2tensor.utils import registry

import tensorflow as tf

@registry.register_hparams
def ppo_base_v1():
  """Set of hyperparameters."""
  hparams = common_hparams.basic_params1()
  hparams.learning_rate = 1e-4
  hparams.add_hparam("init_mean_factor", 0.1)
  hparams.add_hparam("init_logstd", 0.1)
  hparams.add_hparam("policy_layers", (100, 100))
  hparams.add_hparam("value_layers", (100, 100))
  hparams.add_hparam("num_agents", 30)
  hparams.add_hparam("clipping_coef", 0.2)
  hparams.add_hparam("gae_gamma", 0.99)
  hparams.add_hparam("gae_lambda", 0.95)
  hparams.add_hparam("entropy_loss_coef", 0.01)
  hparams.add_hparam("value_loss_coef", 1)
  hparams.add_hparam("optimization_epochs", 15)
  hparams.add_hparam("epoch_length", 200)
  hparams.add_hparam("epochs_num", 2000)
  hparams.add_hparam("eval_every_epochs", 10)
  hparams.add_hparam("num_eval_agents", 3)
  hparams.add_hparam("video_during_eval", True)
  hparams.add_hparam("save_models_every_epochs", 30)
  hparams.add_hparam("optimization_batch_size", 50)
  hparams.add_hparam("max_gradients_norm", 0.5)
  hparams.add_hparam("simulated_environment", False)
  return hparams

@registry.register_hparams
def continuous_action_base():
  hparams = ppo_base_v1()
  hparams.add_hparam("network", feed_forward_gaussian_fun)
  return hparams

@registry.register_hparams
def discrete_action_base():
  hparams = ppo_base_v1()
  hparams.add_hparam("network", feed_forward_categorical_fun)
  return hparams

@registry.register_hparams
def atari_base():
  """Atari base parameters."""
  hparams = discrete_action_base()
  hparams.learning_rate = 16e-5
  hparams.num_agents = 5
  hparams.epoch_length = 200
  hparams.gae_gamma = 0.985
  hparams.gae_lambda = 0.985
  hparams.entropy_loss_coef = 0.002
  hparams.value_loss_coef = 0.025
  hparams.optimization_epochs = 10
  hparams.epochs_num = 10000
  hparams.num_eval_agents = 1
  hparams.network = feed_forward_cnn_small_categorical_fun
  return hparams

NetworkOutput = collections.namedtuple(
    "NetworkOutput", "policy, value, action_postprocessing")

def feed_forward_gaussian_fun(action_space, config, observations):
  """Feed-forward Gaussian."""
  if not isinstance(action_space, gym.spaces.box.Box):
    raise ValueError("Expecting continuous action space.")

  mean_weights_initializer = tf.contrib.layers.variance_scaling_initializer(
      factor=config.init_mean_factor)
  logstd_initializer = tf.random_normal_initializer(config.init_logstd, 1e-10)

  flat_observations = tf.reshape(observations, [
      tf.shape(observations)[0], tf.shape(observations)[1],
      functools.reduce(operator.mul, observations.shape.as_list()[2:], 1)])

  with tf.variable_scope("network_parameters"):
    with tf.variable_scope("policy"):
      x = flat_observations
      for size in config.policy_layers:
        x = tf.contrib.layers.fully_connected(x, size, tf.nn.relu)
      mean = tf.contrib.layers.fully_connected(
          x, action_space.shape[0], tf.tanh,
          weights_initializer=mean_weights_initializer)
      logstd = tf.get_variable(
          "logstd", mean.shape[2:], tf.float32, logstd_initializer)
      logstd = tf.tile(
          logstd[None, None],
          [tf.shape(mean)[0], tf.shape(mean)[1]] + [1] * (mean.shape.ndims - 2))
    with tf.variable_scope("value"):
      x = flat_observations
      for size in config.value_layers:
        x = tf.contrib.layers.fully_connected(x, size, tf.nn.relu)
      value = tf.contrib.layers.fully_connected(x, 1, None)[..., 0]
  mean = tf.check_numerics(mean, "mean")
  logstd = tf.check_numerics(logstd, "logstd")
  value = tf.check_numerics(value, "value")

  policy = tf.contrib.distributions.MultivariateNormalDiag(mean,
                                                           tf.exp(logstd))

  return NetworkOutput(policy, value, lambda a: tf.clip_by_value(a, -2., 2))

def feed_forward_categorical_fun(action_space, config, observations):
  """Feed-forward categorical."""
  if not isinstance(action_space, gym.spaces.Discrete):
    raise ValueError("Expecting discrete action space.")
  flat_observations = tf.reshape(observations, [
      tf.shape(observations)[0], tf.shape(observations)[1],
      functools.reduce(operator.mul, observations.shape.as_list()[2:], 1)])
  with tf.variable_scope("network_parameters"):
    with tf.variable_scope("policy"):
      x = flat_observations
      for size in config.policy_layers:
        x = tf.contrib.layers.fully_connected(x, size, tf.nn.relu)
      logits = tf.contrib.layers.fully_connected(x, action_space.n,
                                                 activation_fn=None)
    with tf.variable_scope("value"):
      x = flat_observations
      for size in config.value_layers:
        x = tf.contrib.layers.fully_connected(x, size, tf.nn.relu)
      value = tf.contrib.layers.fully_connected(x, 1, None)[..., 0]
  policy = tf.contrib.distributions.Categorical(logits=logits)
  return NetworkOutput(policy, value, lambda a: a)

def feed_forward_cnn_small_categorical_fun(action_space, config, observations):
  """Small cnn network with categorical output."""
  del config

  obs_shape = observations.shape.as_list()
  x = tf.reshape(observations, [-1] + obs_shape[2:])

  with tf.variable_scope("network_parameters"):
    x = tf.to_float(x) / 255.0
    x = tf.contrib.layers.conv2d(x, 32, [5, 5], [2, 2],
                                 activation_fn=tf.nn.relu, padding="SAME")
    x = tf.contrib.layers.conv2d(x, 32, [5, 5], [2, 2],
                                 activation_fn=tf.nn.relu, padding="SAME")

    flat_x = tf.reshape(
        x, [tf.shape(observations)[0], tf.shape(observations)[1],
            functools.reduce(operator.mul, x.shape.as_list()[1:], 1)])

    x = tf.contrib.layers.fully_connected(flat_x, 128, tf.nn.relu)
    logits = tf.contrib.layers.fully_connected(x, action_space.n,
                                               activation_fn=None)

    value = tf.contrib.layers.fully_connected(x, 1, activation_fn=None)[..., 0]
    policy = tf.contrib.distributions.Categorical(logits=logits)

  return NetworkOutput(policy, value, lambda a: a)
<|endoftext|>


<commit_before>import contextlib
import logging

from sqlalchemy.engine import create_engine
from sqlalchemy.engine.url import make_url, URL
from sqlalchemy.orm.session import sessionmaker

log = logging.getLogger(__name__)

def include(app):
    conversion_map = {
        'url': make_url,
        'make_engine': 'object',
        'make_session_factory': 'object',
        'sessionmaker.class_': 'object',
        'sessionmaker.autoflush': 'bool',
        'sessionmaker.autocommit': 'bool',
        'sessionmaker.expire_on_commit': 'bool',
    }
    defaults = {
        'make_engine': make_engine,
        'make_session_factory': make_session_factory,
    }
    settings = app.parse_settings(
        app.settings, conversion_map=conversion_map, defaults=defaults,
        prefix='sqlalchemy.')

    engine = settings['make_engine'](app, settings)
    app['sqlalchemy.engine'] = engine

    session_factory = settings['make_session_factory'](app, settings)
    app['sqlalchemy.session_factory'] = session_factory

    def make_db_session(request):
        return request.app['sqlalchemy.session_factory']()

    def db_session(request):
        session = request.make_db_session()
        request.on_finished(close, session)
        return session

    def close(app, request, response, session):
        if response is None or (400 <= response.status_int < 599):
            session.rollback()
            log.debug('Session rolled back')
        else:
            session.commit()
            log.debug('Session committed')
        session.close()
        log.debug('Session closed')

    @property
    @contextlib.contextmanager
    def managed_db_session(request):
        session = request.make_db_session()
        try:
            yield session
        except:
            session.rollback()
            raise
        else:
            session.commit()
        finally:
            session.close()

    app.add_request_attribute(make_db_session)
    app.add_request_attribute(db_session, reify=True)
    app.add_request_attribute(managed_db_session)

def make_engine(app, settings):
    url = settings.get('url')
    url_args = app.get_settings(settings, 'url.')
    if url:
        for k, v in url_args.items():
            setattr(url, k, v)
    else:
        url = URL(**url_args)
    engine_args = app.get_settings(settings, 'engine.')
    engine = create_engine(url, **engine_args)
    return engine

def make_session_factory(app, settings):
    engine = app['sqlalchemy.engine']
    sessionmaker_args = app.get_settings(settings, 'sessionmaker.')
    session_factory = sessionmaker(engine, **sessionmaker_args)
    return session_factory
<commit_msg>Use tangled.settings.parse_settings() directly<commit_after>import contextlib
import logging

from sqlalchemy.engine import create_engine
from sqlalchemy.engine.url import make_url, URL
from sqlalchemy.orm.session import sessionmaker

from tangled.settings import parse_settings

log = logging.getLogger(__name__)

def include(app):
    conversion_map = {
        'url': make_url,
        'make_engine': 'object',
        'make_session_factory': 'object',
        'sessionmaker.class_': 'object',
        'sessionmaker.autoflush': 'bool',
        'sessionmaker.autocommit': 'bool',
        'sessionmaker.expire_on_commit': 'bool',
    }
    defaults = {
        'make_engine': make_engine,
        'make_session_factory': make_session_factory,
    }
    settings = parse_settings(
        app.settings, conversion_map=conversion_map, defaults=defaults,
        prefix='sqlalchemy.')

    engine = settings['make_engine'](app, settings)
    app['sqlalchemy.engine'] = engine

    session_factory = settings['make_session_factory'](app, settings)
    app['sqlalchemy.session_factory'] = session_factory

    def make_db_session(request):
        return request.app['sqlalchemy.session_factory']()

    def db_session(request):
        session = request.make_db_session()
        request.on_finished(close, session)
        return session

    def close(app, request, response, session):
        if response is None or (400 <= response.status_int < 599):
            session.rollback()
            log.debug('Session rolled back')
        else:
            session.commit()
            log.debug('Session committed')
        session.close()
        log.debug('Session closed')

    @property
    @contextlib.contextmanager
    def managed_db_session(request):
        session = request.make_db_session()
        try:
            yield session
        except:
            session.rollback()
            raise
        else:
            session.commit()
        finally:
            session.close()

    app.add_request_attribute(make_db_session)
    app.add_request_attribute(db_session, reify=True)
    app.add_request_attribute(managed_db_session)

def make_engine(app, settings):
    url = settings.get('url')
    url_args = app.get_settings(settings, 'url.')
    if url:
        for k, v in url_args.items():
            setattr(url, k, v)
    else:
        url = URL(**url_args)
    engine_args = app.get_settings(settings, 'engine.')
    engine = create_engine(url, **engine_args)
    return engine

def make_session_factory(app, settings):
    engine = app['sqlalchemy.engine']
    sessionmaker_args = app.get_settings(settings, 'sessionmaker.')
    session_factory = sessionmaker(engine, **sessionmaker_args)
    return session_factory
<|endoftext|>


<commit_before>import re
import six

try:
    import urllib.parse as urlparse
except ImportError:
    import urlparse

import bleach
import markdown

from django.template.defaultfilters import urlizetrunc
from django.utils.encoding import force_text
from django.utils.safestring import SafeData, mark_safe
from django.utils.html import TRAILING_PUNCTUATION, WRAPPING_PUNCTUATION, \
    escape, word_split_re, simple_url_re, simple_url_2_re, smart_urlquote, simple_email_re

# These are based on defaults from bleach.
ALLOWED_TAGS = (
    'a',
    'b',
    'blockquote',
    'br',
    'em',
    'i',
    'img',
    'li',
    'ol',
    'strong',
    'ul',
    'hr',
    'p',
    'h3',
    'video',
    'span',
    'iframe'
)

ALLOWED_ATTRIBUTES = {
    'a':   ('href', 'title', 'rel'),
    'img': ('src', 'alt'),
    'iframe': ('src', 'height', 'width'),
    '*':   ('class',),
}

PROFANITY_REPLACEMENTS = (
    ('fuck',    'smurf'),
    ('FUCK',    'SMURF'),
    ('Fuck',    'Smurf'),
    ('fucked',  'smurfed'),
    ('fucker',  'smurfer'),
    ('Fucker',  'Smurfer'),
    ('fucking', 'smurfing'),
    ('Fucking', 'Smurfing'),
)

BBCODE_REPLACEMENTS = (
    (r'\[url\](.+?)\[/url\]',       r'\[link\]\(\1\)'),
    (r'\[url=(.+?)\](.+?)\[/url\]', r'\[\2\]\(\1\)'),
    (r'\[img\](.+?)\[/img\]',       r'\1'),
    (r'\[IMG\](.+?)\[/IMG\]',       r'\1'),
    (r'\[img=(.+?)\](.+?)\[/img\]', r'\1'),
    (r'\[b\](.+?)\[/b\]',           r'**\1**'),
    (r'\[i\](.+?)\[/i\]',           r'*\1*'),
    (r'\[quote\](.+?)\[/quote\]',   r'
\1
'),
)

EMOTICON_REPLACEMENTS = (
    (' :) ',       'happy'),
    (':-)',        'happy'),
    (':smile:',    'happy'),
    (' :( ',       'unhappy'),
    (':-(',        'unhappy'),
    (':sad:',      'unhappy'),
    (' ;) ',       'wink'),
    (';-)',        'wink'),
    (':wink:',     'wink'),
    (' :P ',       'tongue'),
    (':-P',        'tongue'),
    (':razz:',     'tongue'),
    (' x( ',       'angry'),
    ('x-(',        'angry'),
    (':angry:',    'angry'),
    (' :x ',       'angry'),
    (':-x',        'angry'),
    (':mad:',      'angry'),
    (' 8) ',       'cool'),
    ('8-)',        'cool'),
    (' B) ',       'cool'),
    ('B-)',        'cool'),
    (':cool:',     'cool'),
    (' :D ',       'grin'),
    (':-D',        'grin'),
    (':grin:',     'grin'),
    ('8-o',        'surprised'),
    (' :o ',       'surprised'),
    (':-0',        'surprised'),
    ('8-0:',       'surprised'),
    (':shock:',    'surprised'),
    (':eek:',      'surprised'),
    (':|',         'displeased'),
    (' :/ ',       'displeased'),
    (':-/',        'displeased'),
    (':thumbs:',   'thumbsup'),
    (':thumbsup:', 'thumbsup'),
    (':up:',       'thumbsup'),
    (':devil:',    'devil'),
    (':evil:',     'devil'),
    (':beer:',     'beer'),
    (':cry:',      'cry'),
    (":'(",        'cry'),
    (":'-(",       'cry'),
    (':laugh:',    'laugh'),
    (':lol:',      'laugh'),
    ('^^',         'laugh'),
    ('^_^',        'laugh'),
)

def clean_text(value, topic=False):
    """
    Replaces "profane" words with more suitable ones.
    Uses bleach to strip all but whitelisted html.
    Converts bbcode to Markdown
    """
    for x in PROFANITY_REPLACEMENTS:
        value = value.replace(x[0], x[1])

    for bbset in BBCODE_REPLACEMENTS:
        p = re.compile(bbset[0], re.DOTALL)
        value = p.sub(bbset[1], value)

    bleached = bleach.clean(value, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, strip=True)

    # We want to retain markdown quotes and we'll be running bleach again in format_post.
    bleached = bleached.replace('>', '>').replace('&', '&')
    return bleached

def convert_links(text, trim_url_limit=None, nofollow=False, autoescape=False):
    """
    Finds URLs in text and attempts to handle correctly.
    Heavily based on django.utils.html.urlize
    With the additions of attempting to embed media links, particularly images.

    Works on http://, https://, www. links, and also on links ending in one of
    the original seven gTLDs (.com, .edu, .gov, .int, .mil, .net, and .org).

    Links can have trailing punctuation (periods, commas, close-parens) and
    leading punctuation (opening parens) and it'll still do the right thing.

    """

    safe_input = isinstance(text, SafeData)
    words = word_split_re.split(force_text(text))
    for i, word in enumerate(words):
        if '.' in word or ':' in word:
            # Deal with punctuation.
            lead, middle, trail = '', word, ''
            for punctuation in TRAILING_PUNCTUATION:
                if middle.endswith(punctuation):
                    middle = middle[:-len(punctuation)]
                    trail = punctuation + trail
            for opening, closing in WRAPPING_PUNCTUATION:
                if middle.startswith(opening):
                    middle = middle[len(opening):]
                    lead = lead + opening
                # Keep parentheses at the end only if they're balanced.
                if (middle.endswith(closing)
                    and middle.count(closing) == middle.count(opening) + 1):
                    middle = middle[:-len(closing)]
                    trail = closing + trail

            # Make URL we want to point to.
            url = None
            if simple_url_re.match(middle):
                url = smart_urlquote(middle)
            elif simple_url_2_re.match(middle):
                url = smart_urlquote('http://%s' % middle)
            elif not ':' in middle and simple_email_re.match(middle):
                local, domain = middle.rsplit('@', 1)
                try:
                    domain = domain.encode('idna').decode('ascii')
                except UnicodeError:
                    continue
            if url:
                u = url.lower()
                if autoescape and not safe_input:
                    lead, trail = escape(lead), escape(trail)
                    url = escape(url)

                # Photos
                if u.endswith('.jpg') or u.endswith('.gif') or u.endswith('.png'):
                    middle = '' % url

                # Youtube
                #'https://www.youtube.com/watch?v=gkqXgaUuxZg'
                elif 'youtube.com/watch' in url:
                    parsed = urlparse.urlsplit(url)
                    query  = urlparse.parse_qs(parsed.query)
                    token  = query.get('v')
                    if token and len(token) > 0:
                        middle = '' % token[0]
                    else:
                        middle = url
                elif 'youtu.be/' in url:
                    try:
                        token = url.rsplit('/', 1)[1]
                        middle = '' % token
                    except IndexError:
                        middle = six.u(url)

                words[i] = mark_safe('%s%s%s' % (lead, middle, trail))
            else:
                if safe_input:
                    words[i] = mark_safe(word)
                elif autoescape:
                    words[i] = escape(word)
        elif safe_input:
            words[i] = mark_safe(word)
        elif autoescape:
            words[i] = escape(word)
    return ''.join(words)

def format_text(value):
    """
    Takes cleaned text and creates HTML-formatted, web-friendly version.
    - converts links to images and video to actual media objects.
    - Make raw links clickable (and truncated, if necessary).
    - Converts smilies and markdown to valid HTML.
    _ Bleaches output (again) to catch any stray HTML inserted by markdown.

    ALWAYS PASS CLEANED TEXT.

    """
    # convert media links
    value = convert_links(value)
    value = urlizetrunc(value, 30)

    for x in EMOTICON_REPLACEMENTS:
        value = value.replace(x[0], ''.format(x[1]))

    markedup = markdown.markdown(value).replace('
\n
', '

')
    with_linebreaks = markedup.replace('\n* ', '
* ')
    bleached = bleach.clean(with_linebreaks, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, strip=True)
    return mark_safe(bleached)

def sanetize_text(text):
    """
    Takes raw user input text, performs various cleaning and formatting functions,
    and returns a clean, sane, HTML-formatted version.
    """
    cleaned_text = clean_text(text)
    return format_text(cleaned_text)
<commit_msg>Added more comments to sanetize.py<commit_after>import re
import six

try:
    import urllib.parse as urlparse
except ImportError:
    import urlparse

import bleach
import markdown

from django.template.defaultfilters import urlizetrunc
from django.utils.encoding import force_text
from django.utils.safestring import SafeData, mark_safe
from django.utils.html import TRAILING_PUNCTUATION, WRAPPING_PUNCTUATION, \
    escape, word_split_re, simple_url_re, simple_url_2_re, smart_urlquote, simple_email_re

# These are tags we will allow in our santized version.
# Based on defaults from Bleach.
# See https://bleach.readthedocs.io/en/latest/
ALLOWED_TAGS = (
    'a',
    'b',
    'blockquote',
    'br',
    'em',
    'i',
    'img',
    'li',
    'ol',
    'strong',
    'ul',
    'hr',
    'p',
    'h3',
    'video',
    'span',
    'iframe'
)

# These are the only valid attributes.
# Again, this is based on Bleach defaults.
ALLOWED_ATTRIBUTES = {
    'a':   ('href', 'title', 'rel'),
    'img': ('src', 'alt'),
    'iframe': ('src', 'height', 'width'),
    '*':   ('class',),
}

# Be cautious with profanity replacements.
# False positives are a mothersmurfer.
PROFANITY_REPLACEMENTS = (
    ('fuck',    'smurf'),
    ('FUCK',    'SMURF'),
    ('Fuck',    'Smurf'),
    ('fucked',  'smurfed'),
    ('fucker',  'smurfer'),
    ('Fucker',  'Smurfer'),
    ('fucking', 'smurfing'),
    ('Fucking', 'Smurfing'),
)

# Simple regexes to allow the use of most BBCODE.
# If you don't support BBCODE, then this isn't particularly useful
# but I find it helpful if things just work as the user expects them to.
BBCODE_REPLACEMENTS = (
    (r'\[url\](.+?)\[/url\]',       r'\[link\]\(\1\)'),
    (r'\[url=(.+?)\](.+?)\[/url\]', r'\[\2\]\(\1\)'),
    (r'\[img\](.+?)\[/img\]',       r'\1'),
    (r'\[IMG\](.+?)\[/IMG\]',       r'\1'),
    (r'\[img=(.+?)\](.+?)\[/img\]', r'\1'),
    (r'\[b\](.+?)\[/b\]',           r'**\1**'),
    (r'\[i\](.+?)\[/i\]',           r'*\1*'),
    (r'\[quote\](.+?)\[/quote\]',   r'
\1
'),
)

# Simple smiley/emoticon/emoji integration.
# To-Do: allow more Slack-ish options.
# note that the actual integration with emojis still has to be handled.
EMOTICON_REPLACEMENTS = (
    (' :) ',       'happy'),
    (':-)',        'happy'),
    (':smile:',    'happy'),
    (' :( ',       'unhappy'),
    (':-(',        'unhappy'),
    (':sad:',      'unhappy'),
    (' ;) ',       'wink'),
    (';-)',        'wink'),
    (':wink:',     'wink'),
    (' :P ',       'tongue'),
    (':-P',        'tongue'),
    (':razz:',     'tongue'),
    (' x( ',       'angry'),
    ('x-(',        'angry'),
    (':angry:',    'angry'),
    (' :x ',       'angry'),
    (':-x',        'angry'),
    (':mad:',      'angry'),
    (' 8) ',       'cool'),
    ('8-)',        'cool'),
    (' B) ',       'cool'),
    ('B-)',        'cool'),
    (':cool:',     'cool'),
    (' :D ',       'grin'),
    (':-D',        'grin'),
    (':grin:',     'grin'),
    ('8-o',        'surprised'),
    (' :o ',       'surprised'),
    (':-0',        'surprised'),
    ('8-0:',       'surprised'),
    (':shock:',    'surprised'),
    (':eek:',      'surprised'),
    (':|',         'displeased'),
    (' :/ ',       'displeased'),
    (':-/',        'displeased'),
    (':thumbs:',   'thumbsup'),
    (':thumbsup:', 'thumbsup'),
    (':up:',       'thumbsup'),
    (':devil:',    'devil'),
    (':evil:',     'devil'),
    (':beer:',     'beer'),
    (':cry:',      'cry'),
    (":'(",        'cry'),
    (":'-(",       'cry'),
    (':laugh:',    'laugh'),
    (':lol:',      'laugh'),
    ('^^',         'laugh'),
    ('^_^',        'laugh'),
)

def clean_text(value, topic=False):
    """
    Replaces "profane" words with more suitable ones.
    Uses bleach to strip all but whitelisted html.
    Converts bbcode to Markdown
    """
    for x in PROFANITY_REPLACEMENTS:
        value = value.replace(x[0], x[1])

    for bbset in BBCODE_REPLACEMENTS:
        p = re.compile(bbset[0], re.DOTALL)
        value = p.sub(bbset[1], value)

    bleached = bleach.clean(value, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, strip=True)

    # We want to retain markdown quotes and we'll be running bleach again in format_post.
    bleached = bleached.replace('>', '>').replace('&', '&')
    return bleached

def convert_links(text, trim_url_limit=None, nofollow=False, autoescape=False):
    """
    Finds URLs in text and attempts to handle correctly.
    Heavily based on django.utils.html.urlize
    With the additions of attempting to embed media links, particularly images.

    Works on http://, https://, www. links, and also on links ending in one of
    the original seven gTLDs (.com, .edu, .gov, .int, .mil, .net, and .org).

    Links can have trailing punctuation (periods, commas, close-parens) and
    leading punctuation (opening parens) and it'll still do the right thing.

    """

    safe_input = isinstance(text, SafeData)
    words = word_split_re.split(force_text(text))
    for i, word in enumerate(words):
        if '.' in word or ':' in word:
            # Deal with punctuation.
            lead, middle, trail = '', word, ''
            for punctuation in TRAILING_PUNCTUATION:
                if middle.endswith(punctuation):
                    middle = middle[:-len(punctuation)]
                    trail = punctuation + trail
            for opening, closing in WRAPPING_PUNCTUATION:
                if middle.startswith(opening):
                    middle = middle[len(opening):]
                    lead = lead + opening
                # Keep parentheses at the end only if they're balanced.
                if (middle.endswith(closing)
                    and middle.count(closing) == middle.count(opening) + 1):
                    middle = middle[:-len(closing)]
                    trail = closing + trail

            # Make URL we want to point to.
            url = None
            if simple_url_re.match(middle):
                url = smart_urlquote(middle)
            elif simple_url_2_re.match(middle):
                url = smart_urlquote('http://%s' % middle)
            elif not ':' in middle and simple_email_re.match(middle):
                local, domain = middle.rsplit('@', 1)
                try:
                    domain = domain.encode('idna').decode('ascii')
                except UnicodeError:
                    continue
            if url:
                u = url.lower()
                if autoescape and not safe_input:
                    lead, trail = escape(lead), escape(trail)
                    url = escape(url)

                # Photos
                if u.endswith('.jpg') or u.endswith('.gif') or u.endswith('.png'):
                    middle = '' % url

                # Youtube
                #'https://www.youtube.com/watch?v=gkqXgaUuxZg'
                elif 'youtube.com/watch' in url:
                    parsed = urlparse.urlsplit(url)
                    query  = urlparse.parse_qs(parsed.query)
                    token  = query.get('v')
                    if token and len(token) > 0:
                        middle = '' % token[0]
                    else:
                        middle = url
                elif 'youtu.be/' in url:
                    try:
                        token = url.rsplit('/', 1)[1]
                        middle = '' % token
                    except IndexError:
                        middle = six.u(url)

                words[i] = mark_safe('%s%s%s' % (lead, middle, trail))
            else:
                if safe_input:
                    words[i] = mark_safe(word)
                elif autoescape:
                    words[i] = escape(word)
        elif safe_input:
            words[i] = mark_safe(word)
        elif autoescape:
            words[i] = escape(word)
    return ''.join(words)

def format_text(value):
    """
    Takes cleaned text and creates HTML-formatted, web-friendly version.
    - converts links to images and video to actual media objects.
    - Make raw links clickable (and truncated, if necessary).
    - Converts smilies and markdown to valid HTML.
    _ Bleaches output (again) to catch any stray HTML inserted by markdown.

    ALWAYS PASS CLEANED TEXT.

    """
    # convert media links
    value = convert_links(value)
    value = urlizetrunc(value, 30)

    for x in EMOTICON_REPLACEMENTS:
        value = value.replace(x[0], ''.format(x[1]))

    markedup = markdown.markdown(value).replace('
\n
', '

')
    with_linebreaks = markedup.replace('\n* ', '
* ')
    bleached = bleach.clean(with_linebreaks, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, strip=True)
    return mark_safe(bleached)

def sanetize_text(text):
    """
    Takes raw user input text, performs various cleaning and formatting functions,
    and returns a clean, sane, HTML-formatted version.
    """
    cleaned_text = clean_text(text)
    return format_text(cleaned_text)
<|endoftext|>


<commit_before>#! /usr/bin/python

#
# FUSION.PY
#
# Description: Sensor fusion for the sonic anemometer
#
# (C) 2016 Visaoni
# Licensed under the MIT License.
#

# TODO: Figure out if DHT or BMP is better for temp

import zmq
import Adafruit_DHT
import Adafruit_BMP.BMP085 as Adafruit_BMP085
import time

# TODO: Take port as input
zmq_tof_host = "tcp://localhost:5555"

zmq_context = zmq.Context()
zmq_tof_subscriber = zmq_context.socket( zmq.SUB )
zmq_tof_subscriber.setsockopt( zmq.CONFLATE, 1 )           # Keeps only latest message
zmq_tof_subscriber.connect( zmq_tof_host )
zmq_tof_subscriber.setsockopt( zmq.SUBSCRIBE, "" )  # Listen to all messages on socket

# TODO: Deal with the fact there are actually two axes
def get_tof():
   return float( zmq_tof_subscriber.recv() )

DHT_sensor = Adafruit_DHT.AM2302
DHT_pin = "UART2_RXD"

# Adafruit_DHT does offer read_retry but has no way to specify infinite retries
# Also returns an error code on failure rather than an exception, hence this construct
def get_rel_humidity_temp():
   result = (None, None)
   while result == (None, None):
      result = Adafruit_DHT.read( DHT_sensor, DHT_pin )
      time.sleep(2)
   return result

BMP_busnum = 2
BMP_sensor = Adafruit_BMP085.BMP085( busnum = BMP_busnum, mode = Adafruit_BMP085.BMP085_ULTRAHIGHRES )

def get_temp():
   return BMP_sensor.read_temperature()

def get_pressure():
   return BMP_sensor.read_pressure()

# TODO: Add more frequent updates (IE avoid having to wait on the DHT)
# Most basic setup possible
# Loop through and grab all sensor readings then analyze
while True:
   rel_humidity, temp = get_rel_humidity_temp()
   pressure = get_pressure()
   tof = get_tof()

   # TODO: calculate wind speed

   print( "Humidity:    " + str(rel_humidity) )
   print( "Temperature: " + str(temp) )
   print( "ToF:         " + str(tof) )

   time.sleep(2) # Wait for DHT<commit_msg>Added stubs for wind speed calculations<commit_after>#! /usr/bin/python

#
# FUSION.PY
#
# Description: Sensor fusion for the sonic anemometer
#
# (C) 2016 Visaoni
# Licensed under the MIT License.
#

# TODO: Figure out if DHT or BMP is better for temp

import zmq
import Adafruit_DHT
import Adafruit_BMP.BMP085 as Adafruit_BMP085
import time

# TODO: Take port as input
zmq_tof_host = "tcp://localhost:5555"

zmq_context = zmq.Context()
zmq_tof_subscriber = zmq_context.socket( zmq.SUB )
zmq_tof_subscriber.setsockopt( zmq.CONFLATE, 1 )           # Keeps only latest message
zmq_tof_subscriber.connect( zmq_tof_host )
zmq_tof_subscriber.setsockopt( zmq.SUBSCRIBE, "" )  # Listen to all messages on socket

# TODO: Deal with the fact there are actually two axes
def get_tof():
   return float( zmq_tof_subscriber.recv() )

DHT_sensor = Adafruit_DHT.AM2302
DHT_pin = "UART2_RXD"

# Adafruit_DHT does offer read_retry but has no way to specify infinite retries
# Also returns an error code on failure rather than an exception, hence this construct
def get_rel_humidity_temp():
   result = (None, None)
   while result == (None, None):
      result = Adafruit_DHT.read( DHT_sensor, DHT_pin )
      time.sleep(2)
   return result

BMP_busnum = 2
BMP_sensor = Adafruit_BMP085.BMP085( busnum = BMP_busnum, mode = Adafruit_BMP085.BMP085_ULTRAHIGHRES )

def get_temp():
   return BMP_sensor.read_temperature()

def get_pressure():
   return BMP_sensor.read_pressure()

# TODO: Take as input, or better yet get from configuration process
#     If windspeed is assumed to be 0, distance can be calculated from tof
axis_distance = 0.5

def find_windspeed( rel_humidity, temp, pressure, tof ):
   pulse_speed = axis_distance / tof
   return pulse_speed - est_speed_of_sound( rel_humidity, temp, pressure )

# TODO: fill in... and figure out how
def est_speed_of_sound( rel_humidity, temp, pressure ):
   return 322.0

# TODO: Add more frequent updates (IE avoid having to wait on the DHT)
# Most basic setup possible
# Loop through and grab all sensor readings then analyze
while True:
   rel_humidity, temp = get_rel_humidity_temp()
   pressure = get_pressure()
   tof = get_tof()

   # TODO: calculate wind speed

   print( "Humidity:    " + str(rel_humidity) )
   print( "Temperature: " + str(temp) )
   print( "ToF:         " + str(tof) )
   print( "Wind speed:  " + str(find_windspeed( rel_humidity, temp, pressure, tof )) )

   time.sleep(2) # Wait for DHT<|endoftext|>


<commit_before>#!/usr/bin/python

import sys, getopt
import thread
import time
import logging
sys.path.append('.')
import RTIMU
import os.path
import math
from Adafruit_PWM_Servo_Driver import PWM
import socket

class PropCalc():
	def __init__(self, prop, status):
		print("Class Initiated to calc prop value")
		self.prop = prop
		self.servoMin = 230  # Min pulse length out of 4096 == 1ms Pulse
		self.servoMax = 650  # Max pulse length out of 4096 == 2ms Pulse
		self.gyro = [[0,0,0,0,0,0],[0,0,0,0,0,0]]
		self.status = status

	def filterValues(self, axis):
		del self.gyro[axis][0]
		self.gyro[axis].append(self.status["gyro-y"])
		y = 0
		for i in self.gyro[axis]:
			y += i
		filteredValue /= len(self.gyro[axis])
		return filteredValue

	def propValue(self, measurement, filtered)
		if measurement < filtered:
			filtered /= 4
		else:
			if fltered > 0:
				filtered /= 2
			else:
				filtered = 0
		return filtered

	def value(self):
		print("Give back values")
		print "CalcProps gestartet"

		xFiltered = propValue(filterValues(0), status["gyro-x"])
		yFiltered = propValue(filterValues(1), status["gyro-y"])

		if self.prop == 0:
			self.status["PropValue"][self.prop] = ((xFiltered + yFiltered)/2) + self.servoMin + self.status["throttle"]
		elif self.prop == 1:
			self.status["PropValue"][self.prop] = ((xFiltered + (yFiltered*-1))/2) + self.servoMin + self.status["throttle"]
		elif self.prop == 2:
			self.status["PropValue"][self.prop] = (((xFiltered*-1) + yFiltered)/2) + self.servoMin + self.status["throttle"]
		elif self.prop == 3:
			self.status["PropValue"][self.prop] = (((xFiltered*-1) + (yFiltered*-1))/2) + self.servoMin + self.status["throttle"]
		return

def ReadSensor(status):
	SETTINGS_FILE = "RTIMULib"

	#print("Using settings file " + SETTINGS_FILE + ".ini")
	#if not os.path.exists(SETTINGS_FILE + ".ini"):
	  #print("Settings file does not exist, will be created")

	s = RTIMU.Settings(SETTINGS_FILE)
	imu = RTIMU.RTIMU(s)

	#print("IMU Name: " + imu.IMUName())

	if (not imu.IMUInit()):
	    #print("IMU Init Failed");
	    sys.exit(1)
	#else:
	    #print("IMU Init Succeeded");

	poll_interval = imu.IMUGetPollInterval()
	#print("Recommended Poll Interval: %dmS\n" % poll_interval)
	logging.debug('ReadSensor, Thread started')
	if imu.IMURead(): #check if gyro can be read
		pass
	else:
		return
	while status["start"]:
		# x, y, z = imu.getFusionData()
		status["gyro-x"] = math.degrees(imu.getIMUData()["fusionPose"][0])
		status["gyro-y"] = math.degrees(imu.getIMUData()["fusionPose"][1])
		status["gyro-z"] = math.degrees(imu.getIMUData()["fusionPose"][2])
		if (status["gyro-y"] > 35) or status["gyro-y"] < -35 or status["gyro-x"] > 35 or status["gyro-x"] < -35:
			status["start"] = 0
			print "Emergency stop!"

		#time.sleep(0.001)
		#logging.debug("Sensor Loop: x: " + str(status["gyro-x"]) + " y: " + str(status["gyro-y"]) + " z: " + str(status["gyro-z"]))
		#GPS
		#altitude by barometric sensor
		#read altitude by google api
		#roll & pitch
		#yaw compass
		# read distance sensor
		# set ready preflight check 1
		# read sonsors for ~30sec if no change set ready for flight 1
		# update sonsors roll & pitch every 1ms
		# update gps barometric sensor, yaw, distane sensor 100ms
		# update altitude over ground every 1sec
		# if something missing, set emergency state (hold position as possible)
	return

def PrintData(status):
	logging.debug('ReadSensor, Thread PrintData')
	while status["start"]:
		time.sleep(1)
		#logging.debug('PrintData, Loop')
		#for i in status:
			#print str(i) + " :" + str(status[i])

def CalcProps(status):

	#
	# This is the chema of the Quadcopter. This can be used for all copters with Prop count div by 4
	#
	#	flight direction
	#	 0		 1
	#	   \          /
	#            \      /
	#		Q
	#            /      \
	#	   /          \
	#	 2		 3
	#		rear
	#
	#

	CountOfProps = 4
	propInstances = []
	for i in range(CountOfProps): #will return 0-3
		propInstances.appen(PropCalc(i, status))
	while status["start"]:
		for i in propInstances:
			i.value()
		time.sleep(0.001)

def ControlProps(status):
	logging.debug('ReadSensor, Thread ControlProps')
	pwm = PWM(0x40)
	pwm.setPWMFreq(50)                        # Set frequency to 60 Hz
	print "Starting Control of Props according Input"
	time.sleep(2)
	print "Starting Loop to Control Sleep"
	try:
		while status["start"]:
			if status["debug"]:
				print("Propeller 1:" + str(status["PropValue"][0]) + " Propeller 2:" + str(status["PropValue"][1]) + " Propeller 3:" + str(status["PropValue"][2]) + " Propeller 4:" + str(status["PropValue"][3]))
				time.sleep(0.1)
			else:
				time.sleep(0.002) #makes no sence to control prop more often with 50Hz
				pwm.setPWM(0, 0, int(status["PropValue"][0]))
				pwm.setPWM(1, 0, int(status["PropValue"][1]))
				#pwm.setPWM(2, 0, int(status["PropValue"][2]))
				#pwm.setPWM(3, 0, int(status["PropValue"][3]))

		print "Stopping Motors"
		pwm.setPWM(0, 0, 180)
		pwm.setPWM(1, 0, 180)
		pwm.setPWM(2, 0, 180)
		pwm.setPWM(3, 0, 180)
	except KeyboardInterrupt:
		pwm.setPWM(0, 0, 180)
		pwm.setPWM(1, 0, 180)
		pwm.setPWM(2, 0, 180)
		pwm.setPWM(3, 0, 180)
		print "Reset Servo!!"
		raise
	except:
		raise
	return

def ReadInput(status):
	logging.debug('ReadInput, Thread ReadInput')
	while status["start"]:
		logging.debug('ReadInput, Loop')
		input = raw_input("Please insert your command: ")
		print "Got command: " + str(input)

		if input == "stop":
			status["start"] = 0
		elif input == "t":
			throttle = raw_input("Insert throttle position: ")
			print "Got throttle: " + str(input)
			status["throttle"] = int(throttle)
		elif input == "throttle":
			throttle = raw_input("Insert throttle position: ")
			print "Got throttle: " + str(input)
			status["throttle"] = int(throttle)
		elif input == "a":
			status["pitch"] += 1
		elif input == "d":
			status["pitch"] -= 1
		elif input == "w":
			status["throttle"] += 1
		elif input == "s":
			status["throttle"] -= 1
		elif input == "test":
			status["throttle"] = 10
			print "Set Trottle to 10"
			time.sleep(0.5)
			status["throttle"] = 100
			print "Set Trottle to 100"
			time.sleep(0.5)
			status["throttle"] = 10
			print "Set Trottle to 10"
			time.sleep(0.5)
			status["throttle"] = 100
			print "Set Trottle to 100"
			time.sleep(0.5)
			status["throttle"] = 10
			print "Set Trottle to 10"
			time.sleep(0.5)
			status["throttle"] = 100
			print "Set Trottle to 100"
			time.sleep(0.5)
			status["throttle"] = 10
			print "Set Trottle to 10"
			time.sleep(1)
		elif input == "esc-reset":
			print "Disconnect ESC from Controller and power"
			print "Press 'y' when done"
			input = raw_input("Done?: ")
			status["start"] = 0
			time.sleep(2)
			if input == "y":
				pwm = PWM(0x40)
				pwm.setPWMFreq(50)                        # Set frequency to 60 Hz
				print "now enter the % of throttle you need"
				while 1:
					try:
						foo = raw_input("procent: ")
						if int(foo) > 100:
							break
						value = (int(foo) * 450 / 100) + 150
						pwm.setPWM(0, 0, value)
						pwm.setPWM(1, 0, value)
						pwm.setPWM(2, 0, value)
						pwm.setPWM(3, 0, value)
						del foo
					except:
						print "Try again"
				print "Done...."

def NetworkSoket(status):
	logging.debug('Starting Up Networks socket')
	HOST = ''   # Symbolic name, meaning all available interfaces
	PORT = 16016 # Arbitrary non-privileged port

	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	logging.debug('Socket Created')

	#Bind socket to local host and port
	try:
		s.bind((HOST, PORT))
	except socket.error as msg:
		logging.debug('Bind failed. Error Code : ' + str(msg[0]) + ' Message ' + msg[1])
		status["start"] = 0
		raise

	print 'Socket bind complete'
	#Start listening on socket
	s.listen(10)
	logging.debug('Socket now listening')

	#now keep talking with the client
	conn, addr = s.accept()
	logging.debug('Connected with ' + addr[0] + ':' + str(addr[1]))
	while status["start"]:
		#Receiving from client
		data = conn.recv(1024)
		data = data.rstrip()
		print data
		if not data:
		    break
		reply = "invalid\n\n"
		if data == "stop":
			reply = "OK...\n\n"
			s.close()
			status["start"] = 0
		if data == "status":
			reply = "Running: " + str(status["start"]) + "\nPropeller 1:" + str(status["PropValue"][0]) + "\nPropeller 2:" + str(status["PropValue"][1]) + "\nPropeller 3:" + str(status["PropValue"][2]) + "\nPropeller 4:" + str(status["PropValue"][3]) + "\n...End\n\n"
		conn.sendall(reply)

	S.close()

def CAM(status):
	logging.debug('CAM, Thread ReadInput')
	while status["start"]:
		time.sleep(1)
		#logging.debug('CAM, Loop')
	return

status={}
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', filename='/var/log/copter.log',level=logging.DEBUG)
logging.debug('Main: Started Up')
status["debug"] = 0
status["start"] = 1
status["throttle"] = 15
status["pitch"] = 0
# check stare of vpn tunnel MUST be online

try:
	thread.start_new_thread( ReadSensor, (status, ) )
	thread.start_new_thread( CalcProps, (status, ) )
	#thread.start_new_thread( PrintData, (status, ) )
	thread.start_new_thread( ControlProps, (status, ) )
	thread.start_new_thread( ReadInput, (status, ) )
	thread.start_new_thread( CAM, (status, ) )
	thread.start_new_thread( NetworkSoket, (status, ) )
	while status["start"]:
		time.sleep(5)
	time.sleep(1)
	quit()
except:
	logging.warning('Main: WARNNG a thread gave back some error')
	quit()

<commit_msg>some things<commit_after>#!/usr/bin/python

import sys, getopt
import thread
import time
import logging
import RTIMU
import os.path
import math
from Adafruit_PWM_Servo_Driver import PWM
import socket
sys.path.append('.')

class PropCalc():
	def __init__(self, prop, status):
		print("Class Initiated to calc prop value")
		self.prop = prop
		self.servoMin = 230  # Min pulse length out of 4096 == 1ms Pulse
		self.servoMax = 650  # Max pulse length out of 4096 == 2ms Pulse
		self.gyro = [[0,0,0,0,0,0],[0,0,0,0,0,0]]
		self.status = status

	def __filterValues(self, axis):
		del self.gyro[axis][0]
		self.gyro[axis].append(self.status["gyro-y"])
		y = 0
		for i in self.gyro[axis]:
			y += i
		filteredValue /= len(self.gyro[axis])
		return filteredValue

	def __propValue(self, measurement, filtered):
		if measurement < filtered:
			filtered /= 4
		else:
			if fltered > 0:
				filtered /= 2
			else:
				filtered = 0
		return filtered

	def value(self):
		print("Give back values")
		print "CalcProps gestartet"

		xFiltered = __propValue(__filterValues(0), status["gyro-x"])
		yFiltered = __propValue(__filterValues(1), status["gyro-y"])

		if self.prop == 0:
			self.status["PropValue"][self.prop] = ((xFiltered + yFiltered)/2) + self.servoMin + self.status["throttle"]
		elif self.prop == 1:
			self.status["PropValue"][self.prop] = ((xFiltered + (yFiltered*-1))/2) + self.servoMin + self.status["throttle"]
		elif self.prop == 2:
			self.status["PropValue"][self.prop] = (((xFiltered*-1) + yFiltered)/2) + self.servoMin + self.status["throttle"]
		elif self.prop == 3:
			self.status["PropValue"][self.prop] = (((xFiltered*-1) + (yFiltered*-1))/2) + self.servoMin + self.status["throttle"]
		return

def ReadSensor(status):
	SETTINGS_FILE = "RTIMULib"

	#print("Using settings file " + SETTINGS_FILE + ".ini")
	#if not os.path.exists(SETTINGS_FILE + ".ini"):
	  #print("Settings file does not exist, will be created")

	s = RTIMU.Settings(SETTINGS_FILE)
	imu = RTIMU.RTIMU(s)

	#print("IMU Name: " + imu.IMUName())

	if (not imu.IMUInit()):
	    #print("IMU Init Failed");
	    sys.exit(1)
	#else:
	    #print("IMU Init Succeeded");

	poll_interval = imu.IMUGetPollInterval()
	#print("Recommended Poll Interval: %dmS\n" % poll_interval)
	logging.debug('ReadSensor, Thread started')
	if imu.IMURead(): #check if gyro can be read
		pass
	else:
		return
	while status["start"]:
		# x, y, z = imu.getFusionData()
		status["gyro-x"] = math.degrees(imu.getIMUData()["fusionPose"][0])
		status["gyro-y"] = math.degrees(imu.getIMUData()["fusionPose"][1])
		status["gyro-z"] = math.degrees(imu.getIMUData()["fusionPose"][2])
		if (status["gyro-y"] > 35) or status["gyro-y"] < -35 or status["gyro-x"] > 35 or status["gyro-x"] < -35:
			status["start"] = 0
			print "Emergency stop!"

		#time.sleep(0.001)
		#logging.debug("Sensor Loop: x: " + str(status["gyro-x"]) + " y: " + str(status["gyro-y"]) + " z: " + str(status["gyro-z"]))
		#GPS
		#altitude by barometric sensor
		#read altitude by google api
		#roll & pitch
		#yaw compass
		# read distance sensor
		# set ready preflight check 1
		# read sonsors for ~30sec if no change set ready for flight 1
		# update sonsors roll & pitch every 1ms
		# update gps barometric sensor, yaw, distane sensor 100ms
		# update altitude over ground every 1sec
		# if something missing, set emergency state (hold position as possible)
	return

def PrintData(status):
	logging.debug('ReadSensor, Thread PrintData')
	while status["start"]:
		time.sleep(1)
		#logging.debug('PrintData, Loop')
		#for i in status:
			#print str(i) + " :" + str(status[i])

def CalcProps(status):

	#
	# This is the chema of the Quadcopter. This can be used for all copters with Prop count div by 4
	#
	#	flight direction
	#	 0		 1
	#	   \          /
	#            \      /
	#		Q
	#            /      \
	#	   /          \
	#	 2		 3
	#		rear
	#
	#

	CountOfProps = 4
	propInstances = []
	for i in range(CountOfProps): #will return 0-3
		propInstances.appen(PropCalc(i, status))
	while status["start"]:
		for i in propInstances:
			i.value()
		time.sleep(0.001)

def ControlProps(status):
	logging.debug('ReadSensor, Thread ControlProps')
	pwm = PWM(0x40)
	pwm.setPWMFreq(50)                        # Set frequency to 60 Hz
	print "Starting Control of Props according Input"
	time.sleep(2)
	print "Starting Loop to Control Sleep"
	try:
		while status["start"]:
			if status["debug"]:
				print("Propeller 1:" + str(status["PropValue"][0]) + " Propeller 2:" + str(status["PropValue"][1]) + " Propeller 3:" + str(status["PropValue"][2]) + " Propeller 4:" + str(status["PropValue"][3]))
				time.sleep(0.1)
			else:
				time.sleep(0.002) #makes no sence to control prop more often with 50Hz
				pwm.setPWM(0, 0, int(status["PropValue"][0]))
				pwm.setPWM(1, 0, int(status["PropValue"][1]))
				pwm.setPWM(2, 0, int(status["PropValue"][2]))
				pwm.setPWM(3, 0, int(status["PropValue"][3]))

		print "Stopping Motors"
		pwm.setPWM(0, 0, 180)
		pwm.setPWM(1, 0, 180)
		pwm.setPWM(2, 0, 180)
		pwm.setPWM(3, 0, 180)
	except KeyboardInterrupt:
		pwm.setPWM(0, 0, 180)
		pwm.setPWM(1, 0, 180)
		pwm.setPWM(2, 0, 180)
		pwm.setPWM(3, 0, 180)
		print "Reset Servo!!"
		raise
	except:
		raise
	return

def ReadInput(status):
	logging.debug('ReadInput, Thread ReadInput')
	while status["start"]:
		logging.debug('ReadInput, Loop')
		input = raw_input("Please insert your command: ")
		print "Got command: " + str(input)

		if input == "stop":
			status["start"] = 0
		elif input == "t":
			throttle = raw_input("Insert throttle position: ")
			print "Got throttle: " + str(input)
			status["throttle"] = int(throttle)
		elif input == "throttle":
			throttle = raw_input("Insert throttle position: ")
			print "Got throttle: " + str(input)
			status["throttle"] = int(throttle)
		elif input == "a":
			status["pitch"] += 1
		elif input == "d":
			status["pitch"] -= 1
		elif input == "w":
			status["throttle"] += 1
		elif input == "s":
			status["throttle"] -= 1
		elif input == "test":
			status["throttle"] = 10
			print "Set Trottle to 10"
			time.sleep(0.5)
			status["throttle"] = 100
			print "Set Trottle to 100"
			time.sleep(0.5)
			status["throttle"] = 10
			print "Set Trottle to 10"
			time.sleep(0.5)
			status["throttle"] = 100
			print "Set Trottle to 100"
			time.sleep(0.5)
			status["throttle"] = 10
			print "Set Trottle to 10"
			time.sleep(0.5)
			status["throttle"] = 100
			print "Set Trottle to 100"
			time.sleep(0.5)
			status["throttle"] = 10
			print "Set Trottle to 10"
			time.sleep(1)
		elif input == "esc-reset":
			print "Disconnect ESC from Controller and power"
			print "Press 'y' when done"
			input = raw_input("Done?: ")
			status["start"] = 0
			time.sleep(2)
			if input == "y":
				pwm = PWM(0x40)
				pwm.setPWMFreq(50)                        # Set frequency to 60 Hz
				print "now enter the % of throttle you need"
				while 1:
					try:
						foo = raw_input("procent: ")
						if int(foo) > 100:
							break
						value = (int(foo) * 450 / 100) + 150
						pwm.setPWM(0, 0, value)
						pwm.setPWM(1, 0, value)
						pwm.setPWM(2, 0, value)
						pwm.setPWM(3, 0, value)
						del foo
					except:
						print "Try again"
				print "Done...."

def NetworkSoket(status):
	logging.debug('Starting Up Networks socket')
	HOST = ''   # Symbolic name, meaning all available interfaces
	PORT = 16016 # Arbitrary non-privileged port

	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	logging.debug('Socket Created')

	#Bind socket to local host and port
	try:
		s.bind((HOST, PORT))
	except socket.error as msg:
		logging.debug('Bind failed. Error Code : ' + str(msg[0]) + ' Message ' + msg[1])
		status["start"] = 0
		raise

	print 'Socket bind complete'
	#Start listening on socket
	s.listen(10)
	logging.debug('Socket now listening')

	#now keep talking with the client
	conn, addr = s.accept()
	logging.debug('Connected with ' + addr[0] + ':' + str(addr[1]))
	while status["start"]:
		#Receiving from client
		data = conn.recv(1024)
		data = data.rstrip()
		print data
		if not data:
            break
		reply = "invalid\n\n"
		if data == "stop":
			reply = "OK...\n\n"
			s.close()
			status["start"] = 0
		if data == "status":
			reply = "Running: " + str(status["start"]) + "\nPropeller 1:" + str(status["PropValue"][0]) + "\nPropeller 2:" + str(status["PropValue"][1]) + "\nPropeller 3:" + str(status["PropValue"][2]) + "\nPropeller 4:" + str(status["PropValue"][3]) + "\n...End\n\n"
		conn.sendall(reply)

	S.close()

def CAM(status):
	logging.debug('CAM, Thread ReadInput')
	while status["start"]:
		time.sleep(1)
		#logging.debug('CAM, Loop')
	return

def __init__():

	status={}
	logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', filename='/var/log/copter.log',level=logging.DEBUG)
	logging.debug('Main: Started Up')
	status["debug"] = 0
	status["start"] = 1
	status["throttle"] = 15
	status["pitch"] = 0
	# check stare of vpn tunnel MUST be online

	try:
		thread.start_new_thread( ReadSensor, (status, ) )
		thread.start_new_thread( CalcProps, (status, ) )
		#thread.start_new_thread( PrintData, (status, ) )
		thread.start_new_thread( ControlProps, (status, ) )
		thread.start_new_thread( ReadInput, (status, ) )
		thread.start_new_thread( CAM, (status, ) )
		thread.start_new_thread( NetworkSoket, (status, ) )
		while status["start"]:
			time.sleep(5)
		time.sleep(1)
		quit()
	except:
		logging.warning('Main: WARNNG a thread gave back some error')
		quit()

<|endoftext|>


<commit_before>import tclab
import numpy as np
import time
import matplotlib.pyplot as plt
from gekko import GEKKO

# Connect to Arduino
a = tclab.TCLab()

# Get Version
print(a.version)

# Turn LED on
print('LED On')
a.LED(100)

# Run time in minutes
run_time = 5.0

# Number of cycles with 3 second intervals
loops = int(20.0*run_time)
tm = np.zeros(loops)

# Temperature (K)
T1 = np.ones(loops) * a.T1 # temperature (degC)
Tsp1 = np.ones(loops) * 35.0 # set point (degC)
T2 = np.ones(loops) * a.T2 # temperature (degC)
Tsp2 = np.ones(loops) * 23.0 # set point (degC)

# Set point changes
Tsp1[50:] = 40.0
Tsp2[10:] = 30.0

# heater values
Q1s = np.ones(loops) * 0.0
Q2s = np.ones(loops) * 0.0

#########################################################
# Initialize Model as Estimator
#########################################################
m = GEKKO(name='tclab-mpc')

# 60 second time horizon, steps of 3 sec
m.time = np.linspace(0,60,21)

# Parameters
U = m.FV(value=10,name='u')
tau = m.FV(value=5,name='tau')
alpha1 = m.FV(value=0.01)   # W / % heater
alpha2 = m.FV(value=0.0075) # W / % heater

Kp = m.Param(value=0.5)
tau = m.Param(value=50.0)
zeta = m.Param(value=1.5)

# Manipulated variables
Q1 = m.MV(value=0)
Q1.STATUS = 1  # use to control temperature
Q1.FSTATUS = 0 # no feedback measurement
Q1.LOWER = 0.0
Q1.UPPER = 100.0
Q1.DMAX = 40.0
Q1.COST = 0.0
Q1.DCOST = 0.0

Q2 = m.MV(value=0)
Q2.STATUS = 1  # use to control temperature
Q2.FSTATUS = 0 # no feedback measurement
Q2.LOWER = 0.0
Q2.UPPER = 100.0
Q2.DMAX = 40.0
Q2.COST = 0.0
Q2.DCOST = 0.0

# Controlled variable
TC1 = m.CV(value=T1[0])
TC1.STATUS = 1     # minimize error with setpoint range
TC1.FSTATUS = 1    # receive measurement
TC1.TR_INIT = 1    # reference trajectory
TC1.TAU = 10       # time constant for response

# Controlled variable
TC2 = m.CV(value=T2[0])
TC2.STATUS = 1     # minimize error with setpoint range
TC2.FSTATUS = 1    # receive measurement
TC2.TR_INIT = 1    # reference trajectory
TC2.TAU = 10       # time constant for response

# State variables
TH1 = m.SV(value=T1[0])
TH2 = m.SV(value=T2[0])

Ta = m.Param(value=23.0+273.15)     # K
mass = m.Param(value=4.0/1000.0)    # kg
Cp = m.Param(value=0.5*1000.0)      # J/kg-K
A = m.Param(value=10.0/100.0**2)    # Area not between heaters in m^2
As = m.Param(value=2.0/100.0**2)    # Area between heaters in m^2
eps = m.Param(value=0.9)            # Emissivity
sigma = m.Const(5.67e-8)            # Stefan-Boltzmann

# Heater temperatures
T1i = m.Intermediate(TH1+273.15)
T2i = m.Intermediate(TH2+273.15)

# Heat transfer between two heaters
Q_C12 = m.Intermediate(U*As*(T2i-T1i)) # Convective
Q_R12 = m.Intermediate(eps*sigma*As*(T2i**4-T1i**4)) # Radiative

# Semi-fundamental correlations (energy balances)
m.Equation(TH1.dt() == (1.0/(mass*Cp))*(U*A*(Ta-T1i) \
                    + eps * sigma * A * (Ta**4 - T1i**4) \
                    + Q_C12 + Q_R12 \
                    + alpha1*Q1))

m.Equation(TH2.dt() == (1.0/(mass*Cp))*(U*A*(Ta-T2i) \
                    + eps * sigma * A * (Ta**4 - T2i**4) \
                    - Q_C12 - Q_R12 \
                    + alpha2*Q2))

# Empirical correlations (lag equations to emulate conduction)
m.Equation(tau * TC1.dt() == -TC1 + TH1)
m.Equation(tau * TC2.dt() == -TC2 + TH2)

# Global Options
#m.server = 'http://127.0.0.1' # local server, if available
m.options.IMODE   = 6 # MPC
m.options.CV_TYPE = 1 # Objective type
m.options.NODES   = 3 # Collocation nodes
m.options.SOLVER  = 3 # 1=APOPT, 3=IPOPT
##################################################################

# Create plot
plt.figure()
plt.ion()
plt.show()

# Main Loop
start_time = time.time()
prev_time = start_time
try:
    for i in range(1,loops):
        # Sleep time
        sleep_max = 3.0
        sleep = sleep_max - (time.time() - prev_time)
        if sleep>=0.01:
            time.sleep(sleep)
        else:
            time.sleep(0.01)

        # Record time and change in time
        t = time.time()
        dt = t - prev_time
        prev_time = t
        tm[i] = t - start_time

        # Read temperatures in Kelvin
        T1[i] = a.T1
        T2[i] = a.T2

        ###############################
        ### MPC CONTROLLER          ###
        ###############################
        TC1.MEAS = T1[i]
        TC2.MEAS = T2[i]
        # input setpoint with deadband +/- DT
        DT = 0.1
        TC1.SPHI = Tsp1[i] + DT
        TC1.SPLO = Tsp1[i] - DT
        TC2.SPHI = Tsp2[i] + DT
        TC2.SPLO = Tsp2[i] - DT
        # solve MPC
        m.solve(disp=False)
        # test for successful solution
        if (m.options.APPSTATUS==1):
            # retrieve the first Q value
            Q1s[i] = Q1.NEWVAL
            Q2s[i] = Q2.NEWVAL
        else:
            # not successful, set heater to zero
            Q1s[i] = 0
            Q2s[i] = 0

        # Write output (0-100)
        a.Q1(Q1s[i])
        a.Q2(Q2s[i])

        # Plot
        plt.clf()
        ax=plt.subplot(3,1,1)
        ax.grid()
        plt.plot(tm[0:i],T1[0:i],'ro',MarkerSize=3,label=r'$T_1$')
        plt.plot(tm[0:i],Tsp1[0:i],'k-',LineWidth=2,label=r'$T_1 Setpoint$')
        plt.ylabel('Temperature (degC)')
        plt.legend(loc='best')
        ax=plt.subplot(3,1,2)
        ax.grid()
        plt.plot(tm[0:i],T2[0:i],'ro',MarkerSize=3,label=r'$T_2$')
        plt.plot(tm[0:i],Tsp2[0:i],'g-',LineWidth=2,label=r'$T_2 Setpoint$')
        plt.ylabel('Temperature (degC)')
        plt.legend(loc='best')
        ax=plt.subplot(3,1,3)
        ax.grid()
        plt.plot(tm[0:i],Q1s[0:i],'r-',LineWidth=3,label=r'$Q_1$')
        plt.plot(tm[0:i],Q2s[0:i],'b:',LineWidth=3,label=r'$Q_2$')
        plt.ylabel('Heaters')
        plt.xlabel('Time (sec)')
        plt.legend(loc='best')
        plt.draw()
        plt.pause(0.05)

# Allow user to end loop with Ctrl-C
except KeyboardInterrupt:
    # Disconnect from Arduino
    a.Q1(0)
    a.Q2(0)
    print('Shutting down')
    a.close()

# Make sure serial connection still closes when there's an error
except:
    # Disconnect from Arduino
    a.Q1(0)
    a.Q2(0)
    print('Error: Shutting down')
    a.close()
    raise
<commit_msg>Model Predictive Control<commit_after>import tclab
import numpy as np
import time
import matplotlib.pyplot as plt
from gekko import GEKKO

# Connect to Arduino
a = tclab.TCLab()

# Get Version
print(a.version)

# Turn LED on
print('LED On')
a.LED(100)

# Run time in minutes
run_time = 5.0

# Number of cycles with 3 second intervals
loops = int(20.0*run_time)
tm = np.zeros(loops)

# Temperature (K)
T1 = np.ones(loops) * a.T1 # temperature (degC)
Tsp1 = np.ones(loops) * 35.0 # set point (degC)
T2 = np.ones(loops) * a.T2 # temperature (degC)
Tsp2 = np.ones(loops) * 23.0 # set point (degC)

# Set point changes
Tsp1[50:] = 40.0
Tsp2[10:] = 30.0

# heater values
Q1s = np.ones(loops) * 0.0
Q2s = np.ones(loops) * 0.0

#########################################################
# Initialize Model as Estimator
#########################################################
m = GEKKO(name='tclab-mpc')

# 60 second time horizon, steps of 3 sec
m.time = np.linspace(0,60,21)

# Parameters
U = m.FV(value=10,name='u')
tau = m.FV(value=5,name='tau')
alpha1 = m.FV(value=0.01)   # W / % heater
alpha2 = m.FV(value=0.0075) # W / % heater

Kp = m.Param(value=0.5)
tau = m.Param(value=50.0)
zeta = m.Param(value=1.5)

# Manipulated variables
Q1 = m.MV(value=0)
Q1.STATUS = 1  # use to control temperature
Q1.FSTATUS = 0 # no feedback measurement
Q1.LOWER = 0.0
Q1.UPPER = 100.0
Q1.DMAX = 40.0
Q1.COST = 0.0
Q1.DCOST = 0.0

Q2 = m.MV(value=0)
Q2.STATUS = 1  # use to control temperature
Q2.FSTATUS = 0 # no feedback measurement
Q2.LOWER = 0.0
Q2.UPPER = 100.0
Q2.DMAX = 40.0
Q2.COST = 0.0
Q2.DCOST = 0.0

# Controlled variable
TC1 = m.CV(value=T1[0])
TC1.STATUS = 1     # minimize error with setpoint range
TC1.FSTATUS = 1    # receive measurement
TC1.TR_INIT = 1    # reference trajectory
TC1.TAU = 10       # time constant for response

# Controlled variable
TC2 = m.CV(value=T2[0])
TC2.STATUS = 1     # minimize error with setpoint range
TC2.FSTATUS = 1    # receive measurement
TC2.TR_INIT = 1    # reference trajectory
TC2.TAU = 10       # time constant for response

# State variables
TH1 = m.SV(value=T1[0])
TH2 = m.SV(value=T2[0])

Ta = m.Param(value=23.0+273.15)     # K
mass = m.Param(value=4.0/1000.0)    # kg
Cp = m.Param(value=0.5*1000.0)      # J/kg-K
A = m.Param(value=10.0/100.0**2)    # Area not between heaters in m^2
As = m.Param(value=2.0/100.0**2)    # Area between heaters in m^2
eps = m.Param(value=0.9)            # Emissivity
sigma = m.Const(5.67e-8)            # Stefan-Boltzmann

# Heater temperatures
T1i = m.Intermediate(TH1+273.15)
T2i = m.Intermediate(TH2+273.15)

# Heat transfer between two heaters
Q_C12 = m.Intermediate(U*As*(T2i-T1i)) # Convective
Q_R12 = m.Intermediate(eps*sigma*As*(T2i**4-T1i**4)) # Radiative

# Semi-fundamental correlations (energy balances)
m.Equation(TH1.dt() == (1.0/(mass*Cp))*(U*A*(Ta-T1i) \
                    + eps * sigma * A * (Ta**4 - T1i**4) \
                    + Q_C12 + Q_R12 \
                    + alpha1*Q1))

m.Equation(TH2.dt() == (1.0/(mass*Cp))*(U*A*(Ta-T2i) \
                    + eps * sigma * A * (Ta**4 - T2i**4) \
                    - Q_C12 - Q_R12 \
                    + alpha2*Q2))

# Empirical correlations (lag equations to emulate conduction)
m.Equation(tau * TC1.dt() == -TC1 + TH1)
m.Equation(tau * TC2.dt() == -TC2 + TH2)

# Global Options
#m.server = 'http://127.0.0.1' # local server, if available
m.options.IMODE   = 6 # MPC
m.options.CV_TYPE = 1 # Objective type
m.options.NODES   = 3 # Collocation nodes
m.options.SOLVER  = 3 # 1=APOPT, 3=IPOPT
##################################################################

# Create plot
plt.figure()
plt.ion()
plt.show()

# Main Loop
start_time = time.time()
prev_time = start_time
try:
    for i in range(1,loops):
        # Sleep time
        sleep_max = 3.0
        sleep = sleep_max - (time.time() - prev_time)
        if sleep>=0.01:
            time.sleep(sleep)
        else:
            time.sleep(0.01)

        # Record time and change in time
        t = time.time()
        dt = t - prev_time
        prev_time = t
        tm[i] = t - start_time

        # Read temperatures in Kelvin
        T1[i] = a.T1
        T2[i] = a.T2

        ###############################
        ### MPC CONTROLLER          ###
        ###############################
        TC1.MEAS = T1[i]
        TC2.MEAS = T2[i]
        # input setpoint with deadband +/- DT
        DT = 0.1
        TC1.SPHI = Tsp1[i] + DT
        TC1.SPLO = Tsp1[i] - DT
        TC2.SPHI = Tsp2[i] + DT
        TC2.SPLO = Tsp2[i] - DT
        # solve MPC
        m.solve(disp=False)
        # test for successful solution
        if (m.options.APPSTATUS==1):
            # retrieve the first Q value
            Q1s[i] = Q1.NEWVAL
            Q2s[i] = Q2.NEWVAL
        else:
            # not successful, set heater to zero
            Q1s[i] = 0
            Q2s[i] = 0

        # Write output (0-100)
        a.Q1(Q1s[i])
        a.Q2(Q2s[i])

        # Plot
        plt.clf()
        ax=plt.subplot(3,1,1)
        ax.grid()
        plt.plot(tm[0:i],T1[0:i],'ro',MarkerSize=3,label=r'$T_1$')
        plt.plot(tm[0:i],Tsp1[0:i],'k-',LineWidth=2,label=r'$T_1 Setpoint$')
        plt.ylabel('Temperature (degC)')
        plt.legend(loc='best')
        ax=plt.subplot(3,1,2)
        ax.grid()
        plt.plot(tm[0:i],T2[0:i],'ro',MarkerSize=3,label=r'$T_2$')
        plt.plot(tm[0:i],Tsp2[0:i],'g-',LineWidth=2,label=r'$T_2 Setpoint$')
        plt.ylabel('Temperature (degC)')
        plt.legend(loc='best')
        ax=plt.subplot(3,1,3)
        ax.grid()
        plt.plot(tm[0:i],Q1s[0:i],'r-',LineWidth=3,label=r'$Q_1$')
        plt.plot(tm[0:i],Q2s[0:i],'b:',LineWidth=3,label=r'$Q_2$')
        plt.ylabel('Heaters')
        plt.xlabel('Time (sec)')
        plt.legend(loc='best')
        plt.draw()
        plt.pause(0.05)

    # Turn off heaters
    a.Q1(0)
    a.Q2(0)
    print('Shutting down')

# Allow user to end loop with Ctrl-C
except KeyboardInterrupt:
    # Disconnect from Arduino
    a.Q1(0)
    a.Q2(0)
    print('Shutting down')
    a.close()

# Make sure serial connection still closes when there's an error
except:
    # Disconnect from Arduino
    a.Q1(0)
    a.Q2(0)
    print('Error: Shutting down')
    a.close()
    raise
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
""" Basic Newton methods using Krylov subspace method.

Options
--------
newton_tol : float
    Tolerrance of Newton step
newton_maxiter : int
    Max iteration number of Newton method
jacobi_alpha : float
    Infinitesimal for calculating Jacobian matrix
trusted_region : float
    Radius in model trusted region approach
hook_maxiter : int
    Max iteration number of hook step
hook_tol : int
    Relative tolerance of hook step iteration

Their default values are set in :py:data:`.default_options`
"""

import numpy as np
import scipy.sparse.linalg as linalg
from itertools import count as icount
from .logger import Logger
from . import krylov

default_options = {
    "newton_tol": 1e-7,
    "newton_maxiter": 100,
    "jacobi_alpha": 1e-7,
    "trusted_region": 1e-1,
    "hook_maxiter": 100,
    "hook_tol": 0.1,
}
""" default values of options

You can get these values through :py:func:`continuate.get_default_options`
"""

def Jacobi(func, x0, jacobi_alpha, fx=None, **opt):
    """
    Jacobi oprator :math:`DF(x0)`, where

    .. math::
        DF(x0)dx = (F(x0 + \\alpha dx / |dx|) - F(x0))/(\\alpha/|dx|)

    Parameters
    ----------
    func: numpy.array -> numpy.array
        A function to be differentiated
    x0: numpy.array
        A position where the Jacobian is evaluated
    alpha: float

    fx: numpy.array, optional
        func(x0)

    Examples
    --------
    >>> import numpy as np
    >>> f = lambda x: np.array([x[1]**2, x[0]**2])
    >>> x0 = np.array([1, 2])
    >>> J = Jacobi(f, x0, 1e-7)

    """
    if fx is None:
        fx = func(x0)

    def wrap(v):
        norm = np.linalg.norm(v)
        r = jacobi_alpha / norm
        return (func(x0 + r * v) - fx) / r

    return linalg.LinearOperator(
        (len(x0), len(x0)),
        matvec=wrap,
        dtype=x0.dtype
    )

class Hessian(object):
    """ Calculate the deviation from linear approximation """

    logger = Logger(__name__, "Hessian")

    def __init__(self, func, x0, jacobi_alpha, **opt):
        self.fx0 = func(x0)
        self.A = Jacobi(func, x0, fx=self.fx0, jacobi_alpha=1e-7)
        self.f = func
        self.x0 = x0
        self.alpha = jacobi_alpha

    def __call__(self, v):
        fxv = self.f(self.x0 + v)
        Av = self.A * v
        nrm = max(map(np.linalg.norm, [self.fx0, fxv, self.fx0 + Av]))
        return np.linalg.norm(fxv - (self.fx0 + Av)) / nrm

    def deviation(self, v):
        return self.f(self.x0 + v) - (self.fx0 + self.A * v)

    def trusted_region(self, v, eps, r0=None, p=2):
        """ Estimate the trusted region in which the deviation is smaller than `eps`.

        Parameters
        ------------
        eps : float
            Destination value of deviation
        p : float, optional (default=2)
            Iteration will end if the deviation is in `[eps/p, eps*p]`.

        Return
        -------
        r : float
            radius of the trusted region

        """
        if type(r0) is float:
            r = r0
        else:
            r = 100*self.alpha
        v = v / np.linalg.norm(v)
        p = max(p, 1.0/p)
        for c in icount():
            e = self(r*v)
            self.logger.info({
                "count": c,
                "deviation": e,
            })
            if (e > eps/p) and (e < eps*p):
                return r
            r = r * np.sqrt(eps / e)

def newton(func, x0, newton_tol, newton_maxiter, **opt):
    """
    solve multi-dimensional equation `F(x) = 0` using Newton-Krylov method.

    Parameters
    -----------
    func: numpy.array -> numpy.array
        `F`
    x0: numpy.array
        initial guess of the solution
    ftol: float, optional
        tolerance of the iteration
    maxiter: int, optional
        maximum number of trial
    inner_tol: float, optional
        tolerance of linear solver (use scipy.sparse.linalg.gmres)

    Returns
    --------
    numpy.array
        The solution `x` satisfies `F(x)=0`

    """
    logger = Logger(__name__, "Newton")
    for t in range(newton_maxiter):
        fx = func(x0)
        res = np.linalg.norm(fx)
        logger.info({
            "count": t,
            "residual": res,
        })
        if res <= newton_tol:
            return x0
        A = Jacobi(func, x0, fx=fx, **opt)
        dx = krylov.gmres(A, -fx, **opt)
        x0 = x0 + dx
    raise RuntimeError("Not convergent (Newton)")

def hook_step(A, b, trusted_region, hook_maxiter, hook_tol, nu=0, **opt):
    """
    optimal hook step based on trusted region approach

    Parameters
    ----------
    A : numpy.array
        square matrix
    b : numpy.array
    r : float
        trusted region
    nu : float, optional (default=0.0)
        initial value of Lagrange multiplier
    e : float, optional (default=0.1)
        relative tolerance of residue form r

    Returns
    --------
    (numpy.array, float)
        argmin of xi, and nu (Lagrange multiplier)

    References
    ----------
    - Numerical Methods for Unconstrained Optimization and Nonlinear Equations
      J. E. Dennis, Jr. and Robert B. Schnabel
      http://dx.doi.org/10.1137/1.9781611971200
      Chapter 6.4: THE MODEL-TRUST REGION APPROACH

    """
    logger = Logger(__name__, "Hook step")
    r = trusted_region
    logger.debug({"nu0": nu, "trusted_region": r, })
    I = np.matrix(np.identity(len(b), dtype=b.dtype))
    AA = np.dot(A.T, A)
    Ab = np.dot(A.T, b)
    xi = np.linalg.solve(AA, Ab)
    if np.linalg.norm(xi) < r:
        return xi, 0
    for t in range(hook_maxiter):
        xi = np.linalg.solve(AA+nu*I, Ab)
        xi_norm = np.linalg.norm(xi)
        Psi = xi_norm - r
        logger.info({
            "count": t,
            "1-|xi|/r": Psi / r,
        })
        if abs(Psi) < hook_tol * r:
            return xi, nu
        dPsi = -np.dot(xi, np.linalg.solve(AA+nu*I, xi)) / xi_norm
        nu = nu - (xi_norm*Psi) / (r*dPsi)
        if nu < 0:
            logger.debug("Negative nu, reset to zero")
            nu = 0
        logger.debug({
            "count": t,
            "Psi": Psi,
            "dPsi": dPsi,
            "nu": nu,
        })
    raise RuntimeError("Not convergent (hook-step)")

def newton_krylov_hook_gen(func, x0, trusted_region, **opt):
    """ Generator of Newton-Krylov-hook iteration

    Parameters
    -----------
    r : float, optional(default=1e-2)
        Initial guess of the radius of trusted region

    Returns
    --------
    generator
        yields `(x, |func(x)|, func(x))`
    """
    logger = Logger(__name__, "NewtonKrylovHook")
    nu = 0.0
    for t in icount():
        fx = func(x0)
        res = np.linalg.norm(fx)
        logger.info({
            "count": t,
            "residual": res,
        })
        yield x0, res, fx
        A = Jacobi(func, x0, fx=fx, **opt)
        b = -fx
        H, V = krylov.arnoldi(A, b, **opt)
        g = krylov.solve_Hessenberg(H, np.linalg.norm(b))
        dx = np.dot(V[:, :len(g)], g)
        dx_norm = np.linalg.norm(dx)
        logger.info({"|dx|": dx_norm, })
        if dx_norm < trusted_region:
            logger.info('in Trusted region')
            x0 = x0 + dx
        else:
            logger.info('hook step')
            beta = np.zeros(H.shape[0])
            beta[0] = np.linalg.norm(b)
            xi, nu = hook_step(H, beta, trusted_region, nu=nu, **opt)
            dx = np.dot(V[:, :len(xi)], xi)
            x0 = x0 + dx

def newton_krylov_hook(func, x0, **opt):
    tol = opt["newton_tol"]
    maxiter = opt["newton_maxiter"]
    gen = newton_krylov_hook_gen(func, x0, **opt)
    for t, (x, res, _) in enumerate(gen):
        if res < tol:
            return x
        if t > maxiter:
            raise RuntimeError("Not Convergent (Newton-Krylov-hook)")
<commit_msg>Update documents<commit_after># -*- coding: utf-8 -*-
""" Basic Newton methods using Krylov subspace method.

Options
--------
newton_tol : float
    Tolerrance of Newton step
newton_maxiter : int
    Max iteration number of Newton method
jacobi_alpha : float
    Infinitesimal for calculating Jacobian matrix
trusted_region : float
    Radius in model trusted region approach
hook_maxiter : int
    Max iteration number of hook step
hook_tol : int
    Relative tolerance of hook step iteration

Their default values are set in :py:data:`.default_options`
"""

import numpy as np
import scipy.sparse.linalg as linalg
from itertools import count as icount
from .logger import Logger
from . import krylov

default_options = {
    "newton_tol": 1e-7,
    "newton_maxiter": 100,
    "jacobi_alpha": 1e-7,
    "trusted_region": 1e-1,
    "hook_maxiter": 100,
    "hook_tol": 0.1,
}
""" default values of options

You can get these values through :py:func:`continuate.get_default_options`
"""

def Jacobi(func, x0, jacobi_alpha, fx=None, **opt):
    """
    Jacobi oprator :math:`DF(x0)`, where

    .. math::
        DF(x0)dx = (F(x0 + \\alpha dx / |dx|) - F(x0))/(\\alpha/|dx|)

    Parameters
    ----------
    func: numpy.array -> numpy.array
        A function to be differentiated
    x0: numpy.array
        A position where the Jacobian is evaluated
    fx: numpy.array, optional
        func(x0)

    Examples
    --------
    >>> import numpy as np
    >>> f = lambda x: np.array([x[1]**2, x[0]**2])
    >>> x0 = np.array([1, 2])
    >>> J = Jacobi(f, x0, 1e-7)

    """
    if fx is None:
        fx = func(x0)

    def wrap(v):
        norm = np.linalg.norm(v)
        r = jacobi_alpha / norm
        return (func(x0 + r * v) - fx) / r

    return linalg.LinearOperator(
        (len(x0), len(x0)),
        matvec=wrap,
        dtype=x0.dtype
    )

class Hessian(object):
    """ Calculate the deviation from linear approximation """

    logger = Logger(__name__, "Hessian")

    def __init__(self, func, x0, jacobi_alpha, **opt):
        self.fx0 = func(x0)
        self.A = Jacobi(func, x0, fx=self.fx0, jacobi_alpha=1e-7)
        self.f = func
        self.x0 = x0
        self.alpha = jacobi_alpha

    def __call__(self, v):
        fxv = self.f(self.x0 + v)
        Av = self.A * v
        nrm = max(map(np.linalg.norm, [self.fx0, fxv, self.fx0 + Av]))
        return np.linalg.norm(fxv - (self.fx0 + Av)) / nrm

    def deviation(self, v):
        return self.f(self.x0 + v) - (self.fx0 + self.A * v)

    def trusted_region(self, v, eps, r0=None, p=2):
        """ Estimate the trusted region in which the deviation is smaller than `eps`.

        Parameters
        ------------
        eps : float
            Destination value of deviation
        p : float, optional (default=2)
            Iteration will end if the deviation is in `[eps/p, eps*p]`.

        Return
        -------
        r : float
            radius of the trusted region

        """
        if type(r0) is float:
            r = r0
        else:
            r = 100*self.alpha
        v = v / np.linalg.norm(v)
        p = max(p, 1.0/p)
        for c in icount():
            e = self(r*v)
            self.logger.info({
                "count": c,
                "deviation": e,
            })
            if (e > eps/p) and (e < eps*p):
                return r
            r = r * np.sqrt(eps / e)

def newton(func, x0, newton_tol, newton_maxiter, **opt):
    """
    solve multi-dimensional equation :math:`F(x) = 0`
    using Newton-Krylov method.

    Parameters
    -----------
    func: numpy.array -> numpy.array
        `F`
    x0: numpy.array
        initial guess of the solution

    Returns
    --------
    x : numpy.array
        The solution :math:`x` satisfies :math:`F(x)=0`

    """
    logger = Logger(__name__, "Newton")
    for t in range(newton_maxiter):
        fx = func(x0)
        res = np.linalg.norm(fx)
        logger.info({
            "count": t,
            "residual": res,
        })
        if res <= newton_tol:
            return x0
        A = Jacobi(func, x0, fx=fx, **opt)
        dx = krylov.gmres(A, -fx, **opt)
        x0 = x0 + dx
    raise RuntimeError("Not convergent (Newton)")

def hook_step(A, b, trusted_region, hook_maxiter, hook_tol, nu=0, **opt):
    """
    optimal hook step based on trusted region approach

    Parameters
    ----------
    A : numpy.array
        square matrix
    b : numpy.array
    r : float
        trusted region
    nu : float, optional (default=0.0)
        initial value of Lagrange multiplier
    e : float, optional (default=0.1)
        relative tolerance of residue form r

    Returns
    --------
    (numpy.array, float)
        argmin of xi, and nu (Lagrange multiplier)

    References
    ----------
    - Numerical Methods for Unconstrained Optimization and Nonlinear Equations
      J. E. Dennis, Jr. and Robert B. Schnabel
      http://dx.doi.org/10.1137/1.9781611971200
      Chapter 6.4: THE MODEL-TRUST REGION APPROACH

    """
    logger = Logger(__name__, "Hook step")
    r = trusted_region
    logger.debug({"nu0": nu, "trusted_region": r, })
    I = np.matrix(np.identity(len(b), dtype=b.dtype))
    AA = np.dot(A.T, A)
    Ab = np.dot(A.T, b)
    xi = np.linalg.solve(AA, Ab)
    if np.linalg.norm(xi) < r:
        return xi, 0
    for t in range(hook_maxiter):
        xi = np.linalg.solve(AA+nu*I, Ab)
        xi_norm = np.linalg.norm(xi)
        Psi = xi_norm - r
        logger.info({
            "count": t,
            "1-|xi|/r": Psi / r,
        })
        if abs(Psi) < hook_tol * r:
            return xi, nu
        dPsi = -np.dot(xi, np.linalg.solve(AA+nu*I, xi)) / xi_norm
        nu = nu - (xi_norm*Psi) / (r*dPsi)
        if nu < 0:
            logger.debug("Negative nu, reset to zero")
            nu = 0
        logger.debug({
            "count": t,
            "Psi": Psi,
            "dPsi": dPsi,
            "nu": nu,
        })
    raise RuntimeError("Not convergent (hook-step)")

def newton_krylov_hook_gen(func, x0, trusted_region, **opt):
    """ Generator of Newton-Krylov-hook iteration

    Yields
    -------
    x : numpy.array
        :math:`x_n`
    residual : float
        :math:`|F(x_n)|`
    fx : numpy.array
        :math:`F(x_n)`
    """
    logger = Logger(__name__, "NewtonKrylovHook")
    nu = 0.0
    for t in icount():
        fx = func(x0)
        res = np.linalg.norm(fx)
        logger.info({
            "count": t,
            "residual": res,
        })
        yield x0, res, fx
        A = Jacobi(func, x0, fx=fx, **opt)
        b = -fx
        H, V = krylov.arnoldi(A, b, **opt)
        g = krylov.solve_Hessenberg(H, np.linalg.norm(b))
        dx = np.dot(V[:, :len(g)], g)
        dx_norm = np.linalg.norm(dx)
        logger.info({"|dx|": dx_norm, })
        if dx_norm < trusted_region:
            logger.info('in Trusted region')
            x0 = x0 + dx
        else:
            logger.info('hook step')
            beta = np.zeros(H.shape[0])
            beta[0] = np.linalg.norm(b)
            xi, nu = hook_step(H, beta, trusted_region, nu=nu, **opt)
            dx = np.dot(V[:, :len(xi)], xi)
            x0 = x0 + dx

def newton_krylov_hook(func, x0, **opt):
    """ Solve multi-dimensional nonlinear equation :math:`F(x)=0`

    Parameters
    -----------
    func : numpy.array -> numpy.array
        The function :math:`F` of the problem.
    x0 : numpy.array
        Initial guess of the solution
    opt : dict
        Options for calculation.
        Please generate by :py:func:`continuate.get_default_options`

    """
    tol = opt["newton_tol"]
    maxiter = opt["newton_maxiter"]
    gen = newton_krylov_hook_gen(func, x0, **opt)
    for t, (x, res, _) in enumerate(gen):
        if res < tol:
            return x
        if t > maxiter:
            raise RuntimeError("Not Convergent (Newton-Krylov-hook)")
<|endoftext|>


<commit_before># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Operations for working with string Tensors.

See the @{$python/string_ops} guide.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import sparse_tensor
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_string_ops
from tensorflow.python.ops import math_ops

# go/tf-wildcard-import
# pylint: disable=wildcard-import
from tensorflow.python.ops.gen_string_ops import *
from tensorflow.python.util import deprecation
from tensorflow.python.util.tf_export import tf_export
# pylint: enable=wildcard-import

# Expose regex_full_match in strings namespace
tf_export("strings.regex_full_match")(regex_full_match)

@tf_export("string_split")
def string_split(source, delimiter=" ", skip_empty=True):  # pylint: disable=invalid-name
  """Split elements of `source` based on `delimiter` into a `SparseTensor`.

  Let N be the size of source (typically N will be the batch size). Split each
  element of `source` based on `delimiter` and return a `SparseTensor`
  containing the split tokens. Empty tokens are ignored.

  If `delimiter` is an empty string, each element of the `source` is split
  into individual strings, each containing one byte. (This includes splitting
  multibyte sequences of UTF-8.) If delimiter contains multiple bytes, it is
  treated as a set of delimiters with each considered a potential split point.

  For example:
  N = 2, source[0] is 'hello world' and source[1] is 'a b c', then the output
  will be

  st.indices = [0, 0;
                0, 1;
                1, 0;
                1, 1;
                1, 2]
  st.shape = [2, 3]
  st.values = ['hello', 'world', 'a', 'b', 'c']

  Args:
    source: `1-D` string `Tensor`, the strings to split.
    delimiter: `0-D` string `Tensor`, the delimiter character, the string should
      be length 0 or 1.
    skip_empty: A `bool`. If `True`, skip the empty strings from the result.

  Raises:
    ValueError: If delimiter is not a string.

  Returns:
    A `SparseTensor` of rank `2`, the strings split according to the delimiter.
    The first column of the indices corresponds to the row in `source` and the
    second column corresponds to the index of the split component in this row.
  """
  delimiter = ops.convert_to_tensor(delimiter, dtype=dtypes.string)
  source = ops.convert_to_tensor(source, dtype=dtypes.string)

  indices, values, shape = gen_string_ops.string_split(
      source, delimiter=delimiter, skip_empty=skip_empty)
  indices.set_shape([None, 2])
  values.set_shape([None])
  shape.set_shape([2])
  return sparse_tensor.SparseTensor(indices, values, shape)

def _reduce_join_reduction_dims(x, axis, reduction_indices):
  """Returns range(rank(x) - 1, 0, -1) if reduction_indices is None."""
  # TODO(aselle): Remove this after deprecation
  if reduction_indices is not None:
    if axis is not None:
      raise ValueError("Can't specify both 'axis' and 'reduction_indices'.")
    axis = reduction_indices
  if axis is not None:
    return axis
  else:
    # Fast path: avoid creating Rank and Range ops if ndims is known.
    if x.get_shape().ndims is not None:
      return constant_op.constant(
          np.arange(x.get_shape().ndims - 1, -1, -1), dtype=dtypes.int32)

    # Otherwise, we rely on Range and Rank to do the right thing at run-time.
    return math_ops.range(array_ops.rank(x) - 1, -1, -1)

@tf_export("reduce_join")
def reduce_join(inputs, axis=None,
                keep_dims=False,
                separator="",
                name=None,
                reduction_indices=None):
  inputs_t = ops.convert_to_tensor(inputs)
  reduction_indices = _reduce_join_reduction_dims(
      inputs_t, axis, reduction_indices)
  return gen_string_ops.reduce_join(
      inputs=inputs_t,
      reduction_indices=reduction_indices,
      keep_dims=keep_dims,
      separator=separator,
      name=name)

reduce_join.__doc__ = deprecation.rewrite_argument_docstring(
    gen_string_ops.reduce_join.__doc__, "reduction_indices", "axis")

ops.NotDifferentiable("RegexReplace")
ops.NotDifferentiable("StringToHashBucket")
ops.NotDifferentiable("StringToHashBucketFast")
ops.NotDifferentiable("StringToHashBucketStrong")
ops.NotDifferentiable("ReduceJoin")
ops.NotDifferentiable("StringJoin")
ops.NotDifferentiable("StringSplit")
ops.NotDifferentiable("AsString")
ops.NotDifferentiable("EncodeBase64")
ops.NotDifferentiable("DecodeBase64")
<commit_msg>Expose `tf.strings.split` with the new implementation<commit_after># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Operations for working with string Tensors.

See the @{$python/string_ops} guide.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import sparse_tensor
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_string_ops
from tensorflow.python.ops import math_ops

# go/tf-wildcard-import
# pylint: disable=wildcard-import
from tensorflow.python.ops.gen_string_ops import *
from tensorflow.python.util import deprecation
from tensorflow.python.util.tf_export import tf_export
# pylint: enable=wildcard-import

# Expose regex_full_match in strings namespace
tf_export("strings.regex_full_match")(regex_full_match)

@tf_export("string_split")
def string_split(source, delimiter=" ", skip_empty=True):  # pylint: disable=invalid-name
  """Split elements of `source` based on `delimiter` into a `SparseTensor`.

  Let N be the size of source (typically N will be the batch size). Split each
  element of `source` based on `delimiter` and return a `SparseTensor`
  containing the split tokens. Empty tokens are ignored.

  If `delimiter` is an empty string, each element of the `source` is split
  into individual strings, each containing one byte. (This includes splitting
  multibyte sequences of UTF-8.) If delimiter contains multiple bytes, it is
  treated as a set of delimiters with each considered a potential split point.

  For example:
  N = 2, source[0] is 'hello world' and source[1] is 'a b c', then the output
  will be

  st.indices = [0, 0;
                0, 1;
                1, 0;
                1, 1;
                1, 2]
  st.shape = [2, 3]
  st.values = ['hello', 'world', 'a', 'b', 'c']

  Args:
    source: `1-D` string `Tensor`, the strings to split.
    delimiter: `0-D` string `Tensor`, the delimiter character, the string should
      be length 0 or 1.
    skip_empty: A `bool`. If `True`, skip the empty strings from the result.

  Raises:
    ValueError: If delimiter is not a string.

  Returns:
    A `SparseTensor` of rank `2`, the strings split according to the delimiter.
    The first column of the indices corresponds to the row in `source` and the
    second column corresponds to the index of the split component in this row.
  """
  delimiter = ops.convert_to_tensor(delimiter, dtype=dtypes.string)
  source = ops.convert_to_tensor(source, dtype=dtypes.string)

  indices, values, shape = gen_string_ops.string_split(
      source, delimiter=delimiter, skip_empty=skip_empty)
  indices.set_shape([None, 2])
  values.set_shape([None])
  shape.set_shape([2])
  return sparse_tensor.SparseTensor(indices, values, shape)

@tf_export("strings.split")
def string_split_v2(source, sep=None):
  if sep is None:
    sep = ''
  sep = ops.convert_to_tensor(sep, dtype=dtypes.string)
  source = ops.convert_to_tensor(source, dtype=dtypes.string)

  indices, values, shape = gen_string_ops.string_split_v2(
      source, sep=sep)
  indices.set_shape([None, 2])
  values.set_shape([None])
  shape.set_shape([2])
  return sparse_tensor.SparseTensor(indices, values, shape)

def _reduce_join_reduction_dims(x, axis, reduction_indices):
  """Returns range(rank(x) - 1, 0, -1) if reduction_indices is None."""
  # TODO(aselle): Remove this after deprecation
  if reduction_indices is not None:
    if axis is not None:
      raise ValueError("Can't specify both 'axis' and 'reduction_indices'.")
    axis = reduction_indices
  if axis is not None:
    return axis
  else:
    # Fast path: avoid creating Rank and Range ops if ndims is known.
    if x.get_shape().ndims is not None:
      return constant_op.constant(
          np.arange(x.get_shape().ndims - 1, -1, -1), dtype=dtypes.int32)

    # Otherwise, we rely on Range and Rank to do the right thing at run-time.
    return math_ops.range(array_ops.rank(x) - 1, -1, -1)

@tf_export("reduce_join")
def reduce_join(inputs, axis=None,
                keep_dims=False,
                separator="",
                name=None,
                reduction_indices=None):
  inputs_t = ops.convert_to_tensor(inputs)
  reduction_indices = _reduce_join_reduction_dims(
      inputs_t, axis, reduction_indices)
  return gen_string_ops.reduce_join(
      inputs=inputs_t,
      reduction_indices=reduction_indices,
      keep_dims=keep_dims,
      separator=separator,
      name=name)

reduce_join.__doc__ = deprecation.rewrite_argument_docstring(
    gen_string_ops.reduce_join.__doc__, "reduction_indices", "axis")

ops.NotDifferentiable("RegexReplace")
ops.NotDifferentiable("StringToHashBucket")
ops.NotDifferentiable("StringToHashBucketFast")
ops.NotDifferentiable("StringToHashBucketStrong")
ops.NotDifferentiable("ReduceJoin")
ops.NotDifferentiable("StringJoin")
ops.NotDifferentiable("StringSplit")
ops.NotDifferentiable("AsString")
ops.NotDifferentiable("EncodeBase64")
ops.NotDifferentiable("DecodeBase64")
<|endoftext|>