<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2005-2009 Edgewall Software
# Copyright (C) 2005 Christopher Lenz <cmlenz@gmx.de>
# Copyright (C) 2006 Matthew Good <trac@matt-good.net>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

def sql_escape_percent(sql):
    import re
    return re.sub("'((?:[^']|(?:''))*)'",
                  lambda m: m.group(0).replace('%', '%%'), sql)

class IterableCursor(object):
    """Wrapper for DB-API cursor objects that makes the cursor iterable
    and escapes all "%"s used inside literal strings with parameterized
    queries.

    Iteration will generate the rows of a SELECT query one by one.
    """
    __slots__ = ['cursor', 'log']

    def __init__(self, cursor, log=None):
        self.cursor = cursor
        self.log = log

    def __getattr__(self, name):
        return getattr(self.cursor, name)

    def __iter__(self):
        while True:
            row = self.cursor.fetchone()
            if not row:
                return
            yield row

    def execute(self, sql, args=None):
        if self.log:
            self.log.debug('SQL: %r', sql)
            try:
                if args:
                    self.log.debug('args: %r', args)
                    r = self.cursor.execute(sql_escape_percent(sql), args)
                else:
                    r = self.cursor.execute(sql)
                rows = getattr(self.cursor, 'rows', None)
                if rows is not None:
                    self.log.debug("prefetch: %d rows", len(rows))
                return r
            except Exception, e:
                self.log.debug('execute exception: %r', e)
                raise
        if args:
            return self.cursor.execute(sql_escape_percent(sql), args)
        return self.cursor.execute(sql)

    def executemany(self, sql, args):
        if self.log:
            self.log.debug('SQL: %r', sql)
            self.log.debug('args: %r', args)
            if not args:
                return
            try:
                if args[0]:
                    return self.cursor.executemany(sql_escape_percent(sql),
                                                   args)
                return self.cursor.executemany(sql, args)
            except Exception, e:
                self.log.debug('executemany exception: %r', e)
                raise
        if not args:
            return
        if args[0]:
            return self.cursor.executemany(sql_escape_percent(sql), args)
        return self.cursor.executemany(sql, args)

class ConnectionWrapper(object):
    """Generic wrapper around connection objects.

    :since 0.12: This wrapper no longer makes cursors produced by the
    connection iterable using `IterableCursor`.

    :since 0.13: added a 'readonly' flag preventing the forwarding of
                 `commit` and `rollback`
    """
    __slots__ = ('cnx', 'log', 'readonly')

    def __init__(self, cnx, log=None, readonly=False):
        self.cnx = cnx
        self.log = log
        self.readonly = readonly

    def __getattr__(self, name):
        if self.readonly and name in ('commit', 'rollback'):
            raise AttributeError
        return getattr(self.cnx, name)

    def __call__(self, query, params=None):
        """Execute an SQL `query`.

        The optional `params` are either a tuple containing the parameter
        values expected by the query, or a sequence of such tuples. In the
        latter case, an "executemany" will be performed.

        If the query is a SELECT, return all the rows ("fetchall").
        When more control is needed, use `cursor()`.
        """
        dql = query.lstrip().startswith('SELECT')
        if self.readonly and not dql:
            raise ValueError("a 'readonly' connection can only do a SELECT")
        cursor = self.cnx.cursor()
        if params and isinstance(params[0], tuple):
            cursor.executemany(query, params)
        else:
            cursor.execute(query, params)
        rows = cursor.fetchall() if dql else None
        cursor.close()
        return rows

<commit_msg>db: in `[trac] debug_sql` mode, show multiline statements expanded.<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2005-2009 Edgewall Software
# Copyright (C) 2005 Christopher Lenz <cmlenz@gmx.de>
# Copyright (C) 2006 Matthew Good <trac@matt-good.net>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

def sql_escape_percent(sql):
    import re
    return re.sub("'((?:[^']|(?:''))*)'",
                  lambda m: m.group(0).replace('%', '%%'), sql)

class IterableCursor(object):
    """Wrapper for DB-API cursor objects that makes the cursor iterable
    and escapes all "%"s used inside literal strings with parameterized
    queries.

    Iteration will generate the rows of a SELECT query one by one.
    """
    __slots__ = ['cursor', 'log']

    def __init__(self, cursor, log=None):
        self.cursor = cursor
        self.log = log

    def __getattr__(self, name):
        return getattr(self.cursor, name)

    def __iter__(self):
        while True:
            row = self.cursor.fetchone()
            if not row:
                return
            yield row

    def execute(self, sql, args=None):
        if self.log:
            self.log.debug('SQL: %s', sql)
            try:
                if args:
                    self.log.debug('args: %r', args)
                    r = self.cursor.execute(sql_escape_percent(sql), args)
                else:
                    r = self.cursor.execute(sql)
                rows = getattr(self.cursor, 'rows', None)
                if rows is not None:
                    self.log.debug("prefetch: %d rows", len(rows))
                return r
            except Exception, e:
                self.log.debug('execute exception: %r', e)
                raise
        if args:
            return self.cursor.execute(sql_escape_percent(sql), args)
        return self.cursor.execute(sql)

    def executemany(self, sql, args):
        if self.log:
            self.log.debug('SQL: %r', sql)
            self.log.debug('args: %r', args)
            if not args:
                return
            try:
                if args[0]:
                    return self.cursor.executemany(sql_escape_percent(sql),
                                                   args)
                return self.cursor.executemany(sql, args)
            except Exception, e:
                self.log.debug('executemany exception: %r', e)
                raise
        if not args:
            return
        if args[0]:
            return self.cursor.executemany(sql_escape_percent(sql), args)
        return self.cursor.executemany(sql, args)

class ConnectionWrapper(object):
    """Generic wrapper around connection objects.

    :since 0.12: This wrapper no longer makes cursors produced by the
    connection iterable using `IterableCursor`.

    :since 0.13: added a 'readonly' flag preventing the forwarding of
                 `commit` and `rollback`
    """
    __slots__ = ('cnx', 'log', 'readonly')

    def __init__(self, cnx, log=None, readonly=False):
        self.cnx = cnx
        self.log = log
        self.readonly = readonly

    def __getattr__(self, name):
        if self.readonly and name in ('commit', 'rollback'):
            raise AttributeError
        return getattr(self.cnx, name)

    def __call__(self, query, params=None):
        """Execute an SQL `query`.

        The optional `params` are either a tuple containing the parameter
        values expected by the query, or a sequence of such tuples. In the
        latter case, an "executemany" will be performed.

        If the query is a SELECT, return all the rows ("fetchall").
        When more control is needed, use `cursor()`.
        """
        dql = query.lstrip().startswith('SELECT')
        if self.readonly and not dql:
            raise ValueError("a 'readonly' connection can only do a SELECT")
        cursor = self.cnx.cursor()
        if params and isinstance(params[0], tuple):
            cursor.executemany(query, params)
        else:
            cursor.execute(query, params)
        rows = cursor.fetchall() if dql else None
        cursor.close()
        return rows

<|endoftext|>


<commit_before>import sys
import decimal
from decimal import Decimal
import copy
import PyQt4
from PyQt4 import QtCore
from PyQt4.QtCore import pyqtSlot
from PyQt4.QtCore import pyqtSignal
import arbmath

class Tops(object):
  ask1 = Decimal('0')
  ask1_amount = Decimal('0')
  bid1 = Decimal('0')
  bid1_amount = Decimal('0')
  ask2 = Decimal('0')
  ask2_amount = Decimal('0')
  bid2 = Decimal('0')
  bid2_amount = Decimal('0')
  ask3 = Decimal('0')
  ask3_amount = Decimal('0')
  bid3 = Decimal('0')
  bid3_amount = Decimal('0')

class ArbData(object):
  def __init__(self, direction = "Forward", p = 0, usdProfit = 0, usdInvestment = 0):
    self.tradeDirection = direction
    self.profit = p
    self.usdProfit = usdProfit
    self.usdInvestment = usdInvestment

class Balance(object):
  balance_usd = Decimal('0')
  balance_ltc = Decimal('0')
  balance_btc = Decimal('0')

class TimerRestarter(object):
  def __init__(self, timer):
    self.timer = timer
  def __enter__(self):
    return self
  def __exit__(self, type, value, traceback):
    self.timer.start()

class TraderThread(QtCore.QThread):
  timer = QtCore.QTimer()
  stopwatch = QtCore.QTime()
  tradeTimer = QtCore.QTimer()
  updateData = pyqtSignal(Tops, ArbData, ArbData)
  updateLag = pyqtSignal(int)
  def __init__(self, parent, tradeAPI, p1='btcusd', p2='ltcbtc', p3 = 'ltcusd', refreshInterval=100, tradeInterval=15000,
  s1ToSpend = 20, s2ToSpend = 0.03, s3ToSpend = 4, minProfit = 0.01, maxLag = 900):
    super(TraderThread, self).__init__(parent)
    self.tradeAPI = tradeAPI
    self.k = Decimal('1') - (tradeAPI.Comission() / Decimal('100.0'))
    self.p1 = p1
    self.p2 = p2
    self.p3 = p3
    self.s1 = p1[3:]
    self.s2 = p1[:3]
    self.s3 = p3[:3]
    self.minProfit = minProfit
    self.maxLag = maxLag
    self.s1ToSpend = s1ToSpend
    self.s2ToSpend = s2ToSpend
    self.s3ToSpend = s3ToSpend
    self.balance = {}
    self.remainders = {}
    self.timer.setInterval(refreshInterval)
    self.timer.setSingleShot(True)
    self.tradeTimer.setInterval(tradeInterval)
    self.tradeTimer.setSingleShot(True)
    self.timer.timeout.connect(self.onTimer)

  @pyqtSlot()
  def onTimer(self):
    with TimerRestarter(self.timer) as timerRestarter:
      self.stopwatch.start()
      t = self.GetTops()
      self.tops = t
      elapsed = self.stopwatch.elapsed()
      self.lag = elapsed
      self.updateLag.emit(elapsed)

      a1, b1 = t.ask1, t.bid1
      a2, b2 = t.ask2, t.bid2
      a3, b3 = t.ask3, t.bid3

      k3 = self.k * self.k * self.k

      profit1 = k3 * (b3) / ((a1) * (a2)) - Decimal('1.0')
      profit2 = k3 * (b1) * (b2) / (a3)  - Decimal('1.0')

      balance = { self.s1: min(self.balance[self.s1], self.s1ToSpend), \
        self.s2: min(self.balance[self.s2], self.s2ToSpend), \
        self.s3: min(self.balance[self.s3], self.s3ToSpend) }

      orders, maxDepth = arbmath.CalculateArbOrders(self.depths, self.p1, self.p2, self.p3, self.k, balance, self.tradeAPI)

      usdProfit = Decimal('0.0')

      if orders:
        print "%s: Exploring arbitrage opportunity: %s" % (str(QtCore.QDateTime.currentDateTime().toString()), orders)

        exchangeModel = arbmath.ExchangeModel(self.depths, self.tradeAPI)

        newBalance = copy.deepcopy(balance)
        for order in orders:
          newBalance, pendingOrder = exchangeModel.ModelTrade(newBalance, *order)

        usdProfit = newBalance[self.s1] - balance[self.s1]

        print "Profit: %s" % usdProfit

        s2Change = newBalance[self.s2] - balance[self.s2]
        if s2Change > 0:
          usdProfit += s2Change * b1
        else:
          usdProfit += s2Change * a1

        s3Change = newBalance[self.s3] - balance[self.s3]
        if s3Change > 0:
          usdProfit += s3Change * b3
        else:
          usdProfit += s3Change * a3

        print "%s gain: %s" % (self.s2, s2Change)
        print "%s gain: %s" % (self.s3, s3Change)

        print "Overall profit: %s" % usdProfit

        if not self.tradeTimer.isActive():
          if usdProfit > self.minProfit:
            for order in orders:
              self.tradeAPI.EnqueueOrder(*order)

            print "Placing orders..."
            r1, r2, r3 = self.tradeAPI.ProcessPendingOrders()

            print "Received from trade 1: %s" % r1
            print "Received from trade 2: %s" % r2
            print "Received from trade 3: %s" % r3

            self.tradeTimer.start()
          else:
            print "Profit %s less than min profit %s" % (usdProfit, self.minProfit)
        else:
          print "Trade timer is active, not trading"

      a1 = ArbData("Forward", profit1, usdProfit)
      a2 = ArbData("Backward", profit2, usdProfit)

      self.updateData.emit(t, a1, a2)

  def run(self):
    self.timer.start()
    self.exec_()

  def GetTop(self, pair):
    r = self.tradeAPI.GetDepth(pair, 1, 1)
    return ((r['ask'][0]['price'], r['ask'][0]['amount']), (r['bid'][0]['price'], r['bid'][0]['amount']))

  def GetTops(self):
    result = Tops()
    depthsAndBalance  = self.tradeAPI.GetDepths([self.p1, self.p2, self.p3], 50, 50, 5)
    r = depthsAndBalance['depths']
    self.balance = depthsAndBalance['balance']
    p1Top = ((r[self.p1]['ask'][0]['price'], r[self.p1]['ask'][0]['amount']), (r[self.p1]['bid'][0]['price'], r[self.p1]['bid'][0]['amount']))
    p2Top = ((r[self.p2]['ask'][0]['price'], r[self.p2]['ask'][0]['amount']), (r[self.p2]['bid'][0]['price'], r[self.p2]['bid'][0]['amount']))
    p3Top = ((r[self.p3]['ask'][0]['price'], r[self.p3]['ask'][0]['amount']), (r[self.p3]['bid'][0]['price'], r[self.p3]['bid'][0]['amount']))

    self.depths = r

    result.ask1 = p1Top[0][0]
    result.ask1_amount = p1Top[0][1]
    result.bid1 = p1Top[1][0]
    result.bid1_amount = p1Top[1][1]
    result.ask2 = p2Top[0][0]
    result.ask2_amount = p2Top[0][1]
    result.bid2 = p2Top[1][0]
    result.bid2_amount = p2Top[1][1]
    result.ask3 = p3Top[0][0]
    result.ask3_amount = p3Top[0][1]
    result.bid3 = p3Top[1][0]
    result.bid3_amount = p3Top[1][1]

    return result<commit_msg>traderthread: fix syntax error<commit_after>import sys
import decimal
from decimal import Decimal
import copy
import PyQt4
from PyQt4 import QtCore
from PyQt4.QtCore import pyqtSlot
from PyQt4.QtCore import pyqtSignal
import arbmath

class Tops(object):
  ask1 = Decimal('0')
  ask1_amount = Decimal('0')
  bid1 = Decimal('0')
  bid1_amount = Decimal('0')
  ask2 = Decimal('0')
  ask2_amount = Decimal('0')
  bid2 = Decimal('0')
  bid2_amount = Decimal('0')
  ask3 = Decimal('0')
  ask3_amount = Decimal('0')
  bid3 = Decimal('0')
  bid3_amount = Decimal('0')

class ArbData(object):
  def __init__(self, direction = "Forward", p = 0, usdProfit = 0, usdInvestment = 0):
    self.tradeDirection = direction
    self.profit = p
    self.usdProfit = usdProfit
    self.usdInvestment = usdInvestment

class Balance(object):
  balance_usd = Decimal('0')
  balance_ltc = Decimal('0')
  balance_btc = Decimal('0')

class TimerRestarter(object):
  def __init__(self, timer):
    self.timer = timer
  def __enter__(self):
    return self
  def __exit__(self, type, value, traceback):
    self.timer.start()

class TraderThread(QtCore.QThread):
  timer = QtCore.QTimer()
  stopwatch = QtCore.QTime()
  tradeTimer = QtCore.QTimer()
  updateData = pyqtSignal(Tops, ArbData, ArbData)
  updateLag = pyqtSignal(int)
  def __init__(self, parent, tradeAPI, p1='btcusd', p2='ltcbtc', p3 = 'ltcusd', refreshInterval=100, tradeInterval=15000,
  s1ToSpend = 20, s2ToSpend = 0.03, s3ToSpend = 4, minProfit = 0.01, maxLag = 900):
    super(TraderThread, self).__init__(parent)
    self.tradeAPI = tradeAPI
    self.k = Decimal('1') - (tradeAPI.Comission() / Decimal('100.0'))
    self.p1 = p1
    self.p2 = p2
    self.p3 = p3
    self.s1 = p1[3:]
    self.s2 = p1[:3]
    self.s3 = p3[:3]
    self.minProfit = minProfit
    self.maxLag = maxLag
    self.s1ToSpend = s1ToSpend
    self.s2ToSpend = s2ToSpend
    self.s3ToSpend = s3ToSpend
    self.balance = {}
    self.remainders = {}
    self.timer.setInterval(refreshInterval)
    self.timer.setSingleShot(True)
    self.tradeTimer.setInterval(tradeInterval)
    self.tradeTimer.setSingleShot(True)
    self.timer.timeout.connect(self.onTimer)

  @pyqtSlot()
  def onTimer(self):
    with TimerRestarter(self.timer) as timerRestarter:
      self.stopwatch.start()
      t = self.GetTops()
      self.tops = t
      elapsed = self.stopwatch.elapsed()
      self.lag = elapsed
      self.updateLag.emit(elapsed)

      a1, b1 = t.ask1, t.bid1
      a2, b2 = t.ask2, t.bid2
      a3, b3 = t.ask3, t.bid3

      k3 = self.k * self.k * self.k

      profit1 = k3 * (b3) / ((a1) * (a2)) - Decimal('1.0')
      profit2 = k3 * (b1) * (b2) / (a3)  - Decimal('1.0')

      balance = { self.s1: min(self.balance[self.s1], self.s1ToSpend), \
        self.s2: min(self.balance[self.s2], self.s2ToSpend), \
        self.s3: min(self.balance[self.s3], self.s3ToSpend) }

      orders, maxDepth = arbmath.CalculateArbOrders(self.depths, self.p1, self.p2, self.p3, self.k, balance, self.tradeAPI)

      usdProfit = Decimal('0.0')

      if orders:
        print "%s: Exploring arbitrage opportunity: %s" % (str(QtCore.QDateTime.currentDateTime().toString()), orders)

        exchangeModel = arbmath.ExchangeModel(self.depths, self.tradeAPI)

        newBalance = copy.deepcopy(balance)
        for order in orders:
          newBalance, pendingOrder = exchangeModel.ModelTrade(newBalance, *order)

        usdProfit = newBalance[self.s1] - balance[self.s1]

        print "Profit: %s" % usdProfit

        s2Change = newBalance[self.s2] - balance[self.s2]
        if s2Change > 0:
          usdProfit += s2Change * b1
        else:
          usdProfit += s2Change * a1

        s3Change = newBalance[self.s3] - balance[self.s3]
        if s3Change > 0:
          usdProfit += s3Change * b3
        else:
          usdProfit += s3Change * a3

        print "%s gain: %s" % (self.s2, s2Change)
        print "%s gain: %s" % (self.s3, s3Change)

        print "Overall profit: %s" % usdProfit

        if not self.tradeTimer.isActive():
          if usdProfit > self.minProfit:
            for order in orders:
              self.tradeAPI.EnqueueOrder(*order)

            print "Placing orders..."
            r1, r2, r3 = self.tradeAPI.PlacePendingOrders()

            print "Received from trade 1: %s" % r1
            print "Received from trade 2: %s" % r2
            print "Received from trade 3: %s" % r3

            self.tradeTimer.start()
          else:
            print "Profit %s less than min profit %s" % (usdProfit, self.minProfit)
        else:
          print "Trade timer is active, not trading"

      a1 = ArbData("Forward", profit1, usdProfit)
      a2 = ArbData("Backward", profit2, usdProfit)

      self.updateData.emit(t, a1, a2)

  def run(self):
    self.timer.start()
    self.exec_()

  def GetTop(self, pair):
    r = self.tradeAPI.GetDepth(pair, 1, 1)
    return ((r['ask'][0]['price'], r['ask'][0]['amount']), (r['bid'][0]['price'], r['bid'][0]['amount']))

  def GetTops(self):
    result = Tops()
    depthsAndBalance  = self.tradeAPI.GetDepths([self.p1, self.p2, self.p3], 50, 50, 5)
    r = depthsAndBalance['depths']
    self.balance = depthsAndBalance['balance']
    p1Top = ((r[self.p1]['ask'][0]['price'], r[self.p1]['ask'][0]['amount']), (r[self.p1]['bid'][0]['price'], r[self.p1]['bid'][0]['amount']))
    p2Top = ((r[self.p2]['ask'][0]['price'], r[self.p2]['ask'][0]['amount']), (r[self.p2]['bid'][0]['price'], r[self.p2]['bid'][0]['amount']))
    p3Top = ((r[self.p3]['ask'][0]['price'], r[self.p3]['ask'][0]['amount']), (r[self.p3]['bid'][0]['price'], r[self.p3]['bid'][0]['amount']))

    self.depths = r

    result.ask1 = p1Top[0][0]
    result.ask1_amount = p1Top[0][1]
    result.bid1 = p1Top[1][0]
    result.bid1_amount = p1Top[1][1]
    result.ask2 = p2Top[0][0]
    result.ask2_amount = p2Top[0][1]
    result.bid2 = p2Top[1][0]
    result.bid2_amount = p2Top[1][1]
    result.ask3 = p3Top[0][0]
    result.ask3_amount = p3Top[0][1]
    result.bid3 = p3Top[1][0]
    result.bid3_amount = p3Top[1][1]

    return result<|endoftext|>


<commit_before>from reader_model import *
import sys
import shutil

atoms, word_true, keep_prob = get_placeholders()
word_predict = reader_model(atoms, keep_prob)
loss, optim = loss_optim(word_predict, word_true)

WORDS, WORD_TO_NUM = getAllWords()
ATOMS, ATOM_TO_NUM = getAllAtoms()
EMBEDDINGS = loadFromFile("savedEmbeddings/embeddings.pkl")
ATOM_EMBED = loadFromFile("savedEmbeddings/atom_embeddings.pkl")

global_counter = 0
def next_batch(num):
    global global_counter
    a_batch = None
    w_batch = []
    i = 0
    while i < num:
        w = WORDS[(global_counter+num)%len(WORDS)]
        global_counter = (global_counter+1)%len(WORDS)
        a_set = []
        for a in ATOMS:
            if a in w and len(a_set) < ATOM_NUM:
                a_set.append(ATOM_EMBED[ATOM_TO_NUM[a]])

        if len(a_set) == 0:
            continue
        while len(a_set) < ATOM_NUM:
            a_set += a_set

        if len(a_set) > ATOM_NUM:
            a_set = a_set[:ATOM_NUM]

        arr = np.reshape(np.array(a_set), [1,-1])
        if a_batch is None:
            a_batch = arr
        else:
            a_batch = np.concatenate([a_batch, arr], axis = 0)

        w_batch.append(EMBEDDINGS[WORD_TO_NUM[w]])
        i += 1

    return [a_batch, w_batch]

merged_summary = tf.summary.merge_all()
steps = 100000
logStep = steps//1000
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    # loadModel(sess)
    shutil.rmtree(LOG_DIR, ignore_errors=True)
    train_writer, test_writer = getSummaryWriters(sess)

    startTime = time.time()
    for i in range(steps):
        batch = next_batch(batch_size)
        _, _loss, _summary = sess.run([optim, loss, merged_summary],
        feed_dict={
            atoms: batch[0],
            word_true: batch[1],
            keep_prob: 1.0
        })

        if i % logStep == 0:
            print("Loss: %s"%_loss)
            if i > 0:
                saveModel(sess, silent=True)
            train_writer.add_summary(_summary, i)

        sys.stdout.write("...Training-%s-(%s / %s)-%s\r"%(
            progress_bar(20, (i%20)+1),
            i, steps,
            estimate_time(startTime, steps, i)
        ))
    saveModel(sess)<commit_msg>now trying to have training and testing batches separate<commit_after>from reader_model import *
import sys
import shutil
import random

atoms, word_true, keep_prob = get_placeholders()
word_predict = reader_model(atoms, keep_prob)
loss, optim = loss_optim(word_predict, word_true)

WORDS, WORD_TO_NUM = getAllWords()
ATOMS, ATOM_TO_NUM = getAllAtoms()

TEST_WORDS = random.sample(WORDS, batch_size)

TRAIN_WORDS = [w for w in WORDS if not w in TEST_WORDS]

EMBEDDINGS = loadFromFile("savedEmbeddings/embeddings.pkl")
ATOM_EMBED = loadFromFile("savedEmbeddings/atom_embeddings.pkl")

def prepare_word(w):
    a_set = []
    for a in ATOMS:
        if a in w and len(a_set) < ATOM_NUM:
            a_set.append(ATOM_EMBED[ATOM_TO_NUM[a]])

    if len(a_set) == 0:
        return [None, None]
    while len(a_set) < ATOM_NUM:
        a_set += a_set

    if len(a_set) > ATOM_NUM:
        a_set = a_set[:ATOM_NUM]

    arr = np.reshape(np.array(a_set), [1, -1])
    return [arr,EMBEDDINGS[WORD_TO_NUM[w]]]

global_counter = 0
def next_batch(num):
    global global_counter
    a_batch = None
    w_batch = []
    i = 0
    while i < num:
        w = TRAIN_WORDS[(global_counter+num)%len(TRAIN_WORDS)]
        global_counter = (global_counter+1)%len(TRAIN_WORDS)
        arr, word_embed = prepare_word(w)
        if arr is None or word_embed is None:
            continue

        if a_batch is None:
            a_batch = arr
        else:
            a_batch = np.concatenate([a_batch, arr], axis = 0)

        w_batch.append(word_embed)
        i += 1

    return [a_batch, w_batch]

def make_test_batch():
    a_batch = None
    w_batch = []
    for w in TEST_WORDS:
        arr, word_embed = prepare_word(w)
        if arr is None or word_embed is None:
            continue

        if a_batch is None:
            a_batch = arr
        else:
            a_batch = np.concatenate([a_batch, arr], axis=0)

        w_batch.append(word_embed)

    return [a_batch, w_batch]

TEST_BATCH = make_test_batch()

merged_summary = tf.summary.merge_all()
steps = 100000
logStep = steps//1000
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    # loadModel(sess)
    shutil.rmtree(LOG_DIR, ignore_errors=True)
    train_writer, test_writer = getSummaryWriters(sess)

    startTime = time.time()
    for i in range(steps):
        batch = next_batch(batch_size)
        _, _loss, _summary = sess.run([optim, loss, merged_summary],
        feed_dict={
            atoms: batch[0],
            word_true: batch[1],
            keep_prob: 0.75
        })

        if i % logStep == 0:
            print("Loss: %s"%_loss)
            if i > 0:
                saveModel(sess, silent=True)
            train_writer.add_summary(_summary, i)

        if i % (2*logStep) == 0:
            _loss, _summary = sess.run([loss, merged_summary], feed_dict={
                atoms: TEST_BATCH[0],
                word_true:TEST_BATCH[1],
                keep_prob:1.0
            })
            test_writer.add_summary(_summary, i)

        sys.stdout.write("...Training-%s-(%s / %s)-%s\r"%(
            progress_bar(20, (i%20)+1),
            i, steps,
            estimate_time(startTime, steps, i)
        ))
    saveModel(sess)<|endoftext|>


<commit_before>import argparse,logging,os
import mxnet as mx
from symbol_resnet import resnet

logger = logging.getLogger()
logger.setLevel(logging.INFO)

def multi_factor_scheduler(begin_epoch, epoch_size, step=[60, 75, 90], factor=0.1):
    step_ = [epoch_size * (x-begin_epoch) for x in step if x-begin_epoch > 0]
    return mx.lr_scheduler.MultiFactorScheduler(step=step_, factor=factor) if len(step_) else None

def main():
    if args.data_type == "cifar10":
        args.aug_level = 1
        # depth should be one of 110, 164, 1001,...,which is should fit (args.depth-2)%9 == 0
        if((args.depth-2)%9 == 0 and args.depth >= 164):
            per_unit = [(args.depth-2)/9]
            filter_list = [16, 64, 128, 256]
            bottle_neck = True
        elif((args.depth-2)%6 == 0 and args.depth < 164):
            per_unit = [(args.depth-2)/6]
            filter_list = [16, 16, 32, 64]
            bottle_neck = False
        else:
            raise ValueError("no experiments done on detph {}, you can do it youself".format(args.depth))
        units = per_unit*3
        symbol = resnet(units=units, num_stage=3, filter_list=filter_list, num_class=10, data_type="cifar10",
                        bottle_neck = bottle_neck, bn_mom=args.bn_mom, workspace=512)
    elif args.data_type == "imagenet":
        if args.depth == 18:
            units = [2, 2, 2, 2]
        elif args.depth == 34:
            units = [3, 4, 6, 3]
        elif args.depth == 50:
            units = [3, 4, 6, 3]
        elif args.depth == 101:
            units = [3, 4, 23, 3]
        elif args.depth == 152:
            units = [3, 8, 36, 3]
        elif args.depth == 200:
            units = [3, 24, 36, 3]
        else:
            raise ValueError("no experiments done on detph {}, you can do it youself".format(args.depth))
        symbol = resnet(units=units, num_stage=4, filter_list=[64, 256, 512, 1024, 2048] if args.depth >=50
                        else [64, 64, 128, 256, 512], num_class=1000, data_type="imagenet", bottle_neck = True
                        if args.depth >= 50 else False, bn_mom=args.bn_mom, workspace=512)
    else:
         raise ValueError("do not support {} yet".format(args.data_type))
    devs = mx.cpu() if args.gpus is None else [mx.gpu(int(i)) for i in args.gpus.split(',')]
    epoch_size = max(int(args.num_examples / args.batch_size), 1)
    begin_epoch = args.model_load_epoch if args.model_load_epoch else 0
    if not os.path.exists("./model"):
        os.mkdir("./model")
    checkpoint = mx.callback.do_checkpoint("model/resnet-{}-{}".format(args.data_type, args.depth))
    kv = mx.kvstore.create(args.kv_store)
    arg_params = None
    aux_params = None
    if args.retrain:
        _, arg_params, aux_params = mx.model.load_checkpoint("model/resnet-{}-{}".format(args.data_type, args.depth),
                                                             args.model_load_epoch)
    import pdb; pdb.set_trace()
    train = mx.io.ImageRecordIter(
        path_imgrec         = os.path.join(args.data_dir, "train.rec") if args.data_type == 'cifar10' else
                              os.path.join(args.data_dir, "train_256_q90.rec") if args.aug_level == 1
                              else os.path.join(args.data_dir, "train_480_q90.rec"),
        label_width         = 1,
        data_name           = 'data',
        label_name          = 'softmax_label',
        data_shape          = (3, 32, 32) if args.data_type=="cifar10" else (3, 224, 224),
        batch_size          = args.batch_size,
        pad                 = 4 if args.data_type == "cifar10" else 0,
        fill_value          = 127,  # only used when pad is valid
        rand_crop           = True,
        max_random_scale    = 1.0,  # 480 with imagnet, 32 with cifar10
        min_random_scale    = 1.0 if args.data_type == "cifar10" else 1.0 if args.aug_level == 1 else 0.533,  # 256.0/480.0
        max_aspect_ratio    = 0 if args.data_type == "cifar10" else 0 if args.aug_level == 1 else 0.25,
        random_h            = 0 if args.data_type == "cifar10" else 0 if args.aug_level == 1 else 36,  # 0.4*90
        random_s            = 0 if args.data_type == "cifar10" else 0 if args.aug_level == 1 else 50,  # 0.4*127
        random_l            = 0 if args.data_type == "cifar10" else 0 if args.aug_level == 1 else 50,  # 0.4*127
        max_rotate_angle    = 0 if args.aug_level <= 2 else 10,
        max_shear_ratio     = 0 if args.aug_level <= 2 else 0.1,
        rand_mirror         = True,
        shuffle             = True,
        num_parts           = kv.num_workers,
        part_index          = kv.rank)
    val = mx.io.ImageRecordIter(
        path_imgrec         = os.path.join(args.data_dir, "val.rec") if args.data_type == 'cifar10' else
                              os.path.join(args.data_dir, "val_256_q90.rec"),
        label_width         = 1,
        data_name           = 'data',
        label_name          = 'softmax_label',
        batch_size          = args.batch_size,
        data_shape          = (3, 32, 32) if args.data_type=="cifar10" else (3, 224, 224),
        rand_crop           = False,
        rand_mirror         = False,
        num_parts           = kv.num_workers,
        part_index          = kv.rank)
    model = mx.model.FeedForward(
        ctx                 = devs,
        symbol              = symbol,
        arg_params          = arg_params,
        aux_params          = aux_params,
        num_epoch           = 200 if args.data_type == "cifar10" else 120,
        begin_epoch         = begin_epoch,
        learning_rate       = args.lr,
        momentum            = args.mom,
        wd                  = args.wd,
        optimizer           = 'nag',
        # optimizer          = 'sgd',
        initializer         = mx.init.Xavier(rnd_type='gaussian', factor_type="in", magnitude=2),
        lr_scheduler        = multi_factor_scheduler(begin_epoch, epoch_size, step=[120, 160], factor=0.1)
                             if args.data_type=='cifar10' else
                             multi_factor_scheduler(begin_epoch, epoch_size, step=[30, 60, 90], factor=0.1),
        )
    model.fit(
        X                  = train,
        eval_data          = val,
        eval_metric        = ['acc'] if args.data_type=='cifar10' else
                             ['acc', mx.metric.create('top_k_accuracy', top_k = 5)],
        kvstore            = kv,
        batch_end_callback = mx.callback.Speedometer(args.batch_size, 50),
        epoch_end_callback = checkpoint)
    # logging.info("top-1 and top-5 acc is {}".format(model.score(X = val,
    #               eval_metric = ['acc', mx.metric.create('top_k_accuracy', top_k = 5)])))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="command for training resnet-v2")
    parser.add_argument('--gpus', type=str, default='0', help='the gpus will be used, e.g "0,1,2,3"')
    parser.add_argument('--data-dir', type=str, default='./data/imagenet/', help='the input data directory')
    parser.add_argument('--data-type', type=str, default='imagenet', help='the dataset type')
    parser.add_argument('--list-dir', type=str, default='./',
                        help='the directory which contain the training list file')
    parser.add_argument('--lr', type=float, default=0.1, help='initialization learning reate')
    parser.add_argument('--mom', type=float, default=0.9, help='momentum for sgd')
    parser.add_argument('--bn-mom', type=float, default=0.9, help='momentum for batch normlization')
    parser.add_argument('--wd', type=float, default=0.0001, help='weight decay for sgd')
    parser.add_argument('--batch-size', type=int, default=256, help='the batch size')
    parser.add_argument('--depth', type=int, default=50, help='the depth of resnet')
    parser.add_argument('--aug-level', type=int, default=2, choices=[1, 2, 3],
                        help='level 1: use only random crop and random mirror\n'
                             'level 2: add scale/aspect/hsv augmentation based on level 1\n'
                             'level 3: add rotation/shear augmentation based on level 2')
    parser.add_argument('--num-examples', type=int, default=1281167, help='the number of training examples')
    parser.add_argument('--kv-store', type=str, default='device', help='the kvstore type')
    parser.add_argument('--model-load-epoch', type=int, default=0,
                        help='load the model on an epoch using the model-load-prefix')
    parser.add_argument('--retrain', action='store_true', default=False, help='true means continue training')
    args = parser.parse_args()
    logging.info(args)
    main()

<commit_msg>remove pdb<commit_after>import argparse,logging,os
import mxnet as mx
from symbol_resnet import resnet

logger = logging.getLogger()
logger.setLevel(logging.INFO)

def multi_factor_scheduler(begin_epoch, epoch_size, step=[60, 75, 90], factor=0.1):
    step_ = [epoch_size * (x-begin_epoch) for x in step if x-begin_epoch > 0]
    return mx.lr_scheduler.MultiFactorScheduler(step=step_, factor=factor) if len(step_) else None

def main():
    if args.data_type == "cifar10":
        args.aug_level = 1
        # depth should be one of 110, 164, 1001,...,which is should fit (args.depth-2)%9 == 0
        if((args.depth-2)%9 == 0 and args.depth >= 164):
            per_unit = [(args.depth-2)/9]
            filter_list = [16, 64, 128, 256]
            bottle_neck = True
        elif((args.depth-2)%6 == 0 and args.depth < 164):
            per_unit = [(args.depth-2)/6]
            filter_list = [16, 16, 32, 64]
            bottle_neck = False
        else:
            raise ValueError("no experiments done on detph {}, you can do it youself".format(args.depth))
        units = per_unit*3
        symbol = resnet(units=units, num_stage=3, filter_list=filter_list, num_class=10, data_type="cifar10",
                        bottle_neck = bottle_neck, bn_mom=args.bn_mom, workspace=512)
    elif args.data_type == "imagenet":
        if args.depth == 18:
            units = [2, 2, 2, 2]
        elif args.depth == 34:
            units = [3, 4, 6, 3]
        elif args.depth == 50:
            units = [3, 4, 6, 3]
        elif args.depth == 101:
            units = [3, 4, 23, 3]
        elif args.depth == 152:
            units = [3, 8, 36, 3]
        elif args.depth == 200:
            units = [3, 24, 36, 3]
        else:
            raise ValueError("no experiments done on detph {}, you can do it youself".format(args.depth))
        symbol = resnet(units=units, num_stage=4, filter_list=[64, 256, 512, 1024, 2048] if args.depth >=50
                        else [64, 64, 128, 256, 512], num_class=1000, data_type="imagenet", bottle_neck = True
                        if args.depth >= 50 else False, bn_mom=args.bn_mom, workspace=512)
    else:
         raise ValueError("do not support {} yet".format(args.data_type))
    devs = mx.cpu() if args.gpus is None else [mx.gpu(int(i)) for i in args.gpus.split(',')]
    epoch_size = max(int(args.num_examples / args.batch_size), 1)
    begin_epoch = args.model_load_epoch if args.model_load_epoch else 0
    if not os.path.exists("./model"):
        os.mkdir("./model")
    checkpoint = mx.callback.do_checkpoint("model/resnet-{}-{}".format(args.data_type, args.depth))
    kv = mx.kvstore.create(args.kv_store)
    arg_params = None
    aux_params = None
    if args.retrain:
        _, arg_params, aux_params = mx.model.load_checkpoint("model/resnet-{}-{}".format(args.data_type, args.depth),
                                                             args.model_load_epoch)
    train = mx.io.ImageRecordIter(
        path_imgrec         = os.path.join(args.data_dir, "train.rec") if args.data_type == 'cifar10' else
                              os.path.join(args.data_dir, "train_256_q90.rec") if args.aug_level == 1
                              else os.path.join(args.data_dir, "train_480_q90.rec"),
        label_width         = 1,
        data_name           = 'data',
        label_name          = 'softmax_label',
        data_shape          = (3, 32, 32) if args.data_type=="cifar10" else (3, 224, 224),
        batch_size          = args.batch_size,
        pad                 = 4 if args.data_type == "cifar10" else 0,
        fill_value          = 127,  # only used when pad is valid
        rand_crop           = True,
        max_random_scale    = 1.0,  # 480 with imagnet, 32 with cifar10
        min_random_scale    = 1.0 if args.data_type == "cifar10" else 1.0 if args.aug_level == 1 else 0.533,  # 256.0/480.0
        max_aspect_ratio    = 0 if args.data_type == "cifar10" else 0 if args.aug_level == 1 else 0.25,
        random_h            = 0 if args.data_type == "cifar10" else 0 if args.aug_level == 1 else 36,  # 0.4*90
        random_s            = 0 if args.data_type == "cifar10" else 0 if args.aug_level == 1 else 50,  # 0.4*127
        random_l            = 0 if args.data_type == "cifar10" else 0 if args.aug_level == 1 else 50,  # 0.4*127
        max_rotate_angle    = 0 if args.aug_level <= 2 else 10,
        max_shear_ratio     = 0 if args.aug_level <= 2 else 0.1,
        rand_mirror         = True,
        shuffle             = True,
        num_parts           = kv.num_workers,
        part_index          = kv.rank)
    val = mx.io.ImageRecordIter(
        path_imgrec         = os.path.join(args.data_dir, "val.rec") if args.data_type == 'cifar10' else
                              os.path.join(args.data_dir, "val_256_q90.rec"),
        label_width         = 1,
        data_name           = 'data',
        label_name          = 'softmax_label',
        batch_size          = args.batch_size,
        data_shape          = (3, 32, 32) if args.data_type=="cifar10" else (3, 224, 224),
        rand_crop           = False,
        rand_mirror         = False,
        num_parts           = kv.num_workers,
        part_index          = kv.rank)
    model = mx.model.FeedForward(
        ctx                 = devs,
        symbol              = symbol,
        arg_params          = arg_params,
        aux_params          = aux_params,
        num_epoch           = 200 if args.data_type == "cifar10" else 120,
        begin_epoch         = begin_epoch,
        learning_rate       = args.lr,
        momentum            = args.mom,
        wd                  = args.wd,
        optimizer           = 'nag',
        # optimizer          = 'sgd',
        initializer         = mx.init.Xavier(rnd_type='gaussian', factor_type="in", magnitude=2),
        lr_scheduler        = multi_factor_scheduler(begin_epoch, epoch_size, step=[120, 160], factor=0.1)
                             if args.data_type=='cifar10' else
                             multi_factor_scheduler(begin_epoch, epoch_size, step=[30, 60, 90], factor=0.1),
        )
    model.fit(
        X                  = train,
        eval_data          = val,
        eval_metric        = ['acc'] if args.data_type=='cifar10' else
                             ['acc', mx.metric.create('top_k_accuracy', top_k = 5)],
        kvstore            = kv,
        batch_end_callback = mx.callback.Speedometer(args.batch_size, 50),
        epoch_end_callback = checkpoint)
    # logging.info("top-1 and top-5 acc is {}".format(model.score(X = val,
    #               eval_metric = ['acc', mx.metric.create('top_k_accuracy', top_k = 5)])))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="command for training resnet-v2")
    parser.add_argument('--gpus', type=str, default='0', help='the gpus will be used, e.g "0,1,2,3"')
    parser.add_argument('--data-dir', type=str, default='./data/imagenet/', help='the input data directory')
    parser.add_argument('--data-type', type=str, default='imagenet', help='the dataset type')
    parser.add_argument('--list-dir', type=str, default='./',
                        help='the directory which contain the training list file')
    parser.add_argument('--lr', type=float, default=0.1, help='initialization learning reate')
    parser.add_argument('--mom', type=float, default=0.9, help='momentum for sgd')
    parser.add_argument('--bn-mom', type=float, default=0.9, help='momentum for batch normlization')
    parser.add_argument('--wd', type=float, default=0.0001, help='weight decay for sgd')
    parser.add_argument('--batch-size', type=int, default=256, help='the batch size')
    parser.add_argument('--depth', type=int, default=50, help='the depth of resnet')
    parser.add_argument('--aug-level', type=int, default=2, choices=[1, 2, 3],
                        help='level 1: use only random crop and random mirror\n'
                             'level 2: add scale/aspect/hsv augmentation based on level 1\n'
                             'level 3: add rotation/shear augmentation based on level 2')
    parser.add_argument('--num-examples', type=int, default=1281167, help='the number of training examples')
    parser.add_argument('--kv-store', type=str, default='device', help='the kvstore type')
    parser.add_argument('--model-load-epoch', type=int, default=0,
                        help='load the model on an epoch using the model-load-prefix')
    parser.add_argument('--retrain', action='store_true', default=False, help='true means continue training')
    args = parser.parse_args()
    logging.info(args)
    main()

<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

import threading, time
import rapidsms
from training.models import *
from rapidsms.message import Message, StatusCodes
from rapidsms.connection import Connection

class App (rapidsms.App):
    '''The training app saves error messages in a queue before they go out
       and waits for someone to either flag them as "ready to go out" or
       override the default response by adding their own.  This is meant
       to be used in trainings, to provide live feedback via SMS'''

    PRIORITY = "last"

    def ajax_GET_pending_count(self, params):

        # returns JUST the number of messages in waiting, which
        # indicates whether anyone is waiting for attention
        return MessageInWaiting.objects.filter(status="P").count()

    def ajax_GET_pending(self, params):

        # return all of the messages in waiting,
        # each of which contain their responses
        return MessageInWaiting.objects.filter(status="P")

    def ajax_POST_accept(self, params, form):
        msg = MessageInWaiting.objects.get(pk=int(form["msg_pk"]))

        # there might be one response, or there
        # might be many -- make it iterable
        responses = form.get("responses", [])
        if not type(responses) == list:
            responses = [responses]

        for resp in responses:

            # if this response was present in the original
            # set, and remains unchanged, just "confirm" it
            originals = msg.responses.filter(text=resp, type="O")
            if len(originals):
                originals[0].type = "C"
                originals[0].save()

            # this response is new, or changed!
            # create a new ResponseInWaiting object
            else:
                ResponseInWaiting(
                    originator=msg,
                    text=resp,
                    type="A").save()

        # find any remaining original responses, which
        # were removed in the webui, and delete them
        msg.responses.filter(type="O").delete()

        # mark the incoming message as "handled", so
        # it will be picked up by the responder_loop
        msg.status = "H"
        msg.save()

        # TODO: send something more useful
        # back to the browser to confirm
        return True

    def start (self):
        """Configure your app in the start phase."""
        # Start the Responder Thread -----------------------------------------

        self.info("[responder] Starting up...")
        # interval to check for responding (in seconds)
        response_interval = 10
        # start a thread for responding
        responder_thread = threading.Thread(
                target=self.responder_loop,
                args=(response_interval,))
        responder_thread.daemon = True
        responder_thread.start()

    def parse (self, message):
        """Parse and annotate messages in the parse phase."""
        pass

    def cleanup (self, message):
        if (self._requires_action(message)):
            # create the message in waiting object and response
            # in waiting objects for this, and assume someone will
            # deal with them.
            # This app should never be running in a production environment
            # unless someone is very carefully tracking the incoming messages
            self.info("Queueing up %s for further handling" % message)
            in_waiting = MessageInWaiting.from_message(message)
            in_waiting.save()
            for response in message.responses:
                resp_in_waiting = ResponseInWaiting.objects.create(type='O', originator=in_waiting,text=response.text)

            # blank out the responses, they will be sent by the responder thread
            # after moderation
            message.responses = []

    def outgoing (self, message):
        """Handle outgoing message notifications."""
        pass

    def stop (self):
        """Perform global app cleanup when the application is stopped."""
        pass

    def _requires_action(self, message):
        # this message requires action if and only if
        # 1. No response is indicated "ok"
        #   "ok" by any app should override any other errors
        # 2. Some message is indicated "app error" or "generic errror"
        #   "none" is the default, and we haven't updated every app to correctly
        #   say "ok" on real responses, so only do this for known errors
        to_return = False
        for response in message.responses:
            if response.status == StatusCodes.OK:
                return False
            elif response.status == StatusCodes.APP_ERROR or response.status == StatusCodes.GENERIC_ERROR:
                to_return = True
        return to_return

    # Responder Thread --------------------
    def responder_loop(self, seconds=10):
        self.info("Starting responder...")
        while True:
            # look for any new handled messages
            # in the database, and send the responses
            for msg_in_waiting in MessageInWaiting.objects.filter(status="H"):
                self.info("Responding to %s.", msg_in_waiting)
                for response in msg_in_waiting.responses.all():
                    self.info("Response: %s.", response)
                    # only send confirmed or added responses
                    if response.type != "O":
                        db_connection = msg_in_waiting.get_connection()
                        if db_connection is not None:
                              db_backend = db_connection.backend
                              # we need to get the real backend from the router
                              # to properly send it
                              real_backend = self.router.get_backend(db_backend.slug)
                              if real_backend:
                                  connection = Connection(real_backend, db_connection.identity)
                                  response_msg = Message(connection, response.text)
                                  self.router.outgoing(response_msg)
                              else:
                                  # TODO: should we fail harder here?  This will permanently
                                  # disable responses to this message which is bad.
                                  self.error("Can't find backend %s.  Messages will not be sent")
                # mark the original message as responded to
                msg_in_waiting.status="R"
                msg_in_waiting.save()
            # wait until it's time to check again
            time.sleep(seconds)
<commit_msg>refactored rapidsms.message into incoming and outgoing<commit_after>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

import threading, time
import rapidsms
from training.models import *
from rapidsms.messages import IncomingMessage, OutgoingMessage, ErrorMessage
from rapidsms.connection import Connection

class App (rapidsms.App):
    '''The training app saves error messages in a queue before they go out
       and waits for someone to either flag them as "ready to go out" or
       override the default response by adding their own.  This is meant
       to be used in trainings, to provide live feedback via SMS'''

    PRIORITY = "last"

    def ajax_GET_pending_count(self, params):

        # returns JUST the number of messages in waiting, which
        # indicates whether anyone is waiting for attention
        return MessageInWaiting.objects.filter(status="P").count()

    def ajax_GET_pending(self, params):

        # return all of the messages in waiting,
        # each of which contain their responses
        return MessageInWaiting.objects.filter(status="P")

    def ajax_POST_accept(self, params, form):
        msg = MessageInWaiting.objects.get(pk=int(form["msg_pk"]))

        # there might be one response, or there
        # might be many -- make it iterable
        responses = form.get("responses", [])
        if not type(responses) == list:
            responses = [responses]

        for resp in responses:

            # if this response was present in the original
            # set, and remains unchanged, just "confirm" it
            originals = msg.responses.filter(text=resp, type="O")
            if len(originals):
                originals[0].type = "C"
                originals[0].save()

            # this response is new, or changed!
            # create a new ResponseInWaiting object
            else:
                ResponseInWaiting(
                    originator=msg,
                    text=resp,
                    type="A").save()

        # find any remaining original responses, which
        # were removed in the webui, and delete them
        msg.responses.filter(type="O").delete()

        # mark the incoming message as "handled", so
        # it will be picked up by the responder_loop
        msg.status = "H"
        msg.save()

        # TODO: send something more useful
        # back to the browser to confirm
        return True

    def start (self):
        """Configure your app in the start phase."""
        # Start the Responder Thread -----------------------------------------

        self.info("[responder] Starting up...")
        # interval to check for responding (in seconds)
        response_interval = 10
        # start a thread for responding
        responder_thread = threading.Thread(
                target=self.responder_loop,
                args=(response_interval,))
        responder_thread.daemon = True
        responder_thread.start()

    def parse (self, message):
        """Parse and annotate messages in the parse phase."""
        pass

    def cleanup (self, message):
        if (self._requires_action(message)):
            # create the message in waiting object and response
            # in waiting objects for this, and assume someone will
            # deal with them.
            # This app should never be running in a production environment
            # unless someone is very carefully tracking the incoming messages
            self.info("Queueing up %s for further handling" % message)
            in_waiting = MessageInWaiting.from_message(message)
            in_waiting.save()
            for response in message.responses:
                resp_in_waiting = ResponseInWaiting.objects.create(type='O', originator=in_waiting,text=response.text)

            # blank out the responses, they will be sent by the responder thread
            # after moderation
            message.responses = []

    def outgoing (self, message):
        """Handle outgoing message notifications."""
        pass

    def stop (self):
        """Perform global app cleanup when the application is stopped."""
        pass

    def _requires_action(self, message):
        # this message requires action if and only if
        # 1. No response is indicated "ok"
        #   "ok" by any app should override any other errors
        # 2. Some message is indicated "app error" or "generic errror"
        #   "none" is the default, and we haven't updated every app to correctly
        #   say "ok" on real responses, so only do this for known errors
        to_return = False
        for response in message.responses:
            if isinstance(response, ErrorMessage):
                to_return = True
            elif isinstance(response, OutgoingMessage):
                return False
        return to_return

    # Responder Thread --------------------
    def responder_loop(self, seconds=10):
        self.info("Starting responder...")
        while True:
            # look for any new handled messages
            # in the database, and send the responses
            for msg_in_waiting in MessageInWaiting.objects.filter(status="H"):
                self.info("Responding to %s.", msg_in_waiting)
                for response in msg_in_waiting.responses.all():
                    self.info("Response: %s.", response)
                    # only send confirmed or added responses
                    if response.type != "O":
                        db_connection = msg_in_waiting.get_connection()
                        if db_connection is not None:
                              db_backend = db_connection.backend
                              # we need to get the real backend from the router
                              # to properly send it
                              real_backend = self.router.get_backend(db_backend.slug)
                              if real_backend:
                                  connection = Connection(real_backend, db_connection.identity)
                                  response_msg = OutgoingMessage(connection, response.text)
                                  self.router.outgoing(response_msg)
                              else:
                                  # TODO: should we fail harder here?  This will permanently
                                  # disable responses to this message which is bad.
                                  self.error("Can't find backend %s.  Messages will not be sent")
                # mark the original message as responded to
                msg_in_waiting.status="R"
                msg_in_waiting.save()
            # wait until it's time to check again
            time.sleep(seconds)
<|endoftext|>


<commit_before>"""fo2 URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import include, url
from django.contrib import admin
from django.contrib.auth import views as auth_views
from django.conf import settings
from django.conf.urls.static import static

from .admin import intr_adm_site
from .views import index_view, test_view, IntranetView, myip_view, \
    OficialSystextilView, ApoioAoErpView

urlpatterns = [
    url(r'^$', index_view, name='index'),
    url(r'^t$', test_view, name='test'),
    url(r'^apoio_ao_erp/', ApoioAoErpView.as_view(), name='apoio_ao_erp'),
    url(r'^intranet/', IntranetView.as_view(), name='intranet'),
    url(r'^accounts/login/$',
        auth_views.login, {'template_name': 'login.html'},
        name='login'),
    url(r'^encerrar/', auth_views.logout, {'next_page': '/'}, name='encerrar'),
    url(r'^myip/', myip_view, name='myip'),
    url(r'^oficial.systextil/', OficialSystextilView.as_view(),
        name='oficial_systextil'),
    # url(r'^', admin.site.urls),
    url(r'^rootadm/', admin.site.urls),
    url(r'^intradm/', intr_adm_site.urls),
    url(r'^lotes/', include('lotes.urls', namespace='producao')),
    url(r'^produto/', include('produto.urls')),
    url(r'^insumo/', include('insumo.urls')),
    url(r'^geral/', include('geral.urls')),
    url(r'^contabil/', include('contabil.urls')),
    url(r'^comercial/', include('comercial.urls')),
    url(r'^logistica/', include('logistica.urls')),
    url(r'^rh/', include('rh.urls')),
    url(r'^cd/', include('cd.urls')),
  ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
<commit_msg>criado url namespace insumo<commit_after>"""fo2 URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import include, url
from django.contrib import admin
from django.contrib.auth import views as auth_views
from django.conf import settings
from django.conf.urls.static import static

from .admin import intr_adm_site
from .views import index_view, test_view, IntranetView, myip_view, \
    OficialSystextilView, ApoioAoErpView

urlpatterns = [
    url(r'^$', index_view, name='index'),
    url(r'^t$', test_view, name='test'),
    url(r'^apoio_ao_erp/', ApoioAoErpView.as_view(), name='apoio_ao_erp'),
    url(r'^intranet/', IntranetView.as_view(), name='intranet'),
    url(r'^accounts/login/$',
        auth_views.login, {'template_name': 'login.html'},
        name='login'),
    url(r'^encerrar/', auth_views.logout, {'next_page': '/'}, name='encerrar'),
    url(r'^myip/', myip_view, name='myip'),
    url(r'^oficial.systextil/', OficialSystextilView.as_view(),
        name='oficial_systextil'),
    # url(r'^', admin.site.urls),
    url(r'^rootadm/', admin.site.urls),
    url(r'^intradm/', intr_adm_site.urls),
    url(r'^lotes/', include('lotes.urls', namespace='producao')),
    url(r'^produto/', include('produto.urls')),
    url(r'^insumo/', include('insumo.urls', namespace='insumo')),
    url(r'^geral/', include('geral.urls')),
    url(r'^contabil/', include('contabil.urls')),
    url(r'^comercial/', include('comercial.urls')),
    url(r'^logistica/', include('logistica.urls')),
    url(r'^rh/', include('rh.urls')),
    url(r'^cd/', include('cd.urls')),
  ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
<|endoftext|>


<commit_before>from __future__ import absolute_import

from .widget import Widget

class WidgetList(object):
    """WidgetList

    Args:
        widgets (dict, list, Widget): The list of widgets for a layer.

    Example:

    .. code::python

        from cartoframes.viz import Widget

        WidgetList([{
            type: 'formula',
            value: 'viewportSum($amount)'
            title: '...',
            description: '...',
            footer: '...'
        }, {
            'type': 'default',
            'value': '"Custom Info"',
        }])
    """

    def __init__(self, widgets=None):
        self._widgets = self._init_widgets(widgets)

    def _init_widgets(self, widgets):
        if isinstance(widgets, list):
            widget_list = []
            for widget in widgets:
                if isinstance(widget, dict):
                    widget_list.append(Widget(widget))
                elif isinstance(widget, Widget):
                    widget_list.append(widget)
            return widget_list
        if isinstance(widgets, dict):
            return [Widget(widgets)]
        elif isinstance(widgets, Widget):
            return [widgets]
        else:
            return []

    def get_widgets_info(self):
        widgets_info = []
        for widget in self._widgets:
            if widget:
                widgets_info.append(widget.get_info())

        return widgets_info

    def get_variables(self):
        output = {}
        for widget in self._widgets:
            if widget._variable_name:
                output[widget._variable_name] = '$' + widget._value if widget.has_bridge() else widget._value

        return output
<commit_msg>Remove whitespace<commit_after>from __future__ import absolute_import

from .widget import Widget

class WidgetList(object):
    """WidgetList

    Args:
        widgets (dict, list, Widget): The list of widgets for a layer.

    Example:

    .. code::python

        from cartoframes.viz import Widget

        WidgetList([{
            type: 'formula',
            value: 'viewportSum($amount)'
            title: '...',
            description: '...',
            footer: '...'
        }, {
            'type': 'default',
            'value': '"Custom Info"',
        }])
    """

    def __init__(self, widgets=None):
        self._widgets = self._init_widgets(widgets)

    def _init_widgets(self, widgets):
        if isinstance(widgets, list):
            widget_list = []
            for widget in widgets:
                if isinstance(widget, dict):
                    widget_list.append(Widget(widget))
                elif isinstance(widget, Widget):
                    widget_list.append(widget)
            return widget_list
        if isinstance(widgets, dict):
            return [Widget(widgets)]
        elif isinstance(widgets, Widget):
            return [widgets]
        else:
            return []

    def get_widgets_info(self):
        widgets_info = []
        for widget in self._widgets:
            if widget:
                widgets_info.append(widget.get_info())

        return widgets_info

    def get_variables(self):
        output = {}
        for widget in self._widgets:
            if widget._variable_name:
                output[widget._variable_name] = '$' + widget._value if widget.has_bridge() else widget._value

        return output
<|endoftext|>


<commit_before>
import logging
from lofarpipe.support.parset import parameterset
from tkp.database import DataBase
from tkp.quality.statistics import rms_with_clipped_subregion
from tkp.lofar.noise import noise_level
from tkp.utility.accessors import FITSImage
import tkp.database.quality
import tkp.quality
from tkp.database.orm import Image
from contextlib import closing

logger = logging.getLogger(__name__)

def parse_parset(parset_file):
    parset = parameterset(parset_file)
    result = {}
    result['sigma'] = parset.getInt('sigma', 3)
    result['f'] = parset.getInt('f', 4)
    result['low_bound'] = parset.getFloat('low_bound', 1)
    result['high_bound'] = parset.getInt('high_bound', 50)
    result['frequency'] = parset.getInt('frequency', 45*10**6)
    result['subbandwidth'] = parset.getInt('subbandwidth', 200*10**3)
    result['intgr_time'] = parset.getFloat('intgr_time', 18654)
    result['configuration'] = parset.getString('configuration', "LBA_INNER")
    result['subbands'] = parset.getInt('subbands', 10)
    result['channels'] = parset.getInt('channels', 64)
    result['ncore'] = parset.getInt('ncore', 24)
    result['nremote'] = parset.getInt('nremote',16)
    result['nintl'] = parset.getInt('nintl', 8)
    return result

def noise(image_id, parset_file):
    """ checks if an image passes the RMS quality check. If not, a rejection entry is added to the database.
    args:
        image_id: id of image in database
        parset_file: parset file location with quality check parameters
    Returns:
        True if image passes tests, false if not
    """

    with closing(DataBase()) as database:
        db_image = Image(database=database, id=image_id)
        fitsimage = FITSImage(db_image.url)
        p = parse_parset(parset_file)

        rms = rms_with_clipped_subregion(fitsimage.data, sigma=p['sigma'], f=p['f'])
        noise = noise_level(p['frequency'], p['subbandwidth'], p['intgr_time'],
            p['configuration'], p['subbands'], p['channels'],
            p['ncore'], p['nremote'], p['nintl'])

        if tkp.quality.rms_valid(rms, noise, low_bound=p['low_bound'], high_bound=p['high_bound']):
            return True
        else:
            ratio = rms / noise
            reason = "noise level is %s times theoretical value" % ratio
            logger.info("image %s invalid: %s " % (db_image.id, reason) )
            tkp.database.quality.reject(database.connection, db_image.id,
                tkp.database.quality.reason['rms'], reason)
            return False<commit_msg>used wrong logger<commit_after>
import logging
from lofarpipe.support.parset import parameterset
from tkp.database import DataBase
from tkp.quality.statistics import rms_with_clipped_subregion
from tkp.lofar.noise import noise_level
from tkp.utility.accessors import FITSImage
import tkp.database.quality
import tkp.quality
from tkp.database.orm import Image

logger = logging.getLogger(__name__)

def parse_parset(parset_file):
    parset = parameterset(parset_file)
    result = {}
    result['sigma'] = parset.getInt('sigma', 3)
    result['f'] = parset.getInt('f', 4)
    result['low_bound'] = parset.getFloat('low_bound', 1)
    result['high_bound'] = parset.getInt('high_bound', 50)
    result['frequency'] = parset.getInt('frequency', 45*10**6)
    result['subbandwidth'] = parset.getInt('subbandwidth', 200*10**3)
    result['intgr_time'] = parset.getFloat('intgr_time', 18654)
    result['configuration'] = parset.getString('configuration', "LBA_INNER")
    result['subbands'] = parset.getInt('subbands', 10)
    result['channels'] = parset.getInt('channels', 64)
    result['ncore'] = parset.getInt('ncore', 24)
    result['nremote'] = parset.getInt('nremote',16)
    result['nintl'] = parset.getInt('nintl', 8)
    return result

def nice_format(f):
    if f > 9999 or f < 0.01:
        return "%.2e" % f
    else:
        return "%.2f" % f

def noise(image_id, parset_file):
    """ checks if an image passes the RMS quality check. If not, a rejection entry is added to the database.
    args:
        image_id: id of image in database
        parset_file: parset file location with quality check parameters
    Returns:
        True if image passes tests, false if not
    """

    database = DataBase()
    db_image = Image(database=database, id=image_id)
    fitsimage = FITSImage(db_image.url)
    p = parse_parset(parset_file)

    rms = rms_with_clipped_subregion(fitsimage.data, sigma=p['sigma'], f=p['f'])
    noise = noise_level(p['frequency'], p['subbandwidth'], p['intgr_time'],
        p['configuration'], p['subbands'], p['channels'],
        p['ncore'], p['nremote'], p['nintl'])

    if tkp.quality.rms_valid(rms, noise, low_bound=p['low_bound'], high_bound=p['high_bound']):
        logger.info("image %i accepted: rms: %s, theoretical noise: %s" % (db_image.id, nice_format(rms), nice_format(noise)))
        return True
    else:
        ratio = rms / noise
        reason = "rms value (%s) is %s times theoretical noise (%s)" % (nice_format(rms), nice_format(ratio), nice_format(noise))
        logger.info("image %s REJECTED: %s " % (db_image.id, reason) )
        tkp.database.quality.reject(database.connection, db_image.id,
            tkp.database.quality.reason['rms'], reason)
        return False<|endoftext|>


<commit_before>import argparse
import logging
import sys
from multiprocessing import Process
from typing import List, Text

import rasa.cli.run
from rasa.cli.utils import print_error, print_success
from rasa.core import utils, cli
from rasa.core.run import serve_application
from rasa.core.utils import AvailableEndpoints, EndpointConfig
from rasa.utils import configure_colored_logging

# noinspection PyProtectedMember
def add_subparser(subparsers: argparse._SubParsersAction,
                  parents: List[argparse.ArgumentParser]):
    shell_parser = subparsers.add_parser(
        "up",
        parents=parents,
        conflict_handler="resolve",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        help="Run the Rasa Interface")

    shell_parser.add_argument(
        "--production",
        action="store_true",
        help="Run Rasa in a production environment")

    shell_parser.add_argument(
        "--auth_token",
        type=str,
        help="Rasa API auth token")

    shell_parser.add_argument(
        "--nlg",
        type=str,
        default="http://localhost:5002/api/nlg",
        help="Rasa NLG endpoint")

    shell_parser.add_argument(
        "--model_endpoint_url",
        type=str,
        default=("http://localhost:5002/api/projects/"
                 "default/models/tags/production"),
        help="Rasa Stack model endpoint URL")

    shell_parser.add_argument(
        "--project_path",
        type=str,
        default=".",
        help="Path to the Rasa project directory"
    )
    shell_parser.add_argument(
        "--data_path",
        type=str,
        default="data",
        help="Path to the directory containing Rasa NLU training data "
             "and Rasa Core stories"
    )

    rasa.cli.run.add_run_arguments(shell_parser)

    shell_parser.set_defaults(func=up)

    cli.arguments.add_logging_option_arguments(shell_parser)

def start_core(args: argparse.Namespace,
               endpoints: AvailableEndpoints = None):
    """Starts the Rasa Core application."""

    if endpoints is None:
        endpoints = AvailableEndpoints.read_endpoints(args.endpoints)

    serve_application(endpoints=endpoints,
                      port=args.port,
                      credentials=args.credentials,
                      cors=args.cors,
                      auth_token=args.auth_token,
                      enable_api=True,
                      jwt_secret=args.jwt_secret,
                      jwt_method=args.jwt_method)

def start_core_for_local_platform(args: argparse.Namespace,
                                  platform_token: Text):
    """Starts the Rasa API with Rasa Core as a background process."""

    endpoints = AvailableEndpoints(
        model=EndpointConfig(args.model_endpoint_url,
                             token=platform_token,
                             wait_time_between_pulls=1),
        event_broker=EndpointConfig(**{"type": "file"}),
        nlg=EndpointConfig(args.nlg, token=platform_token))

    vars(args).update(dict(nlu_model=None,
                           channel="rasa",
                           credentials="credentials.yml",
                           cors="*",
                           auth_token=args.auth_token,
                           enable_api=True,
                           endpoints=endpoints))

    p = Process(target=start_core, args=(args, endpoints))
    p.start()

def up(args: argparse.Namespace):
    logging.getLogger('werkzeug').setLevel(logging.WARN)
    logging.getLogger('engineio').setLevel(logging.WARN)
    logging.getLogger('matplotlib').setLevel(logging.WARN)
    logging.getLogger('socketio').setLevel(logging.ERROR)

    configure_colored_logging(args.loglevel)
    utils.configure_file_logging(args.loglevel,
                                 args.log_file)

    if args.production:
        print_success("Starting Rasa Core")
        start_core(args)
    else:
        try:
            from rasa_platform import config
            from rasa_platform.api.server import main_local
            from rasa_platform.services.event_service import main
        except ImportError as e:
            print_error("Rasa Platform is not installed. The `rasa up` "
                        "command requires an installation of Rasa Platform."
                        "Error:\n{}".format(e))
            sys.exit()

        print_success("Starting Rasa Event Service")
        print_success("Starting Rasa Core")
        print_success("Starting Rasa Interface")

        p = Process(target=main, args=("rasa_event.log",))
        p.start()

        start_core_for_local_platform(args, config.platform_token)

        main_local(args.project_path, args.data_path)
<commit_msg>increase pull interval<commit_after>import argparse
import logging
import sys
from multiprocessing import Process
from typing import List, Text

import rasa.cli.run
from rasa.cli.utils import print_error, print_success
from rasa.core import utils, cli
from rasa.core.run import serve_application
from rasa.core.utils import AvailableEndpoints, EndpointConfig
from rasa.utils import configure_colored_logging

# noinspection PyProtectedMember
def add_subparser(subparsers: argparse._SubParsersAction,
                  parents: List[argparse.ArgumentParser]):
    shell_parser = subparsers.add_parser(
        "up",
        parents=parents,
        conflict_handler="resolve",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        help="Run the Rasa Interface")

    shell_parser.add_argument(
        "--production",
        action="store_true",
        help="Run Rasa in a production environment")

    shell_parser.add_argument(
        "--auth_token",
        type=str,
        help="Rasa API auth token")

    shell_parser.add_argument(
        "--nlg",
        type=str,
        default="http://localhost:5002/api/nlg",
        help="Rasa NLG endpoint")

    shell_parser.add_argument(
        "--model_endpoint_url",
        type=str,
        default=("http://localhost:5002/api/projects/"
                 "default/models/tags/production"),
        help="Rasa Stack model endpoint URL")

    shell_parser.add_argument(
        "--project_path",
        type=str,
        default=".",
        help="Path to the Rasa project directory"
    )
    shell_parser.add_argument(
        "--data_path",
        type=str,
        default="data",
        help="Path to the directory containing Rasa NLU training data "
             "and Rasa Core stories"
    )

    rasa.cli.run.add_run_arguments(shell_parser)

    shell_parser.set_defaults(func=up)

    cli.arguments.add_logging_option_arguments(shell_parser)

def start_core(args: argparse.Namespace,
               endpoints: AvailableEndpoints = None):
    """Starts the Rasa Core application."""

    if endpoints is None:
        endpoints = AvailableEndpoints.read_endpoints(args.endpoints)

    serve_application(endpoints=endpoints,
                      port=args.port,
                      credentials=args.credentials,
                      cors=args.cors,
                      auth_token=args.auth_token,
                      enable_api=True,
                      jwt_secret=args.jwt_secret,
                      jwt_method=args.jwt_method)

def start_core_for_local_platform(args: argparse.Namespace,
                                  platform_token: Text):
    """Starts the Rasa API with Rasa Core as a background process."""

    endpoints = AvailableEndpoints(
        model=EndpointConfig(args.model_endpoint_url,
                             token=platform_token,
                             wait_time_between_pulls=5),
        event_broker=EndpointConfig(**{"type": "file"}),
        nlg=EndpointConfig(args.nlg, token=platform_token))

    vars(args).update(dict(nlu_model=None,
                           channel="rasa",
                           credentials="credentials.yml",
                           cors="*",
                           auth_token=args.auth_token,
                           enable_api=True,
                           endpoints=endpoints))

    p = Process(target=start_core, args=(args, endpoints))
    p.start()

def up(args: argparse.Namespace):
    logging.getLogger('werkzeug').setLevel(logging.WARN)
    logging.getLogger('engineio').setLevel(logging.WARN)
    logging.getLogger('matplotlib').setLevel(logging.WARN)
    logging.getLogger('socketio').setLevel(logging.ERROR)

    configure_colored_logging(args.loglevel)
    utils.configure_file_logging(args.loglevel,
                                 args.log_file)

    if args.production:
        print_success("Starting Rasa Core")
        start_core(args)
    else:
        try:
            from rasa_platform import config
            from rasa_platform.api.server import main_local
            from rasa_platform.services.event_service import main
        except ImportError as e:
            print_error("Rasa Platform is not installed. The `rasa up` "
                        "command requires an installation of Rasa Platform."
                        "Error:\n{}".format(e))
            sys.exit()

        print_success("Starting Rasa Event Service")
        print_success("Starting Rasa Core")
        print_success("Starting Rasa Interface")

        p = Process(target=main, args=("rasa_event.log",))
        p.start()

        start_core_for_local_platform(args, config.platform_token)

        main_local(args.project_path, args.data_path)
<|endoftext|>


<commit_before>import sys
import math
import wave
import struct
import pyaudio
import numpy as np
import matplotlib.pyplot as plt

# PyAudio object variable
pa = pyaudio.PyAudio()

# Size of each read-in chunk
CHUNK = 1
# Amount of channels of the live recording
CHANNELS = 2
# Sample width of the live recording
WIDTH = 2
# Sample rate in Hz of the live recording
SAMPLE_RATE = 44100
# Set how often plot data will be saved (every nth CHUNK) - the lower the number, the more precise the result
try:
    NTH_ITERATION = int(sys.argv[2])
except (ValueError, IndexError):
    print('The second argument has to be a number')
    sys.exit()

def main():
    # Mode the program was run in
    mode = sys.argv[1]

    # Execute the chosen mode
    if mode == '--file' or mode == '-f':
        filemode()
    elif mode == '--live' or mode == '-l':
        livemode()
    else:
        print('Please either choose file-mode or live-mode with the first argument')

def filemode():
    # Read in the given file
    (waveform, stream) = readin(sys.argv[3])

    # Give some feedback
    print('Now noise-cancelling the file')

    # Collecting the volume levels in decibels in a list
    decibel_levels = []

    # Counting the iterations of the while-loop
    iteration = 0

    # Read a first chunk and continue to do so for as long as there is a stream to read in
    original = waveform.readframes(CHUNK)
    while original != b'':
        try:
            # Invert the original audio
            inverted = invert(original)

            # Calculate the difference of the source and the inverted audio
            difference = calculate_difference(original, inverted)

            # Play back both audios
            stream.write(original)
            stream.write(inverted)

            # On every nth iteration append the difference between the level of the source audio and the inverted one
            if iteration % NTH_ITERATION == 0:
                # Print the current difference
                print('Difference (in dB): {}'.format(difference))
                # Append the difference to the list used for the plot
                decibel_levels.append(difference)

            # Read in the next chunk of data
            original = waveform.readframes(CHUNK)

            # Add up one to the iterations
            iteration += 1
        except (KeyboardInterrupt, SystemExit):
            break

    # Stop the stream after there is no more data to read
    stream.stop_stream()
    stream.close()

    # Outputting feedback regarding the end of the file
    print('Finished noise-cancelling the file')

    # Calculate and output the absolute median difference level
    print('Difference - Median (in dB): {}'.format(np.fabs(np.median(decibel_levels))))

    # Plot the results
    plot_results(decibel_levels, NTH_ITERATION)

    # Terminate PyAudio as well as the program
    pa.terminate()
    sys.exit()

def livemode():
    # Start live recording
    print('Now noise-cancelling live')

    # Create a new PyAudio object using the preset constants
    stream = pa.open(
        format=pa.get_format_from_width(WIDTH),
        channels=CHANNELS,
        rate=SAMPLE_RATE,
        frames_per_buffer=CHUNK,
        input=True,
        output=True
    )

    # Collecting the volume levels in decibels in a list
    decibel_levels = []

    # Grab a chunk of data in iterations according to the preset constants
    try:
        for i in range(0, int(SAMPLE_RATE / CHUNK * sys.maxunicode)):
            # Read in a chunk of live audio on each iteration
            original = stream.read(CHUNK)

            # Invert the original audio
            inverted = invert(original)

            # Calculate the difference of the source and the inverted audio
            difference = calculate_difference(original, inverted)

            # Play back the inverted audio
            stream.write(inverted, CHUNK)

            # On every nth iteration append the difference between the level of the source audio and the inverted one
            if i % NTH_ITERATION == 0:
                # Print the current difference
                print('Difference (in dB): {}'.format(difference))
                # Append the difference to the list used for the plot
                decibel_levels.append(difference)
    except (KeyboardInterrupt, SystemExit):
        # Outputting feedback regarding the end of the file
        print('Finished noise-cancelling the file')

        # Calculate and output the absolute median difference level
        print('Difference - Median (in dB): {}'.format(np.fabs(np.median(decibel_levels))))

        # Plot the results
        plot_results(decibel_levels, NTH_ITERATION)

        # Terminate the program
        stream.stop_stream()
        stream.close()
        pa.terminate()
        sys.exit()

def readin(file):
    """
    Reads in the given wave file and returns a new PyAudio stream object from it.

    :param file: The path to the file to read in
    :return (waveform, stream): (The actual audio data as a waveform, the PyAudio object for said data)
    """

    # Open the waveform from the command argument
    try:
        waveform = wave.open(file, 'r')
    except wave.Error:
        print('The program can only process wave audio files (.wav)')
        sys.exit()
    except FileNotFoundError:
        print('The chosen file does not exist')
        sys.exit()

    # Load PyAudio and create a useable waveform object
    stream = pa.open(
        format=pa.get_format_from_width(waveform.getsampwidth()),
        channels=waveform.getnchannels(),
        rate=waveform.getframerate(),
        output=True
    )

    # Return the waveform as well as the generated PyAudio stream object
    return waveform, stream

def invert(data):
    """
    Inverts the byte data it received utilizing an XOR operation.

    :param data: A chunk of byte data
    :return inverted: The same size of chunked data inverted bitwise
    """

    # Convert the bytestring into an integer
    intwave = int.from_bytes(data, byteorder='big')
    # Invert the integer
    intwave ^= 2147483647
    # Convert the integer back into a bytestring
    inverted = intwave.to_bytes(4, byteorder='big')
    # Return the inverted audio data
    return inverted

def calculate_decibel(data):
    """
    Calculates the volume level in decibel of the given data

    :param data: A bytestring used to calculate the decibel level
    :return db: The calculated volume level in decibel
    """

    count = len(data) / 2
    form = "%dh" % count
    shorts = struct.unpack(form, data)
    sum_squares = 0.0
    for sample in shorts:
        n = sample * (1.0 / 32768)
        sum_squares += n * n
    rms = math.sqrt(sum_squares / count) + 0.0001
    db = 20 * math.log10(rms)
    return db

def calculate_difference(value_1, value_2):
    difference = calculate_decibel(value_1) - calculate_decibel(value_2)
    return difference

def plot_results(data, nth_iteration):
    """
    Plots the list it receives and cuts off the first ten entries to circumvent the plotting of initial silence

    :param data: A list of data to be plotted
    :param nth_iteration: Used for the label of the x axis
    """

    plt.plot(data[10:])
    plt.xlabel('Time (every {}th {} byte)'.format(nth_iteration, CHUNK))
    plt.ylabel('Volume level (in dB)')
    plt.show()

main()
<commit_msg>Added a docstring to the new function from commit 5c0c11f<commit_after>import sys
import math
import wave
import struct
import pyaudio
import numpy as np
import matplotlib.pyplot as plt

# PyAudio object variable
pa = pyaudio.PyAudio()

# Size of each read-in chunk
CHUNK = 1
# Amount of channels of the live recording
CHANNELS = 2
# Sample width of the live recording
WIDTH = 2
# Sample rate in Hz of the live recording
SAMPLE_RATE = 44100
# Set how often plot data will be saved (every nth CHUNK) - the lower the number, the more precise the result
try:
    NTH_ITERATION = int(sys.argv[2])
except (ValueError, IndexError):
    print('The second argument has to be a number')
    sys.exit()

def main():
    # Mode the program was run in
    mode = sys.argv[1]

    # Execute the chosen mode
    if mode == '--file' or mode == '-f':
        filemode()
    elif mode == '--live' or mode == '-l':
        livemode()
    else:
        print('Please either choose file-mode or live-mode with the first argument')

def filemode():
    # Read in the given file
    (waveform, stream) = readin(sys.argv[3])

    # Give some feedback
    print('Now noise-cancelling the file')

    # Collecting the volume levels in decibels in a list
    decibel_levels = []

    # Counting the iterations of the while-loop
    iteration = 0

    # Read a first chunk and continue to do so for as long as there is a stream to read in
    original = waveform.readframes(CHUNK)
    while original != b'':
        try:
            # Invert the original audio
            inverted = invert(original)

            # Calculate the difference of the source and the inverted audio
            difference = calculate_difference(original, inverted)

            # Play back both audios
            stream.write(original)
            stream.write(inverted)

            # On every nth iteration append the difference between the level of the source audio and the inverted one
            if iteration % NTH_ITERATION == 0:
                # Print the current difference
                print('Difference (in dB): {}'.format(difference))
                # Append the difference to the list used for the plot
                decibel_levels.append(difference)

            # Read in the next chunk of data
            original = waveform.readframes(CHUNK)

            # Add up one to the iterations
            iteration += 1
        except (KeyboardInterrupt, SystemExit):
            break

    # Stop the stream after there is no more data to read
    stream.stop_stream()
    stream.close()

    # Outputting feedback regarding the end of the file
    print('Finished noise-cancelling the file')

    # Calculate and output the absolute median difference level
    print('Difference - Median (in dB): {}'.format(np.fabs(np.median(decibel_levels))))

    # Plot the results
    plot_results(decibel_levels, NTH_ITERATION)

    # Terminate PyAudio as well as the program
    pa.terminate()
    sys.exit()

def livemode():
    # Start live recording
    print('Now noise-cancelling live')

    # Create a new PyAudio object using the preset constants
    stream = pa.open(
        format=pa.get_format_from_width(WIDTH),
        channels=CHANNELS,
        rate=SAMPLE_RATE,
        frames_per_buffer=CHUNK,
        input=True,
        output=True
    )

    # Collecting the volume levels in decibels in a list
    decibel_levels = []

    # Grab a chunk of data in iterations according to the preset constants
    try:
        for i in range(0, int(SAMPLE_RATE / CHUNK * sys.maxunicode)):
            # Read in a chunk of live audio on each iteration
            original = stream.read(CHUNK)

            # Invert the original audio
            inverted = invert(original)

            # Calculate the difference of the source and the inverted audio
            difference = calculate_difference(original, inverted)

            # Play back the inverted audio
            stream.write(inverted, CHUNK)

            # On every nth iteration append the difference between the level of the source audio and the inverted one
            if i % NTH_ITERATION == 0:
                # Print the current difference
                print('Difference (in dB): {}'.format(difference))
                # Append the difference to the list used for the plot
                decibel_levels.append(difference)
    except (KeyboardInterrupt, SystemExit):
        # Outputting feedback regarding the end of the file
        print('Finished noise-cancelling the file')

        # Calculate and output the absolute median difference level
        print('Difference - Median (in dB): {}'.format(np.fabs(np.median(decibel_levels))))

        # Plot the results
        plot_results(decibel_levels, NTH_ITERATION)

        # Terminate the program
        stream.stop_stream()
        stream.close()
        pa.terminate()
        sys.exit()

def readin(file):
    """
    Reads in the given wave file and returns a new PyAudio stream object from it.

    :param file: The path to the file to read in
    :return (waveform, stream): (The actual audio data as a waveform, the PyAudio object for said data)
    """

    # Open the waveform from the command argument
    try:
        waveform = wave.open(file, 'r')
    except wave.Error:
        print('The program can only process wave audio files (.wav)')
        sys.exit()
    except FileNotFoundError:
        print('The chosen file does not exist')
        sys.exit()

    # Load PyAudio and create a useable waveform object
    stream = pa.open(
        format=pa.get_format_from_width(waveform.getsampwidth()),
        channels=waveform.getnchannels(),
        rate=waveform.getframerate(),
        output=True
    )

    # Return the waveform as well as the generated PyAudio stream object
    return waveform, stream

def invert(data):
    """
    Inverts the byte data it received utilizing an XOR operation.

    :param data: A chunk of byte data
    :return inverted: The same size of chunked data inverted bitwise
    """

    # Convert the bytestring into an integer
    intwave = int.from_bytes(data, byteorder='big')
    # Invert the integer
    intwave ^= 2147483647
    # Convert the integer back into a bytestring
    inverted = intwave.to_bytes(4, byteorder='big')
    # Return the inverted audio data
    return inverted

def calculate_decibel(data):
    """
    Calculates the volume level in decibel of the given data

    :param data: A bytestring used to calculate the decibel level
    :return db: The calculated volume level in decibel
    """

    count = len(data) / 2
    form = "%dh" % count
    shorts = struct.unpack(form, data)
    sum_squares = 0.0
    for sample in shorts:
        n = sample * (1.0 / 32768)
        sum_squares += n * n
    rms = math.sqrt(sum_squares / count) + 0.0001
    db = 20 * math.log10(rms)
    return db

def calculate_difference(data_1, data_2):
    """
    Calculates the difference level in decibel between the received binary inputs

    :param data_1: The first binary digit
    :param data_2: The second binary digit
    :return difference: The calculated difference level (in dB)
    """
    difference = calculate_decibel(data_1) - calculate_decibel(data_2)
    return difference

def plot_results(data, nth_iteration):
    """
    Plots the list it receives and cuts off the first ten entries to circumvent the plotting of initial silence

    :param data: A list of data to be plotted
    :param nth_iteration: Used for the label of the x axis
    """

    plt.plot(data[10:])
    plt.xlabel('Time (every {}th {} byte)'.format(nth_iteration, CHUNK))
    plt.ylabel('Volume level (in dB)')
    plt.show()

main()
<|endoftext|>