<commit_before># Python Security Project (PySec) and its related class files.
#
# PySec is a set of tools for secure application development under Linux
#
# Copyright 2014 PySec development team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# -*- coding: ascii -*-
"""Implementations of POP protocols"""
import socket
import select
from functools import partial
import errno

from pysec.core import memory
from pysec.xsplit import xbounds
from pysec.net.error import TooBigReply, TooManyFlushData

EOL = '\r\n'
MULTI_END = '.%s' % EOL

POP3_PORT = 110

OK_REPLY = 1
ERR_REPLY = 0
UNKNOWN_REPLY = -1

def is_ok(reply):
    return reply.startswith('+OK')

def is_err(reply):
    return reply.startswith('+ERR')

QUIT = 0
# TODO other commands
CMDS = 'QUIT',
MULTI = ()

def poplines(fd, bufsize, timeout):
    mem = memory.Memory(bufsize)
    timeout = int(timeout)
    rem = mstart = mend = 0
    while 1:
        # move
        if mstart > 0:
            mem[0:mend-mstart] = mem[mstart:mend]
        # fill
        if not select.select((fd,), (), (), timeout)[0]:
            raise socket.timeout
        mend += mem.read(fd, mend, mem.size - mend)
        if not mend:
            yield ''
            break
        #
        for start, end in xbounds(mem, EOL, 1, 0, mend):
            if mem[end-len(EOL):end] == EOL:
                yield mem[start:end]
                rem = end
            else:
                if (end-start) == mem.size:
                    raise TooBigReply()
        mem[0:mend-rem] = mem[rem:mend]
        mend -= rem

class POP3_Session(object):

    def __init__(self, host, port=POP3_PORT, timeout=60,
                 bufsize=4096, maxflush=4096):
        self.host = str(host)
        self.port = int(port)
        self.bufsize = int(bufsize)
        self.timeout = int(timeout)
        socket.setdefaulttimeout(self.timeout)
        self.sock = socket.socket()
        self.sock.settimeout(self.timeout)
        self.lines = poplines(self.sock.fileno(), self.bufsize, self.timeout)
        self.maxflush = int(maxflush)

    def get_reply(self):
        return self.lines.next()

    def get_multi_reply(self):
        for line in self.lines:
            if line == MULTI_END:
                yield line
                break
            yield line

    @property
    def is_open(self):
        return self.sock.fileno() >= 0

    @property
    def can_read(self):
        return bool(select.select((self.sock.fileno(),), (), (), 0)[0])

    def connect(self):
        self.sock.connect((self.host, self.port))
        return self.get_reply()

    def close(self):
        sock = self.sock
        try:
            sock.shutdown(socket.SHUT_RDWR)
        except IOError, ex:
            if ex.args[0] != errno.ENOTCONN:
                raise
        finally:
            sock.close()

    def flush(self):
        size  = 0
        while self.can_read:
            chunk = self.sock.recv(self.bufsize)
            size += len(chunk)
            if size > self.maxflush:
                raise TooManyFlushData()
            yield self.sock.recv(self.bufsize)

    def flush_all(self):
        for _ in self.flush():
            pass

    def putcmd(self, cmd, args=None):
        self.sock.sendall('%s%s%s' % (cmd, ' ' if args is None
                                               else ' %s ' % args, EOL))

    def cmd(self, cmd, args=None):
        self.flush_all()
        cmd = str(cmd).upper()
        if cmd not in CMDS:
            raise ValueError("command %r unknown" % cmd)
        self.putcmd(cmd, args)
        if cmd in MULTI:
            return self.get_multi_reply()
        else:
            return self.get_reply()

    def __enter__(self):
        return self, self.connect()

    def __exit__(self, exc_type, exc_val, exc_trace):
        self.close()
        return 0
<commit_msg>add log in pop lib<commit_after># Python Security Project (PySec) and its related class files.
#
# PySec is a set of tools for secure application development under Linux
#
# Copyright 2014 PySec development team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# -*- coding: ascii -*-
"""Implementations of POP protocols"""
import errno
from functools import partial
import select
import socket
from pysec.core import memory
from pysec.xsplit import xbounds
from pysec.net.error import TooBigReply, TooManyFlushData
from pysec import log

__name__ = 'pysec.net.pop'

# set actions
log.register_actions('POP3_NEW_SESSION',
                      'POP3_CONNECT',
                      'POP3_CLOSE',
                      'POP3_FLUSH',
                      'POP3_CMD',
                      'POP3_PUTCMD',
                      'POP3_SIMPLEREPLY',
                      'POP3_MULREPLY')

EOL = '\r\n'
MULTI_END = '.%s' % EOL

POP3_PORT = 110

OK_REPLY = 1
ERR_REPLY = 0
UNKNOWN_REPLY = -1

def is_ok(reply):
    return reply.startswith('+OK')

def is_err(reply):
    return reply.startswith('+ERR')

QUIT = 0
# TODO other commands
CMDS = 'QUIT',
MULTI = ()

def poplines(fd, bufsize, timeout):
    mem = memory.Memory(bufsize)
    timeout = int(timeout)
    rem = mstart = mend = 0
    while 1:
        # move
        if mstart > 0:
            mem[0:mend-mstart] = mem[mstart:mend]
        # fill
        if not select.select((fd,), (), (), timeout)[0]:
            raise socket.timeout
        mend += mem.read(fd, mend, mem.size - mend)
        if not mend:
            yield ''
            break
        #
        for start, end in xbounds(mem, EOL, 1, 0, mend):
            if mem[end-len(EOL):end] == EOL:
                yield mem[start:end]
                rem = end
            else:
                if (end-start) == mem.size:
                    raise TooBigReply()
        mem[0:mend-rem] = mem[rem:mend]
        mend -= rem

class POP3_Session(object):

    @log.wrap(log.actions.POP3_NEW_SESSION,
              fields=('host', 'port', 'timeout', 'bufsize', 'maxflush'),
              lib=__name__)
    def __init__(self, host, port=POP3_PORT, timeout=60,
                 bufsize=4096, maxflush=4096):
        self.host = str(host)
        self.port = int(port)
        self.bufsize = int(bufsize)
        self.timeout = int(timeout)
        socket.setdefaulttimeout(self.timeout)
        self.sock = socket.socket()
        self.sock.settimeout(self.timeout)
        self.lines = poplines(self.sock.fileno(), self.bufsize, self.timeout)
        self.maxflush = int(maxflush)

    @log.wrap(log.actions.POP3_SIMPLEREPLY, result='reply', lib=__name__)
    def get_reply(self):
        return self.lines.next()

    @log.wrap(log.actions.POP3_MULREPLY, result='reply', lib=__name__)
    def get_multi_reply(self):
        for line in self.lines:
            if line == MULTI_END:
                yield line
                break
            yield line

    @property
    def is_open(self):
        return self.sock.fileno() >= 0

    @property
    def can_read(self):
        return bool(select.select((self.sock.fileno(),), (), (), 0)[0])

    def connect(self):
        self.sock.connect((self.host, self.port))
        return self.get_reply()

    @log.wrap(log.actions.POP3_CLOSE, lib=__name__)
    def close(self):
        sock = self.sock
        try:
            sock.shutdown(socket.SHUT_RDWR)
        except IOError, ex:
            if ex.args[0] != errno.ENOTCONN:
                raise
        finally:
            sock.close()

    def flush(self):
        size  = 0
        while self.can_read:
            chunk = self.sock.recv(self.bufsize)
            size += len(chunk)
            if size > self.maxflush:
                raise TooManyFlushData()
            yield self.sock.recv(self.bufsize)

    @log.wrap(log.actions.POP3_FLUSH, lib=__name__)
    def flush_all(self):
        for _ in self.flush():
            pass

    @log.wrap(log.actions.POP3_PUTCMD, fields=('cmd', 'args'), lib=__name__)
    def putcmd(self, cmd, args=None):
        self.sock.sendall('%s%s%s' % (cmd, ' ' if args is None
                                               else ' %s ' % args, EOL))

    @log.wrap(log.actions.POP3_CMD, fields=('cmd', 'args'), lib=__name__)
    def cmd(self, cmd, args=None):
        self.flush_all()
        cmd = str(cmd).upper()
        if cmd not in CMDS:
            raise ValueError("command %r unknown" % cmd)
        self.putcmd(cmd, args)
        if cmd in MULTI:
            return self.get_multi_reply()
        else:
            return self.get_reply()

    def __enter__(self):
        return self, self.connect()

    def __exit__(self, exc_type, exc_val, exc_trace):
        self.close()
        return 0
<|endoftext|>


<commit_before>#!/usr/bin/python

import os
import sys
import time
import json
import tempfile
import signal
import subprocess
import threading
import logging
import logging.handlers
import socket
from math import ceil
from exchanges import *
from trading import *
from utils import *

if len(sys.argv) < 2:
  print "usage:", sys.argv[0], "server[:port] [users.dat]"
  sys.exit(1)

if not os.path.isdir('logs'):
  os.makedirs('logs')

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
sh = logging.handlers.SocketHandler('', logging.handlers.DEFAULT_TCP_LOGGING_PORT)
sh.setLevel(logging.DEBUG)
fh = logging.FileHandler('logs/%d.log' % time.time())
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter(fmt = '%(asctime)s %(levelname)s: %(message)s', datefmt="%Y/%m/%d-%H:%M:%S")
sh.setFormatter(formatter)
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(sh)
logger.addHandler(fh)
logger.addHandler(ch)

userfile = 'users.dat' if len(sys.argv) == 2 else sys.argv[2]
if userfile == "-":
  userdata = [ line.strip().split() for line in sys.stdin.readlines() if len(line.strip().split('#')[0].split()) >= 5 ]
else:
  try:
    userdata = [ line.strip().split() for line in open(userfile).readlines() if len(line.strip().split('#')[0].split()) >= 5 ] # address units exchange key secret [trader]
  except:
    logger.error("%s could not be read", userfile)
    sys.exit(1)

_server = sys.argv[1]
_wrappers = { 'poloniex' : Poloniex(), 'ccedk' : CCEDK(), 'bitcoincoid' : BitcoinCoId(), 'bter' : BTER(), 'testing' : Peatio() }

# one request signer thread for each key and unit
class RequestThread(ConnectionThread):
  def __init__(self, conn, key, secret, exchange, unit, address, sampling, cost, logger = None):
    super(RequestThread, self).__init__(conn, logger)
    self.key = key
    self.secret = secret
    self.exchange = exchange
    self.unit = unit
    self.initsampling = sampling
    self.sampling = sampling
    self.address = address
    self.errorflag = False
    self.trials = 0
    self.exchangeupdate = 0
    self.cost = cost.copy()

  def register(self):
    response = self.conn.post('register', { 'address' : self.address, 'key' : self.key, 'name' : repr(self.exchange) })
    if response['code'] == 0: # reset sampling in case of server restart
      self.sampling = self.initsampling
    return response

  def submit(self):
    data, sign = self.exchange.create_request(self.unit, self.key, self.secret)
    params = { 'unit' : self.unit, 'user' : self.key, 'sign' : sign }
    params.update(data)
    params.update(self.cost)
    curtime = time.time()
    ret = self.conn.post('liquidity', params, trials = 1, timeout = 10)
    if ret['code'] != 0:
      self.trials += time.time() - curtime + 60.0 / self.sampling
      self.logger.error("submit: %s" % ret['message'])
      if ret['code'] == 11: # user unknown, just register again
        self.register()
    else:
      self.trials = 0
    self.errorflag = self.trials >= 120 # notify that something is wrong after 2 minutes of failures

  def run(self):
    ret = self.register()
    if ret['code'] != 0: self.logger.error("register: %s" % ret['message'])
    while self.active:
      curtime = time.time()
      self.submit()
      time.sleep(max(60.0 / self.sampling - time.time() + curtime, 0))

# retrieve initial data
conn = Connection(_server, logger)
basestatus = conn.get('status')
exchangeinfo = conn.get('exchanges')
sampling = min(240, 3 * basestatus['sampling'] / 2)

# parse user data
users = {}
for user in userdata:
  key = user[3]
  secret = user[4]
  name = user[2].lower()
  if not name in _wrappers:
    logger.error("unknown exchange: %s", user[2])
    sys.exit(2)
  if not name in exchangeinfo:
    logger.error("exchange not supported by pool: %s", name)
    sys.exit(2)
  units = [ unit.lower() for unit in user[1].split(',') ]
  exchange = _wrappers[name]
  users[key] = {}
  for unit in user[1].split(','):
    unit = unit.lower()
    if not unit in exchangeinfo[name]:
      logger.error("unit %s on %s not supported by pool", unit, name)
      logger.info("supported units on %s are: %s", name, " ".join(exchangeinfo[name]))
      sys.exit(2)
    cost = { 'bid' : exchangeinfo[name][unit]['bid']['rate'], 'ask' : exchangeinfo[name][unit]['ask']['rate'] }
    if len(user) >= 6 and float(user[5]) != 0.0:
      cost['bid'] = float(user[5]) / 100.0
      cost['ask'] = float(user[5]) / 100.0
    if len(user) >= 7 and float(user[6]) != 0.0:
      cost['ask'] = float(user[6]) / 100.0
    bot = 'pybot' if len(user) < 8 else user[7]
    users[key][unit] = { 'request' : RequestThread(conn, key, secret, exchange, unit, user[0], sampling, cost, logger) }
    users[key][unit]['request'].start()
    target = { 'bid': exchangeinfo[name][unit]['bid']['target'], 'ask': exchangeinfo[name][unit]['ask']['target'] }
    if bot == 'none':
      users[key][unit]['order'] = None
    elif bot == 'nubot':
      users[key][unit]['order'] = NuBot(conn, users[key][unit]['request'], key, secret, exchange, unit, target, logger)
    elif bot == 'pybot':
      users[key][unit]['order'] = PyBot(conn, users[key][unit]['request'], key, secret, exchange, unit, target, logger)
    else:
      logger.error("unknown order handler: %s", bot)
      users[key][unit]['order'] = None
    if users[key][unit]['order']:
      if users[key][unit]['order']:
        users[key][unit]['order'].start()

logger.debug('starting liquidity propagation with sampling %d' % sampling)
starttime = time.time()
curtime = time.time()
effs = []

while True: # print some info every minute until program terminates
  try:
    time.sleep(max(60 - time.time() + curtime, 0))
    curtime = time.time()
    for user in users: # post some statistics
      response = conn.get(user, trials = 1)
      if 'error' in response:
        logger.error('unable to receive statistics for user %s: %s', user, response['message'])
        users[user].values()[0]['request'].register() # reassure to be registered if
        newstatus = conn.get('status', trials = 3)
        if not 'error' in newstatus:
          basestatus = newstatus
          sampling = min(240, 3 * basestatus['sampling'] / 2)
      else:
        # collect user information
        effective_rate = 0.0
        total = 0.0
        for unit in response['units']:
          for side in [ 'bid', 'ask' ]:
            effective_rate += float(sum([ o['amount'] * o['cost'] for o in response['units'][unit][side] ]))
            total += float(sum([ o['amount'] for o in response['units'][unit][side] ]))
        if total > 0.0: effective_rate /= total
        orderstring = ""
        for unit in response['units']:
          unitstring = ""
          for side in ['bid', 'ask']:
            market = response['units'][unit][side]
            coststring = ""
            for order in response['units'][unit][side]:
              if order['amount'] > 0:
                coststring += " %.4f x %.2f%%," % (order['amount'], order['cost'] * 100.0)
            if len(coststring):
              unitstring += " - %s:%s" % (side, coststring[:-1])
          if len(unitstring):
            orderstring += " - %s%s" % (unit, unitstring)
        # print user information
        logger.info('%s - balance: %.8f rate: %.2f%% ppm: %.8f efficiency: %.2f%% rejects: %d missing: %d%s - %s', repr(users[user].values()[0]['request'].exchange),
          response['balance'], effective_rate * 100, effective_rate * total / float(60 * 24), response['efficiency'] * 100, response['rejects'], response['missing'], orderstring, user)
        if not effs:
          effs = [ response['efficiency'] for i in xrange(5) ]
        if curtime - starttime > 90:
          effs = effs[1:] + [response['efficiency']]
          if sorted(effs)[2] < 0.95:
            for unit in response['units']:
              if response['units'][unit]['rejects'] > 1 and response['units'][unit]['rejects'] / float(basestatus['sampling']) >= 0.05: # look for valid error and adjust nonce shift
                if response['units'][unit]['last_error'] != "":
                  if 'deviates too much from current price' in response['units'][unit]['last_error']:
                    PyBot.pricefeed.price(unit, True) # force a price update
                    if users[user][unit]['order']: users[user][unit]['order'].shutdown()
                    logger.warning('price missmatch for %s on %s, forcing price update', unit, repr(users[user][unit]['request'].exchange))
                  else:
                    shift = users[user][unit]['request'].exchange._shift
                    users[user][unit]['request'].exchange.adjust(response['units'][unit]['last_error'])
                    if shift != users[user][unit]['request'].exchange._shift:
                      logger.warning('too many rejected requests for %s on %s, adjusting nonce shift to %d',
                        unit, repr(users[user][unit]['request'].exchange), users[user][unit]['request'].exchange._shift)
                else:
                  if users[user][unit]['request'].sampling < 2 * sampling: # just send more requests
                    users[user][unit]['request'].sampling = users[user][unit]['request'].sampling + 1
                    logger.warning('increasing sampling to %d',
                      unit, repr(users[user][unit]['request'].exchange), users[user][unit]['request'].sampling)
              if response['units'][unit]['missing'] / float(basestatus['sampling']) >= 0.05: # look for missing error and adjust sampling
                if users[user][unit]['request'].sampling < 2 * sampling: # just send more requests
                  users[user][unit]['request'].sampling = users[user][unit]['request'].sampling + 1
                  logger.warning('too many missing requests for %s on %s, increasing sampling to %d',
                    unit, repr(users[user][unit]['request'].exchange), users[user][unit]['request'].sampling)
                else: # just wait a little bit
                  logger.warning('too many missing requests, sleeping a short while to synchronize')
                  curtime += 0.7

  except KeyboardInterrupt: break
  except Exception as e:
    logger.error('exception caught in main loop: %s', sys.exc_info()[1])

logger.info('stopping trading bots, please allow the client up to 1 minute to terminate')
while True:
  try:
    for user in users:
      for unit in users[user]:
        users[user][unit]['request'].stop()
        if users[user][unit]['order']:
          users[user][unit]['order'].stop()
    for user in users:
      for unit in users[user]:
        users[user][unit]['request'].join()
        if users[user][unit]['order']:
          users[user][unit]['order'].join()
  except KeyboardInterrupt: continue
  break<commit_msg>sampling fix<commit_after>#!/usr/bin/python

import os
import sys
import time
import json
import tempfile
import signal
import subprocess
import threading
import logging
import logging.handlers
import socket
from math import ceil
from exchanges import *
from trading import *
from utils import *

if len(sys.argv) < 2:
  print "usage:", sys.argv[0], "server[:port] [users.dat]"
  sys.exit(1)

if not os.path.isdir('logs'):
  os.makedirs('logs')

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
sh = logging.handlers.SocketHandler('', logging.handlers.DEFAULT_TCP_LOGGING_PORT)
sh.setLevel(logging.DEBUG)
fh = logging.FileHandler('logs/%d.log' % time.time())
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter(fmt = '%(asctime)s %(levelname)s: %(message)s', datefmt="%Y/%m/%d-%H:%M:%S")
sh.setFormatter(formatter)
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(sh)
logger.addHandler(fh)
logger.addHandler(ch)

userfile = 'users.dat' if len(sys.argv) == 2 else sys.argv[2]
if userfile == "-":
  userdata = [ line.strip().split() for line in sys.stdin.readlines() if len(line.strip().split('#')[0].split()) >= 5 ]
else:
  try:
    userdata = [ line.strip().split() for line in open(userfile).readlines() if len(line.strip().split('#')[0].split()) >= 5 ] # address units exchange key secret [trader]
  except:
    logger.error("%s could not be read", userfile)
    sys.exit(1)

if not userdata:
  logger.error('no valid users could be found')
  sys.exit(1)

_server = sys.argv[1]
_wrappers = { 'poloniex' : Poloniex(), 'ccedk' : CCEDK(), 'bitcoincoid' : BitcoinCoId(), 'bter' : BTER(), 'testing' : Peatio() }

# one request signer thread for each key and unit
class RequestThread(ConnectionThread):
  def __init__(self, conn, key, secret, exchange, unit, address, sampling, cost, logger = None):
    super(RequestThread, self).__init__(conn, logger)
    self.key = key
    self.secret = secret
    self.exchange = exchange
    self.unit = unit
    self.initsampling = sampling
    self.sampling = sampling
    self.address = address
    self.errorflag = False
    self.trials = 0
    self.exchangeupdate = 0
    self.cost = cost.copy()

  def register(self):
    response = self.conn.post('register', { 'address' : self.address, 'key' : self.key, 'name' : repr(self.exchange) })
    if response['code'] == 0: # reset sampling in case of server restart
      self.sampling = self.initsampling
    return response

  def submit(self):
    data, sign = self.exchange.create_request(self.unit, self.key, self.secret)
    params = { 'unit' : self.unit, 'user' : self.key, 'sign' : sign }
    params.update(data)
    params.update(self.cost)
    curtime = time.time()
    ret = self.conn.post('liquidity', params, trials = 1, timeout = 10)
    if ret['code'] != 0:
      self.trials += time.time() - curtime + 60.0 / self.sampling
      self.logger.error("submit: %s" % ret['message'])
      if ret['code'] == 11: # user unknown, just register again
        self.register()
    else:
      self.trials = 0
    self.errorflag = self.trials >= 120 # notify that something is wrong after 2 minutes of failures

  def run(self):
    ret = self.register()
    if ret['code'] != 0: self.logger.error("register: %s" % ret['message'])
    while self.active:
      curtime = time.time()
      self.submit()
      time.sleep(max(60.0 / self.sampling - time.time() + curtime, 0))

# retrieve initial data
conn = Connection(_server, logger)
basestatus = conn.get('status')
exchangeinfo = conn.get('exchanges')
sampling = min(240, 3 * basestatus['sampling'] / 2)

# parse user data
users = {}
for user in userdata:
  key = user[3]
  secret = user[4]
  name = user[2].lower()
  if not name in _wrappers:
    logger.error("unknown exchange: %s", user[2])
    sys.exit(2)
  if not name in exchangeinfo:
    logger.error("exchange not supported by pool: %s", name)
    sys.exit(2)
  units = [ unit.lower() for unit in user[1].split(',') ]
  exchange = _wrappers[name]
  users[key] = {}
  for unit in user[1].split(','):
    unit = unit.lower()
    if not unit in exchangeinfo[name]:
      logger.error("unit %s on %s not supported by pool", unit, name)
      logger.info("supported units on %s are: %s", name, " ".join(exchangeinfo[name]))
      sys.exit(2)
    cost = { 'bid' : exchangeinfo[name][unit]['bid']['rate'], 'ask' : exchangeinfo[name][unit]['ask']['rate'] }
    if len(user) >= 6 and float(user[5]) != 0.0:
      cost['bid'] = float(user[5]) / 100.0
      cost['ask'] = float(user[5]) / 100.0
    if len(user) >= 7 and float(user[6]) != 0.0:
      cost['ask'] = float(user[6]) / 100.0
    bot = 'pybot' if len(user) < 8 else user[7]
    users[key][unit] = { 'request' : RequestThread(conn, key, secret, exchange, unit, user[0], sampling, cost, logger) }
    users[key][unit]['request'].start()
    target = { 'bid': exchangeinfo[name][unit]['bid']['target'], 'ask': exchangeinfo[name][unit]['ask']['target'] }
    if bot == 'none':
      users[key][unit]['order'] = None
    elif bot == 'nubot':
      users[key][unit]['order'] = NuBot(conn, users[key][unit]['request'], key, secret, exchange, unit, target, logger)
    elif bot == 'pybot':
      users[key][unit]['order'] = PyBot(conn, users[key][unit]['request'], key, secret, exchange, unit, target, logger)
    else:
      logger.error("unknown order handler: %s", bot)
      users[key][unit]['order'] = None
    if users[key][unit]['order']:
      if users[key][unit]['order']:
        users[key][unit]['order'].start()

logger.debug('starting liquidity propagation with sampling %d' % sampling)
starttime = time.time()
curtime = time.time()
effs = []

while True: # print some info every minute until program terminates
  try:
    time.sleep(max(60 - time.time() + curtime, 0))
    curtime = time.time()
    for user in users: # post some statistics
      response = conn.get(user, trials = 1)
      if 'error' in response:
        logger.error('unable to receive statistics for user %s: %s', user, response['message'])
        users[user].values()[0]['request'].register() # reassure to be registered if
        newstatus = conn.get('status', trials = 3)
        if not 'error' in newstatus:
          basestatus = newstatus
          sampling = min(240, 3 * basestatus['sampling'] / 2)
      else:
        # collect user information
        effective_rate = 0.0
        total = 0.0
        for unit in response['units']:
          for side in [ 'bid', 'ask' ]:
            effective_rate += float(sum([ o['amount'] * o['cost'] for o in response['units'][unit][side] ]))
            total += float(sum([ o['amount'] for o in response['units'][unit][side] ]))
        if total > 0.0: effective_rate /= total
        orderstring = ""
        for unit in response['units']:
          unitstring = ""
          for side in ['bid', 'ask']:
            market = response['units'][unit][side]
            coststring = ""
            for order in response['units'][unit][side]:
              if order['amount'] > 0:
                coststring += " %.4f x %.2f%%," % (order['amount'], order['cost'] * 100.0)
            if len(coststring):
              unitstring += " - %s:%s" % (side, coststring[:-1])
          if len(unitstring):
            orderstring += " - %s%s" % (unit, unitstring)
        # print user information
        logger.info('%s - balance: %.8f rate: %.2f%% ppm: %.8f efficiency: %.2f%% rejects: %d missing: %d%s - %s', repr(users[user].values()[0]['request'].exchange),
          response['balance'], effective_rate * 100, effective_rate * total / float(60 * 24), response['efficiency'] * 100, response['rejects'], response['missing'], orderstring, user)
        if not effs:
          effs = [ response['efficiency'] for i in xrange(5) ]
        if curtime - starttime > 90:
          effs = effs[1:] + [response['efficiency']]
          if sorted(effs)[2] < 0.95:
            for unit in response['units']:
              if response['units'][unit]['rejects'] > 1 and response['units'][unit]['rejects'] / float(basestatus['sampling']) >= 0.05: # look for valid error and adjust nonce shift
                if response['units'][unit]['last_error'] != "":
                  if 'deviates too much from current price' in response['units'][unit]['last_error']:
                    PyBot.pricefeed.price(unit, True) # force a price update
                    if users[user][unit]['order']: users[user][unit]['order'].shutdown()
                    logger.warning('price missmatch for %s on %s, forcing price update', unit, repr(users[user][unit]['request'].exchange))
                  else:
                    shift = users[user][unit]['request'].exchange._shift
                    users[user][unit]['request'].exchange.adjust(response['units'][unit]['last_error'])
                    if shift != users[user][unit]['request'].exchange._shift:
                      logger.warning('too many rejected requests for %s on %s, adjusting nonce shift to %d',
                        unit, repr(users[user][unit]['request'].exchange), users[user][unit]['request'].exchange._shift)
                else:
                  if users[user][unit]['request'].sampling < 2 * sampling: # just send more requests
                    users[user][unit]['request'].sampling = users[user][unit]['request'].sampling + 1
                    logger.warning('increasing sampling to %d',
                      unit, repr(users[user][unit]['request'].exchange), users[user][unit]['request'].sampling)
              if response['units'][unit]['missing'] / float(basestatus['sampling']) >= 0.05: # look for missing error and adjust sampling
                if users[user][unit]['request'].sampling < 2 * sampling: # just send more requests
                  users[user][unit]['request'].sampling = users[user][unit]['request'].sampling + 1
                  logger.warning('too many missing requests for %s on %s, increasing sampling to %d',
                    unit, repr(users[user][unit]['request'].exchange), users[user][unit]['request'].sampling)
                else: # just wait a little bit
                  logger.warning('too many missing requests, sleeping a short while to synchronize')
                  curtime += 0.7

  except KeyboardInterrupt: break
  except Exception as e:
    logger.error('exception caught in main loop: %s', sys.exc_info()[1])

logger.info('stopping trading bots, please allow the client up to 1 minute to terminate')
while True:
  try:
    for user in users:
      for unit in users[user]:
        users[user][unit]['request'].stop()
        if users[user][unit]['order']:
          users[user][unit]['order'].stop()
    for user in users:
      for unit in users[user]:
        users[user][unit]['request'].join()
        if users[user][unit]['order']:
          users[user][unit]['order'].join()
  except KeyboardInterrupt: continue
  break<|endoftext|>


<commit_before>#
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# author: Reza Hosseini

## create a directory if it does not exist
def CreateDir(directory):
  if not os.path.exists(directory):
    os.makedirs(directory)

#### this function creates a latex file skin. i.e. the main document
def LatexSkin(fn, docType='article', figsPath='{figs/}',
              latexInputFn='Input.tex'):
  latex1 = '\\documentclass{' + docType + '}'
  latex2 = ('\n \\usepackage{enumerate,amsmath,graphics,amssymb,graphicx,' +
            'amscd,amscd,amsbsy,multirow,float,booktabs,verbatim,xy,' +
            'geometry,import} \n')
  latex3 = '\n \\graphicspath{' + figsPath + '} \n'
  latex4 = ('\n \\begin{document} \n \\input{' + latexInputFn +
            '} \n \\end{document}')

  with open(fn, 'w') as f:
    f.write(latex1)
    f.write(latex2)
    f.write(latex3)
    f.write(latex4)

#### this function writes the text in the center for a given latex file
def WriteLatex(fn, text):
    with open(fn, 'a') as f:
        f.write('\n' + text + '\n')

### this begins a frame in latex
def BeginFrame(fn, frameTitle=''):
    with open(fn, 'a') as f:
        f.write('\n' + '\\begin{frame}' + '\n')
        f.write('\n' + '\\frametitle{' + frameTitle +'}' + '\n')

### this ends a frame in latex
def EndFrame(fn):
    with open(fn, 'a') as f:
        f.write('\n' + '\\end{frame}' + '\n')

### this function writes the text in the center for a given latex file
def WriteLatexCenter(fn, text):
    with open(fn, 'a') as f:
        f.write('\\begin{center} \n ')
        f.write(text + '\n')
        f.write('\\end{center} \n ')

### this function adds a figure with caption and label to a latex file
def LatexFig(latexFn, figFn, figLabel=None, figCaption=None, scale=str(0.5)):
  with open(latexFn, 'a') as f:
    f.write('\n \\begin{figure}[H] \n')
    f.write('\\centering \n')
    f.write('\\includegraphics[scale=' + scale + ']{' + figFn + '} \n')
    if figCaption is not None:
      f.write('\\caption{' + figCaption + '} \n')
    if figLabel is not None:
      f.write('\\label{' + figLabel + '} \n')
    f.write('\\end{figure} \n')

############################ Creating figures ################
def Pdff(fn, PlotFcn, plotTitle):

  with PdfPages(fn) as pdf:
    plt.figure(figsize=(3, 3))
    PlotFcn()
    plt.title(pltTitle)
    pdf.savefig()  # saves the current figure into a pdf page
    plt.close()

def SaveFig(fn, PlotFcn, plotTitle='', format='png', dpi=100):
  PlotFcn()
  import matplotlib
  if plotTitle != '':
    matplotlib.pylab.title(plotTitle)
  matplotlib.pylab.savefig(fn, format=format, dpi=dpi)

'''def PlotFcn(): pylab.plot(range(7), [3, 1, 4, 1, 5, 9, 2], 'r-o')
PlotFcn()
figFn = ofigs+'fig1.pdf'
plotTitle = 'maast & kashk'
pdff(figFn,PlotFcn,plotTitle)
fn = odir + 'skin.tex'
LatexSkin(fn,docType='article')
fn = odir + 'Input.tex'
with open(fn, 'a') as f: f.write('This is an automatically generated EDA')
fn = odir + 'Input.tex'
LatexFig(fn,'fig1.pdf','figLabel','mofo')'''

### save fig and add to input latex file
def SaveFig_addLatex(figCaption, latexFn, ofigs, figFn='', format='png',
                     dpi=100, skipCaption=False, skipLabel=False):
  import matplotlib

  if figFn == '':
    figFn=figCaption.replace(' ','_')

  figFn = ofigs + figFn + '.png'
  matplotlib.pylab.savefig(figFn, format=format, dpi=dpi)
  LatexFig(latexFn=latexFn, figFn=figFn, figLabel=figFn,
           figCaption=figCaption, skipCaption=skipCaption,
           skipLabel=skipLabel)

def SaveFig_addSlide(figCaption, latexFn, ofigs, figFn='', format='png',
                     dpi=100, skipCaption=False, skipLabel=False):

  import matplotlib
  if figFn == '':
    figFn=figCaption.replace(' ','_')
  figFn = ofigs + figFn + '.png'
  matplotlib.pylab.savefig(figFn,format=format,dpi=dpi)

  with open(latexFn, 'a') as f:
    f.write('\n \\begin{frame} \n')
    f.write('\n \\frametitle{ \n')
    f.write(figCaption)
    f.write('}')

  LatexFig(latexFn=latexFn, figFn=figFn, figLabel=figFn,
           figCaption=figCaption, skipCaption=skipCaption,
           skipLabel=skipLabel)
  with open(latexFn, 'a') as f:
      f.write('\n \\end{frame} \n')

def Excise(filename, start, end):

  with open(filename) as infile, open(filename + ".out666", "w") as outfile:

    for line in infile:
      if line.strip() == start:
        print('zereshk')
        break
      outfile.write(line)

    for line in infile:
      if line.strip() == end:
        print('albaloo!')
        break
    for line in infile:
      outfile.write(line)

  os.remove(filename)
  os.rename(filename + ".out", filename)

def PltCloseStat(stat=True):
  if stat:
    plt.close()

  return None

def SaveFigStat(stat=False, figCaption='test caption'):

  if stat:
    SaveFig_addSlide(
        figCaption=figCaption,
        latexFn=latexInputFn, ofigs=ofigs,
        figFn='', format='png', dpi=200)

  return None

### this function adds a figure with caption and label to a latex file
def DfLatexTable(df):

  table = ''
  n = len(df)
  colNames = df.columns
  top = ' & '.join(colNames)
  table = top + ' \\\\' + '  \n' + '    '
  table = table + '\hline' + '\n'
  table = table + '\hline' + '\n'
  for i in range(n):
    dfRow = df.iloc[i,:]
    tableRow = ' & '.join(dfRow)
    table = table + tableRow + '\\\\' + '\n' + '    '
    table = table + '\hline' + '\n'

  table = table + '\hline' + '\n'

  return table
<commit_msg>Update output.py<commit_after>#
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# author: Reza Hosseini

## create a directory if it does not exist
def CreateDir(directory):
  if not os.path.exists(directory):
    os.makedirs(directory)

#### this function creates a latex file skin. i.e. the main document
def LatexSkin(fn, docType='article', figsPath='{figs/}',
              latexInputFn='Input.tex'):
  latex1 = '\\documentclass{' + docType + '}'
  latex2 = ('\n \\usepackage{enumerate,amsmath,graphics,amssymb,graphicx,' +
            'amscd,amscd,amsbsy,multirow,float,booktabs,verbatim,xy,' +
            'geometry,import} \n')
  latex3 = '\n \\graphicspath{' + figsPath + '} \n'
  latex4 = ('\n \\begin{document} \n \\input{' + latexInputFn +
            '} \n \\end{document}')

  with open(fn, 'w') as f:
    f.write(latex1)
    f.write(latex2)
    f.write(latex3)
    f.write(latex4)

#### this function writes the text in the center for a given latex file
def WriteLatex(fn, text):
    with open(fn, 'a') as f:
        f.write('\n' + text + '\n')

### this begins a frame in latex
def BeginFrame(fn, frameTitle=''):
    with open(fn, 'a') as f:
        f.write('\n' + '\\begin{frame}' + '\n')
        f.write('\n' + '\\frametitle{' + frameTitle +'}' + '\n')

### this ends a frame in latex
def EndFrame(fn):
    with open(fn, 'a') as f:
        f.write('\n' + '\\end{frame}' + '\n')

### this function writes the text in the center for a given latex file
def WriteLatexCenter(fn, text):
    with open(fn, 'a') as f:
        f.write('\\begin{center} \n ')
        f.write(text + '\n')
        f.write('\\end{center} \n ')

### this function adds a figure with caption and label to a latex file
def LatexFig(latexFn, figFn, figLabel=None, figCaption=None, scale=str(0.5)):
  with open(latexFn, 'a') as f:
    f.write('\n \\begin{figure}[H] \n')
    f.write('\\centering \n')
    f.write('\\includegraphics[scale=' + scale + ']{' + figFn + '} \n')
    if figCaption is not None:
      f.write('\\caption{' + figCaption + '} \n')
    if figLabel is not None:
      f.write('\\label{' + figLabel + '} \n')
    f.write('\\end{figure} \n')

############################ Creating figures ################
def Pdff(fn, PlotFcn, plotTitle):

  with PdfPages(fn) as pdf:
    plt.figure(figsize=(3, 3))
    PlotFcn()
    plt.title(pltTitle)
    pdf.savefig()  # saves the current figure into a pdf page
    plt.close()

def SaveFig(fn, PlotFcn, plotTitle='', format='png', dpi=100):
  PlotFcn()
  import matplotlib
  if plotTitle != '':
    matplotlib.pylab.title(plotTitle)
  matplotlib.pylab.savefig(fn, format=format, dpi=dpi)

'''def PlotFcn(): pylab.plot(range(7), [3, 1, 4, 1, 5, 9, 2], 'r-o')
PlotFcn()
figFn = ofigs+'fig1.pdf'
plotTitle = 'maast & kashk'
pdff(figFn,PlotFcn,plotTitle)
fn = odir + 'skin.tex'
LatexSkin(fn,docType='article')
fn = odir + 'Input.tex'
with open(fn, 'a') as f: f.write('This is an automatically generated EDA')
fn = odir + 'Input.tex'
LatexFig(fn,'fig1.pdf','figLabel','mofo')'''

### save fig and add to input latex file
def SaveFig_addLatex(figCaption, latexFn, ofigs, figFn='', format='png',
                     dpi=100, skipCaption=False, skipLabel=False):
  import matplotlib

  if figFn == '':
    figFn=figCaption.replace(' ','_')

  figFn = ofigs + figFn + '.png'
  matplotlib.pylab.savefig(figFn, format=format, dpi=dpi)
  LatexFig(latexFn=latexFn, figFn=figFn, figLabel=figFn,
           figCaption=figCaption, skipCaption=skipCaption,
           skipLabel=skipLabel)

def SaveFig_addSlide(figCaption, latexFn, ofigs, figFn='', format='png',
                     dpi=100, skipCaption=False, skipLabel=False):

  import matplotlib
  if figFn == '':
    figFn=figCaption.replace(' ','_')
  figFn = ofigs + figFn + '.png'
  matplotlib.pylab.savefig(figFn,format=format,dpi=dpi)

  with open(latexFn, 'a') as f:
    f.write('\n \\begin{frame} \n')
    f.write('\n \\frametitle{ \n')
    f.write(figCaption)
    f.write('}')

  LatexFig(latexFn=latexFn, figFn=figFn, figLabel=figFn,
           figCaption=figCaption, skipCaption=skipCaption,
           skipLabel=skipLabel)
  with open(latexFn, 'a') as f:
      f.write('\n \\end{frame} \n')

def Excise(filename, start, end):

  with open(filename) as infile, open(filename + ".out666", "w") as outfile:

    for line in infile:
      if line.strip() == start:
        print('zereshk')
        break
      outfile.write(line)

    for line in infile:
      if line.strip() == end:
        print('albaloo!')
        break
    for line in infile:
      outfile.write(line)

  os.remove(filename)
  os.rename(filename + ".out", filename)

def PltCloseStat(stat=True):
  if stat:
    plt.close()

  return None

def SaveFigStat(stat=False, figCaption='test caption'):

  if stat:
    SaveFig_addSlide(
        figCaption=figCaption,
        latexFn=latexInputFn, ofigs=ofigs,
        figFn='', format='png', dpi=200)

  return None

### this function adds a figure with caption and label to a latex file
def DfLatexTable(df):

  table = ''
  n = len(df)
  colNames = df.columns
  top = ' & '.join(colNames)
  table = top + ' \\\\' + '  \n' + '    '
  table = table + '\hline' + '\n'
  table = table + '\hline' + '\n'
  for i in range(n):
    dfRow = df.iloc[i,:]
    tableRow = ' & '.join(dfRow)
    table = table + tableRow + '\\\\' + '\n' + '    '
    table = table + '\hline' + '\n'

  table = table + '\hline' + '\n'

  return table

## creates a latex file which would create a pdf after running
# the pdf has all the photos found in path/figs/
# the latex files will be created in path with the name latexFn
def ConcatPhotos_viaLatex(path, latexFn):
    figsPath = path + "/figs/"
    files = os.listdir(figsPath)

    fn = path + latexFn
    LatexSkin(fn=fn, docType='article', figsPath='{figs/}', latexInputFn='Input.tex')
    latexFn0 = path + "input.tex"
    for file in files:
        LatexFig(
            latexFn=latexFn0,
            figFn=file,
            figLabel=None, figCaption=None, scale=str(0.5))

"""
path = "/Users/rz13/Dropbox/Reza_Docs/morgage_application2/rent/"
latexFn = "rent_boa_copies.tex"
ConcatPhotos_viaLatex(path=path, latexFn=latexFn)
"""

<|endoftext|>


<commit_before>"""
Node Prep/Release script entry points for jobs ran with `Render.exe -r sw ...`
"""
import os
from shutil import copyfile
from subprocess import call
from ringling.hpc import env
from ringling.hpc.scripts import LOG
ENV = env()

def _setup_node_project():
    LOG.info("Setting up node project directory: %s" % ENV['NODE_PROJECT'])
    if not os.path.isdir(ENV['NODE_PROJECT']):
        os.makedirs(ENV['NODE_PROJECT'])
    src = os.path.join(ENV['PROJECT'],'workspace.mel')
    dst = os.path.join(ENV['NODE_PROJECT'],'workspace.mel')
    copyfile(src,dst)
    LOG.debug("File copied: %s -> %s" % (src,dst))
    LOG.debug("node local workspace:")
    LOG.debug(open(dst).read())

def _cleanup_node_project():
    LOG.info("Cleaning up node project: %s" % ENV['NODE_PROJECT'])
    LOG.info("\tCalculating size...")
    LOG.info("\tThis could take a while...")
    size = sum([os.path.getsize(os.path.join(root,f)) for root,dirs,files in os.walk(ENV['NODE_PROJECT']) for f in files])
    LOG.info("\t%0.1f MB" % (size/(1024*1024.0)))
    call('rmdir /S /Q %s' % ENV['NODE_PROJECT'], shell=True)

def _setup_dirmaps():
    dst = os.path.join(ENV['NODE_PROJECT'],'scripts','userSetup.mel')
    LOG.info("Writing Dirmaps to: %s" % dst)
    if not os.path.isdir(os.path.dirname(dst)):
        os.makedirs(os.path.dirname(dst))
    with open(dst,'wb') as fh:
        fh.write("""
dirmap -m "S:/" "//desmond/spool/";
dirmap -m "{node_project}" "{project}";
dirmap -m "{project_name}//" "{project}/";
dirmap -en on;
""".format(project_name=os.path.basename(ENV['PROJECT'])[-1],node_project=ENV['NODE_PROJECT'].replace('\\', '/'),
           project=ENV['PROJECT'].replace('\\', '/')))
    with open(dst) as fh:
        LOG.debug(fh.read())

def prep():
    _setup_node_project()
    _setup_dirmaps()

def release():
    _cleanup_node_project()
<commit_msg>bringing the render runtime scripts up to speed with the new helper<commit_after>"""
Node Prep/Release script entry points for jobs ran with `Render.exe -r sw ...`
"""
import os
from shutil import copyfile
from subprocess import call
from ringling.hpc import env
from ringling.hpc.scripts import LOG
from ringling.maya.helpers import Path
ENV = env()

def _setup_node_project():
    LOG.info("Setting up node project directory: %s" % ENV['NODE_PROJECT'])
    if not os.path.isdir(ENV['NODE_PROJECT']):
        os.makedirs(ENV['NODE_PROJECT'])
    src = os.path.join(ENV['PROJECT'],'workspace.mel')
    dst = os.path.join(ENV['NODE_PROJECT'],'workspace.mel')
    copyfile(src,dst)
    LOG.debug("File copied: %s -> %s" % (src,dst))
    LOG.debug("node local workspace:")
    LOG.debug(open(dst).read())

def _cleanup_node_project():
    LOG.info("Cleaning up node project: %s" % ENV['NODE_PROJECT'])
    LOG.info("\tCalculating size...")
    LOG.info("\tThis could take a while...")
    size = sum([os.path.getsize(os.path.join(root,f)) for root,dirs,files in os.walk(ENV['NODE_PROJECT']) for f in files])
    LOG.info("\t%0.1f MB" % (size/(1024*1024.0)))
    call('rmdir /S /Q %s' % ENV['NODE_PROJECT'], shell=True)

def _setup_dirmaps():
    dst = os.path.join(ENV['NODE_PROJECT'],'scripts','userSetup.mel')
    LOG.info("Writing Dirmaps to: %s" % dst)
    if not os.path.isdir(os.path.dirname(dst)):
        os.makedirs(os.path.dirname(dst))
    with open(dst,'wb') as fh:
        fh.write("""
dirmap -m "S:/" "//desmond/spool/";
dirmap -m "{node_project}" "{project}";
dirmap -m "{project_name}//" "{project}/";
dirmap -en on;
""".format(project_name=Path(ENV['PROJECT']).name, node_project=ENV['NODE_PROJECT'].replace('\\', '/'),
           project=Path(ENV['PROJECT']).punc))
    with open(dst) as fh:
        LOG.debug(fh.read())

def prep():
    _setup_node_project()
    _setup_dirmaps()

def release():
    _cleanup_node_project()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interface to ElasticSearch for serving search requests
#
# Tuan Tran (ttran@l3s.de)
#

import sys
from conn import connect,close as es_close

QUERY = { "query":{ "multi_match" : {"query": "%s", "fields": [ "text", "title", "contributor" ]}}}

def search(client, term, k):
	'''
	Get the top-k results from ES using default ranking function. The results are cached into
	main memory
	'''
	res = client.search(index='wiki',body= {
		"query": {
		    "multi_match" : {
		        "query": "%s" % term,
		        "fields": [ "text", "title", "contributor" ]
		    }
		}
    })

    if res == None or len(res) == 0: return [];
    for hit in res['hits']['hits']:
    	print("%(title)s %(contributor)s: %(text)s" % hit["_source"])

if __name__ == "__main__":
	client = connect()
	search(client,sys.argv[1])
	es_close(client)

<commit_msg>fix ident typos<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interface to ElasticSearch for serving search requests
#
# Tuan Tran (ttran@l3s.de)
#

import sys
from conn import connect,close as es_close

QUERY = { "query":{ "multi_match" : {"query": "%s", "fields": [ "text", "title", "contributor" ]}}}

def search(client, term, k):
    '''
    Get the top-k results from ES using default ranking function. The results are cached into
    main memory
    '''
    res = client.search(index='wiki',body= {
        "query": {
            "multi_match" : {
                "query": "%s" % term,
                "fields": [ "text", "title", "contributor" ]
            }
        }
    })

    if res == None or len(res) == 0: return [];
    for hit in res['hits']['hits']:
        print("%(title)s %(contributor)s: %(text)s" % hit["_source"])

if __name__ == "__main__":
    client = connect()
    search(client,sys.argv[1])
    es_close(client)

<|endoftext|>


<commit_before>import copy
import json
import os
from pprint import pprint as pp
from pytest import symbols
import requests

requests.packages.urllib3.disable_warnings()

opd = os.path.dirname
DISTRIBUTIONROOT = opd(opd(opd(opd(opd(opd(__file__))))))
del opd
SERVICELIBDIR = os.path.join(DISTRIBUTIONROOT,
                             'devtools',
                             'sample_data',
                             'service_library')
CREATE = json.load(open(os.path.join(SERVICELIBDIR, 'create.json'), 'r'))

def test__common_service_handler(bigip, neutronless_wrappedicontroldriver):
    mgmt_rt = bigip
    nless_wicd = neutronless_wrappedicontroldriver
    print(type(symbols))
    print(symbols.debug)
    print(nless_wicd.hostnames)
    partition_names = [x.name for x in mgmt_rt.tm.sys.folders.get_collection()]
    pp(partition_names)
    copy.deepcopy(CREATE)
    nless_wicd._common_service_handler(CREATE)
    try:
        print('after _common_service_handler: %r' % CREATE['traffic_group'])
    except KeyError:
        pass
    pp(nless_wicd.plugin_rpc.call_args_list)
<commit_msg>Removed common_service_handler test<commit_after><|endoftext|>


<commit_before>"""
Listens events:

forget (string)

    Given string can be task name, remembered field (url, imdb_url) or a title. If given value is a
    task name then everything in that task will be forgotten. With title all learned fields from it and the
    title will be forgotten. With field value only that particular field is forgotten.
"""

import logging
from datetime import datetime, timedelta
from sqlalchemy import Column, Integer, String, DateTime, Unicode, Boolean, asc, or_, select, update, Index
from sqlalchemy.schema import ForeignKey
from sqlalchemy.orm import relation
from flexget.manager import Session
from flexget.event import event
from flexget.plugin import register_plugin, priority, register_parser_option
from flexget import schema
from flexget.utils.sqlalchemy_utils import table_schema, table_add_column
from flexget.utils.imdb import is_imdb_url, extract_id

log = logging.getLogger('seen')
Base = schema.versioned_base('seen', 3)

@schema.upgrade('seen')
def upgrade(ver, session):
    if ver is None:
        log.info('Converting seen imdb_url to imdb_id for seen movies.')
        field_table = table_schema('seen_field', session)
        for row in session.execute(select([field_table.c.id, field_table.c.value], field_table.c.field == 'imdb_url')):
            new_values = {'field': 'imdb_id', 'value': extract_id(row['value'])}
            session.execute(update(field_table, field_table.c.id == row['id'], new_values))
        ver = 1
    if ver == 1:
        field_table = table_schema('seen_field', session)
        log.info('Adding index to seen_field table.')
        Index('ix_seen_field_seen_entry_id', field_table.c.seen_entry_id).create(bind=session.bind)
        ver = 2
    if ver == 2:
        log.info('Adding local column to seen_entry table')
        table_add_column('seen_entry', 'local', Boolean, session, default=False)
        ver = 3

    return ver

class SeenEntry(Base):

    __tablename__ = 'seen_entry'

    id = Column(Integer, primary_key=True)
    title = Column(Unicode)
    reason = Column(Unicode)
    task = Column('feed', Unicode)
    added = Column(DateTime)
    local = Column(Boolean)

    fields = relation('SeenField', backref='seen_entry', cascade='all, delete, delete-orphan')

    def __init__(self, title, task, reason=None, local=False):
        self.title = title
        self.reason = reason
        self.task = task
        self.added = datetime.now()
        self.local = local

    def __str__(self):
        return '<SeenEntry(title=%s,reason=%s,task=%s,added=%s)>' % (self.title, self.reason, self.task, self.added)

class SeenField(Base):

    __tablename__ = 'seen_field'

    id = Column(Integer, primary_key=True)
    seen_entry_id = Column(Integer, ForeignKey('seen_entry.id'), nullable=False, index=True)
    field = Column(Unicode)
    value = Column(Unicode, index=True)
    added = Column(DateTime)

    def __init__(self, field, value):
        self.field = field
        self.value = value
        self.added = datetime.now()

    def __str__(self):
        return '<SeenField(field=%s,value=%s,added=%s)>' % (self.field, self.value, self.added)

@event('forget')
def forget(value):
    """
    See module docstring
    :param string value: Can be task name, entry title or field value
    :return: count, field_count where count is number of entries removed and field_count number of fields
    """
    log.debug('forget called with %s' % value)
    session = Session()

    try:
        count = 0
        field_count = 0
        for se in session.query(SeenEntry).filter(or_(SeenEntry.title == value, SeenEntry.task == value)).all():
            field_count += len(se.fields)
            count += 1
            log.debug('forgetting %s' % se)
            session.delete(se)

        for sf in session.query(SeenField).filter(SeenField.value == value).all():
            se = session.query(SeenEntry).filter(SeenEntry.id == sf.seen_entry_id).first()
            field_count += len(se.fields)
            count += 1
            log.debug('forgetting %s' % se)
            session.delete(se)
        return count, field_count
    finally:
        session.commit()
        session.close()

class MigrateSeen(object):

    def migrate2(self):
        session = Session()

        try:
            from progressbar import ProgressBar, Percentage, Bar, ETA
        except:
            print 'Critical: progressbar library not found, try running `bin/easy_install progressbar` ?'
            return

        class Seen(Base):

            __tablename__ = 'seen'

            id = Column(Integer, primary_key=True)
            field = Column(String)
            value = Column(String, index=True)
            task = Column('feed', String)
            added = Column(DateTime)

            def __init__(self, field, value, task):
                self.field = field
                self.value = value
                self.task = task
                self.added = datetime.now()

            def __str__(self):
                return '<Seen(%s=%s)>' % (self.field, self.value)

        print ''

        # REPAIR / REMOVE DUPLICATES
        index = 0
        removed = 0
        total = session.query(Seen).count() + 1

        widgets = ['Repairing - ', ETA(), ' ', Percentage(), ' ', Bar(left='[', right=']')]
        bar = ProgressBar(widgets=widgets, maxval=total).start()

        for seen in session.query(Seen).all():
            index += 1
            if index % 10 == 0:
                bar.update(index)
            amount = 0
            for dupe in session.query(Seen).filter(Seen.value == seen.value):
                amount += 1
                if amount > 1:
                    removed += 1
                    session.delete(dupe)
        bar.finish()

        # MIGRATE
        total = session.query(Seen).count() + 1
        widgets = ['Upgrading - ', ETA(), ' ', Percentage(), ' ', Bar(left='[', right=']')]
        bar = ProgressBar(widgets=widgets, maxval=total).start()

        index = 0
        for seen in session.query(Seen).all():
            index += 1
            if not index % 10:
                bar.update(index)
            se = SeenEntry(u'N/A', seen.task, u'migrated')
            se.added = seen.added
            se.fields.append(SeenField(seen.field, seen.value))
            session.add(se)
        bar.finish()

        session.execute('drop table seen;')
        session.commit()

    def on_process_start(self, task):
        # migrate seen to seen_entry
        session = Session()
        from flexget.utils.sqlalchemy_utils import table_exists
        if table_exists('seen', session):
            self.migrate2()
        session.close()

class SeenSearch(object):

    def on_process_start(self, task):
        if not task.manager.options.seen_search:
            return

        task.manager.disable_tasks()

        session = Session()
        shown = []
        for field in session.query(SeenField).\
            filter(SeenField.value.like(unicode('%' + task.manager.options.seen_search + '%'))).\
            order_by(asc(SeenField.added)).all():

            se = session.query(SeenEntry).filter(SeenEntry.id == field.seen_entry_id).first()
            if not se:
                print 'ERROR: <SeenEntry(id=%s)> missing' % field.seen_entry_id
                continue

            # don't show duplicates
            if se.id in shown:
                continue
            shown.append(se.id)

            print 'ID: %s Name: %s Task: %s Added: %s' % (se.id, se.title, se.task, se.added.strftime('%c'))
            for sf in se.fields:
                print ' %s: %s' % (sf.field, sf.value)
            print ''

        if not shown:
            print 'No results'

        session.close()

class SeenForget(object):

    def on_process_start(self, task):
        if not task.manager.options.forget:
            return

        task.manager.disable_tasks()

        forget_name = task.manager.options.forget
        if is_imdb_url(forget_name):
            imdb_id = extract_id(forget_name)
            if imdb_id:
                forget_name = imdb_id

        count, fcount = forget(forget_name)
        log.info('Removed %s titles (%s fields)' % (count, fcount))
        task.manager.config_changed()

class SeenCmd(object):

    def on_process_start(self, task):
        if not task.manager.options.seen:
            return

        task.manager.disable_tasks()

        seen_name = task.manager.options.seen
        if is_imdb_url(seen_name):
            imdb_id = extract_id(seen_name)
            if imdb_id:
                seen_name = imdb_id

        session = Session()
        se = SeenEntry(u'--seen', unicode(task.name))
        sf = SeenField(u'--seen', seen_name)
        se.fields.append(sf)
        session.add(se)
        session.commit()

        log.info('Added %s as seen. This will affect all tasks.' % seen_name)

class FilterSeen(object):
    """
        Remembers previously downloaded content and rejects them in
        subsequent executions. Without this plugin FlexGet would
        download all matching content on every execution.

        This plugin is enabled on all tasks by default.
        See wiki for more information.
    """

    def __init__(self):
        # remember and filter by these fields
        self.fields = ['title', 'url', 'original_url']
        self.keyword = 'seen'

    def validator(self):
        from flexget import validator
        root = validator.factory()
        root.accept('boolean')
        root.accept('choice').accept_choices(['global', 'local'])
        return root

    @priority(255)
    def on_task_filter(self, task, config, remember_rejected=False):
        """Filter seen entries"""
        if config is False:
            log.debug('%s is disabled' % self.keyword)
            return

        fields = self.fields
        local = config == 'local'

        for entry in task.entries:
            # construct list of values looked
            values = []
            for field in fields:
                if field not in entry:
                    continue
                if entry[field] not in values and entry[field]:
                    values.append(unicode(entry[field]))
            if values:
                log.trace('querying for: %s' % ', '.join(values))
                # check if SeenField.value is any of the values
                found = task.session.query(SeenField).join(SeenEntry).filter(SeenField.value.in_(values))
                if local:
                    found = found.filter(SeenEntry.task == task.name)
                else:
                    found = found.filter(SeenEntry.local == False)
                found = found.first()
                if found:
                    log.debug("Rejecting '%s' '%s' because of seen '%s'" % (entry['url'], entry['title'], found.value))
                    task.reject(entry, 'Entry with %s `%s` is already seen' % (found.field, found.value),
                                remember=remember_rejected)

    def on_task_exit(self, task, config):
        """Remember succeeded entries"""
        if config is False:
            log.debug('disabled')
            return

        fields = self.fields
        if isinstance(config, list):
            fields.extend(config)

        for entry in task.accepted:
            self.learn(task, entry, fields=fields, local=config == 'local')
            # verbose if in learning mode
            if task.manager.options.learn:
                log.info("Learned '%s' (will skip this in the future)" % (entry['title']))

    def learn(self, task, entry, fields=None, reason=None, local=False):
        """Marks entry as seen"""
        # no explicit fields given, use default
        if not fields:
            fields = self.fields
        se = SeenEntry(entry['title'], unicode(task.name), reason, local)
        remembered = []
        for field in fields:
            if not field in entry:
                continue
            # removes duplicate values (eg. url, original_url are usually same)
            if entry[field] in remembered:
                continue
            remembered.append(entry[field])
            sf = SeenField(unicode(field), unicode(entry[field]))
            se.fields.append(sf)
            log.debug("Learned '%s' (field: %s)" % (entry[field], field))
        # Only add the entry to the session if it has one of the required fields
        if se.fields:
            task.session.add(se)

    def forget(self, task, title):
        """Forget SeenEntry with :title:. Return True if forgotten."""
        se = task.session.query(SeenEntry).filter(SeenEntry.title == title).first()
        if se:
            log.debug("Forgotten '%s' (%s fields)" % (title, len(se.fields)))
            task.session.delete(se)
            return True

@event('manager.db_cleanup')
def db_cleanup(session):
    log.debug('TODO: Disabled because of ticket #1321')
    return

    # Remove seen fields over a year old
    result = session.query(SeenField).filter(SeenField.added < datetime.now() - timedelta(days=365)).delete()
    if result:
        log.verbose('Removed %d seen fields older than 1 year.' % result)

register_plugin(FilterSeen, 'seen', builtin=True, api_ver=2)
register_plugin(SeenSearch, '--seen-search', builtin=True)
register_plugin(SeenCmd, '--seen', builtin=True)
register_plugin(SeenForget, '--forget', builtin=True)
register_plugin(MigrateSeen, 'migrate_seen', builtin=True)

register_parser_option('--forget', action='store', dest='forget', default=False,
                       metavar='TASK|VALUE', help='Forget task (completely) or given title or url.')
register_parser_option('--seen', action='store', dest='seen', default=False,
                       metavar='VALUE', help='Add title or url to what has been seen in tasks.')
register_parser_option('--seen-search', action='store', dest='seen_search', default=False,
                       metavar='VALUE', help='Search given text from seen database.')
<commit_msg>Fix an issue with seen plugin db migration.<commit_after>"""
Listens events:

forget (string)

    Given string can be task name, remembered field (url, imdb_url) or a title. If given value is a
    task name then everything in that task will be forgotten. With title all learned fields from it and the
    title will be forgotten. With field value only that particular field is forgotten.
"""

import logging
from datetime import datetime, timedelta
from sqlalchemy import Column, Integer, String, DateTime, Unicode, Boolean, asc, or_, select, update, Index
from sqlalchemy.schema import ForeignKey
from sqlalchemy.orm import relation
from flexget.manager import Session
from flexget.event import event
from flexget.plugin import register_plugin, priority, register_parser_option
from flexget import schema
from flexget.utils.sqlalchemy_utils import table_schema, table_add_column
from flexget.utils.imdb import is_imdb_url, extract_id

log = logging.getLogger('seen')
Base = schema.versioned_base('seen', 4)

@schema.upgrade('seen')
def upgrade(ver, session):
    if ver is None:
        log.info('Converting seen imdb_url to imdb_id for seen movies.')
        field_table = table_schema('seen_field', session)
        for row in session.execute(select([field_table.c.id, field_table.c.value], field_table.c.field == 'imdb_url')):
            new_values = {'field': 'imdb_id', 'value': extract_id(row['value'])}
            session.execute(update(field_table, field_table.c.id == row['id'], new_values))
        ver = 1
    if ver == 1:
        field_table = table_schema('seen_field', session)
        log.info('Adding index to seen_field table.')
        Index('ix_seen_field_seen_entry_id', field_table.c.seen_entry_id).create(bind=session.bind)
        ver = 2
    if ver == 2:
        log.info('Adding local column to seen_entry table')
        table_add_column('seen_entry', 'local', Boolean, session, default=False)
        ver = 3
    if ver == 3:
        # setting the default to False in the last migration was broken, fix the data
        log.info('Repairing seen table')
        entry_table = table_schema('seen_entry', session)
        session.execute(update(entry_table, entry_table.c.local == None, {'local': False}))
        ver = 4

    return ver

class SeenEntry(Base):

    __tablename__ = 'seen_entry'

    id = Column(Integer, primary_key=True)
    title = Column(Unicode)
    reason = Column(Unicode)
    task = Column('feed', Unicode)
    added = Column(DateTime)
    local = Column(Boolean)

    fields = relation('SeenField', backref='seen_entry', cascade='all, delete, delete-orphan')

    def __init__(self, title, task, reason=None, local=False):
        self.title = title
        self.reason = reason
        self.task = task
        self.added = datetime.now()
        self.local = local

    def __str__(self):
        return '<SeenEntry(title=%s,reason=%s,task=%s,added=%s)>' % (self.title, self.reason, self.task, self.added)

class SeenField(Base):

    __tablename__ = 'seen_field'

    id = Column(Integer, primary_key=True)
    seen_entry_id = Column(Integer, ForeignKey('seen_entry.id'), nullable=False, index=True)
    field = Column(Unicode)
    value = Column(Unicode, index=True)
    added = Column(DateTime)

    def __init__(self, field, value):
        self.field = field
        self.value = value
        self.added = datetime.now()

    def __str__(self):
        return '<SeenField(field=%s,value=%s,added=%s)>' % (self.field, self.value, self.added)

@event('forget')
def forget(value):
    """
    See module docstring
    :param string value: Can be task name, entry title or field value
    :return: count, field_count where count is number of entries removed and field_count number of fields
    """
    log.debug('forget called with %s' % value)
    session = Session()

    try:
        count = 0
        field_count = 0
        for se in session.query(SeenEntry).filter(or_(SeenEntry.title == value, SeenEntry.task == value)).all():
            field_count += len(se.fields)
            count += 1
            log.debug('forgetting %s' % se)
            session.delete(se)

        for sf in session.query(SeenField).filter(SeenField.value == value).all():
            se = session.query(SeenEntry).filter(SeenEntry.id == sf.seen_entry_id).first()
            field_count += len(se.fields)
            count += 1
            log.debug('forgetting %s' % se)
            session.delete(se)
        return count, field_count
    finally:
        session.commit()
        session.close()

class MigrateSeen(object):

    def migrate2(self):
        session = Session()

        try:
            from progressbar import ProgressBar, Percentage, Bar, ETA
        except:
            print 'Critical: progressbar library not found, try running `bin/easy_install progressbar` ?'
            return

        class Seen(Base):

            __tablename__ = 'seen'

            id = Column(Integer, primary_key=True)
            field = Column(String)
            value = Column(String, index=True)
            task = Column('feed', String)
            added = Column(DateTime)

            def __init__(self, field, value, task):
                self.field = field
                self.value = value
                self.task = task
                self.added = datetime.now()

            def __str__(self):
                return '<Seen(%s=%s)>' % (self.field, self.value)

        print ''

        # REPAIR / REMOVE DUPLICATES
        index = 0
        removed = 0
        total = session.query(Seen).count() + 1

        widgets = ['Repairing - ', ETA(), ' ', Percentage(), ' ', Bar(left='[', right=']')]
        bar = ProgressBar(widgets=widgets, maxval=total).start()

        for seen in session.query(Seen).all():
            index += 1
            if index % 10 == 0:
                bar.update(index)
            amount = 0
            for dupe in session.query(Seen).filter(Seen.value == seen.value):
                amount += 1
                if amount > 1:
                    removed += 1
                    session.delete(dupe)
        bar.finish()

        # MIGRATE
        total = session.query(Seen).count() + 1
        widgets = ['Upgrading - ', ETA(), ' ', Percentage(), ' ', Bar(left='[', right=']')]
        bar = ProgressBar(widgets=widgets, maxval=total).start()

        index = 0
        for seen in session.query(Seen).all():
            index += 1
            if not index % 10:
                bar.update(index)
            se = SeenEntry(u'N/A', seen.task, u'migrated')
            se.added = seen.added
            se.fields.append(SeenField(seen.field, seen.value))
            session.add(se)
        bar.finish()

        session.execute('drop table seen;')
        session.commit()

    def on_process_start(self, task):
        # migrate seen to seen_entry
        session = Session()
        from flexget.utils.sqlalchemy_utils import table_exists
        if table_exists('seen', session):
            self.migrate2()
        session.close()

class SeenSearch(object):

    def on_process_start(self, task):
        if not task.manager.options.seen_search:
            return

        task.manager.disable_tasks()

        session = Session()
        shown = []
        for field in session.query(SeenField).\
            filter(SeenField.value.like(unicode('%' + task.manager.options.seen_search + '%'))).\
            order_by(asc(SeenField.added)).all():

            se = session.query(SeenEntry).filter(SeenEntry.id == field.seen_entry_id).first()
            if not se:
                print 'ERROR: <SeenEntry(id=%s)> missing' % field.seen_entry_id
                continue

            # don't show duplicates
            if se.id in shown:
                continue
            shown.append(se.id)

            print 'ID: %s Name: %s Task: %s Added: %s' % (se.id, se.title, se.task, se.added.strftime('%c'))
            for sf in se.fields:
                print ' %s: %s' % (sf.field, sf.value)
            print ''

        if not shown:
            print 'No results'

        session.close()

class SeenForget(object):

    def on_process_start(self, task):
        if not task.manager.options.forget:
            return

        task.manager.disable_tasks()

        forget_name = task.manager.options.forget
        if is_imdb_url(forget_name):
            imdb_id = extract_id(forget_name)
            if imdb_id:
                forget_name = imdb_id

        count, fcount = forget(forget_name)
        log.info('Removed %s titles (%s fields)' % (count, fcount))
        task.manager.config_changed()

class SeenCmd(object):

    def on_process_start(self, task):
        if not task.manager.options.seen:
            return

        task.manager.disable_tasks()

        seen_name = task.manager.options.seen
        if is_imdb_url(seen_name):
            imdb_id = extract_id(seen_name)
            if imdb_id:
                seen_name = imdb_id

        session = Session()
        se = SeenEntry(u'--seen', unicode(task.name))
        sf = SeenField(u'--seen', seen_name)
        se.fields.append(sf)
        session.add(se)
        session.commit()

        log.info('Added %s as seen. This will affect all tasks.' % seen_name)

class FilterSeen(object):
    """
        Remembers previously downloaded content and rejects them in
        subsequent executions. Without this plugin FlexGet would
        download all matching content on every execution.

        This plugin is enabled on all tasks by default.
        See wiki for more information.
    """

    def __init__(self):
        # remember and filter by these fields
        self.fields = ['title', 'url', 'original_url']
        self.keyword = 'seen'

    def validator(self):
        from flexget import validator
        root = validator.factory()
        root.accept('boolean')
        root.accept('choice').accept_choices(['global', 'local'])
        return root

    @priority(255)
    def on_task_filter(self, task, config, remember_rejected=False):
        """Filter seen entries"""
        if config is False:
            log.debug('%s is disabled' % self.keyword)
            return

        fields = self.fields
        local = config == 'local'

        for entry in task.entries:
            # construct list of values looked
            values = []
            for field in fields:
                if field not in entry:
                    continue
                if entry[field] not in values and entry[field]:
                    values.append(unicode(entry[field]))
            if values:
                log.trace('querying for: %s' % ', '.join(values))
                # check if SeenField.value is any of the values
                found = task.session.query(SeenField).join(SeenEntry).filter(SeenField.value.in_(values))
                if local:
                    found = found.filter(SeenEntry.task == task.name)
                else:
                    found = found.filter(SeenEntry.local == False)
                found = found.first()
                if found:
                    log.debug("Rejecting '%s' '%s' because of seen '%s'" % (entry['url'], entry['title'], found.value))
                    task.reject(entry, 'Entry with %s `%s` is already seen' % (found.field, found.value),
                                remember=remember_rejected)

    def on_task_exit(self, task, config):
        """Remember succeeded entries"""
        if config is False:
            log.debug('disabled')
            return

        fields = self.fields
        if isinstance(config, list):
            fields.extend(config)

        for entry in task.accepted:
            self.learn(task, entry, fields=fields, local=config == 'local')
            # verbose if in learning mode
            if task.manager.options.learn:
                log.info("Learned '%s' (will skip this in the future)" % (entry['title']))

    def learn(self, task, entry, fields=None, reason=None, local=False):
        """Marks entry as seen"""
        # no explicit fields given, use default
        if not fields:
            fields = self.fields
        se = SeenEntry(entry['title'], unicode(task.name), reason, local)
        remembered = []
        for field in fields:
            if not field in entry:
                continue
            # removes duplicate values (eg. url, original_url are usually same)
            if entry[field] in remembered:
                continue
            remembered.append(entry[field])
            sf = SeenField(unicode(field), unicode(entry[field]))
            se.fields.append(sf)
            log.debug("Learned '%s' (field: %s)" % (entry[field], field))
        # Only add the entry to the session if it has one of the required fields
        if se.fields:
            task.session.add(se)

    def forget(self, task, title):
        """Forget SeenEntry with :title:. Return True if forgotten."""
        se = task.session.query(SeenEntry).filter(SeenEntry.title == title).first()
        if se:
            log.debug("Forgotten '%s' (%s fields)" % (title, len(se.fields)))
            task.session.delete(se)
            return True

@event('manager.db_cleanup')
def db_cleanup(session):
    log.debug('TODO: Disabled because of ticket #1321')
    return

    # Remove seen fields over a year old
    result = session.query(SeenField).filter(SeenField.added < datetime.now() - timedelta(days=365)).delete()
    if result:
        log.verbose('Removed %d seen fields older than 1 year.' % result)

register_plugin(FilterSeen, 'seen', builtin=True, api_ver=2)
register_plugin(SeenSearch, '--seen-search', builtin=True)
register_plugin(SeenCmd, '--seen', builtin=True)
register_plugin(SeenForget, '--forget', builtin=True)
register_plugin(MigrateSeen, 'migrate_seen', builtin=True)

register_parser_option('--forget', action='store', dest='forget', default=False,
                       metavar='TASK|VALUE', help='Forget task (completely) or given title or url.')
register_parser_option('--seen', action='store', dest='seen', default=False,
                       metavar='VALUE', help='Add title or url to what has been seen in tasks.')
register_parser_option('--seen-search', action='store', dest='seen_search', default=False,
                       metavar='VALUE', help='Search given text from seen database.')
<|endoftext|>