<commit_before>#!/usr/bin/python
import setproctitle as SP
import ConfigParser
import os
import time
import requests
import re
import logging
import sys
import socket
import signal
import threading
import resource, psutil
__version__ = os.popen("git describe").read().strip()
source_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

#
# This is needed to force ipv4 on ipv6 devices. It's sometimes needed
# if there isn't a clean ipv6 route to get to the big wild internet.
# In these cases, a pure ipv6 route simply will not work.  People aren't
# always in full control of every hop ... so it's much safer to force
# ipv4 then optimistically cross our fingers.
#
origGetAddrInfo = socket.getaddrinfo

def getAddrInfoWrapper(host, port, family=0, socktype=0, proto=0, flags=0):
  attempts = 1
  max_attempts = 10

  while attempts < max_attempts:
    try:
      res = origGetAddrInfo(host, port, socket.AF_INET, socktype, proto, flags)
      return res

    except:
      logging.warn( "[%d/%d] Unable to resolve %s on %d ... sleeping a bit" % (attempts, max_attempts, host, port))
      time.sleep(1)
      attempts += 1

  # If we have tried this a few times and nothing happens, then we just bail
  raise Exception

# Replace the original socket.getaddrinfo by our version
socket.getaddrinfo = getAddrInfoWrapper

import urllib2

import ts as TS
import db as DB
import cloud
from Queue import Queue
from threading import Lock
#from multiprocessing import Queue, Lock

#
# The process delay is used throughout to measure things like the delay in
# forking a subprocesses, waiting for DNS, and then starting a stream or
# waiting for all the sub-processes like the web-server to clean up and free
# the tcp port they are listening on, and shut down.
#
# Making this generous shouldn't be discouraged as it is also used as a metric
# to calculate the number of accomodations that are to be given to make service
# continuous.
#
# Things are specified in multiples of this value ... for instance PROCESS_DELAY
# / 4 or * 2.  4 is a good number.
#
PROCESS_DELAY = 4

#
# Maintain a pidfile for the manager and the webserver (which
# likes to become a zombie ... braaaainnns!) so we have to take
# care of it separately and specially - like a little retard.
#

DIR_BACKUPS = 'backups'
DIR_STREAMS = 'streams'
DIR_SLICES = 'slices'

IS_TEST = True

manager_pid = 0
queue = Queue()

params = {'shutdown_time': None}
start_time = None
config = {}
pid_map = {}
lockMap = {'prune': Lock(), 'main': Lock()}
last_official_query = None

def do_nothing(signal, frame=None):
  # Catches signals that we would rather just ignore
  return True

def base_stats():
  # Reports base-level statistical information about the health of the server.
  # This is used for the /stats and /heartbeat call.
  try:
    # for some reason this can lead to a memory error
    load = [float(unit) for unit in os.popen("uptime | awk -F : ' { print $NF } '").read().split(', ')]

  except:
    load = 0

  return {
    'uptime': TS.uptime(),
    'last_recorded': float(DB.get('last_recorded', use_cache=False) or 0),
    'now': time.time(),
    'now-human': TS.ts_to_name(),
    'version': __version__,
    'load': load,
    'files': psutil.Process().open_files(),
    'mem': [
      # Current memory footprint
      psutil.Process(os.getpid()).memory_info().rss,

      # Maximum lifetime
      resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024
    ],
    'tlist': [ thread.name for thread in threading.enumerate() ],
    'disk': cloud.size('.') / (1024.0 ** 3)
  }

def mail_config(parser):
  cfg = os.environ.get('CLOUD_CFG')

  parser.add_argument("-c", "--config", default=cfg, help="cloud credential file to use")
  args = parser.parse_args()

  if args.config is None:
    sys.stderr.write("Define the cloud configuration location with the CLOUD_CFG environment variable or using the -c option\n")
    return None

  cloud_config = ConfigParser.ConfigParser()
  cloud_config.read(args.config)

  return config_section_map('Mailgun', cloud_config)

# Taken from https://bradgignac.com/2014/05/12/sending-email-with-python-and-the-mailgun-api.html
def send_email(config, who, subject, body, sender='Indycast Reminders <reminders@indycast.net>'):
  key = config['base_key']
  request_url = "%s/%s" % (config['base_url'].strip('/'), 'messages')

  request = requests.post(request_url, auth=('api', key), data={
    'from': sender,
    'to': who,
    'subject': subject,
    'text': re.sub('<[^<]+?>', '', body),
    'html': body
  })

  return request

def am_i_official():
  # Takes the callsign and port and queries the server for its per-instance uuid
  # If those values match our uuid then we claim that we are the official instance
  # and can do various privileged things.  Otherwise, we try not to intrude.
  global config, last_official_query

  # Don't cache a true value ... see https://github.com/kristopolous/DRR/issues/84 for details
  # Actually we'll cache it for a few seconds.
  if 'official' not in config or (config['official'] and (not last_official_query or last_official_query + 10 < time.time())):
    endpoint = "http://%s.indycast.net:%d/uuid" % (config['callsign'], config['port'])

    try:
      stream = urllib2.urlopen(endpoint)
      data = stream.read()
      config['official'] = (data.strip() == config['uuid'])
      last_official_query = time.time()

    except:
      # We can't contact the server so we just return false
      # and set nothing
      return False

  return config['official']

def webserver_shutdown():
  endpoint = "http://127.0.0.1:%d/halt" % (config['port'],)

  try:
    stream = urllib2.urlopen(endpoint)
    data = stream.read()

  except urllib2.URLError:
    logging.info("Webserver already shutdown")

def public_config():
  # Returns a configuration, removing sensitive information
  global config
  return {k: v for k, v in config.items() if k != '_private'}

def shutdown_handler(signal=signal.SIGINT, frame=None):
  # shutdown_handler is hit on the keyboard interrupt
  shutdown()

def shutdown_real(do_restart=False):
  # During a restart shutdown we just kill the webserver.
  # The other processes will die off later.

  logging.info('total: %d' % (threading.activeCount(), ))
  # we release our main lock which signifies that the
  # threads should die off.
  lockMap['main'].release()
  logging.info("Releasing main lock")

  """
  for x in range(50):
    logging.info('total: %d' % (threading.activeCount(), ))
    time.sleep(0.25)
  """

  if not do_restart:
    for key, value in pid_map.items():
      try:
        value.terminate()
      except:
        pass

    webserver_shutdown()
    logging.info("Shutting down")

    logging.info("Uptime: %ds", TS.uptime())

    #if os.path.isfile(PIDFILE_MANAGER):
    #  os.unlink(PIDFILE_MANAGER)

    sys.exit(0)

def shutdown(do_restart=False):
  # All shutdown should be instantiated from the manager thread
  # Make sure that all shutdown happens from the manager
  # thread
  #
  # All we do it put it in our queue ... and then trust the that
  # queue will call the real shutdown
  if do_restart:
    queue.put(('restart', True))

  else:
    queue.put(('shutdown', True))

  return None

def manager_is_running(pid=None):
  if pid is not None:
    lockMap['main'].acquire(False)

  if lockMap['main'].acquire(False):
    lockMap['main'].release()
    return False

  return True

# From https://wiki.python.org/moin/ConfigParserExamples
def config_section_map(section, Config):
  # Takes a section in a config file and makes a dictionary
  # out of it.
  # Returns that dictionary.
  dict1 = {}

  try:
    options = Config.options(section)

  except ConfigParser.NoSectionError as exc:
    return None

  for option in options:
    try:
      dict1[option] = Config.get(section, option)
      if dict1[option] == -1:
        logging.info("skip: %s" % option)

    except Exception as exc:
      logging.warning("exception on %s!" % option)
      dict1[option] = None

  return dict1
<commit_msg>WS cleanup<commit_after>#!/usr/bin/python
import setproctitle as SP
import ConfigParser
import os
import time
import requests
import re
import logging
import sys
import socket
import signal
import threading
import resource, psutil
__version__ = os.popen("git describe").read().strip()
source_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

#
# This is needed to force ipv4 on ipv6 devices. It's sometimes needed
# if there isn't a clean ipv6 route to get to the big wild internet.
# In these cases, a pure ipv6 route simply will not work.  People aren't
# always in full control of every hop ... so it's much safer to force
# ipv4 then optimistically cross our fingers.
#
origGetAddrInfo = socket.getaddrinfo

def getAddrInfoWrapper(host, port, family=0, socktype=0, proto=0, flags=0):
  attempts = 1
  max_attempts = 10

  while attempts < max_attempts:
    try:
      res = origGetAddrInfo(host, port, socket.AF_INET, socktype, proto, flags)
      return res

    except:
      logging.warn( "[%d/%d] Unable to resolve %s on %d ... sleeping a bit" % (attempts, max_attempts, host, port))
      time.sleep(1)
      attempts += 1

  # If we have tried this a few times and nothing happens, then we just bail
  raise Exception

# Replace the original socket.getaddrinfo by our version
socket.getaddrinfo = getAddrInfoWrapper

import urllib2

import ts as TS
import db as DB
import cloud
from Queue import Queue
from threading import Lock
#from multiprocessing import Queue, Lock

#
# The process delay is used throughout to measure things like the delay in
# forking a subprocesses, waiting for DNS, and then starting a stream or
# waiting for all the sub-processes like the web-server to clean up and free
# the tcp port they are listening on, and shut down.
#
# Making this generous shouldn't be discouraged as it is also used as a metric
# to calculate the number of accomodations that are to be given to make service
# continuous.
#
# Things are specified in multiples of this value ... for instance PROCESS_DELAY
# / 4 or * 2.  4 is a good number.
#
PROCESS_DELAY = 4

#
# Maintain a pidfile for the manager and the webserver (which
# likes to become a zombie ... braaaainnns!) so we have to take
# care of it separately and specially - like a little retard.
#

DIR_BACKUPS = 'backups'
DIR_STREAMS = 'streams'
DIR_SLICES = 'slices'

IS_TEST = True

manager_pid = 0
queue = Queue()

params = {'shutdown_time': None}
start_time = None
config = {}
pid_map = {}
lockMap = {'prune': Lock(), 'main': Lock()}
last_official_query = None

def do_nothing(signal, frame=None):
  # Catches signals that we would rather just ignore
  return True

def base_stats():
  # Reports base-level statistical information about the health of the server.
  # This is used for the /stats and /heartbeat call.
  try:
    # for some reason this can lead to a memory error
    load = [float(unit) for unit in os.popen("uptime | awk -F : ' { print $NF } '").read().split(', ')]

  except:
    load = 0

  return {
    'uptime': TS.uptime(),
    'last_recorded': float(DB.get('last_recorded', use_cache=False) or 0),
    'now': time.time(),
    'now-human': TS.ts_to_name(),
    'version': __version__,
    'load': load,
    'files': psutil.Process().open_files(),
    'mem': [
      # Current memory footprint
      psutil.Process(os.getpid()).memory_info().rss,

      # Maximum lifetime
      resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024
    ],
    'tlist': [ thread.name for thread in threading.enumerate() ],
    'disk': cloud.size('.') / (1024.0 ** 3)
  }

def mail_config(parser):
  cfg = os.environ.get('CLOUD_CFG')

  parser.add_argument("-c", "--config", default=cfg, help="cloud credential file to use")
  args = parser.parse_args()

  if args.config is None:
    sys.stderr.write("Define the cloud configuration location with the CLOUD_CFG environment variable or using the -c option\n")
    return None

  cloud_config = ConfigParser.ConfigParser()
  cloud_config.read(args.config)

  return config_section_map('Mailgun', cloud_config)

# Taken from https://bradgignac.com/2014/05/12/sending-email-with-python-and-the-mailgun-api.html
def send_email(config, who, subject, body, sender='Indycast Reminders <reminders@indycast.net>'):
  key = config['base_key']
  request_url = "%s/%s" % (config['base_url'].strip('/'), 'messages')

  request = requests.post(request_url, auth=('api', key), data={
    'from': sender,
    'to': who,
    'subject': subject,
    'text': re.sub('<[^<]+?>', '', body),
    'html': body
  })

  return request

def am_i_official():
  # Takes the callsign and port and queries the server for its per-instance uuid
  # If those values match our uuid then we claim that we are the official instance
  # and can do various privileged things.  Otherwise, we try not to intrude.
  global config, last_official_query

  # Don't cache a true value ... see https://github.com/kristopolous/DRR/issues/84 for details
  # Actually we'll cache it for a few seconds.
  if 'official' not in config or (config['official'] and (not last_official_query or last_official_query + 10 < time.time())):
    endpoint = "http://%s.indycast.net:%d/uuid" % (config['callsign'], config['port'])

    try:
      stream = urllib2.urlopen(endpoint)
      data = stream.read()
      config['official'] = (data.strip() == config['uuid'])
      last_official_query = time.time()

    except:
      # We can't contact the server so we just return false
      # and set nothing
      return False

  return config['official']

def webserver_shutdown():
  endpoint = "http://127.0.0.1:%d/halt" % (config['port'],)

  try:
    stream = urllib2.urlopen(endpoint)
    data = stream.read()

  except urllib2.URLError:
    logging.info("Webserver already shutdown")

def public_config():
  # Returns a configuration, removing sensitive information
  global config
  return {k: v for k, v in config.items() if k != '_private'}

def shutdown_handler(signal=signal.SIGINT, frame=None):
  # shutdown_handler is hit on the keyboard interrupt
  shutdown()

def shutdown_real(do_restart=False):
  # During a restart shutdown we just kill the webserver.
  # The other processes will die off later.

  logging.info('total: %d' % (threading.activeCount(), ))
  # we release our main lock which signifies that the
  # threads should die off.
  lockMap['main'].release()
  logging.info("Releasing main lock")

  """
  for x in range(50):
    logging.info('total: %d' % (threading.activeCount(), ))
    time.sleep(0.25)
  """

  if not do_restart:
    for key, value in pid_map.items():
      try:
        value.terminate()
      except:
        pass

    webserver_shutdown()
    logging.info("Shutting down")
    logging.info("Uptime: %ds", TS.uptime())

    #if os.path.isfile(PIDFILE_MANAGER):
    #  os.unlink(PIDFILE_MANAGER)

    sys.exit(0)

def shutdown(do_restart=False):
  # All shutdown should be instantiated from the manager thread
  # Make sure that all shutdown happens from the manager
  # thread
  #
  # All we do it put it in our queue ... and then trust the that
  # queue will call the real shutdown
  if do_restart:
    queue.put(('restart', True))

  else:
    queue.put(('shutdown', True))

  return None

def manager_is_running(pid=None):
  if pid is not None:
    lockMap['main'].acquire(False)

  if lockMap['main'].acquire(False):
    lockMap['main'].release()
    return False

  return True

# From https://wiki.python.org/moin/ConfigParserExamples
def config_section_map(section, Config):
  # Takes a section in a config file and makes a dictionary
  # out of it.
  # Returns that dictionary.
  dict1 = {}

  try:
    options = Config.options(section)

  except ConfigParser.NoSectionError as exc:
    return None

  for option in options:
    try:
      dict1[option] = Config.get(section, option)
      if dict1[option] == -1:
        logging.info("skip: %s" % option)

    except Exception as exc:
      logging.warning("exception on %s!" % option)
      dict1[option] = None

  return dict1
<|endoftext|>


<commit_before>"""
github.py - Willie Github Module
Copyright 2012, Dimitri Molenaars http://tyrope.nl/
Licensed under the Eiffel Forum License 2.

http://willie.dftba.net/
"""

from datetime import datetime
from urllib2 import HTTPError
import json
from willie import web
from willie.module import commands
import os

def checkConfig(bot):
    if not bot.config.has_option('github', 'oauth_token') or not bot.config.has_option('github', 'repo'):
        return False
    else:
        return [bot.config.github.oauth_token, bot.config.github.repo]

def configure(config):
    """
    | [github] | example | purpose |
    | -------- | ------- | ------- |
    | oauth_token | 5868e7af57496cc3ae255868e7af57496cc3ae25 | The OAuth token to connect to your github repo |
    | repo | embolalia/willie | The GitHub repo you're working from. |
    """
    chunk = ''
    if config.option('Configuring github issue reporting and searching module', False):
        config.interactive_add('github', 'oauth_token', 'Github API Oauth2 token', '')
        config.interactive_add('github', 'repo', 'Github repository', 'embolalia/willie')
    return chunk

@commands('makeissue', 'makebug')
def issue(bot, trigger):
    """Create a GitHub issue, also known as a bug report. Syntax: .makeissue Title of the bug report"""
    # check input
    if not trigger.group(2):
        return bot.say('Please title the issue')

    # Is the Oauth token and repo available?
    gitAPI = checkConfig(bot)
    if gitAPI == False:
        return bot.say('Git module not configured, make sure github.oauth_token and github.repo are defined')

    # parse input
    now = ' '.join(str(datetime.utcnow()).split(' ')).split('.')[0] + ' UTC'
    body = 'Submitted by: %s\nFrom channel: %s\nAt %s' % (trigger.nick, trigger.sender, now)
    data = {"title": trigger.group(2).encode('utf-8'), "body": body}
    # submit
    try:
        raw = web.post('https://api.github.com/repos/' + gitAPI[1] + '/issues?access_token=' + gitAPI[0], json.dumps(data))
    except HTTPError:
        return bot.say('The GitHub API returned an error.')

    data = json.loads(raw)
    bot.say('Issue #%s posted. %s' % (data['number'], data['html_url']))
    bot.debug(__file__, 'Issue #%s created in %s' % (data['number'], trigger.sender), 'warning')

@commands('addtrace', 'addtraceback')
def add_traceback(bot, trigger):
    """Add a traceback to a GitHub issue.

    This pulls the traceback from the exceptions log file. To use, put .addtrace
    followed by the issue number to add the comment to, then the signature of
    the error (the message shown to the channel when the error occured). This
    command will only work for errors from unhandled exceptions."""
    # Make sure the API is set up
    gitAPI = checkConfig(bot)
    if gitAPI == False:
        return bot.say('Git module not configured, make sure github.oauth_token and github.repo are defined')

    # Make sure the input is valid
    args = trigger.group(2).split(None, 1)
    if len(args) != 2:
        bot.say(validate)
        return
    number, trace = args

    # Make sure the given issue number exists
    issue_data = web.get('https://api.github.com/repos/%s/issues/%s' % (gitAPI[1], number))
    issue_data = json.loads(issue_data)
    if 'message' in issue_data and issue_data['message'] == 'Not Found':
        return bot.say("That issue doesn't exist.")

    # Find the relevant lines from the log file
    post = ''
    logfile = os.path.join(bot.config.logdir, 'exceptions.log')
    with open(logfile) as log:
        in_trace = False
        for data in log:
            if data == 'Signature: ' + trace + '\n':
                post = data
                in_trace = True
            elif data == '----------------------------------------\n':
                in_trace = False
            elif in_trace:
                post += data

    # Give an error if we didn't find the traceback
    if not post:
        return bot.say("I don't remember getting that error. Please post it "
                       "yourself at https://github.com/%s/issues/%s"
                       % (gitAPI[1], number))

    # Make the comment
    try:
        raw = web.post('https://api.github.com/repos/' + gitAPI[1] + '/issues/'
                       + number + '/comments?access_token=' + gitAPI[0],
                       json.dumps({'body': '``\n' + post + '``'}))
    except OSError:#HTTPError:
        return bot.say('The GitHub API returned an error.')

    data = json.loads(raw)
    bot.say('Added traceback to issue #%s. %s' % (number, data['html_url']))
    bot.debug(__file__, 'Traceback added to #%s in %s.' % (number, trigger.sender), 'warning')

@commands('findissue', 'findbug')
def findIssue(bot, trigger):
    """Search for a GitHub issue by keyword or ID. usage: .findissue search keywords/ID (optional) You can specify the first keyword as "CLOSED" to search closed issues."""
    if not trigger.group(2):
        return bot.reply('What are you searching for?')

    # Is the Oauth token and repo available?
    gitAPI = checkConfig(bot)
    if gitAPI == False:
        return bot.say('Git module not configured, make sure github.oauth_token and github.repo are defined')
    firstParam = trigger.group(2).split(' ')[0]
    if firstParam.isdigit():
        URL = 'https://api.github.com/repos/%s/issues/%s' % (gitAPI[1], trigger.group(2))
    elif firstParam == 'CLOSED':
        if '%20'.join(trigger.group(2).split(' ')[1:]) not in ('', '\x02', '\x03'):
            URL = 'https://api.github.com/legacy/issues/search/' + gitAPI[1] + '/closed/' + '%20'.join(trigger.group(2).split(' ')[1:])
        else:
            return bot.reply('What are you searching for?')
    else:
        URL = 'https://api.github.com/legacy/issues/search/%s/open/%s' % (gitAPI[1], trigger.group(2))

    try:
        raw = web.get(URL)
    except HTTPError:
        return bot.say('The GitHub API returned an error.')

    try:
        if firstParam.isdigit():
            data = json.loads(raw)
        else:
            data = json.loads(raw)['issues'][-1]
    except (KeyError, IndexError):
        return bot.say('No search results.')
    if len(data['body'].split('\n')) > 1:
        body = data['body'].split('\n')[0] + '...'
    else:
        body = data['body'].split('\n')[0]
    bot.reply('[#%s]\x02title:\x02 %s \x02|\x02 %s' % (data['number'], data['title'], body))
    bot.say(data['html_url'])
<commit_msg>Add debug print for invalid API results.<commit_after>"""
github.py - Willie Github Module
Copyright 2012, Dimitri Molenaars http://tyrope.nl/
Licensed under the Eiffel Forum License 2.

http://willie.dftba.net/
"""

from datetime import datetime
from urllib2 import HTTPError
import json
from willie import web
from willie.module import commands
import os

def checkConfig(bot):
    if not bot.config.has_option('github', 'oauth_token') or not bot.config.has_option('github', 'repo'):
        return False
    else:
        return [bot.config.github.oauth_token, bot.config.github.repo]

def configure(config):
    """
    | [github] | example | purpose |
    | -------- | ------- | ------- |
    | oauth_token | 5868e7af57496cc3ae255868e7af57496cc3ae25 | The OAuth token to connect to your github repo |
    | repo | embolalia/willie | The GitHub repo you're working from. |
    """
    chunk = ''
    if config.option('Configuring github issue reporting and searching module', False):
        config.interactive_add('github', 'oauth_token', 'Github API Oauth2 token', '')
        config.interactive_add('github', 'repo', 'Github repository', 'embolalia/willie')
    return chunk

@commands('makeissue', 'makebug')
def issue(bot, trigger):
    """Create a GitHub issue, also known as a bug report. Syntax: .makeissue Title of the bug report"""
    # check input
    if not trigger.group(2):
        return bot.say('Please title the issue')

    # Is the Oauth token and repo available?
    gitAPI = checkConfig(bot)
    if gitAPI == False:
        return bot.say('Git module not configured, make sure github.oauth_token and github.repo are defined')

    # parse input
    now = ' '.join(str(datetime.utcnow()).split(' ')).split('.')[0] + ' UTC'
    body = 'Submitted by: %s\nFrom channel: %s\nAt %s' % (trigger.nick, trigger.sender, now)
    data = {"title": trigger.group(2).encode('utf-8'), "body": body}
    # submit
    try:
        raw = web.post('https://api.github.com/repos/' + gitAPI[1] + '/issues?access_token=' + gitAPI[0], json.dumps(data))
    except HTTPError:
        return bot.say('The GitHub API returned an error.')

    data = json.loads(raw)
    bot.say('Issue #%s posted. %s' % (data['number'], data['html_url']))
    bot.debug(__file__, 'Issue #%s created in %s' % (data['number'], trigger.sender), 'warning')

@commands('addtrace', 'addtraceback')
def add_traceback(bot, trigger):
    """Add a traceback to a GitHub issue.

    This pulls the traceback from the exceptions log file. To use, put .addtrace
    followed by the issue number to add the comment to, then the signature of
    the error (the message shown to the channel when the error occured). This
    command will only work for errors from unhandled exceptions."""
    # Make sure the API is set up
    gitAPI = checkConfig(bot)
    if gitAPI == False:
        return bot.say('Git module not configured, make sure github.oauth_token and github.repo are defined')

    # Make sure the input is valid
    args = trigger.group(2).split(None, 1)
    if len(args) != 2:
        bot.say(validate)
        return
    number, trace = args

    # Make sure the given issue number exists
    issue_data = web.get('https://api.github.com/repos/%s/issues/%s' % (gitAPI[1], number))
    issue_data = json.loads(issue_data)
    if 'message' in issue_data and issue_data['message'] == 'Not Found':
        return bot.say("That issue doesn't exist.")

    # Find the relevant lines from the log file
    post = ''
    logfile = os.path.join(bot.config.logdir, 'exceptions.log')
    with open(logfile) as log:
        in_trace = False
        for data in log:
            if data == 'Signature: ' + trace + '\n':
                post = data
                in_trace = True
            elif data == '----------------------------------------\n':
                in_trace = False
            elif in_trace:
                post += data

    # Give an error if we didn't find the traceback
    if not post:
        return bot.say("I don't remember getting that error. Please post it "
                       "yourself at https://github.com/%s/issues/%s"
                       % (gitAPI[1], number))

    # Make the comment
    try:
        raw = web.post('https://api.github.com/repos/' + gitAPI[1] + '/issues/'
                       + number + '/comments?access_token=' + gitAPI[0],
                       json.dumps({'body': '``\n' + post + '``'}))
    except OSError:#HTTPError:
        return bot.say('The GitHub API returned an error.')

    data = json.loads(raw)
    bot.say('Added traceback to issue #%s. %s' % (number, data['html_url']))
    bot.debug(__file__, 'Traceback added to #%s in %s.' % (number, trigger.sender), 'warning')

@commands('findissue', 'findbug')
def findIssue(bot, trigger):
    """Search for a GitHub issue by keyword or ID. usage: .findissue search keywords/ID (optional) You can specify the first keyword as "CLOSED" to search closed issues."""
    if not trigger.group(2):
        return bot.reply('What are you searching for?')

    # Is the Oauth token and repo available?
    gitAPI = checkConfig(bot)
    if gitAPI == False:
        return bot.say('Git module not configured, make sure github.oauth_token and github.repo are defined')
    firstParam = trigger.group(2).split(' ')[0]
    if firstParam.isdigit():
        URL = 'https://api.github.com/repos/%s/issues/%s' % (gitAPI[1], trigger.group(2))
    elif firstParam == 'CLOSED':
        if '%20'.join(trigger.group(2).split(' ')[1:]) not in ('', '\x02', '\x03'):
            URL = 'https://api.github.com/legacy/issues/search/' + gitAPI[1] + '/closed/' + '%20'.join(trigger.group(2).split(' ')[1:])
        else:
            return bot.reply('What are you searching for?')
    else:
        URL = 'https://api.github.com/legacy/issues/search/%s/open/%s' % (gitAPI[1], trigger.group(2))

    try:
        raw = web.get(URL)
    except HTTPError:
        return bot.say('The GitHub API returned an error.')

    try:
        if firstParam.isdigit():
            data = json.loads(raw)
        else:
            data = json.loads(raw)['issues'][-1]
    except (KeyError, IndexError):
        return bot.say('No search results.')
    try:
        if len(data['body'].split('\n')) > 1:
            body = data['body'].split('\n')[0] + '...'
        else:
            body = data['body'].split('\n')[0]
    except (KeyError):
        bot.debug('GitHub KeyErr', 'API returned an invalid result on query request '+trigger.group(2), 'always')
        return bot.say('Invalid result, please try again later.')
    bot.reply('[#%s]\x02title:\x02 %s \x02|\x02 %s' % (data['number'], data['title'], body))
    bot.say(data['html_url'])
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import time
import fluidsynth
from ctypes import *

seqduration = 1000

def schedule_next_callback():
	# I want to be called back before the end of the next sequence
	callbackdate = int(now + seqduration/2)
	sequencer.timer(callbackdate, dest=mySeqID)

def schedule_next_sequence():
	global now
	# the sequence to play
	# the beat : 2 beats per sequence
	sequencer.note(int(now + seqduration * 1/2), 0, 60, duration=250, velocity=80, dest=synthSeqID)
	sequencer.note(int(now + seqduration * 2/2), 0, 60, duration=250, velocity=80, dest=synthSeqID)
	# melody
	sequencer.note(int(now + seqduration*1/10), 1, 45, duration=250, velocity=int(127*2/3), dest=synthSeqID)
	sequencer.note(int(now + seqduration*4/10), 1, 50, duration=250, velocity=int(127*2/3), dest=synthSeqID)
	sequencer.note(int(now + seqduration*8/10), 1, 55, duration=250, velocity=int(127*3/3), dest=synthSeqID)
	# so that we are called back early enough to schedule the next sequence
	schedule_next_callback()

	now = now + seqduration

def seq_callback(time, event, seq, data):
	schedule_next_sequence()

if __name__=="__main__":
	global sequencer, fs, mySeqID, synthSeqID, now
	fs = fluidsynth.Synth(gain=1)
	fs.start()
	# you might have to use other drivers:
	#fs.start(driver="alsa", midi_driver="alsa_seq")

	# maybe use better samples. this sounds horrible.
	sfid = fs.sfload("example.sf2")
	fs.program_select(0, sfid, 0, 0)
	fs.program_select(1, sfid, 0, 0) # use the same program for channel 2 for cheapness

	sequencer = fluidsynth.Sequencer()
	synthSeqID = sequencer.register_fluidsynth(fs)
	mySeqID = sequencer.register_client("mycallback", seq_callback)
	now = sequencer.get_tick()
	schedule_next_sequence()

	time.sleep(10)

	sequencer.delete()
	fs.delete()
<commit_msg>Update sequencerTest.py<commit_after>#!/usr/bin/env python3

import time
import fluidsynth
from ctypes import *

seqduration = 1000

def schedule_next_callback():
	# I want to be called back before the end of the next sequence
	callbackdate = int(now + seqduration/2)
	sequencer.timer(callbackdate, dest=mySeqID)

def schedule_next_sequence():
	global now
	# the sequence to play
	# the beat : 2 beats per sequence
	sequencer.note(int(now + seqduration * 1/2), 0, 60, duration=250, velocity=80, dest=synthSeqID)
	sequencer.note(int(now + seqduration * 2/2), 0, 60, duration=250, velocity=80, dest=synthSeqID)
	# melody
	sequencer.note(int(now + seqduration*1/10), 1, 45, duration=250, velocity=int(127*2/3), dest=synthSeqID)
	sequencer.note(int(now + seqduration*4/10), 1, 50, duration=250, velocity=int(127*2/3), dest=synthSeqID)
	sequencer.note(int(now + seqduration*8/10), 1, 55, duration=250, velocity=int(127*3/3), dest=synthSeqID)
	# so that we are called back early enough to schedule the next sequence
	schedule_next_callback()

	now = now + seqduration

def seq_callback(time, event, seq, data):
	schedule_next_sequence()

if __name__=="__main__":
	global sequencer, fs, mySeqID, synthSeqID, now
	fs = fluidsynth.Synth()
	fs.start()
	# you might have to use other drivers:
	#fs.start(driver="alsa", midi_driver="alsa_seq")

	sfid = fs.sfload("example.sf2")
	fs.program_select(0, sfid, 0, 0)
	fs.program_select(1, sfid, 0, 0) # use the same program for channel 2 for cheapness

	sequencer = fluidsynth.Sequencer()
	synthSeqID = sequencer.register_fluidsynth(fs)
	mySeqID = sequencer.register_client("mycallback", seq_callback)
	now = sequencer.get_tick()
	schedule_next_sequence()

	time.sleep(10)

	sequencer.delete()
	fs.delete()
<|endoftext|>


<commit_before>#!/usr/bin/python3

import sys
import os
import logging
from glob import glob
from argparse import ArgumentParser
import tarfile
from io import BytesIO
from lxml import etree

# Files to backup relative to root (/data)
BACKUP = ('conf/settings.xml', )

def setting_files(members, files):
	for info in members:
		if info.name in files:
			yield info

def backup(root, media, files):
	target = os.path.join(media[0], 'venus-cfg.tar')
	with tarfile.open(target, 'w') as tar:
		for name in files:
			tar.add(os.path.join(root, name), name)

def restore(root, media, files):
	# Iterate through all possible media to find a backup to restore
	for m in media:
		target = os.path.join(m, 'venus-cfg.tar')
		if os.path.exists(target):
			# TODO stop localsettings while doing this. Then reboot as some processes
			# don't survive a localsettings outage.
			with tarfile.open(target, 'r') as tar:
				tar.extractall(path=root, members=setting_files(tar, files))
			break

def delete_from_tree(tree, path):
	obj = tree.xpath(path)
	if obj:
		obj[0].getparent().remove(obj[0])

def sanitize_settings(fp):
	parser = etree.XMLParser(remove_blank_text=True)
	tree = etree.parse(fp, parser)

	# Remove device-specific settings (based on serial number)
	delete_from_tree(tree, "/Settings/Devices")
	delete_from_tree(tree, "/Settings/CanBms")
	delete_from_tree(tree, "/Settings/Fronius/InverterIds")
	delete_from_tree(tree, "/Settings/Fronius/Inverters")
	delete_from_tree(tree, "/Settings/Victron/Products")

	# Return result
	return etree.tostring(tree, encoding='UTF-8', pretty_print=True, xml_declaration=True)

def export(root, media, files):
	with open(os.path.join(root, 'conf/settings.xml'), encoding='UTF-8') as fp:
		buf = sanitize_settings(fp)

	target = os.path.join(media[0], 'venus-cfg.tar')
	with tarfile.open(target, 'w') as tar:
		info = tarfile.TarInfo('conf/settings.xml')
		info.size = len(buf)
		tar.addfile(info, BytesIO(buf))
		for name in files:
			tar.add(os.path.join(root, name), name)

def main():
	# Parse arguments
	parser = ArgumentParser(description=sys.argv[0])
	parser.add_argument('--backup', help='Perform a backup of settings',
		default=False, action="store_true")
	parser.add_argument('--restore', help='Perform a restore of settings',
		default=False, action="store_true")
	parser.add_argument('--export',
		help='Export settings to be imported on another GX device',
		default=False, action="store_true")
	parser.add_argument('--target',
		help='Optional path to backup to, if omitted /media will be scanned',
		default=None)
	parser.add_argument('--root',
		help='Optional partition where settings are stored, defaults to /data',
		default='/data')
	args = parser.parse_args()

	if not (args.backup or args.restore or args.export):
		parser.print_help()
		sys.exit(1)

	# Logging
	logging.basicConfig(level=logging.INFO)

	# Find suitable place to put the backup
	if args.target is not None:
		media = [args.target]
	else:
		media = [x for x in glob('/media/*') if os.path.ismount(x)]
		media.sort()

	if not media:
		logging.info("Could not find suitable location for backup")
		sys.exit(1)

	if args.backup:
		backup(args.root, media, BACKUP)
	elif args.restore:
		restore(args.root, media, BACKUP)
	elif args.export:
		export(args.root, media, ())

if __name__ == "__main__":
	main()
<commit_msg>stop localsettings and reboot when doing restore<commit_after>#!/usr/bin/python3

import sys
import os
import logging
from glob import glob
from argparse import ArgumentParser
import tarfile
import struct
from time import sleep
from io import BytesIO
from lxml import etree

# Files to backup relative to root (/data)
BACKUP = ('conf/settings.xml', )

def svstat(service):
	""" Simple python implementation of svstat, because the
	    daemontools version doesn't return useful exit statuses
		and this is less insane than parsing output. """
	f = os.path.join(service, 'supervise', 'status')
	try:
		fd = os.open(f, os.O_RDONLY | os.O_NONBLOCK)
	except OSError:
		return False
	else:
		status = os.read(fd, 18)
		if len(status) == 18:
			pid = struct.unpack('<i', status[12:16])[0]
			return pid > 0
		return False

def svc(service, ops):
	""" Simple python implementation of svc, because it is easy and we already
	    did svstat. """
	f = os.path.join(service, 'supervise', 'control')
	fd = os.open(f, os.O_WRONLY | os.O_NONBLOCK)
	try:
		os.write(fd, ops)
	finally:
		os.close(fd)

def reboot():
	""" Reboot by killing init. """
	os.kill(1, 2) # SIGINT

def setting_files(members, files):
	for info in members:
		if info.name in files:
			yield info

def backup(root, media, files):
	target = os.path.join(media[0], 'venus-cfg.tar')
	with tarfile.open(target, 'w') as tar:
		for name in files:
			tar.add(os.path.join(root, name), name)

def restore(root, media, files):
	# Iterate through all possible media to find a backup to restore
	for m in media:
		target = os.path.join(m, 'venus-cfg.tar')
		if os.path.exists(target):
			# Stop localsettings while doing this, otherwise it will just
			# overwrite the file again on shutdown later.
			svc("/service/localsettings", b"d")

			# Wait for it to stop
			for _ in range(3):
				if not svstat("/service/localsettings"):
					break # range(3)
				sleep(1)
			else:
				logging.error("localsettings failed to stop")
				reboot()
				break # m in media

			# Restore the settings
			with tarfile.open(target, 'r') as tar:
				tar.extractall(path=root, members=setting_files(tar, files))

			# Then reboot as some processes don't survive a localsettings
			# outage.
			reboot()
			break # m in media

def delete_from_tree(tree, path):
	obj = tree.xpath(path)
	if obj:
		obj[0].getparent().remove(obj[0])

def sanitize_settings(fp):
	parser = etree.XMLParser(remove_blank_text=True)
	tree = etree.parse(fp, parser)

	# Remove device-specific settings (based on serial number)
	delete_from_tree(tree, "/Settings/Devices")
	delete_from_tree(tree, "/Settings/CanBms")
	delete_from_tree(tree, "/Settings/Fronius/InverterIds")
	delete_from_tree(tree, "/Settings/Fronius/Inverters")
	delete_from_tree(tree, "/Settings/Victron/Products")

	# Return result
	return etree.tostring(tree, encoding='UTF-8', pretty_print=True, xml_declaration=True)

def export(root, media, files):
	with open(os.path.join(root, 'conf/settings.xml'), encoding='UTF-8') as fp:
		buf = sanitize_settings(fp)

	target = os.path.join(media[0], 'venus-cfg.tar')
	with tarfile.open(target, 'w') as tar:
		info = tarfile.TarInfo('conf/settings.xml')
		info.size = len(buf)
		tar.addfile(info, BytesIO(buf))
		for name in files:
			tar.add(os.path.join(root, name), name)

def main():
	# Parse arguments
	parser = ArgumentParser(description=sys.argv[0])
	parser.add_argument('--backup', help='Perform a backup of settings',
		default=False, action="store_true")
	parser.add_argument('--restore', help='Perform a restore of settings',
		default=False, action="store_true")
	parser.add_argument('--export',
		help='Export settings to be imported on another GX device',
		default=False, action="store_true")
	parser.add_argument('--target',
		help='Optional path to backup to, if omitted /media will be scanned',
		default=None)
	parser.add_argument('--root',
		help='Optional partition where settings are stored, defaults to /data',
		default='/data')
	args = parser.parse_args()

	if not (args.backup or args.restore or args.export):
		parser.print_help()
		sys.exit(1)

	# Logging
	logging.basicConfig(level=logging.INFO)

	# Find suitable place to put the backup
	if args.target is not None:
		media = [args.target]
	else:
		media = [x for x in glob('/media/*') if os.path.ismount(x)]
		media.sort()

	if not media:
		logging.info("Could not find suitable location for backup")
		sys.exit(1)

	if args.backup:
		backup(args.root, media, BACKUP)
	elif args.restore:
		restore(args.root, media, BACKUP)
	elif args.export:
		export(args.root, media, ())

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from st2client import models
from st2client.models.action_alias import ActionAlias
from st2client.commands.action import ActionRunCommandMixin
from st2client.commands import resource
from st2client.formatters import table

__all__ = [
    'ActionAliasBranch'
]

class ActionAliasBranch(resource.ResourceBranch):
    def __init__(self, description, app, subparsers, parent_parser=None):
        super(ActionAliasBranch, self).__init__(
            ActionAlias, description, app, subparsers,
            parent_parser=parent_parser, read_only=False,
            commands={
                'list': ActionAliasListCommand,
                'get': ActionAliasGetCommand
            })

        self.commands['match'] = ActionAliasMatchCommand(
            self.resource, self.app, self.subparsers,
            add_help=False)
        self.commands['execute'] = ActionAliasExecuteCommand(
            self.resource, self.app, self.subparsers,
            add_help=False)

class ActionAliasListCommand(resource.ContentPackResourceListCommand):
    display_attributes = ['ref', 'pack', 'description', 'enabled']

class ActionAliasGetCommand(resource.ContentPackResourceGetCommand):
    display_attributes = ['all']
    attribute_display_order = ['id', 'ref', 'pack', 'name', 'description',
                               'enabled', 'action_ref', 'formats']

class ActionAliasMatchCommand(resource.ResourceCommand):
    display_attributes = ['id', 'name', 'description']

    def __init__(self, resource, *args, **kwargs):
        super(ActionAliasMatchCommand, self).__init__(
            resource, 'match',
            'Get the list of %s that match the command text.' %
            resource.get_plural_display_name().lower(),
            *args, **kwargs)

        self.parser.add_argument('match_text',
                                 metavar='command',
                                 help=help)

        self.parser.add_argument('-a', '--attr', nargs='+',
                                 default=self.display_attributes,
                                 help=('List of attributes to include in the '
                                       'output. "all" will return all '
                                       'attributes.'))
        self.parser.add_argument('-w', '--width', nargs='+', type=int,
                                 default=None,
                                 help=('Set the width of columns in output.'))

    @resource.add_auth_token_to_kwargs_from_cli
    def run(self, args, **kwargs):
        matches = self.manager.match(args.match_text, **kwargs)
        return [match['actionalias'] for match in matches]  # show only alias objects

    def run_and_print(self, args, **kwargs):
        instances = self.run(args, **kwargs)
        self.print_output(instances, table.MultiColumnTable,
                          attributes=args.attr, widths=args.width,
                          json=args.json, yaml=args.yaml)

class ActionAliasExecuteCommand(ActionRunCommandMixin):
    def __init__(self, resource, *args, **kwargs):
        super(ActionAliasExecuteCommand, self).__init__(
            resource, 'execute',
            'Execute the command text by finding a matching ActionAlias.',
            *args, **kwargs)

        self.parser.add_argument('command_text',
                                 metavar='command',
                                 help=help)
        self.parser.add_argument('-h', '--help',
                                 action='store_true', dest='help',
                                 help='Print usage for the given action.')
        self.parser.add_argument('--trace-tag', '--trace_tag',
                                 help='A trace tag string to track execution later.',
                                 dest='trace_tag', required=False)
        self.parser.add_argument('--trace-id',
                                 help='Existing trace id for this execution.',
                                 dest='trace_id', required=False)
        self.parser.add_argument('-a', '--async',
                                 action='store_true', dest='async',
                                 help='Do not wait for action to finish.')
        self.parser.add_argument('-u', '--user', type=str, default=None,
                                 help='User under which to run the action (admins only).')

    @resource.add_auth_token_to_kwargs_from_cli
    def run(self, args, **kwargs):
        matches = self.manager.match(args.match_text, **kwargs)

        match = matches[0]
        action_alias = match['actionalias']

        execution = models.ActionAliasExecution()
        execution.name = action_alias['name']
        execution.format = match['representation']
        execution.command = args.match_text
        execution.source_channel = 'cli'  # ?
        execution.notification_channel = None
        execution.notification_route = None
        execution.user = args.user

        action_exec_mgr = self.app.client.managers['ActionAliasExecution']

        execution = action_exec_mgr.create(execution, **kwargs)

        return execution
<commit_msg>use the correct base class for the command<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from st2client import models
from st2client.models.action_alias import ActionAlias
from st2client.commands.action import ActionRunCommandMixin
from st2client.commands import resource
from st2client.formatters import table

__all__ = [
    'ActionAliasBranch'
]

class ActionAliasBranch(resource.ResourceBranch):
    def __init__(self, description, app, subparsers, parent_parser=None):
        super(ActionAliasBranch, self).__init__(
            ActionAlias, description, app, subparsers,
            parent_parser=parent_parser, read_only=False,
            commands={
                'list': ActionAliasListCommand,
                'get': ActionAliasGetCommand
            })

        self.commands['match'] = ActionAliasMatchCommand(
            self.resource, self.app, self.subparsers,
            add_help=False)
        self.commands['execute'] = ActionAliasExecuteCommand(
            self.resource, self.app, self.subparsers,
            add_help=False)

class ActionAliasListCommand(resource.ContentPackResourceListCommand):
    display_attributes = ['ref', 'pack', 'description', 'enabled']

class ActionAliasGetCommand(resource.ContentPackResourceGetCommand):
    display_attributes = ['all']
    attribute_display_order = ['id', 'ref', 'pack', 'name', 'description',
                               'enabled', 'action_ref', 'formats']

class ActionAliasMatchCommand(resource.ResourceCommand):
    display_attributes = ['id', 'name', 'description']

    def __init__(self, resource, *args, **kwargs):
        super(ActionAliasMatchCommand, self).__init__(
            resource, 'match',
            'Get the list of %s that match the command text.' %
            resource.get_plural_display_name().lower(),
            *args, **kwargs)

        self.parser.add_argument('match_text',
                                 metavar='command',
                                 help=help)

        self.parser.add_argument('-a', '--attr', nargs='+',
                                 default=self.display_attributes,
                                 help=('List of attributes to include in the '
                                       'output. "all" will return all '
                                       'attributes.'))
        self.parser.add_argument('-w', '--width', nargs='+', type=int,
                                 default=None,
                                 help=('Set the width of columns in output.'))

    @resource.add_auth_token_to_kwargs_from_cli
    def run(self, args, **kwargs):
        matches = self.manager.match(args.match_text, **kwargs)
        return [match['actionalias'] for match in matches]  # show only alias objects

    def run_and_print(self, args, **kwargs):
        instances = self.run(args, **kwargs)
        self.print_output(instances, table.MultiColumnTable,
                          attributes=args.attr, widths=args.width,
                          json=args.json, yaml=args.yaml)

class ActionAliasExecuteCommand(resource.ResourceCommand):
    def __init__(self, resource, *args, **kwargs):
        super(ActionAliasExecuteCommand, self).__init__(
            resource, 'execute',
            'Execute the command text by finding a matching ActionAlias.',
            *args, **kwargs)

        self.parser.add_argument('command_text',
                                 metavar='command',
                                 help=help)
        self.parser.add_argument('-h', '--help',
                                 action='store_true', dest='help',
                                 help='Print usage for the given action.')
        self.parser.add_argument('--trace-tag', '--trace_tag',
                                 help='A trace tag string to track execution later.',
                                 dest='trace_tag', required=False)
        self.parser.add_argument('--trace-id',
                                 help='Existing trace id for this execution.',
                                 dest='trace_id', required=False)
        self.parser.add_argument('-a', '--async',
                                 action='store_true', dest='async',
                                 help='Do not wait for action to finish.')
        self.parser.add_argument('-u', '--user', type=str, default=None,
                                 help='User under which to run the action (admins only).')

    @resource.add_auth_token_to_kwargs_from_cli
    def run(self, args, **kwargs):
        matches = self.manager.match(args.match_text, **kwargs)

        match = matches[0]
        action_alias = match['actionalias']

        execution = models.ActionAliasExecution()
        execution.name = action_alias['name']
        execution.format = match['representation']
        execution.command = args.match_text
        execution.source_channel = 'cli'  # ?
        execution.notification_channel = None
        execution.notification_route = None
        execution.user = args.user

        action_exec_mgr = self.app.client.managers['ActionAliasExecution']

        execution = action_exec_mgr.create(execution, **kwargs)

        return execution
<|endoftext|>


<commit_before>#GOPIGO API http://www.dexterindustries.com/GoPiGo/programming/python-programming-for-the-raspberry-pi-gopigo/

#imports
from gopigo import *
import time
import threading

__author__ = 'Robert Hayek and Keith Neyman'

#Global Variable that says how far butler can get to a wall
STOP_DIST = 50

#print variables
ERROR = "ERROR"
VOLT = "HAZARDOUS VOLTAGE"
STOP = "STOPPING"
MOVE = "MOVING"
YES = "CONTINUING"

class Pigo(threading.Thread):

    ############
    ######## BASIC STATUS AND METHODS
    ############
    status = {'isMoving' : False, 'servo': 90, 'leftSpeed' : 175, 'rightSpeed' : 175, "distance" : 100}#gets the status of the robot

    def __init__(self):
        threading.Thread.__init__(self)
        print "BUTLERPI IS NOW ON"
        self.status['distance'] = us_dist(15) #update distance  with the current distance through ultrasonic sensor in (mm)
    def stop(self):
        self.status["isMoving"] = False
        print STOP #debugging message (delete later after)
        for x in range(5):
            stop()
            time.sleep(.05)

    def fwd(self):
        self.status['isMoving'] = True
        print MOVE#debugging message (delete later)
        for x in range(5):
            fwd()
            time.sleep(.1)

    #Check if conditions are safe for ButlerPi to continue
    def keepGoing(self):
        if self.status['distance'] < STOP_DIST:
            print "OBSTACLE FOUND CHECKING WEATHER TO STOP"
            self.checkDistance()
            if self.status['distance'] < STOP_DIST:
                print ERROR
                return False
        elif volt() > 14 or volt() < 6: #check voltage and turn off robot is voltage is under 6V or above 14V
            print VOLT
            return False
        else:
            print YES
            return True
    #always watch for obstacles
    def keepWatch(self):
        while self.keepGoing():
            self.checkDistance()
    #check the distance Butler is away from something
    def checkDistance(self):
        self.status['distance'] = us_dist(15)
        print "CHECKING DISTANCE..." + "SOMETHING IS " + str(self.status['distance']) + "mm away"

    def equalizeSpeed(self):
        set_left_speed(500)
        set_right_speed(500)

    def butlerStatus(self):
        print volt()
        print fw_ver()
        print read_status()
        print read_enc_status()
        print read_timeout_status()

    ############
    ######## Dance Methods
    ############
    def spin(self):
        right_rot()
        time.sleep(6)
        self.stop()
        left_rot()
        time.sleep(6)
        self.stop()

    def strobe(self):#Strobe light using the LEDs
        while self.keepGoing():
            led_on(1)
            time.sleep(.10)
            led_off(1)
            led_on(0)
            time.sleep(.10)
            led_off(1)

    def shuffle(self):
        while self.keepGoing():
            right_rot()
            time.sleep(.10)
            left_rot()
            time.sleep(.10)

    def shakeServo(self):
        enable_servo()
        servo(45)
        servo(-10)
        servo(10)
        servo(-20)
        servo(10)
        diasble_servo()

    def rightTurn(self):
        fwd()
        right_rot(50)
        fwd()

    def leftTurn(self):
        fwd()
        time.sleep(.10)
        left_rot(50)
        fwd()
        time.sleep(.10)

    #############
    ######## COMPLEX METHODS
    #############
    def servoSweep(self):
        for ang in range(20, 160, 5):
            servo(ang)
            time.sleep(.1)

    def dance(self):
        print "STARTING DANCE METHOD" #Dance Method
        self.spin()
        #self.shuffle()
        #self.servoSweep()
       # self.shakeServo()
       # self.rightTurn()
       # self.leftTurn()

############
######## MAIN APP STARTS HERE
############
butler = Pigo()
while butler.keepGoing():
    butler.keepGoing()
    butler.equalizeSpeed()
    butler.dance()
    butler.strobe()
    butler.keepWatch()
butler.stop()
print butler.status
butler.butlerStatus()<commit_msg>stash<commit_after>#GOPIGO API http://www.dexterindustries.com/GoPiGo/programming/python-programming-for-the-raspberry-pi-gopigo/

#imports
from gopigo import *
import time
import threading

__author__ = 'Robert Hayek and Keith Neyman'

#Global Variable that says how far butler can get to a wall
STOP_DIST = 50

#print variables
ERROR = "ERROR"
VOLT = "HAZARDOUS VOLTAGE"
STOP = "STOPPING"
MOVE = "MOVING"
YES = "CONTINUING"

class Pigo(threading.Thread):

    ############
    ######## BASIC STATUS AND METHODS
    ############
    status = {'isMoving' : False, 'servo': 90, 'leftSpeed' : 175, 'rightSpeed' : 175, "distance" : 100}#gets the status of the robot

    def __init__(self):
        threading.Thread.__init__(self)
        print "BUTLERPI IS NOW ON"
        self.status['distance'] = us_dist(15) #update distance  with the current distance through ultrasonic sensor in (mm)
    def stop(self):
        self.status["isMoving"] = False
        print STOP #debugging message (delete later after)
        for x in range(5):
            stop()
            time.sleep(.05)

    def fwd(self):
        self.status['isMoving'] = True
        print MOVE#debugging message (delete later)
        for x in range(5):
            fwd()
            time.sleep(.1)

    #Check if conditions are safe for ButlerPi to continue
    def keepGoing(self):
        if self.status['distance'] < STOP_DIST:
            print "OBSTACLE FOUND CHECKING WEATHER TO STOP"
            self.checkDistance()
            if self.status['distance'] < STOP_DIST:
                print ERROR
                return False
        elif volt() > 14 or volt() < 6: #check voltage and turn off robot is voltage is under 6V or above 14V
            print VOLT
            return False
        else:
            print YES
            return True
    #always watch for obstacles
    def keepWatch(self):
        while self.keepGoing():
            self.checkDistance()
    #check the distance Butler is away from something
    def checkDistance(self):
        self.status['distance'] = us_dist(15)
        print "CHECKING DISTANCE..." + "SOMETHING IS " + str(self.status['distance']) + "mm away"

    def equalizeSpeed(self):
        set_left_speed(500)
        set_right_speed(500)

    def butlerStatus(self):
        print volt()
        print fw_ver()
        print read_status()
        print read_enc_status()
        print read_timeout_status()

    ############
    ######## Dance Methods
    ############
    def spin(self):
        right_rot()
        time.sleep(6)
        self.stop()
        left_rot()
        time.sleep(6)
        self.stop()

    def strobe(self):# Strobe light using the LEDs
        while butler.keepGoing():
            led_on(1)
            time.sleep(.10)
            led_off(1)
            led_on(0)
            time.sleep(.10)
            led_off(1)

    def shuffle(self):
        while butler.keepGoing():
            right_rot()
            time.sleep(.10)
            left_rot()
            time.sleep(.10)

    def shakeServo(self):
        enable_servo()
        servo(45)
        servo(-10)
        servo(10)
        servo(-20)
        servo(10)
        diasble_servo()

    def rightTurn(self):
        fwd()
        right_rot(50)
        fwd()

    def leftTurn(self):
        fwd()
        time.sleep(.10)
        left_rot(50)
        fwd()
        time.sleep(.10)

    #############
    ######## COMPLEX METHODS
    #############
    def servoSweep(self):
        for ang in range(20, 160, 5):
            servo(ang)
            time.sleep(.1)

    def dance(self):
        print "STARTING DANCE METHOD" #Dance Method
        self.spin()
        #self.shuffle()
        #self.servoSweep()
       # self.shakeServo()
       # self.rightTurn()
       # self.leftTurn()

############
######## MAIN APP STARTS HERE
############
butler = Pigo()
while butler.keepGoing():
    butler.keepGoing()
    butler.equalizeSpeed()
    butler.dance()
    butler.strobe()
    butler.keepWatch()
butler.stop()
print butler.status
butler.butlerStatus()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    Treehouse node message models.
'''

# This file is part of treehouse.

# Distributed under the terms of the last AGPL License.
# The full license is in the file LICENCE, distributed as part of this software.

__author__ = 'Team Machine'

import uuid
import arrow
from schematics import models
from schematics import types
from schematics.types import compound
from treehouse.messages import Resource

class Node(models.Model):
    '''
        Thanks for all the fish!
    '''
    uuid = types.UUIDType(default=uuid.uuid4)
    account = types.StringType(required=True)
    checked = types.StringType(default=False)
    status = types.StringType()
    created_at = types.TimestampType(default=arrow.utcnow().timestamp)
    created_by = types.StringType()
    last_update_by = types.StringType()
    last_update_at = types.TimestampType(default=arrow.utcnow().timestamp)
    checksum = types.StringType()
    name = types.StringType()
    description = types.StringType()
    region = types.StringType()
    ranking = types.StringType()
    public = types.StringType()
    uri = types.StringType()
    centers = compound.ListType(types.StringType())
    labels = compound.ListType(types.StringType())
    hashs = compound.ListType(types.StringType())
    resources = compound.ListType(types.StringType())
    units = compound.ListType(types.StringType())
    history = compound.ListType(types.StringType())
    labels_total = types.IntType()
    hashs_total = types.IntType()

class ModifyNode(models.Model):
    '''
        Modify node

        This model is similar to node.

        It lacks of require and default values on it's fields.

        The reason of it existence is that we need to validate
        every input data that came from outside the system, with
        this we prevent users from using PATCH to create fields
        outside the scope of the resource.
    '''
    uuid = types.UUIDType()
    account = types.StringType()
    status = types.StringType()
    centers = compound.ListType(types.StringType())
    created_at = types.TimestampType()
    created_by = types.StringType()
    last_update_by = types.StringType()
    last_update_at = types.TimestampType(default=arrow.utcnow().timestamp)
    checksum = types.StringType()
    name = types.StringType()
    description = types.StringType()
    region = types.StringType()
    ranking = types.StringType()
    public = types.StringType()
    checked = types.StringType()
    uri = types.StringType()
    labels = compound.ListType(types.StringType())
    hashs = compound.ListType(types.StringType())
    resources = compound.ListType(types.StringType())
    units = compound.ListType(types.StringType())
    history = compound.ListType(types.StringType())
    labels_total = types.IntType()
    hashs_total = types.IntType()<commit_msg>update<commit_after># -*- coding: utf-8 -*-
'''
    Treehouse node message models.
'''

# This file is part of treehouse.

# Distributed under the terms of the last AGPL License.
# The full license is in the file LICENCE, distributed as part of this software.

__author__ = 'Team Machine'

import uuid
import arrow
from schematics import models
from schematics import types
from schematics.types import compound
from treehouse.messages import Resource

class Node(models.Model):
    '''
        Thanks for all the fish!
    '''
    uuid = types.UUIDType(default=uuid.uuid4)
    account = types.StringType(required=True)
    checked = types.BooleanType(default=False)
    status = types.StringType()
    created_at = types.TimestampType(default=arrow.utcnow().timestamp)
    created_by = types.StringType()
    last_update_by = types.StringType()
    last_update_at = types.TimestampType(default=arrow.utcnow().timestamp)
    checksum = types.StringType()
    name = types.StringType()
    description = types.StringType()
    region = types.StringType()
    ranking = types.StringType()
    public = types.StringType()
    uri = types.StringType()
    centers = compound.ListType(types.StringType())
    labels = compound.ListType(types.StringType())
    hashs = compound.ListType(types.StringType())
    resources = compound.ListType(types.StringType())
    units = compound.ListType(types.StringType())
    history = compound.ListType(types.StringType())
    labels_total = types.IntType()
    hashs_total = types.IntType()

class ModifyNode(models.Model):
    '''
        Modify node

        This model is similar to node.

        It lacks of require and default values on it's fields.

        The reason of it existence is that we need to validate
        every input data that came from outside the system, with
        this we prevent users from using PATCH to create fields
        outside the scope of the resource.
    '''
    uuid = types.UUIDType()
    account = types.StringType()
    status = types.StringType()
    centers = compound.ListType(types.StringType())
    created_at = types.TimestampType()
    created_by = types.StringType()
    last_update_by = types.StringType()
    last_update_at = types.TimestampType(default=arrow.utcnow().timestamp)
    checksum = types.StringType()
    name = types.StringType()
    description = types.StringType()
    region = types.StringType()
    ranking = types.StringType()
    public = types.StringType()
    checked = types.StringType()
    uri = types.StringType()
    labels = compound.ListType(types.StringType())
    hashs = compound.ListType(types.StringType())
    resources = compound.ListType(types.StringType())
    units = compound.ListType(types.StringType())
    history = compound.ListType(types.StringType())
    labels_total = types.IntType()
    hashs_total = types.IntType()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""

ZUGBRUECKE
Calling routines in Windows DLLs from Python scripts running on unixlike systems
https://github.com/pleiszenburg/zugbruecke

    src/zugbruecke/core/interpreter.py: Class for managing Python interpreter on Wine

    Required to run on platform / side: [UNIX]

    Copyright (C) 2017-2021 Sebastian M. Ernst <ernst@pleiszenburg.de>

<LICENSE_BLOCK>
The contents of this file are subject to the GNU Lesser General Public License
Version 2.1 ("LGPL" or "License"). You may not use this file except in
compliance with the License. You may obtain a copy of the License at
https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt
https://github.com/pleiszenburg/zugbruecke/blob/master/LICENSE

Software distributed under the License is distributed on an "AS IS" basis,
WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
specific language governing rights and limitations under the License.
</LICENSE_BLOCK>

"""

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# IMPORT
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

import os
import queue
import signal
import subprocess
import time
import threading

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# WINE PYTHON INTERPRETER CLASS
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

class Interpreter:
    """
    Class for managing Python interpreter on Wine
    """

    def __init__(self, session_id, parameter, session_log):

        # Set ID, parameters and pointer to log
        self.id = session_id
        self.p = parameter
        self.log = session_log

        # Log status
        self.log.out("[interpreter] STARTING ...")

        # Session is up
        self.up = True

        # Start wine python
        self._python_start()

        # Log status
        self.log.out("[interpreter] STARTED.")

    # session destructor
    def terminate(self):

        if not self.up:
            return

        # Log status
        self.log.out("[interpreter] TERMINATING ...")

        # Shut down wine python
        self._python_stop()

        # Shut down processing thread
        self._thread_stop()

        # Log status
        self.log.out("[interpreter] TERMINATED.")

        # Session is down
        self.up = False

    @staticmethod
    def _stream_worker(in_stream, out_queue):
        """reads lines from stream and puts them into queue"""

        for line in iter(in_stream.readline, b""):
            out_queue.put(line)
        in_stream.close()

    @staticmethod
    def _start_stream_worker(in_stream, worker_function):
        """starts reader thread and returns a thread object and a queue object"""

        out_queue = queue.Queue()
        reader_thread = threading.Thread(
            target=worker_function, args=(in_stream, out_queue)
        )
        reader_thread.daemon = True
        reader_thread.start()
        return reader_thread, out_queue

    @staticmethod
    def _read_stream(in_queue, processing_function):
        """reads lines from queue and processes them"""

        try:
            line = in_queue.get_nowait()
        except queue.Empty:
            pass
        else:
            line = line.decode("utf-8")
            processing_function(line.strip("\n"))
            in_queue.task_done()

    def _processing_worker(self):
        """reads lines from queues and puts them into process methods"""

        # Log status
        self.log.out("[interpreter] Starting log processing thread ...")

        while self._is_alive():
            time.sleep(0.1)
            while not self.stdout_queue.empty():
                self._read_stream(
                    self.stdout_queue, lambda line: self.log.out("[P] " + line)
                )
            while not self.stderr_queue.empty():
                self._read_stream(
                    self.stderr_queue, lambda line: self.log.err("[P] " + line)
                )

        # Log status
        self.log.out("[interpreter] Joining worker threads and queues ...")

        self.stdout_queue.join()
        self.stderr_queue.join()
        self.stdout_thread.join()
        self.stderr_thread.join()

        # Log status
        self.log.out("[interpreter] ... worker threads and queues joined.")

    def _is_alive(self):

        return self.proc_winepython.poll() is None

    def _python_start(self):

        # Log status
        self.log.out(
            "[interpreter] Command: " + " ".join(self.p["server_command_list"])
        )

        # Prepare environment for interpreter - inherit from settings of current session!
        envvar_dict = {
            k: os.environ[k] for k in os.environ.keys()
        }  # HACK Required for Travis CI
        envvar_update_dict = dict(
            WENV_ARCH=self.p["arch"],  # Architecture
            WENV_PYTHONVERSION=self.p["pythonversion"],  # Version of Wine Python
        )
        envvar_dict.update(envvar_update_dict)

        # Log status
        self.log.out("[interpreter] Environment: " + str(envvar_dict))

        # Fire up Wine-Python process
        self.proc_winepython = subprocess.Popen(
            ["wenv", "python", "-u"] + self.p["server_command_list"],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=False,
            start_new_session=True,
            close_fds=True,
            env=envvar_dict,
        )

        # Status log
        self.log.out("[interpreter] Started with PID %d." % self.proc_winepython.pid)

        # Log status
        self.log.out("[interpreter] Starting stream reader threads ...")

        # Start worker threads and queues for reading from streams
        (
            self.stdout_thread,
            self.stdout_queue,
        ) = self._start_stream_worker(self.proc_winepython.stdout, self._stream_worker)
        (
            self.stderr_thread,
            self.stderr_queue,
        ) = self._start_stream_worker(self.proc_winepython.stderr, self._stream_worker)

        # Log status
        self.log.out("[interpreter] Stream reader threads started.")

        # Start processing thread for pushing lines into log
        self.processing_thread = threading.Thread(target=self._processing_worker)
        self.processing_thread.daemon = True
        self.processing_thread.start()

        # Log status
        self.log.out("[interpreter] Log processing thread started.")

    def _python_stop(self):

        self.log.out("[interpreter] Ensure process has terminated, waiting ...")

        # Time-step
        wait_for_seconds = 0.01
        # Timeout
        timeout_after_seconds = self.p["timeout_stop"]

        # Start waiting at ...
        started_waiting_at = time.time()
        # Wait for process
        while (
            self._is_alive()
            and started_waiting_at + timeout_after_seconds > time.time()
        ):
            time.sleep(wait_for_seconds)
        # Is process still alive?
        if self._is_alive():
            self.log.out(
                "[interpreter] ... did not terminate after %d seconds, sending SIGINT ..."
                % timeout_after_seconds
            )
            os.killpg(os.getpgid(self.proc_winepython.pid), signal.SIGINT)
        else:
            self.log.out(
                "[interpreter] ... terminated on its own after %.02f seconds."
                % (time.time() - started_waiting_at)
            )
            return

        # Start waiting at ...
        started_waiting_at = time.time()
        # Wait for process
        while (
            self._is_alive()
            and started_waiting_at + timeout_after_seconds > time.time()
        ):
            time.sleep(wait_for_seconds)
        # Is process still alive?
        if self._is_alive():
            self.log.out(
                "[interpreter] ... did not terminate after %d seconds, sending SIGTERM ..."
                % timeout_after_seconds
            )
            os.killpg(os.getpgid(self.proc_winepython.pid), signal.SIGTERM)
        else:
            self.log.out(
                "[interpreter] ... terminated with SIGINT after %.02f seconds."
                % (time.time() - started_waiting_at)
            )
            return

        # Is process still alive?
        if self._is_alive():
            self.log.out("[interpreter] ... failed to terminate with SIGTERM!")
            raise TimeoutError("interpreter process could not be terminated")
        else:
            self.log.out("[interpreter] ... terminated with SIGTERM.")
            return

    def _thread_stop(self):

        timeout_after_seconds = self.p["timeout_stop"]

        self.log.out("[interpreter] Joining processing thread ...")

        # Joining thread ...
        self.processing_thread.join(timeout=timeout_after_seconds)
        if self.processing_thread.is_alive():
            self.log.out("[interpreter] ... failed to join thread!")
            raise TimeoutError("processing thread could not be terminated")

        # Log status
        self.log.out("[interpreter] ... joined.")
<commit_msg>interpreter cleanup (2)<commit_after># -*- coding: utf-8 -*-

"""

ZUGBRUECKE
Calling routines in Windows DLLs from Python scripts running on unixlike systems
https://github.com/pleiszenburg/zugbruecke

    src/zugbruecke/core/interpreter.py: Class for managing Python interpreter on Wine

    Required to run on platform / side: [UNIX]

    Copyright (C) 2017-2021 Sebastian M. Ernst <ernst@pleiszenburg.de>

<LICENSE_BLOCK>
The contents of this file are subject to the GNU Lesser General Public License
Version 2.1 ("LGPL" or "License"). You may not use this file except in
compliance with the License. You may obtain a copy of the License at
https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt
https://github.com/pleiszenburg/zugbruecke/blob/master/LICENSE

Software distributed under the License is distributed on an "AS IS" basis,
WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
specific language governing rights and limitations under the License.
</LICENSE_BLOCK>

"""

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# IMPORT
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

import os
import queue
import signal
import subprocess
import time
import threading

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# WINE PYTHON INTERPRETER CLASS
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

class Interpreter:
    """
    Class for managing Python interpreter on Wine
    """

    def __init__(self, session_id, parameter, session_log):

        # Set ID, parameters and pointer to log
        self._id = session_id
        self._p = parameter
        self._log = session_log

        # Log status
        self._log.out("[interpreter] STARTING ...")

        # Session is up
        self._up = True

        # Start wine python
        self._python_start()

        # Log status
        self._log.out("[interpreter] STARTED.")

    # session destructor
    def terminate(self):

        if not self._up:
            return

        # Log status
        self._log.out("[interpreter] TERMINATING ...")

        # Shut down wine python
        self._python_stop()

        # Shut down processing thread
        self._thread_stop()

        # Log status
        self._log.out("[interpreter] TERMINATED.")

        # Session is down
        self._up = False

    @staticmethod
    def _stream_worker(in_stream, out_queue):
        """reads lines from stream and puts them into queue"""

        for line in iter(in_stream.readline, b""):
            out_queue.put(line)
        in_stream.close()

    @staticmethod
    def _start_stream_worker(in_stream, worker_function):
        """starts reader thread and returns a thread object and a queue object"""

        out_queue = queue.Queue()
        reader_thread = threading.Thread(
            target=worker_function, args=(in_stream, out_queue)
        )
        reader_thread.daemon = True
        reader_thread.start()
        return reader_thread, out_queue

    @staticmethod
    def _read_stream(in_queue, processing_function):
        """reads lines from queue and processes them"""

        try:
            line = in_queue.get_nowait()
        except queue.Empty:
            pass
        else:
            line = line.decode("utf-8")
            processing_function(line.strip("\n"))
            in_queue.task_done()

    def _processing_worker(self):
        """reads lines from queues and puts them into process methods"""

        # Log status
        self._log.out("[interpreter] Starting log processing thread ...")

        while self._is_alive():
            time.sleep(0.1)
            while not self.stdout_queue.empty():
                self._read_stream(
                    self.stdout_queue, lambda line: self._log.out("[P] " + line)
                )
            while not self.stderr_queue.empty():
                self._read_stream(
                    self.stderr_queue, lambda line: self._log.err("[P] " + line)
                )

        # Log status
        self._log.out("[interpreter] Joining worker threads and queues ...")

        self.stdout_queue.join()
        self.stderr_queue.join()
        self.stdout_thread.join()
        self.stderr_thread.join()

        # Log status
        self._log.out("[interpreter] ... worker threads and queues joined.")

    def _is_alive(self):

        return self._proc_winepython.poll() is None

    def _python_start(self):

        # Log status
        self._log.out(
            "[interpreter] Command: " + " ".join(self._p["server_command_list"])
        )

        # Prepare environment for interpreter - inherit from settings of current session!
        envvar_dict = {
            k: os.environ[k] for k in os.environ.keys()
        }  # HACK Required for Travis CI
        envvar_update_dict = dict(
            WENV_ARCH=self._p["arch"],  # Architecture
            WENV_PYTHONVERSION=self._p["pythonversion"],  # Version of Wine Python
        )
        envvar_dict.update(envvar_update_dict)

        # Log status
        self._log.out("[interpreter] Environment: " + str(envvar_dict))

        # Fire up Wine-Python process
        self._proc_winepython = subprocess.Popen(
            ["wenv", "python", "-u"] + self._p["server_command_list"],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=False,
            start_new_session=True,
            close_fds=True,
            env=envvar_dict,
        )

        # Status log
        self._log.out("[interpreter] Started with PID %d." % self._proc_winepython.pid)

        # Log status
        self._log.out("[interpreter] Starting stream reader threads ...")

        # Start worker threads and queues for reading from streams
        (
            self.stdout_thread,
            self.stdout_queue,
        ) = self._start_stream_worker(self._proc_winepython.stdout, self._stream_worker)
        (
            self.stderr_thread,
            self.stderr_queue,
        ) = self._start_stream_worker(self._proc_winepython.stderr, self._stream_worker)

        # Log status
        self._log.out("[interpreter] Stream reader threads started.")

        # Start processing thread for pushing lines into log
        self._processing_thread = threading.Thread(target=self._processing_worker)
        self._processing_thread.daemon = True
        self._processing_thread.start()

        # Log status
        self._log.out("[interpreter] Log processing thread started.")

    def _python_stop(self):

        self._log.out("[interpreter] Ensure process has terminated, waiting ...")

        # Time-step
        wait_for_seconds = 0.01
        # Timeout
        timeout_after_seconds = self._p["timeout_stop"]

        # Start waiting at ...
        started_waiting_at = time.time()
        # Wait for process
        while (
            self._is_alive()
            and started_waiting_at + timeout_after_seconds > time.time()
        ):
            time.sleep(wait_for_seconds)
        # Is process still alive?
        if self._is_alive():
            self._log.out(
                "[interpreter] ... did not terminate after %d seconds, sending SIGINT ..."
                % timeout_after_seconds
            )
            os.killpg(os.getpgid(self._proc_winepython.pid), signal.SIGINT)
        else:
            self._log.out(
                "[interpreter] ... terminated on its own after %.02f seconds."
                % (time.time() - started_waiting_at)
            )
            return

        # Start waiting at ...
        started_waiting_at = time.time()
        # Wait for process
        while (
            self._is_alive()
            and started_waiting_at + timeout_after_seconds > time.time()
        ):
            time.sleep(wait_for_seconds)
        # Is process still alive?
        if self._is_alive():
            self._log.out(
                "[interpreter] ... did not terminate after %d seconds, sending SIGTERM ..."
                % timeout_after_seconds
            )
            os.killpg(os.getpgid(self._proc_winepython.pid), signal.SIGTERM)
        else:
            self._log.out(
                "[interpreter] ... terminated with SIGINT after %.02f seconds."
                % (time.time() - started_waiting_at)
            )
            return

        # Is process still alive?
        if self._is_alive():
            self._log.out("[interpreter] ... failed to terminate with SIGTERM!")
            raise TimeoutError("interpreter process could not be terminated")
        else:
            self._log.out("[interpreter] ... terminated with SIGTERM.")
            return

    def _thread_stop(self):

        timeout_after_seconds = self._p["timeout_stop"]

        self._log.out("[interpreter] Joining processing thread ...")

        # Joining thread ...
        self._processing_thread.join(timeout=timeout_after_seconds)
        if self._processing_thread.is_alive():
            self._log.out("[interpreter] ... failed to join thread!")
            raise TimeoutError("processing thread could not be terminated")

        # Log status
        self._log.out("[interpreter] ... joined.")
<|endoftext|>


<commit_before><commit_msg>Add refresh_db flag to pkgrepo.managed<commit_after><|endoftext|>


<commit_before># coding=utf8
"""
safety.py - Alerts about malicious URLs
Copyright © 2014, Elad Alfassa, <elad@fedoraproject.org>
Licensed under the Eiffel Forum License 2.

This module uses virustotal.com
"""
from __future__ import unicode_literals
from __future__ import print_function
import willie.web as web
from willie.config import ConfigurationError
from willie.formatting import color, bold
import willie.tools
import willie.module
import sys
import json
import time
import os.path

if sys.version_info.major > 2:
    unicode = str
    from urllib.request import urlretrieve
    from urllib.parse import urlparse
else:
    from urllib import urlretrieve
    from urlparse import urlparse

vt_base_api_url = 'https://www.virustotal.com/vtapi/v2/url/'
malware_domains = []

def configure(config):
    """

    | [safety] | example | purpose |
    | ---- | ------- | ------- |
    | enabled_by_default | True | Enable safety on implicity on channels |
    | vt_api_key | ea4ca709a686edfcc96a144c224935776e2ba46b77 | VirusTotal API key |
    """
    if config.option('Configure malicious URL protection?'):
        config.add_section('safety')
        config.add_option('safety', 'enabled_by_default', 'Enable malicious URL checking for channels by default?', True)
        config.interactive_add('safety', 'vt_api_key', 'VirusTotal API Key (not mandatory)', None)

def setup(bot):
    if not bot.config.has_section('safety'):
        raise ConfigurationError("Safety module not configured")
    if bot.db and not bot.db.preferences.has_columns('safety'):
        bot.db.preferences.add_columns(['safety'])
    bot.memory['safety_cache'] = willie.tools.WillieMemory()
    loc = os.path.join(bot.config.homedir, 'malwaredomains.txt')
    if os.path.isfile(loc):
        if os.path.getmtime(loc) < time.time() - 24 * 60 * 60 * 7:
            # File exists but older than one week, update
            _download_malwaredomains_db(loc)
    else:
        _download_malwaredomains_db(loc)
    with open(loc, 'r') as f:
        for line in f:
            malware_domains.append(unicode(line).strip())

def _download_malwaredomains_db(path):
    print('Downloading malwaredomains db…')
    urlretrieve('http://mirror1.malwaredomains.com/files/justdomains', path)

@willie.module.rule('(?u).*(https?://\S+).*')
@willie.module.priority('high')
def url_handler(bot, trigger):
    """ Check for malicious URLs """
    check = True    # Enable URL checking
    strict = False  # Strict mode: kick on malicious URL
    positives = 0   # Number of engines saying it's malicious
    total = 0       # Number of total engines
    use_vt = True   # Use VirusTotal
    if bot.config.has_section('safety'):
        check = bot.config.safety.enabled_by_default
        if check is None:
            # If not set, assume default
            check = True
        else:
            check = bool(check)
    # DB overrides config:
    if bot.db and trigger.sender.lower() in bot.db.preferences:
        setting = bot.db.preferences.get(trigger.sender.lower(), 'safety')
        if setting == 'off':
            return  # Not checking
        elif setting in ['on', 'strict', 'local', 'local strict']:
            check = True
        if setting == 'strict' or setting == 'local strict':
            strict = True
        if setting == 'local' or setting == 'local strict':
            use_vt = False

    if not check:
        return  # Not overriden by DB, configured default off

    apikey = bot.config.safety.vt_api_key
    try:
        if apikey is not None and use_vt:
            payload = {'resource': unicode(trigger),
                       'apikey': apikey,
                       'scan': '1'}

            if trigger not in bot.memory['safety_cache']:
                result = web.post(vt_base_api_url+'report', payload)
                if sys.version_info.major > 2:
                    result = result.decode('utf-8')
                result = json.loads(result)
                age = time.time()
                data = {'positives': result['positives'],
                        'total': result['total'],
                        'age': age}
                bot.memory['safety_cache'][trigger] = data
                if len(bot.memory['safety_cache']) > 1024:
                    _clean_cache(bot)
            else:
                print('using cache')
                result = bot.memory['safety_cache'][trigger]
            positives = result['positives']
            total = result['total']
    except Exception as e:
        bot.debug('[safety]', e, 'debug')
        pass  # Ignoring exceptions with VT so MalwareDomains will always work

    if unicode(urlparse(trigger).netloc) in malware_domains:
        # malwaredomains is more trustworthy than some VT engines
        # therefor it gets a weight of 10 engines when calculating confidence
        positives += 10
        total += 10

    if positives > 1:
        # Possibly malicious URL detected!
        confidence = '{}%'.format(round((positives / total) * 100))
        msg = 'link posted by %s is possibliy malicious ' % bold(trigger.nick)
        msg += '(confidence %s - %s/%s)' % (confidence, positives, total)
        bot.say('[' + bold(color('WARNING', 'red')) + '] ' + msg)
        if strict:
            bot.write(['KICK', trigger.sender, trigger.nick,
                       'Posted a malicious link'])

@willie.module.commands('safety')
def toggle_safety(bot, trigger):
    """ Set safety setting for channel """
    allowed_states = ['strict', 'on', 'off', 'local', 'local strict']
    if not trigger.group(2) or trigger.group(2).lower() not in allowed_states:
        bot.reply('Available options: strict / on /off')
        return
    if not bot.db:
        bot.reply('No database configured, can\'t modify settings')
        return
    if not trigger.isop and not trigger.admin:
        bot.reply('Only channel operators can change safety settings')

    channel = trigger.sender.lower()
    bot.db.preferences.update(channel, {'safety': trigger.group(2).lower()})
    bot.reply('Safety is now set to %s in this channel' % trigger.group(2))

# Clean the cache every day, also when > 1024 entries
@willie.module.interval(24 * 60 * 60)
def _clean_cache(bot):
    """ Cleanup old entries in URL cache """
    # TODO probably should be using locks here, to make sure stuff doesn't
    # explode
    oldest_key_age = 0
    oldest_key = ''
    for key, data in willie.tools.iteritems(bot.memory['safety_cache']):
        if data['age'] > oldest_key_age:
            oldest_key_age = data['age']
            oldest_key = key
    if oldest_key in bot.memory['safety_cache']:
        del bot.memory['safety_cache'][oldest_key]
<commit_msg>[safety] Support simple whitelist configuration<commit_after># coding=utf8
"""
safety.py - Alerts about malicious URLs
Copyright © 2014, Elad Alfassa, <elad@fedoraproject.org>
Licensed under the Eiffel Forum License 2.

This module uses virustotal.com
"""
from __future__ import unicode_literals
from __future__ import print_function
import willie.web as web
from willie.config import ConfigurationError
from willie.formatting import color, bold
import willie.tools
import willie.module
import sys
import json
import time
import os.path

if sys.version_info.major > 2:
    unicode = str
    from urllib.request import urlretrieve
    from urllib.parse import urlparse
else:
    from urllib import urlretrieve
    from urlparse import urlparse

vt_base_api_url = 'https://www.virustotal.com/vtapi/v2/url/'
malware_domains = []
known_good = []

def configure(config):
    """

    | [safety] | example | purpose |
    | ---- | ------- | ------- |
    | enabled_by_default | True | Enable safety on implicity on channels |
    | vt_api_key | ea4ca709a686edfcc96a144c224935776e2ba46b77 | VirusTotal API key |
    | known_good | youtube.com,vimeo.com | list of "known good" domains to ignore |
    """
    if config.option('Configure malicious URL protection?'):
        config.add_section('safety')
        config.add_option('safety', 'enabled_by_default', 'Enable malicious URL checking for channels by default?', True)
        config.interactive_add('safety', 'vt_api_key', 'VirusTotal API Key (not mandatory)', None)

def setup(bot):
    if not bot.config.has_section('safety'):
        raise ConfigurationError("Safety module not configured")
    if bot.db and not bot.db.preferences.has_columns('safety'):
        bot.db.preferences.add_columns(['safety'])
    bot.memory['safety_cache'] = willie.tools.WillieMemory()
    known_good = bot.confg.get_list('safety', 'known_good')

    loc = os.path.join(bot.config.homedir, 'malwaredomains.txt')
    if os.path.isfile(loc):
        if os.path.getmtime(loc) < time.time() - 24 * 60 * 60 * 7:
            # File exists but older than one week, update
            _download_malwaredomains_db(loc)
    else:
        _download_malwaredomains_db(loc)
    with open(loc, 'r') as f:
        for line in f:
            malware_domains.append(unicode(line).strip())

def _download_malwaredomains_db(path):
    print('Downloading malwaredomains db…')
    urlretrieve('http://mirror1.malwaredomains.com/files/justdomains', path)

@willie.module.rule('(?u).*(https?://\S+).*')
@willie.module.priority('high')
def url_handler(bot, trigger):
    """ Check for malicious URLs """
    check = True    # Enable URL checking
    strict = False  # Strict mode: kick on malicious URL
    positives = 0   # Number of engines saying it's malicious
    total = 0       # Number of total engines
    use_vt = True   # Use VirusTotal
    if bot.config.has_section('safety'):
        check = bot.config.safety.enabled_by_default
        if check is None:
            # If not set, assume default
            check = True
        else:
            check = bool(check)
    # DB overrides config:
    if bot.db and trigger.sender.lower() in bot.db.preferences:
        setting = bot.db.preferences.get(trigger.sender.lower(), 'safety')
        if setting == 'off':
            return  # Not checking
        elif setting in ['on', 'strict', 'local', 'local strict']:
            check = True
        if setting == 'strict' or setting == 'local strict':
            strict = True
        if setting == 'local' or setting == 'local strict':
            use_vt = False

    if not check:
        return  # Not overriden by DB, configured default off

    netloc = urlparse(trigger).netloc
    if netloc in known_good:
        return  # whitelisted

    apikey = bot.config.safety.vt_api_key
    try:
        if apikey is not None and use_vt:
            payload = {'resource': unicode(trigger),
                       'apikey': apikey,
                       'scan': '1'}

            if trigger not in bot.memory['safety_cache']:
                result = web.post(vt_base_api_url+'report', payload)
                if sys.version_info.major > 2:
                    result = result.decode('utf-8')
                result = json.loads(result)
                age = time.time()
                data = {'positives': result['positives'],
                        'total': result['total'],
                        'age': age}
                bot.memory['safety_cache'][trigger] = data
                if len(bot.memory['safety_cache']) > 1024:
                    _clean_cache(bot)
            else:
                print('using cache')
                result = bot.memory['safety_cache'][trigger]
            positives = result['positives']
            total = result['total']
    except Exception as e:
        bot.debug('[safety]', e, 'debug')
        pass  # Ignoring exceptions with VT so MalwareDomains will always work

    if unicode(netloc) in malware_domains:
        # malwaredomains is more trustworthy than some VT engines
        # therefor it gets a weight of 10 engines when calculating confidence
        positives += 10
        total += 10

    if positives > 1:
        # Possibly malicious URL detected!
        confidence = '{}%'.format(round((positives / total) * 100))
        msg = 'link posted by %s is possibliy malicious ' % bold(trigger.nick)
        msg += '(confidence %s - %s/%s)' % (confidence, positives, total)
        bot.say('[' + bold(color('WARNING', 'red')) + '] ' + msg)
        if strict:
            bot.write(['KICK', trigger.sender, trigger.nick,
                       'Posted a malicious link'])

@willie.module.commands('safety')
def toggle_safety(bot, trigger):
    """ Set safety setting for channel """
    allowed_states = ['strict', 'on', 'off', 'local', 'local strict']
    if not trigger.group(2) or trigger.group(2).lower() not in allowed_states:
        bot.reply('Available options: strict / on /off')
        return
    if not bot.db:
        bot.reply('No database configured, can\'t modify settings')
        return
    if not trigger.isop and not trigger.admin:
        bot.reply('Only channel operators can change safety settings')

    channel = trigger.sender.lower()
    bot.db.preferences.update(channel, {'safety': trigger.group(2).lower()})
    bot.reply('Safety is now set to %s in this channel' % trigger.group(2))

# Clean the cache every day, also when > 1024 entries
@willie.module.interval(24 * 60 * 60)
def _clean_cache(bot):
    """ Cleanup old entries in URL cache """
    # TODO probably should be using locks here, to make sure stuff doesn't
    # explode
    oldest_key_age = 0
    oldest_key = ''
    for key, data in willie.tools.iteritems(bot.memory['safety_cache']):
        if data['age'] > oldest_key_age:
            oldest_key_age = data['age']
            oldest_key = key
    if oldest_key in bot.memory['safety_cache']:
        del bot.memory['safety_cache'][oldest_key]
<|endoftext|>