<commit_before>#!/usr/bin/env python3

import socket
import json
import multiprocessing
import sys
import time
import argparse

default_bound_ip = ''
default_bound_port = 53450
default_buffer_size = 1024
default_timeout = 10

def safe_string(dangerous_string):
	return dangerous_string.replace('\n', '\\n').replace('\r', '\\r').replace('\033', '[ESC]')

def client_thread(client, q, buffer_size):
	client_from = client.getpeername()
	try:
		while True:
			client.send('> '.encode('ascii'))
			data = client.recv(buffer_size)		# BLOCKING FUNCTION!!!
			data = data.decode('ascii')
			q.put([json.dumps({'event' : 'receive_data', 'data' : data, 'client_from' : client_from})])
			client.send('< {}'.format(data).encode('ascii'))
			q.put([json.dumps({'event' : 'send_data', 'data' : data, 'client_from' : client_from})])
			if data[:-2] == 'quit':
				break
		client.shutdown(socket.SHUT_RDWR)
		client.close()
	except BrokenPipeError:
		pass
	except socket.timeout:
		client.shutdown(socket.SHUT_RDWR)
		client.close()
	except KeyboardInterrupt:
		pass
	finally:
		q.put([json.dumps({'event' : 'lostclient', 'client_from' : client_from})])

def server_thread(server, q, buffer_size, timeout_seconds):
	try:
		while True:
			active_client_threads = []
			client, client_from = server.accept()		# BLOCKING FUNCTION!!!
			client.settimeout(timeout_seconds)
			thread = multiprocessing.Process(target=client_thread, args=(client, q, buffer_size))
			q.put([json.dumps({'event' : 'newclient', 'client_from' : client_from})])
			active_client_threads.append(thread)
			thread.daemon = True
			thread.start()
	except KeyboardInterrupt:
		pass

def server_setup(bound_ip, bound_port, buffer_size, timeout_seconds):
	try:
		q = multiprocessing.Queue()
		jobs = []
		connected_clients = 0
		server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
		if timeout_seconds < 1:
			timeout_seconds = None
		server.bind((bound_ip, bound_port))
		server.listen(1)
		server_ip, server_port = server.getsockname()
		print('Server started on {}:{}'.format(server_ip, server_port))
		thread = multiprocessing.Process(target=server_thread, args=(server, q, buffer_size, timeout_seconds))
		jobs.append(thread)
		thread.daemon = False
		thread.start()
		while True:
			while not q.empty():
				item = q.get()
				if type(item) is list and type(item[0]) is str:
					data = json.loads(item[0])
					if data['event'] and data['event'] == 'newclient':
						connected_clients += 1
						print('{}: Connected.'.format(data['client_from'][0]))
					elif data['event'] and data['event'] == 'receive_data':
						print(safe_string('{} -> {}'.format(data['client_from'][0], data['data'])))
					elif data['event'] and data['event'] == 'send_data':
						print(safe_string('{} <- {}'.format(data['client_from'][0], data['data'])))
					elif data['event'] and data['event'] == 'lostclient':
						connected_clients -= 1
						print('{}: Disconnected.'.format(data['client_from'][0]))
			time.sleep(0.02)	# 50 times per second.
	finally:
		server.shutdown(socket.SHUT_RDWR)
		server.close()
		print('\nServer closed.')

def main():
	parser = argparse.ArgumentParser(description='(server)^2 Modification Global Backend')
	parser.add_argument('-a', '--address', type=str, metavar='IP', dest='bound_ip', help='address to bind to', action='store', default=default_bound_ip)
	parser.add_argument('-p', '--port', type=int, metavar='PORT', dest='bound_port', help='port to bind to', action='store', default=default_bound_port)
	parser.add_argument('-b', '--buffer', type=int, metavar='BYTES', dest='buffer_size', help='size of network buffer in bytes', action='store', default=default_buffer_size)
	parser.add_argument('-t', '--timeout', type=int, metavar='SECONDS', dest='timeout_seconds', help='timeout in seconds or 0 to block', action='store', default=default_timeout)
	settings = vars(parser.parse_args())
	server_setup(settings['bound_ip'], settings['bound_port'], settings['buffer_size'], settings['timeout_seconds'])

if __name__ == '__main__':
	try:
		main()
	except KeyboardInterrupt:
		print('\nCaught KeyboardInterrupt!')
	finally:
		print('Stopping!')
		sys.exit(0)
<commit_msg>Remove unnecessary job storage.<commit_after>#!/usr/bin/env python3

import socket
import json
import multiprocessing
import sys
import time
import argparse

default_bound_ip = ''
default_bound_port = 53450
default_buffer_size = 1024
default_timeout = 10

def safe_string(dangerous_string):
	return dangerous_string.replace('\n', '\\n').replace('\r', '\\r').replace('\033', '[ESC]')

def client_thread(client, q, buffer_size):
	client_from = client.getpeername()
	try:
		while True:
			client.send('> '.encode('ascii'))
			data = client.recv(buffer_size)		# BLOCKING FUNCTION!!!
			data = data.decode('ascii')
			q.put([json.dumps({'event' : 'receive_data', 'data' : data, 'client_from' : client_from})])
			client.send('< {}'.format(data).encode('ascii'))
			q.put([json.dumps({'event' : 'send_data', 'data' : data, 'client_from' : client_from})])
			if data[:-2] == 'quit':
				break
		client.shutdown(socket.SHUT_RDWR)
		client.close()
	except BrokenPipeError:
		pass
	except socket.timeout:
		client.shutdown(socket.SHUT_RDWR)
		client.close()
	except KeyboardInterrupt:
		pass
	finally:
		q.put([json.dumps({'event' : 'lostclient', 'client_from' : client_from})])

def server_thread(server, q, buffer_size, timeout_seconds):
	try:
		while True:
			client, client_from = server.accept()		# BLOCKING FUNCTION!!!
			client.settimeout(timeout_seconds)
			thread = multiprocessing.Process(target=client_thread, args=(client, q, buffer_size))
			q.put([json.dumps({'event' : 'newclient', 'client_from' : client_from})])
			thread.daemon = True
			thread.start()
	except KeyboardInterrupt:
		pass

def server_setup(bound_ip, bound_port, buffer_size, timeout_seconds):
	try:
		q = multiprocessing.Queue()
		connected_clients = 0
		server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
		if timeout_seconds < 1:
			timeout_seconds = None
		server.bind((bound_ip, bound_port))
		server.listen(1)
		server_ip, server_port = server.getsockname()
		print('Server started on {}:{}'.format(server_ip, server_port))
		thread = multiprocessing.Process(target=server_thread, args=(server, q, buffer_size, timeout_seconds))
		thread.daemon = False
		thread.start()
		while True:
			while not q.empty():
				item = q.get()
				if type(item) is list and type(item[0]) is str:
					data = json.loads(item[0])
					if data['event'] and data['event'] == 'newclient':
						connected_clients += 1
						print('{}: Connected.'.format(data['client_from'][0]))
					elif data['event'] and data['event'] == 'receive_data':
						print(safe_string('{} -> {}'.format(data['client_from'][0], data['data'])))
					elif data['event'] and data['event'] == 'send_data':
						print(safe_string('{} <- {}'.format(data['client_from'][0], data['data'])))
					elif data['event'] and data['event'] == 'lostclient':
						connected_clients -= 1
						print('{}: Disconnected.'.format(data['client_from'][0]))
			time.sleep(0.02)	# 50 times per second.
	finally:
		server.shutdown(socket.SHUT_RDWR)
		server.close()
		print('\nServer closed.')

def main():
	parser = argparse.ArgumentParser(description='(server)^2 Modification Global Backend')
	parser.add_argument('-a', '--address', type=str, metavar='IP', dest='bound_ip', help='address to bind to', action='store', default=default_bound_ip)
	parser.add_argument('-p', '--port', type=int, metavar='PORT', dest='bound_port', help='port to bind to', action='store', default=default_bound_port)
	parser.add_argument('-b', '--buffer', type=int, metavar='BYTES', dest='buffer_size', help='size of network buffer in bytes', action='store', default=default_buffer_size)
	parser.add_argument('-t', '--timeout', type=int, metavar='SECONDS', dest='timeout_seconds', help='timeout in seconds or 0 to block', action='store', default=default_timeout)
	settings = vars(parser.parse_args())
	server_setup(settings['bound_ip'], settings['bound_port'], settings['buffer_size'], settings['timeout_seconds'])

if __name__ == '__main__':
	try:
		main()
	except KeyboardInterrupt:
		print('\nCaught KeyboardInterrupt!')
	finally:
		print('Stopping!')
		sys.exit(0)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import asyncio
import logging
import sys

import discord
from discord.ext.commands import when_mentioned_or
import yaml

from bot import BeattieBot

try:
    import uvloop
except ImportError:
    pass
else:
    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())

with open('config.yaml') as file:
    config = yaml.load(file)

self_bot = 'self' in sys.argv

if self_bot:
    token = config['self']
    bot = BeattieBot('b>', self_bot=True)
else:
    token = config['token']
    bot = BeattieBot(when_mentioned_or('>'))

extensions = ('cat', 'default', 'eddb', 'nsfw', 'repl', 'rpg', 'stats',
              'wolfram', 'xkcd')

for extension in extensions:
    try:
        bot.load_extension(extension)
    except Exception as e:
        print(f'Failed to load extension {extension}\n{type(e).__name__}: {e}')

if not self_bot:
    logger = logging.getLogger('discord')
    logger.setLevel(logging.INFO)
    handler = logging.FileHandler(
        filename='discord.log', encoding='utf-8', mode='w')
    handler.setFormatter(
        logging.Formatter('%(asctime)s:%(levelname)s:%(name)s: %(message)s'))
    logger.addHandler(handler)
    bot.logger = logger

bot.run(token, bot=not self_bot)
<commit_msg>Log at level DEBUG<commit_after>#!/usr/bin/env python
import asyncio
import logging
import sys

import discord
from discord.ext.commands import when_mentioned_or
import yaml

from bot import BeattieBot

try:
    import uvloop
except ImportError:
    pass
else:
    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())

with open('config.yaml') as file:
    config = yaml.load(file)

self_bot = 'self' in sys.argv

if self_bot:
    token = config['self']
    bot = BeattieBot('b>', self_bot=True)
else:
    token = config['token']
    bot = BeattieBot(when_mentioned_or('>'))

extensions = ('cat', 'default', 'eddb', 'nsfw', 'repl', 'rpg', 'stats',
              'wolfram', 'xkcd')

for extension in extensions:
    try:
        bot.load_extension(extension)
    except Exception as e:
        print(f'Failed to load extension {extension}\n{type(e).__name__}: {e}')

if not self_bot:
    logger = logging.getLogger('discord')
    logger.setLevel(logging.DEBUG)
    handler = logging.FileHandler(
        filename='discord.log', encoding='utf-8', mode='w')
    handler.setFormatter(
        logging.Formatter('%(asctime)s:%(levelname)s:%(name)s: %(message)s'))
    logger.addHandler(handler)
    bot.logger = logger

bot.run(token, bot=not self_bot)
<|endoftext|>


<commit_before>import re
import json
import argparse
from mtgz.colors import ColoredManaSymbol
from mtgz.search_engine import SearchEngine
from mtgz.services import DBUploader

def console():
	arguments_parse = argparse.ArgumentParser(description='Magic: The Gathering Search Engine')
	arguments_parse.add_argument('--name', metavar = '-n', nargs='*', help='look up for card\'s name')
	arguments_parse.add_argument('--type', nargs='*', help='look up for card\'s type')
	arguments_parse.add_argument('--text', nargs='*', help='look up for card\'s text')
	arguments_parse.add_argument('--cmc', nargs='?', help='look up for card\'s cmc')
	arguments_parse.add_argument('--rarity', nargs='?', help='look up for card\'s rarity')
	arguments_parse.add_argument('--color', nargs='?', help='look up for card\'s color identity')
	arguments_parse.add_argument('--upgrade', action='store_true', help='upgrade the database')

	return arguments_parse.parse_args()

def print_card(card):
	painter = ColoredManaSymbol()
	print('{0} {1}'.format(card['name'], (painter.color(card['manaCost']) if 'manaCost' in card else '')))
	print('{0} {1}\n'.format(card['type'], '({0}/{1})'.format(card['power'], card['toughness']) if 'power' in card else ''))
	if 'text' in card:
		text = card['text']
		for manacost in re.findall('{\w}|{\w/\w}', text):
			text = text.replace(manacost, painter.color(manacost))
		print(text)
	print('------------------------------------------------')

def main():
	DBUploader().upgrade()

	arguments = console()
	# mapped by card name
	# TODO 'pickle' me
	search_engine = SearchEngine(json.loads(open('AllCards.json').read()))
	if arguments.name is not None:
		search_engine.find_by('name', ' '.join(arguments.name))
	if arguments.type is not None:
		search_engine.find_by('type', ' '.join(arguments.type))
	if arguments.text is not None:
		search_engine.find_by('text', ' '.join(arguments.text))
	if arguments.cmc is not None:
		search_engine.find_by('cmc', arguments.cmc)
	if arguments.color is not None:
		search_engine.find_by('color', arguments.color)
	if arguments.rarity is not None:
		search_engine.find_by('rarity', arguments.rarity)

	filtered_cards = search_engine.filter()

	for card in filtered_cards:	print_card(card)
	print('\n{0} cards found'.format(len(filtered_cards)))

if __name__ == '__main__':
	main()
<commit_msg>Fixing English syntax<commit_after>import re
import json
import argparse
from mtgz.colors import ColoredManaSymbol
from mtgz.search_engine import SearchEngine
from mtgz.services import DBUploader

def console():
	arguments_parse = argparse.ArgumentParser(description='Magic: The Gathering Search Engine')
	arguments_parse.add_argument('--name', metavar = '-n', nargs='*', help='look up for card\'s name')
	arguments_parse.add_argument('--type', nargs='*', help='look up for card\'s type')
	arguments_parse.add_argument('--text', nargs='*', help='look up for card\'s text')
	arguments_parse.add_argument('--cmc', nargs='?', help='look up for card\'s cmc')
	arguments_parse.add_argument('--rarity', nargs='?', help='look up for card\'s rarity')
	arguments_parse.add_argument('--color', nargs='?', help='look up for card\'s color identity')
	arguments_parse.add_argument('--upgrade', action='store_true', help='upgrade database')

	return arguments_parse.parse_args()

def print_card(card):
	painter = ColoredManaSymbol()
	print('{0} {1}'.format(card['name'], (painter.color(card['manaCost']) if 'manaCost' in card else '')))
	print('{0} {1}\n'.format(card['type'], '({0}/{1})'.format(card['power'], card['toughness']) if 'power' in card else ''))
	if 'text' in card:
		text = card['text']
		for manacost in re.findall('{\w}|{\w/\w}', text):
			text = text.replace(manacost, painter.color(manacost))
		print(text)
	print('------------------------------------------------')

def main():
	DBUploader().upgrade()

	arguments = console()
	# mapped by card name
	# TODO 'pickle' me
	search_engine = SearchEngine(json.loads(open('AllCards.json').read()))
	if arguments.name is not None:
		search_engine.find_by('name', ' '.join(arguments.name))
	if arguments.type is not None:
		search_engine.find_by('type', ' '.join(arguments.type))
	if arguments.text is not None:
		search_engine.find_by('text', ' '.join(arguments.text))
	if arguments.cmc is not None:
		search_engine.find_by('cmc', arguments.cmc)
	if arguments.color is not None:
		search_engine.find_by('color', arguments.color)
	if arguments.rarity is not None:
		search_engine.find_by('rarity', arguments.rarity)

	filtered_cards = search_engine.filter()

	for card in filtered_cards:	print_card(card)
	print('\n{0} cards found'.format(len(filtered_cards)))

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>from flask import *
from flask.json import JSONEncoder
from flask.ext.cors import CORS

import simplejson as json
import os, sys
import datetime

app = Flask(__name__)
app.config.from_object(__name__)

app.config['SQLALCHEMY_DATABASE_URI'] = os.getenv("DATABASE_URL")
if app.config['SQLALCHEMY_DATABASE_URI'] == None:
    print "Need database config"
    sys.exit(1)

from models import db, Quote, Vote

db.init_app(app)

@app.route("/")
def hello():
    return "Hello World!"

@app.route("/quote", methods = ['GET'])
def get_quote():
    result = db.session.query(Vote.quote_id, db.func.sum(Vote.value).label("score")).group_by(Vote.quote_id).order_by("score DESC").all()
    return jsonify(result)

@app.route("/quote/<int:id>", methods = ['GET'])
def get_single_quote(id):
    quote = Quote.query.get(id)
    quote.view_count += 1
    db.session.commit()
    return jsonify(quote.serialize)

@app.route("/quote", methods = ['POST'])
def post_new_quote():
    body = request.get_json()
    conditions = {}
    if "conditions" in body:
        conditions = body['conditions']

    quote = Quote(text = body['text'], conditions = json.dumps(conditions), date_created = datetime.datetime.utcnow(), view_count = 1)
    db.session.add(quote)
    db.session.commit()

    vote = Vote(ip = request.remote_addr, value = 1, date_created = datetime.datetime.utcnow(), quote_id = quote.id)        #auto upvote every new quote by 1
    db.session.add(vote)
    db.session.commit()

    return jsonify(quote.serialize)

@app.route("/quote/<int:quote_id>/vote", methods = ['POST'])
def post_new_vote(quote_id):
    body = request.get_json()
    vote = Vote(ip = request.remote_addr, value = body['value'], date_created = datetime.datetime.utcnow(), quote_id = quote_id)
    db.session.add(vote)
    db.session.commit()

    return jsonify(vote.serialize)

cors = CORS(app)
if __name__ == "__main__":
    app.debug = True
    app.run()
<commit_msg>get real remote_addr<commit_after>from flask import *
from flask.json import JSONEncoder
from flask.ext.cors import CORS

import simplejson as json
import os, sys
import datetime

app = Flask(__name__)
app.config.from_object(__name__)

app.config['SQLALCHEMY_DATABASE_URI'] = os.getenv("DATABASE_URL")
if app.config['SQLALCHEMY_DATABASE_URI'] == None:
    print "Need database config"
    sys.exit(1)

from models import db, Quote, Vote

db.init_app(app)

@app.route("/")
def hello():
    return "Hello World!"

@app.route("/quote", methods = ['GET'])
def get_quote():
    result = db.session.query(Vote.quote_id, db.func.sum(Vote.value).label("score")).group_by(Vote.quote_id).order_by("score DESC").all()
    return jsonify(result)

@app.route("/quote/<int:id>", methods = ['GET'])
def get_single_quote(id):
    quote = Quote.query.get(id)
    quote.view_count += 1
    db.session.commit()
    return jsonify(quote.serialize)

@app.route("/quote", methods = ['POST'])
def post_new_quote():
    body = request.get_json()
    conditions = {}
    if "conditions" in body:
        conditions = body['conditions']

    quote = Quote(text = body['text'], conditions = json.dumps(conditions), date_created = datetime.datetime.utcnow(), view_count = 1)
    db.session.add(quote)
    db.session.commit()

    vote = Vote(ip = request.environ.get('X-Forward-For', request.remote_addr), value = 1, date_created = datetime.datetime.utcnow(), quote_id = quote.id)        #auto upvote every new quote by 1
    db.session.add(vote)
    db.session.commit()

    return jsonify(quote.serialize)

@app.route("/quote/<int:quote_id>/vote", methods = ['POST'])
def post_new_vote(quote_id):
    body = request.get_json()

    vote = Vote(ip = request.environ.get('X-Forward-For', request.remote_addr), value = body['value'], date_created = datetime.datetime.utcnow(), quote_id = quote_id)
    db.session.add(vote)
    db.session.commit()

    return jsonify(vote.serialize)

cors = CORS(app)
if __name__ == "__main__":
    app.debug = True
    app.run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from datetime import datetime
import os
import urllib2
import json
import smtplib
import logging
import logging.handlers
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
# Import settings
from settings import SETTINGS

# Setup log
def log_setup():
    rotate_handler = logging.handlers.RotatingFileHandler(SETTINGS['log_file_name'], maxBytes=2048, backupCount=1)
    formatter = logging.Formatter(u'%(filename)s[LINE:%(lineno) d]# %(levelname)-8s [%(asctime)s] %(message)s')
    rotate_handler.setFormatter(formatter)
    logger = logging.getLogger()
    logger.addHandler(rotate_handler)
    logger.setLevel(logging.DEBUG)

# Get data from web resource
def get_data():
    try:
        logging.info(u'Requesting ' + SETTINGS['url'] + u'...')
        response = urllib2.urlopen(SETTINGS['url']).read()
        data = json.loads(response)
        return data
    except Exceptiona as exc:
        logging.error(u'Exception during web request ' + exc.message)
    raise

# Parse received data, search for certain criteria
def parse_data(data):
    logging.info(u'Parsing data...')

    results = []
    # Provide your own code for this
    for planning in data['planning']:
        if not planning['disabled']:
            for interval in planning['intervals']:
                if interval["free"]:
                    results.append(interval["formattedDate"])
    return results

# Format  message with results
def format_message(data):
    try:
        logging.info(u'Formatting message...')

        if len(data) == 0:
            return u''

        mail_template = open(SETTINGS['mail_template']).read()
        results_list = u'
'
        for item in data:
            results_list += u'
' + item + u'
'
        results_list += u'
'

        mail_template = mail_template.replace(u'{{date}}', datetime.now().strftime('%d.%m.%Y %H:%M:%S'))
        mail_template = mail_template.replace(u'{{results}}', results_list)
        mail_template = mail_template.replace(u'{{visit_url}}', SETTINGS['mail_msg_info'])

        return mail_template

    except Exceptiona as exc:
        logging.error(u'Exception while formatting ' + exc.message)
        raise

# Send mail with notification
def send_mail(message):
    try:
        logging.info(u'Sending email...')
        from_ = SETTINGS['mail_sender_address']
        to_ = SETTINGS['mail_receiver_address']

        # Create message container - the correct MIME type is multipart/alternative.
        msg = MIMEMultipart('alternative')
        msg['Subject'] = SETTINGS['mail_subject']
        msg['From'] = from_
        msg['To'] = to_

        # Create the body of the message (a plain-text and an HTML version).
        text = u'See HTML message for more info'
        html1 = message

        # Record the MIME types of both parts - text/plain and text/html.
        part1 = MIMEText(text, 'plain')
        part2 = MIMEText(html1, 'html')

        # Attach parts into message container.
        # According to RFC 2046, the last part of a multipart message, in this case
        # the HTML message, is best and preferred.
        msg.attach(part1)
        msg.attach(part2)
        # Send the message via local SMTP server.
        mail = smtplib.SMTP(SETTINGS['mail_smtp_address'], SETTINGS['mail_smtp_port'])

        mail.ehlo()

        mail.starttls()

        # Note, if you're using Gmail, you should allow unsafe apps to send emails in Google account.
        mail.login(SETTINGS['mail_sender_address'], SETTINGS['mail_sender_password'])
        mail.sendmail(from_, to_, msg.as_string())
        mail.quit()

        return True

    except Exception as exc:
        logging.error(u'Exception during mail sending ' + exc.message)
        raise

# Function that run all job, if previous doesn't create file with results.
def run_task():
    # If result file is exist, it means that script has already ran successfully
    if os.path.isfile(SETTINGS['result_file_name']):
        return
    else:
        try:
            log_setup()
            data = get_data()
            parsed_data = parse_data(data)
            # If there are results, format them and send by email
            if len(parsed_data) > 0:
                formatted_message = format_message(parsed_data)
                send_mail(formatted_message)
                # Create file with results
                file_result = open(SETTINGS['result_file_name'], 'w+')
                file_result.write('Data parsed and sent successfully at ' + datetime.now().strftime('%d.%m.%Y %H:%M:%S'))
                file_result.close()

        except Exception as exc:
            logging.error(u'Exception ' + exc.message)

# Run the script
run_task()
<commit_msg>Convert relative filenames to absolute.<commit_after># -*- coding: utf-8 -*-
from datetime import datetime
import os
import urllib2
import json
import smtplib
import logging
import logging.handlers
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
# Import settings
from settings import SETTINGS

# Returns absolute path to file (used for names from settings)
def abs_path(path):
    return os.path.join(os.path.dirname(__file__), path)

# Setup log
def log_setup():
    rotate_handler = logging.handlers.RotatingFileHandler(abs_path(SETTINGS['log_file_name']), maxBytes=2048, backupCount=1)
    formatter = logging.Formatter(u'%(filename)s[LINE:%(lineno) d]# %(levelname)-8s [%(asctime)s] %(message)s')
    rotate_handler.setFormatter(formatter)
    logger = logging.getLogger()
    logger.addHandler(rotate_handler)
    logger.setLevel(logging.DEBUG)

# Get data from web resource
def get_data():
    try:
        logging.info(u'Requesting ' + SETTINGS['url'] + u'...')
        response = urllib2.urlopen(SETTINGS['url']).read()
        data = json.loads(response)
        return data
    except Exceptiona as exc:
        logging.error(u'Exception during web request ' + exc.message)
    raise

# Parse received data, search for certain criteria
def parse_data(data):
    logging.info(u'Parsing data...')

    results = []
    # Provide your own code for this
    for planning in data['planning']:
        if not planning['disabled']:
            for interval in planning['intervals']:
                if interval["free"]:
                    results.append(interval["formattedDate"])

    if len(results) == 0:
        logging.info(u'There are no results')

    return results

# Format  message with results
def format_message(data):
    try:
        logging.info(u'Formatting message...')

        if len(data) == 0:
            return u''

        mail_template = open(abs_path(SETTINGS['mail_template'])).read()
        results_list = u'
'
        for item in data:
            results_list += u'
' + item + u'
'
        results_list += u'
'

        mail_template = mail_template.replace(u'{{date}}', datetime.now().strftime('%d.%m.%Y %H:%M:%S'))
        mail_template = mail_template.replace(u'{{results}}', results_list)
        mail_template = mail_template.replace(u'{{visit_url}}', SETTINGS['mail_msg_info'])

        return mail_template

    except Exceptiona as exc:
        logging.error(u'Exception while formatting ' + exc.message)
        raise

# Send mail with notification
def send_mail(message):
    try:
        logging.info('Sending email...')
        from_ = SETTINGS['mail_sender_address']
        to_ = SETTINGS['mail_receiver_address']

        # Create message container - the correct MIME type is multipart/alternative.
        msg = MIMEMultipart('alternative')
        msg['Subject'] = SETTINGS['mail_subject']
        msg['From'] = from_
        msg['To'] = to_

        # Create the body of the message (a plain-text and an HTML version).
        text = u'See HTML message for more info'
        html1 = message

        # Record the MIME types of both parts - text/plain and text/html.
        part1 = MIMEText(text, 'plain')
        part2 = MIMEText(html1, 'html')

        # Attach parts into message container.
        # According to RFC 2046, the last part of a multipart message, in this case
        # the HTML message, is best and preferred.
        msg.attach(part1)
        msg.attach(part2)
        # Send the message via local SMTP server.
        mail = smtplib.SMTP(SETTINGS['mail_smtp_address'], SETTINGS['mail_smtp_port'])

        mail.ehlo()

        mail.starttls()

        # Note, if you're using Gmail, you should allow unsafe apps to send emails in Google account.
        mail.login(SETTINGS['mail_sender_address'], SETTINGS['mail_sender_password'])
        mail.sendmail(from_, to_, msg.as_string())
        mail.quit()

        logging.info('Email sent')
        return True

    except Exception as exc:
        logging.error(u'Exception during mail sending ' + exc.message)
        raise

# Function that run all job, if previous doesn't create file with results.
def run_task():
    # If result file is exist, it means that script has already ran successfully
    if os.path.isfile(abs_path(SETTINGS['result_file_name'])):
        print 'Check result here ' + os.path.abspath(abs_path((SETTINGS['result_file_name'])))
        logging.info('Task completed, check results file here ' +
                     os.path.abspath(abs_path((SETTINGS['result_file_name']))))
        return
    else:
        try:
            log_setup()
            data = get_data()
            parsed_data = parse_data(data)
            # If there are results, format them and send by email
            if len(parsed_data) > 0:
                formatted_message = format_message(parsed_data)
                send_mail(formatted_message)
                print 'Mail sent'
                # Create file with results
                file_result = open(abs_path(SETTINGS['result_file_name']), 'w+')
                file_result.write('Data parsed and sent successfully at ' + datetime.now().strftime('%d.%m.%Y %H:%M:%S'))
                file_result.close()

        except Exception as exc:
            logging.error(u'Exception ' + exc.message)

# Run the script
run_task()
<|endoftext|>


<commit_before>from pyA20.gpio import gpio
from pyA20.gpio import port
from time import gmtime, strftime
import time
import requests

start_time = None
final_url = "https://racks.octanis.org/api/v1/racks/1/geography/KEY"
state = 0
channel = port.PA10
gpio.init()
gpio.setcfg(channel, gpio.INPUT)
gpio.pullup(port.PA10, gpio.PULLDOWN)
File = open("roomlog.txt", "a")

try:
  while True:
    time.sleep(2)
    prevstate =  state

    if gpio.input(port.PA10) == 1:
       state = 1
    else:
       state = 0
    if(prevstate != state):
    	print("change detected")
        if (start_time == None)
            start_time = time.time()
        File.write(strftime("%a, %d %b %Y %H:%M:%S", gmtime()) + " MED 3 2215 is " + ("opened." if state == 1 else "closed.") + "\n")

    else
        if (start_time is not None)
            elapsed_time = time.time() - start_time
            if (elapsed_time >= 60):
                start_time = None
                payload = {'accessibleToPublic': state}
            	response = requests.post(final_url, data=payload)

except KeyboardInterrupt:
    print ("Goodbye.")
<commit_msg>adding colons after ifs and elses<commit_after>from pyA20.gpio import gpio
from pyA20.gpio import port
from time import gmtime, strftime
import time
import requests

start_time = None
final_url = "https://racks.octanis.org/api/v1/racks/1/geography/KEY"
state = 0
channel = port.PA10
gpio.init()
gpio.setcfg(channel, gpio.INPUT)
gpio.pullup(port.PA10, gpio.PULLDOWN)
File = open("roomlog.txt", "a")

try:
  while True:
    time.sleep(2)
    prevstate =  state

    if gpio.input(port.PA10) == 1:
       state = 1
    else:
       state = 0
    if(prevstate != state):
    	print("change detected")
        if (start_time == None):
            start_time = time.time()
        File.write(strftime("%a, %d %b %Y %H:%M:%S", gmtime()) + " MED 3 2215 is " + ("opened." if state == 1 else "closed.") + "\n")

    else:
        if (start_time is not None):
            elapsed_time = time.time() - start_time
            if (elapsed_time >= 60):
                start_time = None
                payload = {'accessibleToPublic': state}
            	response = requests.post(final_url, data=payload)

except KeyboardInterrupt:
    print ("Goodbye.")
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- conding: utf-8 -*-
########################
#File Name:main.py
#Author:WmTiger
#Mail:bfstiger@gmail.com
#Created Time:2016-    09-07 10:49:35
########################

import sys
import socket
import cmdhandler
import struct
import conf
import threading
import globalvar

s = socket.socket()

def sendHeartBeat():
    print 'send heart beat'
    cmdhandler.sendMsg(s, 1010, [1])
    global t
    t = threading.Timer(3.0, sendHeartBeat)
    print t.enumerate()
    t.start()
t = threading.Timer(3.0, sendHeartBeat)

def main(host, port):
    print 'mian start', host, port
    databuf = bytes()
    BUFSIZE = 4096
    HEADSIZE = 8    # len: 4, cmd: 2, fmtlen: 2
    global s
    #s = socket.socket()
    connre = s.connect_ex((host,port))
    while 1:
        if connre == 0:
            break
        else:
            connre = s.connect_ex((host, port))
    global t
    t.start()
    #print 'Usage: cmd, [p1,...]'
    while 1:
    #   cmdandparams = raw_input('input cmd and params:')
    #   print cmdandparams
        if globalvar.needReconnect == 1:
            main()
            return
        temp = s.recv(BUFSIZE)
        if temp:
            databuf += temp
            while 1:
                if len(databuf) < HEADSIZE:
                    break
                headpack = struct.unpack('<ihh', databuf[:HEADSIZE])
#               print 'headpack:', headpack
                bodysize = headpack[0]
                cmdId = headpack[1]
                fmtsize = headpack[2]
                if len(databuf) < HEADSIZE + bodysize:
                    break
                fmt = databuf[HEADSIZE:HEADSIZE+fmtsize]
#               print 'fmt:',fmt
                body = databuf[HEADSIZE+fmtsize:HEADSIZE+bodysize]
                bodypack = struct.unpack('<'+fmt,body)
#               print 'bodypack',bodypack
                callCmdHandler(s, cmdId, bodypack)
                databuf = databuf[HEADSIZE+bodysize:]

def callCmdHandler(s, cmd, params):
#   print cmd, params
    if (cmd not in cmdhandler.cmdfmt):
        print cmd, 'not in cmdhander.cmdfmt'
        return 0
    funname = cmdhandler.cmdfmt[cmd][1]
#   print funname
    if (funname):
        fun = getattr(cmdhandler, funname)
        if (callable(fun)):
            fun(s, params)
        else:
            print 'fun not defined'
    else:
        print 'callback name is undefined'

if __name__ == "__main__":
    if (len(sys.argv) < 3):
        print 'usage: python',sys.argv[0], 'host port'

        host = conf.severhost[0]
        #host = '192.168.2.116'
        port = conf.severport[0]
        main(host, port)
    else:
    #   sys.exit()
        main(sys.argv[1], int(sys.argv[2]))
<commit_msg>ddd<commit_after><|endoftext|>


<commit_before>import copy
import glob
import os
import time

import gym
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

import algo
from arguments import get_args
from envs import make_vec_envs
from model import Policy
from storage import RolloutStorage
from visualize import visdom_plot

args = get_args()

assert args.algo in ['a2c', 'ppo', 'acktr']
if args.recurrent_policy:
    assert args.algo in ['a2c', 'ppo'], \
        'Recurrent policy is not implemented for ACKTR'

num_updates = int(args.num_frames) // args.num_steps // args.num_processes

torch.manual_seed(args.seed)
if args.cuda:
    torch.cuda.manual_seed(args.seed)

try:
    os.makedirs(args.log_dir)
except OSError:
    files = glob.glob(os.path.join(args.log_dir, '*.monitor.csv'))
    for f in files:
        os.remove(f)

def main():
    print("#######")
    print("WARNING: All rewards are clipped or normalized so you need to use a monitor (see envs.py) or visdom plot to get true rewards")
    print("#######")

    torch.set_num_threads(1)
    device = torch.device("cuda:0" if args.cuda else "cpu")

    if args.vis:
        from visdom import Visdom
        viz = Visdom(port=args.port)
        win = None

    envs = make_vec_envs(args.env_name, args.seed, args.num_processes,
                        args.gamma, args.log_dir, args.add_timestep, device)

    obs_shape = envs.observation_space.shape

    actor_critic = Policy(obs_shape, envs.action_space,
        base_kwargs={'recurrent': args.recurrent_policy})
    actor_critic.to(device)

    if args.algo == 'a2c':
        agent = algo.A2C_ACKTR(actor_critic, args.value_loss_coef,
                               args.entropy_coef, lr=args.lr,
                               eps=args.eps, alpha=args.alpha,
                               max_grad_norm=args.max_grad_norm)
    elif args.algo == 'ppo':
        agent = algo.PPO(actor_critic, args.clip_param, args.ppo_epoch, args.num_mini_batch,
                         args.value_loss_coef, args.entropy_coef, lr=args.lr,
                               eps=args.eps,
                               max_grad_norm=args.max_grad_norm)
    elif args.algo == 'acktr':
        agent = algo.A2C_ACKTR(actor_critic, args.value_loss_coef,
                               args.entropy_coef, acktr=True)

    rollouts = RolloutStorage(args.num_steps, args.num_processes, obs_shape,
        envs.action_space, actor_critic.recurrent_hidden_state_size)

    obs = envs.reset()
    rollouts.obs[0].copy_(obs)

    # These variables are used to compute average rewards for all processes.
    episode_rewards = torch.zeros([args.num_processes, 1])
    final_rewards = torch.zeros([args.num_processes, 1])

    rollouts.to(device)

    start = time.time()
    for j in range(num_updates):
        for step in range(args.num_steps):
            # Sample actions
            with torch.no_grad():
                value, action, action_log_prob, recurrent_hidden_states = actor_critic.act(
                        rollouts.obs[step],
                        rollouts.recurrent_hidden_states[step],
                        rollouts.masks[step])

            # Obser reward and next obs
            obs, reward, done, info = envs.step(action)
            episode_rewards += reward

            # If done then clean the history of observations.
            masks = torch.FloatTensor([[0.0] if done_ else [1.0] for done_ in done])
            final_rewards *= masks
            final_rewards += (1 - masks) * episode_rewards
            episode_rewards *= masks

            rollouts.insert(obs, recurrent_hidden_states, action, action_log_prob, value, reward, masks)

        with torch.no_grad():
            next_value = actor_critic.get_value(rollouts.obs[-1],
                                                rollouts.recurrent_hidden_states[-1],
                                                rollouts.masks[-1]).detach()

        rollouts.compute_returns(next_value, args.use_gae, args.gamma, args.tau)

        value_loss, action_loss, dist_entropy = agent.update(rollouts)

        rollouts.after_update()

        if j % args.save_interval == 0 and args.save_dir != "":
            save_path = os.path.join(args.save_dir, args.algo)
            try:
                os.makedirs(save_path)
            except OSError:
                pass

            # A really ugly way to save a model to CPU
            save_model = actor_critic
            if args.cuda:
                save_model = copy.deepcopy(actor_critic).cpu()

            save_model = [save_model,
                            hasattr(envs.venv, 'ob_rms') and envs.venv.ob_rms or None]

            torch.save(save_model, os.path.join(save_path, args.env_name + ".pt"))

        if j % args.log_interval == 0:
            end = time.time()
            total_num_steps = (j + 1) * args.num_processes * args.num_steps
            print("Updates {}, num timesteps {}, FPS {}, mean/median reward {:.1f}/{:.1f}, min/max reward {:.1f}/{:.1f}, entropy {:.5f}, value loss {:.5f}, policy loss {:.5f}".
                format(j, total_num_steps,
                       int(total_num_steps / (end - start)),
                       final_rewards.mean(),
                       final_rewards.median(),
                       final_rewards.min(),
                       final_rewards.max(), dist_entropy,
                       value_loss, action_loss))
        if args.vis and j % args.vis_interval == 0:
            try:
                # Sometimes monitor doesn't properly flush the outputs
                win = visdom_plot(viz, win, args.log_dir, args.env_name,
                                  args.algo, args.num_frames)
            except IOError:
                pass

if __name__ == "__main__":
    main()
<commit_msg>Change rewards logging<commit_after>import copy
import glob
import os
import time
from collections import deque

import gym
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

import algo
from arguments import get_args
from envs import make_vec_envs
from model import Policy
from storage import RolloutStorage
from visualize import visdom_plot

args = get_args()

assert args.algo in ['a2c', 'ppo', 'acktr']
if args.recurrent_policy:
    assert args.algo in ['a2c', 'ppo'], \
        'Recurrent policy is not implemented for ACKTR'

num_updates = int(args.num_frames) // args.num_steps // args.num_processes

torch.manual_seed(args.seed)
if args.cuda:
    torch.cuda.manual_seed(args.seed)

try:
    os.makedirs(args.log_dir)
except OSError:
    files = glob.glob(os.path.join(args.log_dir, '*.monitor.csv'))
    for f in files:
        os.remove(f)

def main():
    print("#######")
    print("WARNING: All rewards are clipped or normalized so you need to use a monitor (see envs.py) or visdom plot to get true rewards")
    print("#######")

    torch.set_num_threads(1)
    device = torch.device("cuda:0" if args.cuda else "cpu")

    if args.vis:
        from visdom import Visdom
        viz = Visdom(port=args.port)
        win = None

    envs = make_vec_envs(args.env_name, args.seed, args.num_processes,
                        args.gamma, args.log_dir, args.add_timestep, device)

    obs_shape = envs.observation_space.shape

    actor_critic = Policy(obs_shape, envs.action_space,
        base_kwargs={'recurrent': args.recurrent_policy})
    actor_critic.to(device)

    if args.algo == 'a2c':
        agent = algo.A2C_ACKTR(actor_critic, args.value_loss_coef,
                               args.entropy_coef, lr=args.lr,
                               eps=args.eps, alpha=args.alpha,
                               max_grad_norm=args.max_grad_norm)
    elif args.algo == 'ppo':
        agent = algo.PPO(actor_critic, args.clip_param, args.ppo_epoch, args.num_mini_batch,
                         args.value_loss_coef, args.entropy_coef, lr=args.lr,
                               eps=args.eps,
                               max_grad_norm=args.max_grad_norm)
    elif args.algo == 'acktr':
        agent = algo.A2C_ACKTR(actor_critic, args.value_loss_coef,
                               args.entropy_coef, acktr=True)

    rollouts = RolloutStorage(args.num_steps, args.num_processes, obs_shape,
        envs.action_space, actor_critic.recurrent_hidden_state_size)

    obs = envs.reset()
    rollouts.obs[0].copy_(obs)
    rollouts.to(device)

    episode_rewards = deque(maxlen=10)

    start = time.time()
    for j in range(num_updates):
        for step in range(args.num_steps):
            # Sample actions
            with torch.no_grad():
                value, action, action_log_prob, recurrent_hidden_states = actor_critic.act(
                        rollouts.obs[step],
                        rollouts.recurrent_hidden_states[step],
                        rollouts.masks[step])

            # Obser reward and next obs
            obs, reward, done, infos = envs.step(action)

            for info in infos:
                if 'episode' in info.keys():
                    episode_rewards.append(info['episode']['r'])

            # If done then clean the history of observations.
            masks = torch.FloatTensor([[0.0] if done_ else [1.0] for done_ in done])
            rollouts.insert(obs, recurrent_hidden_states, action, action_log_prob, value, reward, masks)

        with torch.no_grad():
            next_value = actor_critic.get_value(rollouts.obs[-1],
                                                rollouts.recurrent_hidden_states[-1],
                                                rollouts.masks[-1]).detach()

        rollouts.compute_returns(next_value, args.use_gae, args.gamma, args.tau)

        value_loss, action_loss, dist_entropy = agent.update(rollouts)

        rollouts.after_update()

        if j % args.save_interval == 0 and args.save_dir != "":
            save_path = os.path.join(args.save_dir, args.algo)
            try:
                os.makedirs(save_path)
            except OSError:
                pass

            # A really ugly way to save a model to CPU
            save_model = actor_critic
            if args.cuda:
                save_model = copy.deepcopy(actor_critic).cpu()

            save_model = [save_model,
                            hasattr(envs.venv, 'ob_rms') and envs.venv.ob_rms or None]

            torch.save(save_model, os.path.join(save_path, args.env_name + ".pt"))

        if j % args.log_interval == 0:
            end = time.time()
            total_num_steps = (j + 1) * args.num_processes * args.num_steps
            print("Updates {}, num timesteps {}, FPS {} \n Last {} episodes: mean/median reward {:.1f}/{:.1f}, min/max reward {:.1f}/{:.1f}, entropy {:.5f}, value loss {:.5f}, policy loss {:.5f}\n".
                format(j, total_num_steps,
                       int(total_num_steps / (end - start)),
                       len(episode_rewards),
                       np.mean(episode_rewards),
                       np.median(episode_rewards),
                       np.min(episode_rewards),
                       np.max(episode_rewards), dist_entropy,
                       value_loss, action_loss))
        if args.vis and j % args.vis_interval == 0:
            try:
                # Sometimes monitor doesn't properly flush the outputs
                win = visdom_plot(viz, win, args.log_dir, args.env_name,
                                  args.algo, args.num_frames)
            except IOError:
                pass

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import wx
import os
import signal
import xml.etree.ElementTree as ET
import multiprocessing
import argparse
import tempfile
import datetime
import time
import base64
import functools
import sys

import logging

sys.path.append(os.path.dirname(__file__))

import clippacloud
from clippacloud import exceptions
from clippacloud import clipcatcher
from clippacloud import config
from clippacloud import plugin

import icon
import traceback
from clippacloud import clipboard

backend = None

class Settings(object):
    def __init__(self, backend, connectiondata):
        self._backend = backend
        self._connectiondata = connectiondata
        self.xml = ET.Element("Settings")
        self.backend = backend
        self.connectiondata = connectiondata
        pass

    def to_xml(self):
        return self.xml

    @property
    def backend(self):
        return self._backend

    @backend.setter
    def backend(self, value):
        conn = self.xml.find("Connection")
        if conn is not None:
            conn.set("backend",value)
        else:
            conn = ET.Element("Connection",{"backend":str(value)})
            self.xml.append(conn)
        self._backend = value

    @property
    def connectiondata(self):
        return self._connectiondata

    @connectiondata.setter
    def connectiondata(self,value):
        conn = self.xml.find("Connection")
        if conn is None:
            conn = ET.Element("Connection")
            self.xml.append(conn)
        conndata = conn.find("ConnectionData")
        if conndata is not None:
            conn.remove(conndata)
        conn.append(value)

    @classmethod
    def from_xml(cls,settingstext):

        try:
            settingsxml = ET.XML(settingstext)
        except ET.ParseError:
            settingsxml = ET.Element("Settings")
        connectionxml = settingsxml.find("Connection")
        if connectionxml is not None:
            backend = connectionxml.get("backend")
            connectiondata = connectionxml.find("ConnectionData") #Later...
        else:
            backend = ""
            connectiondata = ET.Element("ConnectionData")
        return cls(backend,connectiondata)

projectname = "clippacloud"
if os.name != "posix":
    from win32com.shell import shellcon, shell
    homedir = shell.SHGetFolderPath(0, shellcon.CSIDL_APPDATA, 0, 0)
else:
    homedir = os.path.join(os.path.expanduser("~"), ".config")
settingsdirectory = os.path.join(homedir,projectname)
if not os.path.isdir(settingsdirectory):
    os.makedirs(settingsdirectory)
settingsfilepath = os.path.join(settingsdirectory,"settings.xml")
#Create the settings file, so the read below will work during first run
if not os.path.isfile(settingsfilepath):
    open(settingsfilepath,'w').close()
#Read the settings file
with open(settingsfilepath,'r') as settingsfile:
    settingstext = settingsfile.read()

FORMAT = '%(asctime)-15s %(message)s'
logfilepath = os.path.join(settingsdirectory,"clippacloud.log")
logging.basicConfig(format=FORMAT,filename=logfilepath)
logging.info("Started clippacloud")

class InitBackendWindow(wx.Frame):
    def __init__(self,settings, backends,app):
        wx.Frame.__init__(self,None,-1,"Initialize Backend")
        self.Bind(wx.EVT_CLOSE,self.on_cancel_button_clicked)
        self.app = app
        panel = wx.Panel(self, -1)

        backendlabel = wx.StaticText(panel, -1, "Choose backend:")
        backend_combo = wx.Choice(panel, -1, choices=backends.keys())

        backend_combo.SetStringSelection(iter(backends).next())

        mainvbox = wx.BoxSizer(wx.VERTICAL)

        ok_button = wx.Button(panel, wx.ID_OK)
        ok_button.Bind(wx.EVT_BUTTON,functools.partial(self.on_ok_button_clicked,combo=backend_combo,backends=backends))
        cancel_button = wx.Button(panel, wx.ID_CANCEL)
        cancel_button.Bind(wx.EVT_BUTTON,self.on_cancel_button_clicked)

        #sizer = wx.FlexGridSizer(2, 2, 5, 5)
        #sizer.AddGrowableCol(1)
        sizer = wx.BoxSizer(wx.HORIZONTAL)
        sizer.Add(backendlabel,0,wx.ALIGN_CENTER_VERTICAL|wx.RIGHT,5)
        sizer.Add(backend_combo,1, flag=wx.EXPAND)
        # box = wx.BoxSizer()

        mainvbox.Add(sizer,0,wx.EXPAND|wx.BOTTOM,15)

        # box.Add(ok_button, flag=wx.ALIGN_RIGHT)
        # box.Add(cancel_button, flag=wx.ALIGN_RIGHT)

        button_sizer = wx.StdDialogButtonSizer()
        button_sizer.AddButton(ok_button)
        button_sizer.AddButton(cancel_button)
        button_sizer.Realize()

        #sizer.Add(wx.BoxSizer())
        #sizer.Add(box,flag=wx.ALIGN_RIGHT)

        if button_sizer:
            mainvbox.Add(button_sizer,0,wx.EXPAND)

        border = wx.BoxSizer()
        border.Add(mainvbox, 1, wx.ALL|wx.EXPAND, 15)
        panel.SetSizerAndFit(border)
        self.Fit()

    def on_ok_button_clicked(self,button,combo,backends):
        try:
            text = combo.GetStringSelection()
            #Eww
            globals()["backend"] = backends[(backend for backend in backends if backend == text).next()]()
            self.Show(False)
        except StopIteration:
            raise ValueError("Backend not found")

        try:
            globals()["backend"].create_new()
            #self.DestroyChildren()
            wx.CallAfter(self.Destroy)
            wx.CallAfter(self.app.Exit)
        except Exception as e:
            logging.error(traceback.format_exc())
            self.Show(True)

    def on_cancel_button_clicked(self,button):
        wx.CallAfter(self.Destroy)
        wx.CallAfter(self.app.Exit)

    def run(self,app):
        app.MainLoop()
        return globals()["backend"]

paused = False

def create_menu_item(menu, label, func):
    item = wx.MenuItem(menu, -1, label)
    menu.Bind(wx.EVT_MENU, func, id=item.GetId())
    menu.AppendItem(item)
    return item

class TaskBarIcon(wx.TaskBarIcon):
    def __init__(self, top_window):
        # If we are using the development branch on Mac OSX, we can make it a menu bar item
        self.top_window = top_window
        # Cache
        icon.getTrayIconPausedIcon()
        if sys.platform == "darwin":
            try:
                super(TaskBarIcon, self).__init__(wx.TBI_CUSTOM_STATUSITEM)
                self.SetIcon(icon.getTrayIconIcon(),"Clippacloud")
            except (AttributeError,wx._core.PyAssertionError):
                super(TaskBarIcon, self).__init__(wx.TBI_DOCK)
                self.SetIcon(icon.getTrayIconIcon(),"Clippacloud")
        else:
            super(TaskBarIcon, self).__init__()
            self.SetIcon(icon.getTrayIconIcon(),"Clippacloud")

    def CreatePopupMenu(self):
        global paused
        menu = wx.Menu()
        if paused:
            label = "Resume Clippacloud"
        else:
            label = "Pause Clippacloud"
        create_menu_item(menu, label, self.on_pause)
        menu.AppendSeparator()
        create_menu_item(menu, 'Exit Clippacloud', self.on_exit)
        return menu

    def set_icon(self, path):
        icon = wx.IconFromBitmap(wx.Bitmap(path))
        self.SetIcon(icon, "Clippacloud")

    def on_pause(self, event):
        global paused
        paused = not paused
        if paused:
            self.SetIcon(icon.getTrayIconPausedIcon(),"Clippacloud")
        else:
            self.SetIcon(icon.getTrayIconIcon(),"Clippacloud")

    def on_exit(self, event):
        wx.CallAfter(self.Destroy)
        self.top_window.Destroy()

def make_backend(settings, app):
    initwindow = InitBackendWindow(settings, clippacloud.backends, app)
    initwindow.Show(True)
    app.SetTopWindow(initwindow)
    app.MainLoop()
    backend = globals()["backend"]
    return backend

def main():
    global backend
    parser = argparse.ArgumentParser(description='Cloud based clipboard syncing.')
    parser.add_argument('-c, --config', dest="config", type=str, nargs='?',
                       help='config file for clippacloud')
    args = parser.parse_args()
    clippacloud.init(args)
    app = wx.App()
    app.SetTopWindow(None)
    # wx is a dummy dummy
    frame = wx.Frame(None)
    app.SetTopWindow(frame)
    icon = TaskBarIcon(frame)
    settings = Settings.from_xml(settingstext)
    while True:
        if not settings.backend:
            backend = make_backend(settings,app)
            if backend is None:
                return 0
            settings.backend = backend.name
            settings.connectiondata = ET.Element("ConnectionData",backend.get_connection_data())
            break
        else:
            try:
                backend = clippacloud.backends[settings.backend]()
                backend.resume(settings.connectiondata)
                break
            except Exception as e:
                settings.backend = None
    clippacloud.backend = backend
    global modified
    modified = datetime.datetime.min
    def idle_func():
        global modified
        global paused
        global backend
        try:
            if not paused:
                cp = clipboard.Clipboard()
                clipcatcher.try_catch_clip(cp,backend)
                files = sorted(clippacloud.backend.list_files(), key=lambda x: x.modified)
                if files:
                    filedesc = files[-1]
                    if filedesc.modified > modified:
                        clippacloud.actions.set_clipboard_from_cloud(cp)
                        modified = filedesc.modified
                    cp.flush()
                    totalsize = sum(x.size for x in files)
                    while totalsize > config.max_size:
                        try:
                            clippacloud.backend.remove_file(files[0].path)
                        except:
                            pass
                        files = files[1:]
                        totalsize = sum(x.size for x in files)
        except exceptions.AccessRevokedException:
            dialog = wx.MessageDialog(None,"Clippacloud's access for your backend has been revoked.\nWould you like to reauthenticate?",
                                            "Access revoked",wx.YES_NO|wx.ICON_ERROR)
            response = dialog.ShowModal()
            if response != wx.ID_YES:
                app.Exit()
                return
            backend = None
            backend = make_backend(settings,app)
            clippacloud.backend = backend
            if backend is None:
                app.Exit()
                return
            settings.backend = backend.name
            settings.connectiondata = ET.Element("ConnectionData",backend.get_connection_data())

        except Exception as e:
            logging.error(traceback.format_exc())
        wx.CallLater(1000,idle_func)

    wx.CallLater(1000,idle_func)
    app.MainLoop()
    try:
        icon.Destroy()
    except:
        pass
    with open(settingsfilepath,"w") as outfile:
        outfile.write(ET.tostring(settings.to_xml()))
    return 0

if __name__ == '__main__':
    try:
        exit_code = main()
    except Exception as e:
        exit_code = 1
        logging.error(traceback.format_exc())
    logging.info("Closing clippacloud")
    logging.shutdown()
    exit(exit_code)<commit_msg>Made argv a parameter for main function<commit_after>import wx
import os
import signal
import xml.etree.ElementTree as ET
import multiprocessing
import argparse
import tempfile
import datetime
import time
import base64
import functools
import sys

import logging

sys.path.append(os.path.dirname(__file__))

import clippacloud
from clippacloud import exceptions
from clippacloud import clipcatcher
from clippacloud import config
from clippacloud import plugin

import icon
import traceback
from clippacloud import clipboard

backend = None

class Settings(object):
    def __init__(self, backend, connectiondata):
        self._backend = backend
        self._connectiondata = connectiondata
        self.xml = ET.Element("Settings")
        self.backend = backend
        self.connectiondata = connectiondata
        pass

    def to_xml(self):
        return self.xml

    @property
    def backend(self):
        return self._backend

    @backend.setter
    def backend(self, value):
        conn = self.xml.find("Connection")
        if conn is not None:
            conn.set("backend",value)
        else:
            conn = ET.Element("Connection",{"backend":str(value)})
            self.xml.append(conn)
        self._backend = value

    @property
    def connectiondata(self):
        return self._connectiondata

    @connectiondata.setter
    def connectiondata(self,value):
        conn = self.xml.find("Connection")
        if conn is None:
            conn = ET.Element("Connection")
            self.xml.append(conn)
        conndata = conn.find("ConnectionData")
        if conndata is not None:
            conn.remove(conndata)
        conn.append(value)

    @classmethod
    def from_xml(cls,settingstext):

        try:
            settingsxml = ET.XML(settingstext)
        except ET.ParseError:
            settingsxml = ET.Element("Settings")
        connectionxml = settingsxml.find("Connection")
        if connectionxml is not None:
            backend = connectionxml.get("backend")
            connectiondata = connectionxml.find("ConnectionData") #Later...
        else:
            backend = ""
            connectiondata = ET.Element("ConnectionData")
        return cls(backend,connectiondata)

projectname = "clippacloud"
if os.name != "posix":
    from win32com.shell import shellcon, shell
    homedir = shell.SHGetFolderPath(0, shellcon.CSIDL_APPDATA, 0, 0)
else:
    homedir = os.path.join(os.path.expanduser("~"), ".config")
settingsdirectory = os.path.join(homedir,projectname)
if not os.path.isdir(settingsdirectory):
    os.makedirs(settingsdirectory)
settingsfilepath = os.path.join(settingsdirectory,"settings.xml")
#Create the settings file, so the read below will work during first run
if not os.path.isfile(settingsfilepath):
    open(settingsfilepath,'w').close()
#Read the settings file
with open(settingsfilepath,'r') as settingsfile:
    settingstext = settingsfile.read()

FORMAT = '%(asctime)-15s %(message)s'
logfilepath = os.path.join(settingsdirectory,"clippacloud.log")
logging.basicConfig(format=FORMAT,filename=logfilepath)
logging.info("Started clippacloud")

class InitBackendWindow(wx.Frame):
    def __init__(self,settings, backends,app):
        wx.Frame.__init__(self,None,-1,"Initialize Backend")
        self.Bind(wx.EVT_CLOSE,self.on_cancel_button_clicked)
        self.app = app
        panel = wx.Panel(self, -1)

        backendlabel = wx.StaticText(panel, -1, "Choose backend:")
        backend_combo = wx.Choice(panel, -1, choices=backends.keys())

        backend_combo.SetStringSelection(iter(backends).next())

        mainvbox = wx.BoxSizer(wx.VERTICAL)

        ok_button = wx.Button(panel, wx.ID_OK)
        ok_button.Bind(wx.EVT_BUTTON,functools.partial(self.on_ok_button_clicked,combo=backend_combo,backends=backends))
        cancel_button = wx.Button(panel, wx.ID_CANCEL)
        cancel_button.Bind(wx.EVT_BUTTON,self.on_cancel_button_clicked)

        #sizer = wx.FlexGridSizer(2, 2, 5, 5)
        #sizer.AddGrowableCol(1)
        sizer = wx.BoxSizer(wx.HORIZONTAL)
        sizer.Add(backendlabel,0,wx.ALIGN_CENTER_VERTICAL|wx.RIGHT,5)
        sizer.Add(backend_combo,1, flag=wx.EXPAND)
        # box = wx.BoxSizer()

        mainvbox.Add(sizer,0,wx.EXPAND|wx.BOTTOM,15)

        # box.Add(ok_button, flag=wx.ALIGN_RIGHT)
        # box.Add(cancel_button, flag=wx.ALIGN_RIGHT)

        button_sizer = wx.StdDialogButtonSizer()
        button_sizer.AddButton(ok_button)
        button_sizer.AddButton(cancel_button)
        button_sizer.Realize()

        #sizer.Add(wx.BoxSizer())
        #sizer.Add(box,flag=wx.ALIGN_RIGHT)

        if button_sizer:
            mainvbox.Add(button_sizer,0,wx.EXPAND)

        border = wx.BoxSizer()
        border.Add(mainvbox, 1, wx.ALL|wx.EXPAND, 15)
        panel.SetSizerAndFit(border)
        self.Fit()

    def on_ok_button_clicked(self,button,combo,backends):
        try:
            text = combo.GetStringSelection()
            #Eww
            globals()["backend"] = backends[(backend for backend in backends if backend == text).next()]()
            self.Show(False)
        except StopIteration:
            raise ValueError("Backend not found")

        try:
            globals()["backend"].create_new()
            #self.DestroyChildren()
            wx.CallAfter(self.Destroy)
            wx.CallAfter(self.app.Exit)
        except Exception as e:
            logging.error(traceback.format_exc())
            self.Show(True)

    def on_cancel_button_clicked(self,button):
        wx.CallAfter(self.Destroy)
        wx.CallAfter(self.app.Exit)

    def run(self,app):
        app.MainLoop()
        return globals()["backend"]

paused = False

def create_menu_item(menu, label, func):
    item = wx.MenuItem(menu, -1, label)
    menu.Bind(wx.EVT_MENU, func, id=item.GetId())
    menu.AppendItem(item)
    return item

class TaskBarIcon(wx.TaskBarIcon):
    def __init__(self, top_window):
        # If we are using the development branch on Mac OSX, we can make it a menu bar item
        self.top_window = top_window
        # Cache
        icon.getTrayIconPausedIcon()
        if sys.platform == "darwin":
            try:
                super(TaskBarIcon, self).__init__(wx.TBI_CUSTOM_STATUSITEM)
                self.SetIcon(icon.getTrayIconIcon(),"Clippacloud")
            except (AttributeError,wx._core.PyAssertionError):
                super(TaskBarIcon, self).__init__(wx.TBI_DOCK)
                self.SetIcon(icon.getTrayIconIcon(),"Clippacloud")
        else:
            super(TaskBarIcon, self).__init__()
            self.SetIcon(icon.getTrayIconIcon(),"Clippacloud")

    def CreatePopupMenu(self):
        global paused
        menu = wx.Menu()
        if paused:
            label = "Resume Clippacloud"
        else:
            label = "Pause Clippacloud"
        create_menu_item(menu, label, self.on_pause)
        menu.AppendSeparator()
        create_menu_item(menu, 'Exit Clippacloud', self.on_exit)
        return menu

    def set_icon(self, path):
        icon = wx.IconFromBitmap(wx.Bitmap(path))
        self.SetIcon(icon, "Clippacloud")

    def on_pause(self, event):
        global paused
        paused = not paused
        if paused:
            self.SetIcon(icon.getTrayIconPausedIcon(),"Clippacloud")
        else:
            self.SetIcon(icon.getTrayIconIcon(),"Clippacloud")

    def on_exit(self, event):
        wx.CallAfter(self.Destroy)
        self.top_window.Destroy()

def make_backend(settings, app):
    initwindow = InitBackendWindow(settings, clippacloud.backends, app)
    initwindow.Show(True)
    app.SetTopWindow(initwindow)
    app.MainLoop()
    backend = globals()["backend"]
    return backend

def main(argv=None):
    if argv is None:
        argv = sys.argv
    global backend
    parser = argparse.ArgumentParser(description='Cloud based clipboard syncing.')
    parser.add_argument('-c, --config', dest="config", type=str, nargs='?',
                       help='config file for clippacloud')
    args = parser.parse_args(args=argv[1:])
    clippacloud.init(args)
    app = wx.App()
    app.SetTopWindow(None)
    # wx is a dummy dummy
    frame = wx.Frame(None)
    app.SetTopWindow(frame)
    icon = TaskBarIcon(frame)
    settings = Settings.from_xml(settingstext)
    while True:
        if not settings.backend:
            backend = make_backend(settings,app)
            if backend is None:
                return 0
            settings.backend = backend.name
            settings.connectiondata = ET.Element("ConnectionData",backend.get_connection_data())
            break
        else:
            try:
                backend = clippacloud.backends[settings.backend]()
                backend.resume(settings.connectiondata)
                break
            except Exception as e:
                settings.backend = None
    clippacloud.backend = backend
    global modified
    modified = datetime.datetime.min
    def idle_func():
        global modified
        global paused
        global backend
        try:
            if not paused:
                cp = clipboard.Clipboard()
                clipcatcher.try_catch_clip(cp,backend)
                files = sorted(clippacloud.backend.list_files(), key=lambda x: x.modified)
                if files:
                    filedesc = files[-1]
                    if filedesc.modified > modified:
                        clippacloud.actions.set_clipboard_from_cloud(cp)
                        modified = filedesc.modified
                    cp.flush()
                    totalsize = sum(x.size for x in files)
                    while totalsize > config.max_size:
                        try:
                            clippacloud.backend.remove_file(files[0].path)
                        except:
                            pass
                        files = files[1:]
                        totalsize = sum(x.size for x in files)
        except exceptions.AccessRevokedException:
            dialog = wx.MessageDialog(None,"Clippacloud's access for your backend has been revoked.\nWould you like to reauthenticate?",
                                            "Access revoked",wx.YES_NO|wx.ICON_ERROR)
            response = dialog.ShowModal()
            if response != wx.ID_YES:
                app.Exit()
                return
            backend = None
            backend = make_backend(settings,app)
            clippacloud.backend = backend
            if backend is None:
                app.Exit()
                return
            settings.backend = backend.name
            settings.connectiondata = ET.Element("ConnectionData",backend.get_connection_data())

        except Exception as e:
            logging.error(traceback.format_exc())
        wx.CallLater(1000,idle_func)

    wx.CallLater(1000,idle_func)
    app.MainLoop()
    try:
        icon.Destroy()
    except:
        pass
    with open(settingsfilepath,"w") as outfile:
        outfile.write(ET.tostring(settings.to_xml()))
    return 0

if __name__ == '__main__':
    try:
        exit_code = main()
    except Exception as e:
        exit_code = 1
        logging.error(traceback.format_exc())
    logging.info("Closing clippacloud")
    logging.shutdown()
    exit(exit_code)<|endoftext|>


<commit_before>#!/usr/bin/env python3.3
"""Example HTTP client using yield-from coroutines (PEP 380).

Requires Python 3.3.

There are many micro-optimizations possible here, but that's not the point.

Some incomplete laundry lists:

TODO:
- Cancellation.
- A more varied set of test URLs.
- Profiling.
- Unittests.

PATTERNS TO TRY:
- Wait for all, collate results.
- Wait for first N that are ready.
- Wait until some predicate becomes true.

FUNCTIONALITY:
- Connection pool (keep connection open).
- Chunked encoding (request and response).
- Pipelining, e.g. zlib (request and response).
- Automatic encoding/decoding.
- A write() call that isn't a generator.
"""

__author__ = 'Guido van Rossum <guido@python.org>'

# Standard library imports (keep in alphabetic order).
import collections
import errno
import logging
import re
import select
import socket
import time

# Local imports (keep in alphabetic order).
import polling

class Scheduler:

    def __init__(self, ioloop):
        self.ioloop = ioloop
        self.current = None
        self.current_name = None

    def run(self):
        self.ioloop.run()

    def start(self, task, name=None):
        if name is None:
            name = task.__name__  # If it doesn't have one, pass one.
        self.ioloop.call_soon(self.run_task, task, name)

    def run_task(self, task, name):
        try:
            self.current = task
            self.current_name = name
            next(self.current)
        except StopIteration:
            pass
        except Exception:
            logging.exception('Exception in task %r', name)
        else:
            if self.current is not None:
                self.start(task, name)
        finally:
            self.current = None
            self.current_name = None

    def block_r(self, fd):
        self.block_io(fd, 'r')

    def block_w(self, fd):
        self.block_io(fd, 'w')

    def block_io(self, fd, flag):
        assert isinstance(fd, int), repr(fd)
        assert flag in ('r', 'w'), repr(flag)
        task, name = self.block()
        if flag == 'r':
            method = self.ioloop.add_reader
            callback = self.unblock_r
        else:
            method = self.ioloop.add_writer
            callback = self.unblock_w
        method(fd, callback, fd, task, name)

    def block(self):
        assert self.current
        task = self.current
        self.current = None
        return task, self.current_name

    def unblock_r(self, fd, task, name):
        self.ioloop.remove_reader(fd)
        self.start(task, name)

    def unblock_w(self, fd, task, name):
        self.ioloop.remove_writer(fd)
        self.start(task, name)

ioloop = polling.Pollster()
trunner = polling.ThreadRunner(ioloop)
sched = Scheduler(ioloop)

def call_in_thread(func, *args, **kwds):
    # TODO: Prove there is no race condition here.
    task, name = sched.block()
    future = trunner.submit(func, *args, **kwds)
    future.add_done_callback(lambda _: sched.start(task, name))
    yield
    assert future.done()
    return future.result()

class RawReader:
    # TODO: Merge with send() and newsocket() functions.

    def __init__(self, sock):
        self.sock = sock

    def read(self, n):
        """Read up to n bytes, blocking at most once."""
        assert n >= 0, n
        sched.block_r(self.sock.fileno())
        yield
        return self.sock.recv(n)

class BufferedReader:

    def __init__(self, raw, limit=8192):
        self.raw = raw
        self.limit = limit
        self.buffer = b''
        self.eof = False

    def read(self, n):
        """Read up to n bytes, blocking at most once."""
        assert n >= 0, n
        if not self.buffer and not self.eof:
            yield from self.fillbuffer(max(n, self.limit))
        return self.getfrombuffer(n)

    def readexactly(self, n):
        """Read exactly n bytes, or until EOF."""
        blocks = []
        count = 0
        while n > count:
            block = yield from self.read(n - count)
            blocks.append(block)
            count += len(block)
        return b''.join(blocks)

    def readline(self):
        """Read up to newline or limit, whichever comes first."""
        end = self.buffer.find(b'\n') + 1  # Point past newline, or 0.
        while not end and not self.eof and len(self.buffer) < self.limit:
            anchor = len(self.buffer)
            yield from self.fillbuffer(self.limit)
            end = self.buffer.find(b'\n', anchor) + 1
        if not end:
            end = len(self.buffer)
        if end > self.limit:
            end = self.limit
        return self.getfrombuffer(end)

    def getfrombuffer(self, n):
        """Read up to n bytes without blocking."""
        if n >= len(self.buffer):
            result, self.buffer = self.buffer, b''
        else:
            result, self.buffer = self.buffer[:n], self.buffer[n:]
        return result

    def fillbuffer(self, n):
        """Fill buffer with one (up to) n bytes from raw reader."""
        assert not self.eof, 'fillbuffer called at eof'
        data = yield from self.raw.read(n)
##        print('fillbuffer:', repr(data)[:100])
        if data:
            self.buffer += data
        else:
            self.eof = True

def send(sock, data):
##     print('send:', repr(data))
    while data:
        sched.block_w(sock.fileno())
        yield
        n = sock.send(data)
        assert 0 <= n <= len(data), (n, len(data))
        if n == len(data):
            break
        data = data[n:]

def newsocket(af, socktype, proto):
    sock = socket.socket(af, socktype, proto)
    sock.setblocking(False)
    return sock

def connect(sock, address):
##     print('connect:', address)
    err = sock.connect_ex(address)
    assert err == errno.EINPROGRESS, err
    sched.block_w(sock.fileno())
    yield
    err = sock.connect_ex(address)
    if err == errno.ECONNREFUSED:
        raise IOError('Connection refused')
    if err not in (0, errno.EISCONN):
        raise IOError('Connect error %d: %s' % (err, errno.errorcode.get(err)))

def urlfetch(host, port=80, method='GET', path='/',
             body=None, hdrs=None, encoding='utf-8'):
    t0 = time.time()
    # Must pass in an IP address.  Later we'll call getaddrinfo()
    # using a thread pool.  We'll also support IPv6.
    if not re.match(r'(\d+)(\.\d+)(\.\d+)(\.\d+)\Z', host):
        infos = yield from call_in_thread(socket.getaddrinfo,
                                          host, port, socket.AF_INET,
                                          socket.SOCK_STREAM,
                                          socket.SOL_TCP)
    else:
        infos = [(socket.AF_INET, socket.SOCK_STREAM, socket.SOL_TCP, '',
                  (host, port))]
    assert infos, 'No address info for (%r, %r)' % (host, port)
    exc = None
    for af, socktype, proto, cname, address in infos:
        sock = None
        try:
            sock = newsocket(af, socktype, proto)
            yield from connect(sock, address)
            break
        except socket.error as err:
            if sock is not None:
                sock.close()
            if exc is None:
                exc = err
    else:
        if exc is not None:
            raise exc
    yield from send(sock,
                    method.encode(encoding) + b' ' +
                    path.encode(encoding) + b' HTTP/1.0\r\n')
    if hdrs:
        kwds = dict(hdrs)
    else:
        kwds = {}
    if 'host' not in kwds:
        kwds['host'] = host
    if body is not None:
        kwds['content_length'] = len(body)
    for header, value in kwds.items():
        yield from send(sock,
                        header.replace('_', '-').encode(encoding) + b': ' +
                        value.encode(encoding) + b'\r\n')

    yield from send(sock, b'\r\n')
    if body is not None:
        yield from send(sock, body)
    ##sock.shutdown(1)  # Close the writing end of the socket.

    # Read HTTP response line.
    raw = RawReader(sock)
    buf = BufferedReader(raw)
    resp = yield from buf.readline()
##     print('resp =', repr(resp))
    m = re.match(br'(?ix) http/(\d\.\d) \s+ (\d\d\d) \s+ ([^\r]*)\r?\n\Z', resp)
    if not m:
        sock.close()
        raise IOError('No valid HTTP response: %r' % resp)
    http_version, status, message = m.groups()

    # Read HTTP headers.
    headers = []
    hdict = {}
    while True:
        line = yield from buf.readline()
        if not line.strip():
            break
        m = re.match(br'([^\s:]+):\s*([^\r]*)\r?\n\Z', line)
        if not m:
            raise IOError('Invalid header: %r' % line)
        header, value = m.groups()
        headers.append((header, value))
        hdict[header.decode(encoding).lower()] = value.decode(encoding)

    # Read response body.
    content_length = hdict.get('content-length')
    if content_length is not None:
        size = int(content_length)  # TODO: Catch errors.
        assert size >= 0, size
    else:
        size = 2**20  # Protective limit (1 MB).
    data = yield from buf.readexactly(size)
    sock.close()  # Can this block?
    t1 = time.time()
##     print(http_version, status, message, headers, hdict, len(data))
    print(host, port, path, status, len(data), '{:.3}'.format(t1-t0))

def doit():
    # This references NDB's default test service.
    # (Sadly the service is single-threaded.)
    gen1 = urlfetch('localhost', 8080, path='/')
    sched.start(gen1, 'gen1')

    gen2 = urlfetch('localhost', 8080, path='/home')
    sched.start(gen2, 'gen2')

    # Fetch python.org home page.
    gen3 = urlfetch('python.org', 80, path='/')
    sched.start(gen3, 'gen3')

##     # Fetch many links from python.org (/x.y.z).
##     for x in '123':
##         for y in '0123456789':
##             path = '/{}.{}'.format(x, y)
##             g = urlfetch('82.94.164.162', 80,
##                          path=path, hdrs={'host': 'python.org'})
##             sched.start(g, path)

    sched.run()

def main():
    doit()

if __name__ == '__main__':
    main()
<commit_msg>Test submit.<commit_after>#!/usr/bin/env python3.3
"""Example HTTP client using yield-from coroutines (PEP 380).

Requires Python 3.3.

There are many micro-optimizations possible here, but that's not the point.

Some incomplete laundry lists:

TODO:
- Cancellation?
- A more varied set of test URLs.
- Profiling.
- Unittests.

PATTERNS TO TRY:
- Wait for all, collate results.
- Wait for first N that are ready.
- Wait until some predicate becomes true.

FUNCTIONALITY:
- Connection pool (keep connection open).
- Chunked encoding (request and response).
- Pipelining, e.g. zlib (request and response).
- Automatic encoding/decoding.
- A write() call that isn't a generator.
"""

__author__ = 'Guido van Rossum <guido@python.org>'

# Standard library imports (keep in alphabetic order).
import collections
import errno
import logging
import re
import select
import socket
import time

# Local imports (keep in alphabetic order).
import polling

class Scheduler:

    def __init__(self, ioloop):
        self.ioloop = ioloop
        self.current = None
        self.current_name = None

    def run(self):
        self.ioloop.run()

    def start(self, task, name=None):
        if name is None:
            name = task.__name__  # If it doesn't have one, pass one.
        self.ioloop.call_soon(self.run_task, task, name)

    def run_task(self, task, name):
        try:
            self.current = task
            self.current_name = name
            next(self.current)
        except StopIteration:
            pass
        except Exception:
            logging.exception('Exception in task %r', name)
        else:
            if self.current is not None:
                self.start(task, name)
        finally:
            self.current = None
            self.current_name = None

    def block_r(self, fd):
        self.block_io(fd, 'r')

    def block_w(self, fd):
        self.block_io(fd, 'w')

    def block_io(self, fd, flag):
        assert isinstance(fd, int), repr(fd)
        assert flag in ('r', 'w'), repr(flag)
        task, name = self.block()
        if flag == 'r':
            method = self.ioloop.add_reader
            callback = self.unblock_r
        else:
            method = self.ioloop.add_writer
            callback = self.unblock_w
        method(fd, callback, fd, task, name)

    def block(self):
        assert self.current
        task = self.current
        self.current = None
        return task, self.current_name

    def unblock_r(self, fd, task, name):
        self.ioloop.remove_reader(fd)
        self.start(task, name)

    def unblock_w(self, fd, task, name):
        self.ioloop.remove_writer(fd)
        self.start(task, name)

ioloop = polling.Pollster()
trunner = polling.ThreadRunner(ioloop)
sched = Scheduler(ioloop)

def call_in_thread(func, *args, **kwds):
    # TODO: Prove there is no race condition here.
    task, name = sched.block()
    future = trunner.submit(func, *args, **kwds)
    future.add_done_callback(lambda _: sched.start(task, name))
    yield
    assert future.done()
    return future.result()

class RawReader:
    # TODO: Merge with send() and newsocket() functions.

    def __init__(self, sock):
        self.sock = sock

    def read(self, n):
        """Read up to n bytes, blocking at most once."""
        assert n >= 0, n
        sched.block_r(self.sock.fileno())
        yield
        return self.sock.recv(n)

class BufferedReader:

    def __init__(self, raw, limit=8192):
        self.raw = raw
        self.limit = limit
        self.buffer = b''
        self.eof = False

    def read(self, n):
        """Read up to n bytes, blocking at most once."""
        assert n >= 0, n
        if not self.buffer and not self.eof:
            yield from self.fillbuffer(max(n, self.limit))
        return self.getfrombuffer(n)

    def readexactly(self, n):
        """Read exactly n bytes, or until EOF."""
        blocks = []
        count = 0
        while n > count:
            block = yield from self.read(n - count)
            blocks.append(block)
            count += len(block)
        return b''.join(blocks)

    def readline(self):
        """Read up to newline or limit, whichever comes first."""
        end = self.buffer.find(b'\n') + 1  # Point past newline, or 0.
        while not end and not self.eof and len(self.buffer) < self.limit:
            anchor = len(self.buffer)
            yield from self.fillbuffer(self.limit)
            end = self.buffer.find(b'\n', anchor) + 1
        if not end:
            end = len(self.buffer)
        if end > self.limit:
            end = self.limit
        return self.getfrombuffer(end)

    def getfrombuffer(self, n):
        """Read up to n bytes without blocking."""
        if n >= len(self.buffer):
            result, self.buffer = self.buffer, b''
        else:
            result, self.buffer = self.buffer[:n], self.buffer[n:]
        return result

    def fillbuffer(self, n):
        """Fill buffer with one (up to) n bytes from raw reader."""
        assert not self.eof, 'fillbuffer called at eof'
        data = yield from self.raw.read(n)
##        print('fillbuffer:', repr(data)[:100])
        if data:
            self.buffer += data
        else:
            self.eof = True

def send(sock, data):
##     print('send:', repr(data))
    while data:
        sched.block_w(sock.fileno())
        yield
        n = sock.send(data)
        assert 0 <= n <= len(data), (n, len(data))
        if n == len(data):
            break
        data = data[n:]

def newsocket(af, socktype, proto):
    sock = socket.socket(af, socktype, proto)
    sock.setblocking(False)
    return sock

def connect(sock, address):
##     print('connect:', address)
    err = sock.connect_ex(address)
    assert err == errno.EINPROGRESS, err
    sched.block_w(sock.fileno())
    yield
    err = sock.connect_ex(address)
    if err == errno.ECONNREFUSED:
        raise IOError('Connection refused')
    if err not in (0, errno.EISCONN):
        raise IOError('Connect error %d: %s' % (err, errno.errorcode.get(err)))

def urlfetch(host, port=80, method='GET', path='/',
             body=None, hdrs=None, encoding='utf-8'):
    t0 = time.time()
    # Must pass in an IP address.  Later we'll call getaddrinfo()
    # using a thread pool.  We'll also support IPv6.
    if not re.match(r'(\d+)(\.\d+)(\.\d+)(\.\d+)\Z', host):
        infos = yield from call_in_thread(socket.getaddrinfo,
                                          host, port, socket.AF_INET,
                                          socket.SOCK_STREAM,
                                          socket.SOL_TCP)
    else:
        infos = [(socket.AF_INET, socket.SOCK_STREAM, socket.SOL_TCP, '',
                  (host, port))]
    assert infos, 'No address info for (%r, %r)' % (host, port)
    exc = None
    for af, socktype, proto, cname, address in infos:
        sock = None
        try:
            sock = newsocket(af, socktype, proto)
            yield from connect(sock, address)
            break
        except socket.error as err:
            if sock is not None:
                sock.close()
            if exc is None:
                exc = err
    else:
        if exc is not None:
            raise exc
    yield from send(sock,
                    method.encode(encoding) + b' ' +
                    path.encode(encoding) + b' HTTP/1.0\r\n')
    if hdrs:
        kwds = dict(hdrs)
    else:
        kwds = {}
    if 'host' not in kwds:
        kwds['host'] = host
    if body is not None:
        kwds['content_length'] = len(body)
    for header, value in kwds.items():
        yield from send(sock,
                        header.replace('_', '-').encode(encoding) + b': ' +
                        value.encode(encoding) + b'\r\n')

    yield from send(sock, b'\r\n')
    if body is not None:
        yield from send(sock, body)
    ##sock.shutdown(1)  # Close the writing end of the socket.

    # Read HTTP response line.
    raw = RawReader(sock)
    buf = BufferedReader(raw)
    resp = yield from buf.readline()
##     print('resp =', repr(resp))
    m = re.match(br'(?ix) http/(\d\.\d) \s+ (\d\d\d) \s+ ([^\r]*)\r?\n\Z', resp)
    if not m:
        sock.close()
        raise IOError('No valid HTTP response: %r' % resp)
    http_version, status, message = m.groups()

    # Read HTTP headers.
    headers = []
    hdict = {}
    while True:
        line = yield from buf.readline()
        if not line.strip():
            break
        m = re.match(br'([^\s:]+):\s*([^\r]*)\r?\n\Z', line)
        if not m:
            raise IOError('Invalid header: %r' % line)
        header, value = m.groups()
        headers.append((header, value))
        hdict[header.decode(encoding).lower()] = value.decode(encoding)

    # Read response body.
    content_length = hdict.get('content-length')
    if content_length is not None:
        size = int(content_length)  # TODO: Catch errors.
        assert size >= 0, size
    else:
        size = 2**20  # Protective limit (1 MB).
    data = yield from buf.readexactly(size)
    sock.close()  # Can this block?
    t1 = time.time()
##     print(http_version, status, message, headers, hdict, len(data))
    print(host, port, path, status, len(data), '{:.3}'.format(t1-t0))

def doit():
    # This references NDB's default test service.
    # (Sadly the service is single-threaded.)
    gen1 = urlfetch('localhost', 8080, path='/')
    sched.start(gen1, 'gen1')

    gen2 = urlfetch('localhost', 8080, path='/home')
    sched.start(gen2, 'gen2')

    # Fetch python.org home page.
    gen3 = urlfetch('python.org', 80, path='/')
    sched.start(gen3, 'gen3')

##     # Fetch many links from python.org (/x.y.z).
##     for x in '123':
##         for y in '0123456789':
##             path = '/{}.{}'.format(x, y)
##             g = urlfetch('82.94.164.162', 80,
##                          path=path, hdrs={'host': 'python.org'})
##             sched.start(g, path)

    sched.run()

def main():
    doit()

if __name__ == '__main__':
    main()
<|endoftext|>