<commit_before>import os
import logging
from logging.handlers import RotatingFileHandler

from flask import Flask
from flask_jwt_extended import JWTManager
from flask_sqlalchemy import SQLAlchemy

db = SQLAlchemy()
application = Flask(__name__)
jwt = JWTManager(application)

def create_app(**kwargs):
    from app.config import configs

    environment_state = get_env()

    application.config.from_object(configs[environment_state])
    application.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

    if kwargs:
        application.config.update(kwargs)

    configure_logging()

    db.init_app(application)

    register_blueprint()

    init_app(application)

    return application

def init_app(app):
    @app.before_request
    def check_for_apikey():
        # print("check: ", request)
        pass

def register_blueprint():
    from app.rest import base_blueprint
    from app.routes.articles.rest import article_blueprint, articles_blueprint
    from app.routes.authentication.rest import auth_blueprint
    from app.routes.events.rest import events_blueprint
    from app.routes.fees.rest import fees_blueprint, fee_blueprint
    from app.routes.event_dates.rest import event_dates_blueprint, event_date_blueprint
    from app.routes.event_types.rest import event_types_blueprint, event_type_blueprint
    from app.routes.speakers.rest import speakers_blueprint, speaker_blueprint
    from app.routes.users.rest import users_blueprint, user_blueprint
    from app.routes.venues.rest import venues_blueprint, venue_blueprint
    application.register_blueprint(base_blueprint)
    application.register_blueprint(auth_blueprint)
    application.register_blueprint(article_blueprint)
    application.register_blueprint(articles_blueprint)
    application.register_blueprint(events_blueprint)
    application.register_blueprint(event_date_blueprint)
    application.register_blueprint(event_dates_blueprint)
    application.register_blueprint(event_types_blueprint)
    application.register_blueprint(event_type_blueprint)
    application.register_blueprint(fees_blueprint)
    application.register_blueprint(fee_blueprint)
    application.register_blueprint(speakers_blueprint)
    application.register_blueprint(speaker_blueprint)
    application.register_blueprint(users_blueprint)
    application.register_blueprint(user_blueprint)
    application.register_blueprint(venues_blueprint)
    application.register_blueprint(venue_blueprint)

def get_env():
    if 'www-preview' in get_root_path():
        return 'preview'
    elif 'www-live' in get_root_path():
        return 'live'
    else:
        return os.environ.get('ENVIRONMENT', 'development')

def get_root_path():
    return application.root_path

def configure_logging():
    if not application.config.get('APP_SERVER'):
        return

    setup_gce_logging()

    ch = logging.StreamHandler()
    if ch in application.logger.handlers:
        return

    del application.logger.handlers[:]

    f = LogTruncatingFormatter(
        "{} %(asctime)s;[%(process)d];%(levelname)s;%(message)s".format(get_env()), "%Y-%m-%d %H:%M:%S")
    ch.setFormatter(f)
    application.logger.addHandler(ch)

    rfh = RotatingFileHandler('logs/app.log', maxBytes=10000, backupCount=3)
    rfh.setLevel(logging.DEBUG)
    rfh.setFormatter(f)

    application.logger.addHandler(rfh)

    if application.config.get('APP_SERVER') == 'gunicorn':
        gunicorn_access_logger = logging.getLogger('gunicorn.access')
        application.logger.handlers.extend(gunicorn_access_logger.handlers)

        gunicorn_error_logger = logging.getLogger('gunicorn.error')
        application.logger.handlers.extend(gunicorn_error_logger.handlers)

        gunicorn_access_logger.addHandler(rfh)
        gunicorn_error_logger.addHandler(rfh)

        gunicorn_access_logger.addHandler(ch)
        gunicorn_error_logger.addHandler(ch)

        application.logger.info('Gunicorn logging configured')
    else:
        werkzeug_log = logging.getLogger('werkzeug')
        werkzeug_log.setLevel(logging.DEBUG)

        werkzeug_log.addHandler(ch)
        werkzeug_log.addHandler(rfh)

        application.logger.info('Flask logging configured')

    db_name = application.config.get('SQLALCHEMY_DATABASE_URI').split('/')[-1]
    application.logger.debug("connected to db: {}".format(db_name))

def setup_gce_logging():  # pragma: no cover
    if application.config['SQLALCHEMY_DATABASE_URI'][:22] in ['postgresql://localhost', 'db://localhost/test_db']:
        return

    import google.cloud.logging
    from google.cloud.logging.handlers import CloudLoggingHandler, setup_logging

    client = google.cloud.logging.Client()
    handler = CloudLoggingHandler(client, name=get_env())
    setup_logging(handler)

class LogTruncatingFormatter(logging.Formatter):
    def format(self, record):
        START_LOG = '127.0.0.1 - - ['
        if 'msg' in dir(record) and record.msg[:15] == START_LOG:
            record.msg = record.msg[42:]
        return super(LogTruncatingFormatter, self).format(record)
<commit_msg>Add gunicorn logs to cloud handler<commit_after>import os
import logging
from logging.handlers import RotatingFileHandler

from flask import Flask
from flask_jwt_extended import JWTManager
from flask_sqlalchemy import SQLAlchemy

db = SQLAlchemy()
application = Flask(__name__)
jwt = JWTManager(application)

def create_app(**kwargs):
    from app.config import configs

    environment_state = get_env()

    application.config.from_object(configs[environment_state])
    application.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

    if kwargs:
        application.config.update(kwargs)

    configure_logging()

    db.init_app(application)

    register_blueprint()

    init_app(application)

    return application

def init_app(app):
    @app.before_request
    def check_for_apikey():
        # print("check: ", request)
        pass

def register_blueprint():
    from app.rest import base_blueprint
    from app.routes.articles.rest import article_blueprint, articles_blueprint
    from app.routes.authentication.rest import auth_blueprint
    from app.routes.events.rest import events_blueprint
    from app.routes.fees.rest import fees_blueprint, fee_blueprint
    from app.routes.event_dates.rest import event_dates_blueprint, event_date_blueprint
    from app.routes.event_types.rest import event_types_blueprint, event_type_blueprint
    from app.routes.speakers.rest import speakers_blueprint, speaker_blueprint
    from app.routes.users.rest import users_blueprint, user_blueprint
    from app.routes.venues.rest import venues_blueprint, venue_blueprint
    application.register_blueprint(base_blueprint)
    application.register_blueprint(auth_blueprint)
    application.register_blueprint(article_blueprint)
    application.register_blueprint(articles_blueprint)
    application.register_blueprint(events_blueprint)
    application.register_blueprint(event_date_blueprint)
    application.register_blueprint(event_dates_blueprint)
    application.register_blueprint(event_types_blueprint)
    application.register_blueprint(event_type_blueprint)
    application.register_blueprint(fees_blueprint)
    application.register_blueprint(fee_blueprint)
    application.register_blueprint(speakers_blueprint)
    application.register_blueprint(speaker_blueprint)
    application.register_blueprint(users_blueprint)
    application.register_blueprint(user_blueprint)
    application.register_blueprint(venues_blueprint)
    application.register_blueprint(venue_blueprint)

def get_env():
    if 'www-preview' in get_root_path():
        return 'preview'
    elif 'www-live' in get_root_path():
        return 'live'
    else:
        return os.environ.get('ENVIRONMENT', 'development')

def get_root_path():
    return application.root_path

def configure_logging():
    if not application.config.get('APP_SERVER'):
        return

    ch = logging.StreamHandler()
    if ch in application.logger.handlers:
        return

    del application.logger.handlers[:]

    f = LogTruncatingFormatter(
        "{} %(asctime)s;[%(process)d];%(levelname)s;%(message)s".format(get_env()), "%Y-%m-%d %H:%M:%S")
    ch.setFormatter(f)
    application.logger.addHandler(ch)

    rfh = RotatingFileHandler('logs/app.log', maxBytes=10000, backupCount=3)
    rfh.setLevel(logging.DEBUG)
    rfh.setFormatter(f)

    application.logger.addHandler(rfh)

    if application.config.get('APP_SERVER') == 'gunicorn':
        gunicorn_access_logger = logging.getLogger('gunicorn.access')
        application.logger.handlers.extend(gunicorn_access_logger.handlers)

        gunicorn_error_logger = logging.getLogger('gunicorn.error')
        application.logger.handlers.extend(gunicorn_error_logger.handlers)

        gunicorn_access_logger.addHandler(rfh)
        gunicorn_error_logger.addHandler(rfh)

        gunicorn_access_logger.addHandler(ch)
        gunicorn_error_logger.addHandler(ch)

        setup_gce_logging(gunicorn_access_logger, gunicorn_error_logger)

        application.logger.info('Gunicorn logging configured')
    else:
        werkzeug_log = logging.getLogger('werkzeug')
        werkzeug_log.setLevel(logging.DEBUG)

        werkzeug_log.addHandler(ch)
        werkzeug_log.addHandler(rfh)

        application.logger.info('Flask logging configured')

    db_name = application.config.get('SQLALCHEMY_DATABASE_URI').split('/')[-1]
    application.logger.debug("connected to db: {}".format(db_name))

def setup_gce_logging(gunicorn_access_logger, gunicorn_error_logger):  # pragma: no cover
    if application.config['SQLALCHEMY_DATABASE_URI'][:22] in ['postgresql://localhost', 'db://localhost/test_db']:
        return

    import google.cloud.logging
    from google.cloud.logging.handlers import CloudLoggingHandler, setup_logging

    client = google.cloud.logging.Client()
    handler = CloudLoggingHandler(client, name=get_env())
    setup_logging(handler)

    gunicorn_access_logger.addHandler(handler)
    gunicorn_error_logger.addHandler(handler)

class LogTruncatingFormatter(logging.Formatter):
    def format(self, record):
        START_LOG = '127.0.0.1 - - ['
        if 'msg' in dir(record) and record.msg[:15] == START_LOG:
            record.msg = record.msg[42:]
        return super(LogTruncatingFormatter, self).format(record)
<|endoftext|>


<commit_before>import discord
from discord.ext import commands
import traceback
import inspect

class AdminCog:
    def __init__(self, bot):
        self.bot = bot
        self.last_eval_result = None
        self.previous_eval_code = None

    @commands.is_owner()
    @commands.command(aliases=['echo'], hidden=True)
    async def say(self, ctx, *, the_text: str):
        """Repeats a given text."""
        await ctx.send(the_text)

    @commands.is_owner()
    @commands.command(name='exit', hidden=True)
    async def _exit(self, ctx):
        """Shuts down AveBot, owner only."""
        await ctx.send(":wave: Exiting AveBot, goodbye!")
        await self.bot.logout()

    @commands.is_owner()
    @commands.command(hidden=True)
    async def load(self, ctx, ext: str):
        """Loads a cog, owner only."""
        try:
            self.bot.load_extension("cogs." + ext)
        except:
            await ctx.send(f':x: Cog loading failed, traceback: ```\n{traceback.format_exc()}\n```')
            return
        self.bot.log.info(f'Loaded ext {ext}')
        await ctx.send(f':white_check_mark: `{ext}` successfully loaded.')

    @commands.is_owner()
    @commands.command(hidden=True)
    async def fetchlog(self, ctx):
        """Returns log"""
        await ctx.send(file=discord.File("avebot.log"), content="Here's the current log file:")

    @commands.is_owner()
    @commands.command(name='eval', hidden=True)
    async def _eval(self, ctx, *, code: str):
        """Evaluates some code (Owner only)"""
        try:
            code = code.strip('` ')

            env = {
                'bot': self.bot,
                'ctx': ctx,
                'message': ctx.message,
                'server': ctx.guild,
                'guild': ctx.guild,
                'channel': ctx.message.channel,
                'author': ctx.message.author,

                # modules
                'discord': discord,
                'commands': commands,

                # utilities
                '_get': discord.utils.get,
                '_find': discord.utils.find,

                 # last result
                '_': self.last_eval_result,
                '_p': self.previous_eval_code,
            }
            env.update(globals())

            self.bot.log.info(f"Evaling {repr(code)}:")
            result = eval(code, env)
            if inspect.isawaitable(result):
                result = await result

            if result is not None:
                self.last_eval_result = result

            self.previous_eval_code = code

            for msg in self.bot.slice_message(repr(result), 1994):
                await ctx.send(f"```{msg}```")
        except:
            for msg in self.bot.slice_message(traceback.format_exc(), 1994):
                await ctx.send(f"```{msg}```")

    @commands.is_owner()
    @commands.command(hidden=True)
    async def sh(self, ctx, *, command: str):
        """Runs a command on shell."""
        command = command.strip('`')
        tmp = await ctx.send(f'Running `{command}`...')
        self.bot.log.info(f"Running {command}")
        shell_output = self.bot.call_shell(command)
        shell_output = f"\"{command}\" output:\n{shell_output}"
        self.bot.log.info(shell_output)
        await tmp.delete()
        for msg in self.bot.slice_message(shell_output, 1994):
                await ctx.send(f"```{msg}```")

    @commands.is_owner()
    @commands.command(hidden=True)
    async def pull(self, ctx):
        """Does a git pull (Owner only)."""
        tmp = await ctx.send('Pulling...')
        git_output = self.bot.call_shell("git pull")
        await tmp.edit(content=f"Pull complete. Output: ```{git_output}```")

    @commands.is_owner()
    @commands.command(hidden=True)
    async def unload(self, ctx, ext: str):
        """Unloads a cog, owner only."""
        self.bot.unload_extension("cogs." + ext)
        self.bot.log.info(f'Unloaded ext {ext}')
        await ctx.send(f':white_check_mark: `{ext}` successfully unloaded.')

    @commands.is_owner()
    @commands.command(hidden=True)
    async def reload(self, ctx, ext: str):
        """Reloads a cog, owner only."""
        if ext == "_":
            ext = self.lastreload
        else:
            self.lastreload = ext

        try:
            self.bot.unload_extension("cogs." + ext)
            self.bot.load_extension("cogs." + ext)
        except:
            await ctx.send(f':x: Cog reloading failed, traceback: ```\n{traceback.format_exc()}\n```')
            return
        self.bot.log.info(f'Reloaded ext {ext}')
        await ctx.send(f':white_check_mark: `{ext}` successfully reloaded.')

    @commands.is_owner()
    @commands.command(hidden=True)
    async def log(self, ctx, count: int):
        """Returns a file out of the last N messages submitted in this channel."""
        log_text = "===start of log, exported by avebot===\n"
        async for mlog in ctx.channel.history(limit=count):
            log_text += "[{}]<{}>{}\n".format(str(mlog.created_at), str(mlog.author), mlog.clean_content)
        mlog_file_name = "files/{}.log".format(ctx.channel.id)
        file = open(mlog_file_name, "w")
        file.write(log_text)
        file.write("===end of log, exported by avebot===")
        file.close()
        await ctx.send(file=discord.File(mlog_file_name), content=f"{ctx.message.author.mention}: Here's the log file you requested.")

def setup(bot):
    bot.add_cog(AdminCog(bot))<commit_msg>autopull: add autopull<commit_after>import discord
from discord.ext import commands
import traceback
import inspect
import re

class AdminCog:
    def __init__(self, bot):
        self.bot = bot
        self.last_eval_result = None
        self.previous_eval_code = None

    @commands.is_owner()
    @commands.command(aliases=['echo'], hidden=True)
    async def say(self, ctx, *, the_text: str):
        """Repeats a given text."""
        await ctx.send(the_text)

    @commands.is_owner()
    @commands.command(name='exit', hidden=True)
    async def _exit(self, ctx):
        """Shuts down AveBot, owner only."""
        await ctx.send(":wave: Exiting AveBot, goodbye!")
        await self.bot.logout()

    @commands.is_owner()
    @commands.command(hidden=True)
    async def load(self, ctx, ext: str):
        """Loads a cog, owner only."""
        try:
            self.bot.load_extension("cogs." + ext)
        except:
            await ctx.send(f':x: Cog loading failed, traceback: ```\n{traceback.format_exc()}\n```')
            return
        self.bot.log.info(f'Loaded ext {ext}')
        await ctx.send(f':white_check_mark: `{ext}` successfully loaded.')

    @commands.is_owner()
    @commands.command(hidden=True)
    async def fetchlog(self, ctx):
        """Returns log"""
        await ctx.send(file=discord.File("avebot.log"), content="Here's the current log file:")

    @commands.is_owner()
    @commands.command(name='eval', hidden=True)
    async def _eval(self, ctx, *, code: str):
        """Evaluates some code (Owner only)"""
        try:
            code = code.strip('` ')

            env = {
                'bot': self.bot,
                'ctx': ctx,
                'message': ctx.message,
                'server': ctx.guild,
                'guild': ctx.guild,
                'channel': ctx.message.channel,
                'author': ctx.message.author,

                # modules
                'discord': discord,
                'commands': commands,

                # utilities
                '_get': discord.utils.get,
                '_find': discord.utils.find,

                 # last result
                '_': self.last_eval_result,
                '_p': self.previous_eval_code,
            }
            env.update(globals())

            self.bot.log.info(f"Evaling {repr(code)}:")
            result = eval(code, env)
            if inspect.isawaitable(result):
                result = await result

            if result is not None:
                self.last_eval_result = result

            self.previous_eval_code = code

            for msg in self.bot.slice_message(repr(result), 1994):
                await ctx.send(f"```{msg}```")
        except:
            for msg in self.bot.slice_message(traceback.format_exc(), 1994):
                await ctx.send(f"```{msg}```")

    @commands.is_owner()
    @commands.command(hidden=True)
    async def sh(self, ctx, *, command: str):
        """Runs a command on shell."""
        command = command.strip('`')
        tmp = await ctx.send(f'Running `{command}`...')
        self.bot.log.info(f"Running {command}")
        shell_output = self.bot.call_shell(command)
        shell_output = f"\"{command}\" output:\n{shell_output}"
        self.bot.log.info(shell_output)
        await tmp.delete()
        for msg in self.bot.slice_message(shell_output, 1994):
                await ctx.send(f"```{msg}```")

    @commands.is_owner()
    @commands.command(hidden=True)
    async def pull(self, ctx):
        """Does a git pull (Owner only)."""
        tmp = await ctx.send('Pulling...')
        git_output = self.bot.call_shell("git pull")
        await tmp.edit(content=f"Pull complete. Output: ```{git_output}```")

    @commands.is_owner()
    @commands.command(hidden=True)
    async def autopull(self, ctx):
        """Does a git pull and reloads updated cogs automatically (Owner only)."""
        tmp = await ctx.send('Pulling...')
        git_output = self.bot.call_shell("git pull")
        cogs_to_reload = re.findall('cogs/([a-z]*).py', git_output)
        await tmp.edit(content=f"Pull complete. Output: ```{git_output}```")
        for cog in cogs_to_reload:
            try:
                self.bot.unload_extension("cogs." + cog)
                self.bot.load_extension("cogs." + cog)
                self.bot.log.info(f'Reloaded ext {ext}')
                await ctx.send(f':white_check_mark: `{ext}` successfully reloaded.')
            except:
                await ctx.send(f':x: Cog reloading failed, traceback: ```\n{traceback.format_exc()}\n```')
                return

    @commands.is_owner()
    @commands.command(hidden=True)
    async def unload(self, ctx, ext: str):
        """Unloads a cog, owner only."""
        self.bot.unload_extension("cogs." + ext)
        self.bot.log.info(f'Unloaded ext {ext}')
        await ctx.send(f':white_check_mark: `{ext}` successfully unloaded.')

    @commands.is_owner()
    @commands.command(hidden=True)
    async def reload(self, ctx, ext: str):
        """Reloads a cog, owner only."""
        if ext == "_":
            ext = self.lastreload
        else:
            self.lastreload = ext

        try:
            self.bot.unload_extension("cogs." + ext)
            self.bot.load_extension("cogs." + ext)
        except:
            await ctx.send(f':x: Cog reloading failed, traceback: ```\n{traceback.format_exc()}\n```')
            return
        self.bot.log.info(f'Reloaded ext {ext}')
        await ctx.send(f':white_check_mark: `{ext}` successfully reloaded.')

    @commands.is_owner()
    @commands.command(hidden=True)
    async def log(self, ctx, count: int):
        """Returns a file out of the last N messages submitted in this channel."""
        log_text = "===start of log, exported by avebot===\n"
        async for mlog in ctx.channel.history(limit=count):
            log_text += "[{}]<{}>{}\n".format(str(mlog.created_at), str(mlog.author), mlog.clean_content)
        mlog_file_name = "files/{}.log".format(ctx.channel.id)
        file = open(mlog_file_name, "w")
        file.write(log_text)
        file.write("===end of log, exported by avebot===")
        file.close()
        await ctx.send(file=discord.File(mlog_file_name), content=f"{ctx.message.author.mention}: Here's the log file you requested.")

def setup(bot):
    bot.add_cog(AdminCog(bot))<|endoftext|>


<commit_before>
generated_id_length = 10
force_ipad_mode_on_gae = True

try:
	if force_ipad_mode_on_gae:
		raise ImportError()

	from google.appengine.ext.webapp.util import run_wsgi_app
	from gae import database
	from gae.bottle import Bottle, HTTPError, request, response
	running_in_ipad = False
except ImportError:
	from test import database
	if force_ipad_mode_on_gae:
		from gae.bottle import Bottle, HTTPError, request, response
	else:
		from bottle import run, Bottle, HTTPError, request, response
	running_in_ipad = True

import random

random.seed()

if running_in_ipad:
	bottle = Bottle()
else:
	# adding catchall so that I will get exception stack traces when debugging
	bottle = Bottle(catchall=False)

def random_id():
	id = ''
	for i in range(1,generated_id_length):
		id += str(random.randint(0,9))
	return id

def create_document(content):
	new_doc = random_id()
	database.put(new_doc, content)
	return new_doc

@bottle.route('/share', method=['GET', 'POST'])
def share():
	content = request.params.content
	if content: # TODO: in the future, support proper RESTfulness
		ret = { 'id': create_document(content) }
		return ret
	else:
		ret = ""
		for item in database.all():
			ret += str(item) + "\n"
		return ret

@bottle.get('/share/<id>')
def access(id):
	content = database.get(id)
	if content:
		response.content_type = 'text/html'

		from plaintext2html import plaintext2html
		return "<title>"+str(id)+"</title>"+plaintext2html(content)+""
	else:
		return HTTPError(404)

if running_in_ipad and not force_ipad_mode_on_gae:
	run(host='localhost', port=8080, debug=True, app=bottle)
<commit_msg>Forcing bottle debub mode when running on GAE<commit_after>
generated_id_length = 10
force_ipad_mode_on_gae = False

try:
	if force_ipad_mode_on_gae:
		raise ImportError()

	from google.appengine.ext.webapp.util import run_wsgi_app
	from gae import database
	from gae.bottle import Bottle, HTTPError, request, response
	running_in_ipad = False
except ImportError:
	from test import database
	if force_ipad_mode_on_gae:
		from gae.bottle import Bottle, HTTPError, request, response
	else:
		from bottle import run, Bottle, HTTPError, request, response
	running_in_ipad = True

import random

random.seed()

if running_in_ipad and not force_ipad_mode_on_gae:
	bottle = Bottle()
else:
	# adding catchall so that I will get exception stack traces when debugging
	bottle = Bottle(catchall=False)

def random_id():
	id = ''
	for i in range(1,generated_id_length):
		id += str(random.randint(0,9))
	return id

def create_document(content):
	new_doc = random_id()
	database.put(new_doc, content)
	return new_doc

@bottle.route('/share', method=['GET', 'POST'])
def share():
	content = request.params.content
	if content: # TODO: in the future, support proper RESTfulness
		ret = { 'id': create_document(content) }
		return ret
	else:
		ret = ""
		for item in database.all():
			ret += str(item) + "\n"
		return ret

@bottle.get('/share/<id>')
def access(id):
	content = database.get(id)
	if content:
		response.content_type = 'text/html'

		from plaintext2html import plaintext2html
		return "<title>"+str(id)+"</title>"+plaintext2html(content)+""
	else:
		return HTTPError(404)

if running_in_ipad and not force_ipad_mode_on_gae:
	run(host='localhost', port=8080, debug=True, app=bottle)
<|endoftext|>


<commit_before>
from twisted.python import log
from shtoom.ui.base import ShtoomBaseUI
from twisted.internet import stdio
from twisted.protocols import basic

class ShtoomMain(basic.LineReceiver, ShtoomBaseUI):
    _cookie = None
    _pending = None
    _debug = True
    from os import linesep as delimiter

    def debugMessage(self, msg):
        if self._debug:
            print msg

    def statusMessage(self, msg):
        if self._debug:
            print "status", msg

    def errorMessage(self, message, exc=None):
        log.msg("error %s"%(message))

    def shutdown(self):
        # XXX Hang up any calls
        from twisted.internet import reactor
        reactor.stop()

    def incomingCall(self, description, call, defresp, defsetup):
        self._pending = ( call, defresp, defsetup )
        self.transport.write("INCOMING CALL: %s\n"%description)
        self.transport.write("Type 'accept' to accept, 'reject' to reject\n")

    def connectionMade(self):
        self.transport.write("Welcome to shtoom\n>> ")

    def lineReceived(self, line):
        args = line.strip().split()
        if args:
            cmd = args[0].lower()
            if hasattr(self, "cmd_%s"%cmd):
                getattr(self, "cmd_%s"%(cmd))(line)
            elif cmd == "?":
                self.cmd_help(line)
            else:
                self.transport.write("Unknown command '%s'\n"%(cmd))
        self.transport.write(">> ")

    def cmd_help(self, line):
        methods = [ x[4:] for x in dir(self) if x[:4] == "cmd_" ]
        self.transport.write("Commands: %s\n"%(' '.join(methods)))

    def cmd_call(self, line):
        if self._cookie is not None:
            self.transport.write("error: only one call at a time for now\n")
            return
        args = line.split()
        if len(args) != 2:
            self.transport.write("error: call <sipurl>\n")
            return
        if not args[1].startswith('sip:'):
            self.transport.write("error: call <sipurl>\n")
            return
        self.sipURL = args[1]
        deferred = self.app.placeCall(self.sipURL)
        deferred.addCallbacks(self.callConnected, self.callDisconnected).addErrback(log.err)

    def callConnected(self, cookie):
        log.msg("Call to %s CONNECTED"%(self.sipURL))

    def callDisconnected(self, call):
        log.msg("Call to %s DISCONNECTED"%(self.sipURL))

    def cmd_hangup(self, line):
        if self._cookie is not None:
            self.app.dropCall(self._cookie)
            self._cookie = None
        else:
            self.transport.write("error: no active call\n")

    def cmd_quit(self, line):
        self.shutdown()

    def cmd_accept(self, line):
        if not self._pending:
            self.transport.write("no pending calls")
            return
        self._cookie, resp, setup = self._pending
        self._pending = None
        setup.addCallbacks(self.callConnected, self.callDisconnected).addErrback(log.err)
        resp.callback('yes')

    def cmd_reject(self, line):
        if not self._pending:
            self.transport.write("no pending calls")
            return
        self._cookie, resp, setup = self._pending
        self._pending = None
        resp.errback('no')
<commit_msg>forgot this one<commit_after>
from twisted.python import log
from shtoom.ui.base import ShtoomBaseUI
from twisted.internet import stdio
from twisted.protocols import basic

class ShtoomMain(basic.LineReceiver, ShtoomBaseUI):
    _cookie = None
    _pending = None
    _debug = True
    from os import linesep as delimiter

    def debugMessage(self, msg):
        if self._debug:
            print msg

    def statusMessage(self, msg):
        if self._debug:
            print "status", msg

    def errorMessage(self, message, exc=None):
        log.msg("error %s"%(message))

    def shutdown(self):
        # XXX Hang up any calls
        from twisted.internet import reactor
        reactor.stop()

    def incomingCall(self, description, call, defresp, defsetup):
        self._pending = ( call, defresp, defsetup )
        self.transport.write("INCOMING CALL: %s\n"%description)
        self.transport.write("Type 'accept' to accept, 'reject' to reject\n")

    def connectionMade(self):
        self.transport.write("Welcome to shtoom\n>> ")

    def lineReceived(self, line):
        args = line.strip().split()
        if args:
            cmd = args[0].lower()
            if hasattr(self, "cmd_%s"%cmd):
                getattr(self, "cmd_%s"%(cmd))(line)
            elif cmd == "?":
                self.cmd_help(line)
            else:
                self.transport.write("Unknown command '%s'\n"%(cmd))
        self.transport.write(">> ")

    def cmd_help(self, line):
        methods = [ x[4:] for x in dir(self) if x[:4] == "cmd_" ]
        self.transport.write("Commands: %s\n"%(' '.join(methods)))

    def cmd_call(self, line):
        if self._cookie is not None:
            self.transport.write("error: only one call at a time for now\n")
            return
        args = line.split()
        if len(args) != 2:
            self.transport.write("error: call <sipurl>\n")
            return
        if not args[1].startswith('sip:'):
            self.transport.write("error: call <sipurl>\n")
            return
        self.sipURL = args[1]
        deferred = self.app.placeCall(self.sipURL)
        deferred.addCallbacks(self.callConnected, self.callDisconnected).addErrback(log.err)

    def callStarted(self, cookie):
        self._cookie = cookie
        log.msg("Call to %s STARTED"%(self.sipURL))

    def callConnected(self, cookie):
        log.msg("Call to %s CONNECTED"%(self.sipURL))

    def callFailed(self, e, message=None):
        log.msg("Call to %s FAILED: %r"%(self.sipURL, e))

    def callDisconnected(self, cookie, message):
        log.msg("Call to %s DISCONNECTED"%(self.sipURL))
        self._cookie = None

    def cmd_hangup(self, line):
        if self._cookie is not None:
            self.app.dropCall(self._cookie)
        else:
            self.transport.write("error: no active call\n")

    def cmd_quit(self, line):
        self.shutdown()

    def cmd_accept(self, line):
        if not self._pending:
            self.transport.write("no pending calls")
            return
        self._cookie, resp, setup = self._pending
        self._pending = None
        setup.addCallbacks(self.callConnected, self.callFailed).addErrback(log.err)
        resp.callback('yes')

    def cmd_reject(self, line):
        if not self._pending:
            self.transport.write("no pending calls")
            return
        self._cookie, resp, setup = self._pending
        self._pending = None
        resp.errback('no')
<|endoftext|>


<commit_before>"""
Taken from Andrej Karpathy's min-char-rnn:
    https://gist.github.com/karpathy/d4dee566867f8291f086

Modified in various ways for better introspection / customization, and added
comments.

----

Minimal character-level Vanilla RNN model.
Written by Andrej Karpathy (@karpathy)
BSD License
"""
from __future__ import print_function

import numpy as np
import sys

# Make it possible to provide input file as a command-line argument; input.txt
# is still the default.
if len(sys.argv) > 1:
    filename = sys.argv[1]
else:
    filename = 'input.txt'

with open(filename, 'r') as f:
    data = f.read()

# All unique characters / entities in the data set.
chars = list(set(data))
data_size, vocab_size = len(data), len(chars)
print('data has %d characters, %d unique.' % (data_size, vocab_size))

# Numerical index [0:N) mapping for N chars, to be used as vector indices (for
# one-hot vectors, etc).
char_to_ix = {ch:i for i, ch in enumerate(chars)}
ix_to_char = {i:ch for i, ch in enumerate(chars)}
print('char_to_ix', char_to_ix)
print('ix_to_char', ix_to_char)

# hyperparameters
hidden_size = 100 # size of hidden layer of neurons
seq_length = 16 # number of steps to unroll the RNN for
learning_rate = 1e-1

# Stop when processed this much data
MAX_DATA = 1000000

# Model parameters/weights -- these are shared among all steps. Weights
# initialized randomly; biases initialized to 0.
# Inputs are characters one-hot encoded in a vocab-sized vector.
# Dimensions: H = hidden_size, V = vocab_size
Wxh = np.random.randn(hidden_size, vocab_size) * 0.01       # input to hidden
Whh = np.random.randn(hidden_size, hidden_size) * 0.01      # hidden to hidden
Why = np.random.randn(vocab_size, hidden_size) * 0.01       # hidden to output
bh = np.zeros((hidden_size, 1))     # hidden bias
by = np.zeros((vocab_size, 1))      # output bias

def lossFun(inputs, targets, hprev):
  """Runs forward and backward passes through the RNN.

  inputs,targets are both list of integers.
  hprev is Hx1 array of initial hidden state
  returns the loss, gradients on model parameters, and last hidden state
  """
  # Caches that keep values computed in the forward pass at each time step, to
  # be reused in the backward pass.
  xs, hs, ys, ps = {}, {}, {}, {}
  hs[-1] = np.copy(hprev)
  loss = 0
  # Forward pass
  for t in xrange(len(inputs)):
    # Input at time step t is xs[t] -- a one-hot encoded vector of shape (V, 1).
    xs[t] = np.zeros((vocab_size,1)) # encode in 1-of-k representation
    xs[t][inputs[t]] = 1

    # Compute h[t] from h[t-1] and x[t]
    hs[t] = np.tanh(np.dot(Wxh, xs[t]) + np.dot(Whh, hs[t-1]) + bh)

    # Compute ps[t] - probabilities for output, and the loss using xentropy.
    ys[t] = np.dot(Why, hs[t]) + by
    ps[t] = np.exp(ys[t]) / np.sum(np.exp(ys[t]))

    # Cross-entropy loss for two probability distributions p and q is defined as
    # follows:
    #
    #   xent(q, p) = -Sum q(k)log(p(k))
    #                  k
    #
    # Where k goes over all the possible values of the random variable p and q
    # are defined for.
    # In our case taking q is the "real answer" which is 1-hot encoded; p is the
    # result of softmax (ps). targets[t] has the only index where q is not 0,
    # so the sum simply becomes log of ps at that index.
    loss += -np.log(ps[t][targets[t],0])

  # Backward pass: compute gradients going backwards.
  # Gradients are initialized to 0s, and every time step contributes to them.
  dWxh, dWhh, dWhy = np.zeros_like(Wxh), np.zeros_like(Whh), np.zeros_like(Why)
  dbh, dby = np.zeros_like(bh), np.zeros_like(by)
  dhnext = np.zeros_like(hs[0])

  for t in reversed(xrange(len(inputs))):
    dy = np.copy(ps[t])
    dy[targets[t]] -= 1 # backprop into y. see http://cs231n.github.io/neural-networks-case-study/#grad if confused here

    # Applying chain rule to propagate grad into dWhy and dby.
    #
    dWhy += np.dot(dy, hs[t].T)
    dby += dy

    # TODO: proper Jacobian matmul here would be dy.dot(Why), that would give
    # a [1,T] vector. Since we need [T,1] for h, we flip the dot (we could have
    # transposed after everything, too)
    dh = np.dot(Why.T, dy) + dhnext # backprop into h
    dhraw = (1 - hs[t] * hs[t]) * dh # backprop through tanh nonlinearity
    dbh += dhraw
    dWxh += np.dot(dhraw, xs[t].T)
    dWhh += np.dot(dhraw, hs[t-1].T)
    dhnext = np.dot(Whh.T, dhraw)

  # Gradient clipping
  for dparam in [dWxh, dWhh, dWhy, dbh, dby]:
    np.clip(dparam, -5, 5, out=dparam)

  return loss, dWxh, dWhh, dWhy, dbh, dby, hs[len(inputs)-1]

def sample(h, seed_ix, n):
  """
  sample a sequence of integers from the model
  h is memory state, seed_ix is seed letter for first time step
  """
  x = np.zeros((vocab_size, 1))
  x[seed_ix] = 1
  ixes = []
  for t in xrange(n):
    h = np.tanh(np.dot(Wxh, x) + np.dot(Whh, h) + bh)
    y = np.dot(Why, h) + by
    p = np.exp(y) / np.sum(np.exp(y))
    ix = np.random.choice(range(vocab_size), p=p.ravel())
    x = np.zeros((vocab_size, 1))
    x[ix] = 1
    ixes.append(ix)
  return ixes

n, p = 0, 0
mWxh, mWhh, mWhy = np.zeros_like(Wxh), np.zeros_like(Whh), np.zeros_like(Why)
mbh, mby = np.zeros_like(bh), np.zeros_like(by) # memory variables for Adagrad
smooth_loss = -np.log(1.0/vocab_size)*seq_length # loss at iteration 0

# gradient checking
from random import uniform
def gradCheck(inputs, targets, hprev):
  global Wxh, Whh, Why, bh, by
  num_checks, delta = 10, 1e-5
  _, dWxh, dWhh, dWhy, dbh, dby, _ = lossFun(inputs, targets, hprev)
  for param,dparam,name in zip([Wxh, Whh, Why, bh, by],
                               [dWxh, dWhh, dWhy, dbh, dby],
                               ['Wxh', 'Whh', 'Why', 'bh', 'by']):
    s0 = dparam.shape
    s1 = param.shape
    assert s0 == s1, 'Error dims dont match: %s and %s.' % (`s0`, `s1`)
    print(name)
    for i in xrange(num_checks):
      ri = int(uniform(0,param.size))
      # evaluate cost at [x + delta] and [x - delta]
      old_val = param.flat[ri]
      param.flat[ri] = old_val + delta
      cg0, _, _, _, _, _, _ = lossFun(inputs, targets, hprev)
      param.flat[ri] = old_val - delta
      cg1, _, _, _, _, _, _ = lossFun(inputs, targets, hprev)
      param.flat[ri] = old_val # reset old value for this parameter
      # fetch both numerical and analytic gradient
      grad_analytic = dparam.flat[ri]
      grad_numerical = (cg0 - cg1) / ( 2 * delta )
      rel_error = abs(grad_analytic - grad_numerical) / abs(grad_numerical + grad_analytic)
      print('%f, %f => %e ' % (grad_numerical, grad_analytic, rel_error))
      # rel_error should be on order of 1e-7 or less

def basicGradCheck():
  inputs = [char_to_ix[ch] for ch in data[:seq_length]]
  targets = [char_to_ix[ch] for ch in data[1:seq_length+1]]
  hprev = np.zeros((hidden_size,1)) # reset RNN memory
  gradCheck(inputs, targets, hprev)

basicGradCheck()

while p < MAX_DATA:
  # prepare inputs (we're sweeping from left to right in steps seq_length long)
  if p+seq_length+1 >= len(data) or n == 0:
    hprev = np.zeros((hidden_size,1)) # reset RNN memory
    p = 0 # go from start of data
  inputs = [char_to_ix[ch] for ch in data[p:p+seq_length]]
  targets = [char_to_ix[ch] for ch in data[p+1:p+seq_length+1]]

  # sample from the model now and then
  if n % 1000 == 0:
    sample_ix = sample(hprev, inputs[0], 200)
    txt = ''.join(ix_to_char[ix] for ix in sample_ix)
    print('----\n %s \n----' % (txt, ))

  # forward seq_length characters through the net and fetch gradient
  loss, dWxh, dWhh, dWhy, dbh, dby, hprev = lossFun(inputs, targets, hprev)
  smooth_loss = smooth_loss * 0.999 + loss * 0.001
  if n % 200 == 0: print('iter %d (p=%d), loss: %f' % (n, p, smooth_loss))

  # perform parameter update with Adagrad
  for param, dparam, mem in zip([Wxh, Whh, Why, bh, by],
                                [dWxh, dWhh, dWhy, dbh, dby],
                                [mWxh, mWhh, mWhy, mbh, mby]):
    mem += dparam * dparam
    param += -learning_rate * dparam / np.sqrt(mem + 1e-8) # adagrad update

  p += seq_length # move data pointer
  n += 1 # iteration counter
<commit_msg>Adding more comments and cleaning up<commit_after>"""
Minimal character-based language model learning with RNNs.

Taken from Andrej Karpathy's min-char-rnn:

    https://gist.github.com/karpathy/d4dee566867f8291f086

The companion blog post is TODO.

Modified in various ways for better introspection / customization, and added
comments. I tried to retain the overall structure of this code almost identical
to the original.

----

Original license/copyright blurb:

Minimal character-level Vanilla RNN model.
Written by Andrej Karpathy (@karpathy)
BSD License
"""
from __future__ import print_function

import numpy as np
import sys

# Make it possible to provide input file as a command-line argument; input.txt
# is still the default.
if len(sys.argv) > 1:
    filename = sys.argv[1]
else:
    filename = 'input.txt'

with open(filename, 'r') as f:
    data = f.read()

# All unique characters / entities in the data set.
chars = list(set(data))
data_size, vocab_size = len(data), len(chars)
print('data has %d characters, %d unique.' % (data_size, vocab_size))

# Numerical index [0:N) mapping for N chars, to be used as vector indices (for
# one-hot vectors, etc).
char_to_ix = {ch:i for i, ch in enumerate(chars)}
ix_to_char = {i:ch for i, ch in enumerate(chars)}
print('char_to_ix', char_to_ix)
print('ix_to_char', ix_to_char)

# Hyperparameters
hidden_size = 100 # size of hidden layer of neurons
seq_length = 16 # number of steps to unroll the RNN for
learning_rate = 1e-1

# Stop when processed this much data
MAX_DATA = 1000000

# Model parameters/weights -- these are shared among all steps. Weights
# initialized randomly; biases initialized to 0.
# Inputs are characters one-hot encoded in a vocab-sized vector.
# Dimensions: H = hidden_size, V = vocab_size
Wxh = np.random.randn(hidden_size, vocab_size) * 0.01       # input to hidden
Whh = np.random.randn(hidden_size, hidden_size) * 0.01      # hidden to hidden
Why = np.random.randn(vocab_size, hidden_size) * 0.01       # hidden to output
bh = np.zeros((hidden_size, 1))     # hidden bias
by = np.zeros((vocab_size, 1))      # output bias

def lossFun(inputs, targets, hprev):
  """Runs forward and backward passes through the RNN.

  inputs, targets: Lists of integers. For some i, inputs[i] is the input
                   character (encoded as an index into the ix_to_char map) and
                   targets[i] is the corresponding next character in the
                   training data (similarly encoded).
  hprev: Hx1 array of initial hidden state
  returns: loss, gradients on model parameters, and last hidden state
  """
  # Caches that keep values computed in the forward pass at each time step, to
  # be reused in the backward pass.
  xs, hs, ys, ps = {}, {}, {}, {}

  # Initial incoming state.
  hs[-1] = np.copy(hprev)
  loss = 0
  # Forward pass
  for t in xrange(len(inputs)):
    # Input at time step t is xs[t]. Prepare a one-hot encoded vector of shape
    # (V, 1).
    xs[t] = np.zeros((vocab_size,1)) # encode in 1-of-k representation
    xs[t][inputs[t]] = 1

    # Compute h[t] from h[t-1] and x[t]
    hs[t] = np.tanh(np.dot(Wxh, xs[t]) + np.dot(Whh, hs[t-1]) + bh)

    # Compute ps[t] - softmax probabilities for output.
    ys[t] = np.dot(Why, hs[t]) + by
    ps[t] = np.exp(ys[t]) / np.sum(np.exp(ys[t]))

    # Cross-entropy loss for two probability distributions p and q is defined as
    # follows:
    #
    #   xent(q, p) = -Sum q(k)log(p(k))
    #                  k
    #
    # Where k goes over all the possible values of the random variable p and q
    # are defined for.
    # In our case taking q is the "real answer" which is 1-hot encoded; p is the
    # result of softmax (ps). targets[t] has the only index where q is not 0,
    # so the sum simply becomes log of ps at that index.
    loss += -np.log(ps[t][targets[t],0])

  # Backward pass: compute gradients going backwards.
  # Gradients are initialized to 0s, and every time step contributes to them.
  dWxh, dWhh, dWhy = np.zeros_like(Wxh), np.zeros_like(Whh), np.zeros_like(Why)
  dbh, dby = np.zeros_like(bh), np.zeros_like(by)

  # Initialize the incoming gradient of h to zero; this is a safe assumption for
  # a sufficiently long unrolling.
  dhnext = np.zeros_like(hs[0])

  # The backwards pass iterates over the input sequence backwards.
  for t in reversed(xrange(len(inputs))):
    # Backprop through the gradients of loss and softmax.
    dy = np.copy(ps[t])
    dy[targets[t]] -= 1

    # Compute gradients for the Why and by parameters.
    dWhy += np.dot(dy, hs[t].T)
    dby += dy

    # Backprop through the fully-connected layer (Why, by) to h. Also add up the
    # incoming gradient for h from the next cell.
    # Note: proper Jacobian matmul here would be dy.dot(Why), that would give
    # a [1,T] vector. Since we need [T,1] for h, we flip the dot (we could have
    # transposed after everything, too)
    dh = np.dot(Why.T, dy) + dhnext

    # Backprop through tanh.
    dhraw = (1 - hs[t] * hs[t]) * dh

    # Compute gradients for the dby, dWxh, Whh parameters.
    dbh += dhraw
    dWxh += np.dot(dhraw, xs[t].T)
    dWhh += np.dot(dhraw, hs[t-1].T)

    # Backprop the gradient to the incoming h, which will be used in the
    # previous time step.
    dhnext = np.dot(Whh.T, dhraw)

  # Gradient clipping to the range [-5, 5].
  for dparam in [dWxh, dWhh, dWhy, dbh, dby]:
    np.clip(dparam, -5, 5, out=dparam)

  return loss, dWxh, dWhh, dWhy, dbh, dby, hs[len(inputs)-1]

def sample(h, seed_ix, n):
  """Sample a sequence of integers from the model.

  Runs the RNN in forward mode for n steps; seed_ix is the seed letter for the
  first time step, and h is the memory state. Returns a sequences of letters produced
  by the model (indices).
  """
  # Create a one-hot vector to represent the input.
  x = np.zeros((vocab_size, 1))
  x[seed_ix] = 1
  ixes = []

  for t in xrange(n):
    # Run the forward pass only.
    h = np.tanh(np.dot(Wxh, x) + np.dot(Whh, h) + bh)
    y = np.dot(Why, h) + by
    p = np.exp(y) / np.sum(np.exp(y))

    # Sample from the distribution produced by softmax.
    ix = np.random.choice(range(vocab_size), p=p.ravel())

    # Prepare input for the next cell.
    x = np.zeros((vocab_size, 1))
    x[ix] = 1
    ixes.append(ix)
  return ixes

# Gradient checking (from karpathy's own comment on the gist)
from random import uniform
def gradCheck(inputs, targets, hprev):
  global Wxh, Whh, Why, bh, by
  num_checks, delta = 10, 1e-5
  _, dWxh, dWhh, dWhy, dbh, dby, _ = lossFun(inputs, targets, hprev)
  for param,dparam,name in zip([Wxh, Whh, Why, bh, by],
                               [dWxh, dWhh, dWhy, dbh, dby],
                               ['Wxh', 'Whh', 'Why', 'bh', 'by']):
    s0 = dparam.shape
    s1 = param.shape
    assert s0 == s1, 'Error dims dont match: %s and %s.' % (`s0`, `s1`)
    print(name)
    for i in xrange(num_checks):
      ri = int(uniform(0,param.size))
      # evaluate cost at [x + delta] and [x - delta]
      old_val = param.flat[ri]
      param.flat[ri] = old_val + delta
      cg0, _, _, _, _, _, _ = lossFun(inputs, targets, hprev)
      param.flat[ri] = old_val - delta
      cg1, _, _, _, _, _, _ = lossFun(inputs, targets, hprev)
      param.flat[ri] = old_val # reset old value for this parameter
      # fetch both numerical and analytic gradient
      grad_analytic = dparam.flat[ri]
      grad_numerical = (cg0 - cg1) / ( 2 * delta )
      rel_error = abs(grad_analytic - grad_numerical) / abs(grad_numerical + grad_analytic)
      print('%f, %f => %e ' % (grad_numerical, grad_analytic, rel_error))
      # rel_error should be on order of 1e-7 or less

def basicGradCheck():
  inputs = [char_to_ix[ch] for ch in data[:seq_length]]
  targets = [char_to_ix[ch] for ch in data[1:seq_length+1]]
  hprev = np.zeros((hidden_size,1)) # reset RNN memory
  gradCheck(inputs, targets, hprev)

# Uncomment this to run a basic gradient check.
#basicGradCheck()

# n is the iteration counter; p is the input sequence pointer, at the beginning
# of each step it points at the sequence in the input that will be used for
# training this iteration.
n, p = 0, 0
mWxh, mWhh, mWhy = np.zeros_like(Wxh), np.zeros_like(Whh), np.zeros_like(Why)
mbh, mby = np.zeros_like(bh), np.zeros_like(by) # memory variables for Adagrad
smooth_loss = -np.log(1.0/vocab_size)*seq_length # loss at iteration 0

while p < MAX_DATA:
  # prepare inputs (we're sweeping from left to right in steps seq_length long)
  if p+seq_length+1 >= len(data) or n == 0:
    hprev = np.zeros((hidden_size,1)) # reset RNN memory
    p = 0 # go from start of data
  inputs = [char_to_ix[ch] for ch in data[p:p+seq_length]]
  targets = [char_to_ix[ch] for ch in data[p+1:p+seq_length+1]]

  # sample from the model now and then
  if n % 1000 == 0:
    sample_ix = sample(hprev, inputs[0], 200)
    txt = ''.join(ix_to_char[ix] for ix in sample_ix)
    print('----\n %s \n----' % (txt, ))

  # forward seq_length characters through the net and fetch gradient
  loss, dWxh, dWhh, dWhy, dbh, dby, hprev = lossFun(inputs, targets, hprev)
  smooth_loss = smooth_loss * 0.999 + loss * 0.001
  if n % 200 == 0: print('iter %d (p=%d), loss: %f' % (n, p, smooth_loss))

  # perform parameter update with Adagrad
  for param, dparam, mem in zip([Wxh, Whh, Why, bh, by],
                                [dWxh, dWhh, dWhy, dbh, dby],
                                [mWxh, mWhh, mWhy, mbh, mby]):
    mem += dparam * dparam
    param += -learning_rate * dparam / np.sqrt(mem + 1e-8) # adagrad update

  p += seq_length
  n += 1
<|endoftext|>


<commit_before>from __future__ import print_function, division

from sympy.core import Basic, S, sympify, Expr, Rational, Symbol
from sympy.core import Add, Mul, expand_power_base, expand_log
from sympy.core.cache import cacheit
from sympy.core.compatibility import default_sort_key, is_sequence
from sympy.core.containers import Tuple
from sympy.utilities.iterables import uniq

class Order(Expr):
    r""" Represents the limiting behavior of some function

    The order of a function characterizes the function based on the limiting
    behavior of the function as it goes to some limit. Only taking the limit
    point to be 0 or positive infinity is currently supported. This is
    expressed in big O notation [1]_.

    The formal definition for the order of a function `g(x)` about a point `a`
    is such that `g(x) = O(f(x))` as `x \rightarrow a` if and only if for any
    `\delta > 0` there exists a `M > 0` such that `|g(x)| \leq M|f(x)|` for
    `|x-a| < \delta`. This is equivalent to `\lim_{x \rightarrow a}
    |g(x)/f(x)| < \infty`.

    Let's illustrate it on the following example by taking the expansion of
    `\sin(x)` about 0:

    .. math ::
        \sin(x) = x - x^3/3! + O(x^5)

    where in this case `O(x^5) = x^5/5! - x^7/7! + \cdots`. By the definition
    of `O`, for any `\delta > 0` there is an `M` such that:

    .. math ::
        |x^5/5! - x^7/7! + ....| <= M|x^5| \text{ for } |x| < \delta

    or by the alternate definition:

    .. math ::
        \lim_{x \rightarrow 0} | (x^5/5! - x^7/7! + ....) / x^5| < \infty

    which surely is true, because

    .. math ::
        \lim_{x \rightarrow 0} | (x^5/5! - x^7/7! + ....) / x^5| = 1/5!

    As it is usually used, the order of a function can be intuitively thought
    of representing all terms of powers greater than the one specified. For
    example, `O(x^3)` corresponds to any terms proportional to `x^3,
    x^4,\ldots` and any higher power. For a polynomial, this leaves terms
    proportional to `x^2`, `x` and constants.

    Examples
    ========

    >>> from sympy import O
    >>> from sympy.abc import x
    >>> O(x)
    O(x)
    >>> O(x)*x
    O(x**2)
    >>> O(x)-O(x)
    O(x)

    References
    ==========

    .. [1] `Big O notation <http://en.wikipedia.org/wiki/Big_O_notation>`_

    Notes
    =====

    In ``O(f(x), x)`` the expression ``f(x)`` is assumed to have a leading
    term.  ``O(f(x), x)`` is automatically transformed to
    ``O(f(x).as_leading_term(x),x)``.

        ``O(expr*f(x), x)`` is ``O(f(x), x)``

        ``O(expr, x)`` is ``O(1)``

        ``O(0, x)`` is 0.

    Multivariate O is also supported:

        ``O(f(x, y), x, y)`` is transformed to
        ``O(f(x, y).as_leading_term(x,y).as_leading_term(y), x, y)``

    In the multivariate case, it is assumed the limits w.r.t. the various
    symbols commute.

    If no symbols are passed then all symbols in the expression are used.

    """

    is_Order = True

    __slots__ = []

    @cacheit
    def __new__(cls, expr, *args, **kwargs):
        expr = sympify(expr)

        if not args:
            if expr.is_Order:
                variables = expr.variables
                point = expr.point
            else:
                variables = list(expr.free_symbols)
                point = [S.Zero]*len(variables)
        else:
            args = list(args if is_sequence(args) else [args])
            variables, point = [], []
            if is_sequence(args[0]):
                for a in args:
                    v, p = list(map(sympify, a))
                    variables.append(v)
                    point.append(p)
            else:
                variables = list(map(sympify, args))
                point = [S.Zero]*len(variables)

        if not all(isinstance(v, Symbol) for v in variables):
           raise TypeError('Variables are not symbols, got %s' % variables)

        if len(list(uniq(variables))) != len(variables):
            raise ValueError('Variables are supposed to be unique symbols, got %s' % variables)

        if expr.is_Order:
            expr_vp = dict(expr.args[1:])
            new_vp = dict(expr_vp)
            vp = dict(zip(variables, point))
            for v, p in vp.items():
                if v in new_vp.keys():
                    if p != new_vp[v]:
                        raise NotImplementedError(
                            "Mixing Order at different points is not supported.")
                else:
                    new_vp[v] = p
            if set(expr_vp.keys()) == set(new_vp.keys()):
                return expr
            else:
                variables = list(new_vp.keys())
                point = [new_vp[v] for v in variables]

        if expr is S.NaN:
            return S.NaN

        if not all(p is S.Zero for p in point) and \
           not all(p is S.Infinity for p in point):
            raise NotImplementedError('Order at points other than 0 '
                'or oo not supported, got %s as a point.' % point)

        if variables:
            if len(variables) > 1:
                # XXX: better way?  We need this expand() to
                # workaround e.g: expr = x*(x + y).
                # (x*(x + y)).as_leading_term(x, y) currently returns
                # x*y (wrong order term!).  That's why we want to deal with
                # expand()'ed expr (handled in "if expr.is_Add" branch below).
                expr = expr.expand()

            if expr.is_Add:
                lst = expr.extract_leading_order(variables, point)
                expr = Add(*[f.expr for (e, f) in lst])

            elif expr:
                if point[0] == S.Zero:
                    expr = expr.as_leading_term(*variables)
                expr = expr.as_independent(*variables, as_Add=False)[1]

                expr = expand_power_base(expr)
                expr = expand_log(expr)

                if len(variables) == 1:
                    # The definition of O(f(x)) symbol explicitly stated that
                    # the argument of f(x) is irrelevant.  That's why we can
                    # combine some power exponents (only "on top" of the
                    # expression tree for f(x)), e.g.:
                    # x**p * (-x)**q -> x**(p+q) for real p, q.
                    x = variables[0]
                    margs = list(Mul.make_args(
                        expr.as_independent(x, as_Add=False)[1]))

                    for i, t in enumerate(margs):
                        if t.is_Pow:
                            b, q = t.args
                            if b in (x, -x) and q.is_real and not q.has(x):
                                margs[i] = x**q
                            elif b.is_Pow and not b.exp.has(x):
                                b, r = b.args
                                if b in (x, -x) and r.is_real:
                                    margs[i] = x**(r*q)
                            elif b.is_Mul and b.args[0] is S.NegativeOne:
                                b = -b
                                if b.is_Pow and not b.exp.has(x):
                                    b, r = b.args
                                    if b in (x, -x) and r.is_real:
                                        margs[i] = x**(r*q)

                    expr = Mul(*margs)

        if expr is S.Zero:
            return expr

        if expr.is_Order:
            expr = expr.expr

        if not expr.has(*variables):
            expr = S.One

        # create Order instance:
        variables.sort(key=default_sort_key)
        args = (expr,) + Tuple(*zip(variables, point))
        obj = Expr.__new__(cls, *args)
        return obj

    def _hashable_content(self):
        return self.args

    def oseries(self, order):
        return self

    def _eval_nseries(self, x, n, logx):
        return self

    @property
    def expr(self):
        return self.args[0]

    @property
    def variables(self):
        if self.args[1:]:
            return tuple(x[0] for x in self.args[1:])
        else:
            return ()

    @property
    def point(self):
        if self.args[1:]:
            return tuple(x[1] for x in self.args[1:])
        else:
            return ()

    @property
    def free_symbols(self):
        return self.expr.free_symbols | set(self.variables)

    def _eval_power(b, e):
        if e.is_Number and e.is_nonnegative:
            return b.func(b.expr ** e, *b.args[1:])
        return

    def as_expr_variables(self, order_symbols):
        if order_symbols is None:
            order_symbols = self.args[1:]
        else:
            if not all(o[1] == order_symbols[0][1] for o in order_symbols) and \
               not all(p == self.point[0] for p in self.point):
                raise NotImplementedError('Order at points other than 0 '
                    'or oo not supported, got %s as a point.' % point)
            if order_symbols[0][1] != self.point[0]:
                raise NotImplementedError(
                        "Multiplying Order at different points is not supported.")
            order_symbols = dict(order_symbols)
            for s, p in dict(self.args[1:]).items():
                if s not in order_symbols.keys():
                    order_symbols[s] = p
            order_symbols = sorted(order_symbols.items(), key=lambda x: default_sort_key(x[0]))
        return self.expr, tuple(order_symbols)

    def removeO(self):
        return S.Zero

    def getO(self):
        return self

    @cacheit
    def contains(self, expr):
        """
        Return True if expr belongs to Order(self.expr, \*self.variables).
        Return False if self belongs to expr.
        Return None if the inclusion relation cannot be determined
        (e.g. when self and expr have different symbols).
        """
        from sympy import powsimp, limit
        if expr is S.Zero:
            return True
        if expr is S.NaN:
            return False
        if expr.is_Order:
            if not all(p == expr.point[0] for p in expr.point) and \
               not all(p == self.point[0] for p in self.point):
                raise NotImplementedError('Order at points other than 0 '
                    'or oo not supported, got %s as a point.' % point)
            else:
                # self and/or expr is O(1):
                if any(not p for p in [expr.point, self.point]):
                    point = self.point + expr.point
                    if point:
                        point = point[0]
                    else:
                        point = S.Zero
                else:
                    point = self.point[0]
            if expr.expr == self.expr:
                # O(1) + O(1), O(1) + O(1, x), etc.
                return all([x in self.args[1:] for x in expr.args[1:]])
            if expr.expr.is_Add:
                return all([self.contains(x) for x in expr.expr.args])
            if self.expr.is_Add:
                return any([self.func(x, *self.args[1:]).contains(expr)
                            for x in self.expr.args])
            if self.variables and expr.variables:
                common_symbols = tuple(
                    [s for s in self.variables if s in expr.variables])
            elif self.variables:
                common_symbols = self.variables
            else:
                common_symbols = expr.variables
            if not common_symbols:
                return None
            r = None
            ratio = self.expr/expr.expr
            ratio = powsimp(ratio, deep=True, combine='exp')
            for s in common_symbols:
                l = limit(ratio, s, point) != 0
                if r is None:
                    r = l
                else:
                    if r != l:
                        return
            return r
        obj = self.func(expr, *self.args[1:])
        return self.contains(obj)

    def _eval_subs(self, old, new):
        if old.is_Symbol and old in self.variables:
            i = self.variables.index(old)
            newexpr = self.expr._subs(old, new)
            if isinstance(new, Symbol):
                newvars = list(self.variables)
                newvars[i] = new
                newpt = self.point
            else:
                newvars = tuple(newexpr.free_symbols) + \
                    self.variables[:i] + self.variables[i + 1:]
                newpt = self.point[0]**(new.as_numer_denom()[1].is_number*2 - 1)
                newpt = [newpt]*len(newvars)
            return Order(newexpr, *zip(newvars, newpt))
        return Order(self.expr._subs(old, new), *self.args[1:])

    def _eval_conjugate(self):
        expr = self.expr._eval_conjugate()
        if expr is not None:
            return self.func(expr, *self.args[1:])

    def _eval_derivative(self, x):
        return self.func(self.expr.diff(x), *self.args[1:]) or self

    def _eval_transpose(self):
        expr = self.expr._eval_transpose()
        if expr is not None:
            return self.func(expr, *self.args[1:])

    def _sage_(self):
        #XXX: SAGE doesn't have Order yet. Let's return 0 instead.
        return Rational(0)._sage_()

O = Order
<commit_msg>Add note: all limit points should be either 0 or oo<commit_after>from __future__ import print_function, division

from sympy.core import Basic, S, sympify, Expr, Rational, Symbol
from sympy.core import Add, Mul, expand_power_base, expand_log
from sympy.core.cache import cacheit
from sympy.core.compatibility import default_sort_key, is_sequence
from sympy.core.containers import Tuple
from sympy.utilities.iterables import uniq

class Order(Expr):
    r""" Represents the limiting behavior of some function

    The order of a function characterizes the function based on the limiting
    behavior of the function as it goes to some limit. Only taking all limit
    points to be 0 or positive infinity is currently supported. This is
    expressed in big O notation [1]_.

    The formal definition for the order of a function `g(x)` about a point `a`
    is such that `g(x) = O(f(x))` as `x \rightarrow a` if and only if for any
    `\delta > 0` there exists a `M > 0` such that `|g(x)| \leq M|f(x)|` for
    `|x-a| < \delta`. This is equivalent to `\lim_{x \rightarrow a}
    |g(x)/f(x)| < \infty`.

    Let's illustrate it on the following example by taking the expansion of
    `\sin(x)` about 0:

    .. math ::
        \sin(x) = x - x^3/3! + O(x^5)

    where in this case `O(x^5) = x^5/5! - x^7/7! + \cdots`. By the definition
    of `O`, for any `\delta > 0` there is an `M` such that:

    .. math ::
        |x^5/5! - x^7/7! + ....| <= M|x^5| \text{ for } |x| < \delta

    or by the alternate definition:

    .. math ::
        \lim_{x \rightarrow 0} | (x^5/5! - x^7/7! + ....) / x^5| < \infty

    which surely is true, because

    .. math ::
        \lim_{x \rightarrow 0} | (x^5/5! - x^7/7! + ....) / x^5| = 1/5!

    As it is usually used, the order of a function can be intuitively thought
    of representing all terms of powers greater than the one specified. For
    example, `O(x^3)` corresponds to any terms proportional to `x^3,
    x^4,\ldots` and any higher power. For a polynomial, this leaves terms
    proportional to `x^2`, `x` and constants.

    Examples
    ========

    >>> from sympy import O
    >>> from sympy.abc import x
    >>> O(x)
    O(x)
    >>> O(x)*x
    O(x**2)
    >>> O(x)-O(x)
    O(x)

    References
    ==========

    .. [1] `Big O notation <http://en.wikipedia.org/wiki/Big_O_notation>`_

    Notes
    =====

    In ``O(f(x), x)`` the expression ``f(x)`` is assumed to have a leading
    term.  ``O(f(x), x)`` is automatically transformed to
    ``O(f(x).as_leading_term(x),x)``.

        ``O(expr*f(x), x)`` is ``O(f(x), x)``

        ``O(expr, x)`` is ``O(1)``

        ``O(0, x)`` is 0.

    Multivariate O is also supported:

        ``O(f(x, y), x, y)`` is transformed to
        ``O(f(x, y).as_leading_term(x,y).as_leading_term(y), x, y)``

    In the multivariate case, it is assumed the limits w.r.t. the various
    symbols commute.

    If no symbols are passed then all symbols in the expression are used.

    """

    is_Order = True

    __slots__ = []

    @cacheit
    def __new__(cls, expr, *args, **kwargs):
        expr = sympify(expr)

        if not args:
            if expr.is_Order:
                variables = expr.variables
                point = expr.point
            else:
                variables = list(expr.free_symbols)
                point = [S.Zero]*len(variables)
        else:
            args = list(args if is_sequence(args) else [args])
            variables, point = [], []
            if is_sequence(args[0]):
                for a in args:
                    v, p = list(map(sympify, a))
                    variables.append(v)
                    point.append(p)
            else:
                variables = list(map(sympify, args))
                point = [S.Zero]*len(variables)

        if not all(isinstance(v, Symbol) for v in variables):
           raise TypeError('Variables are not symbols, got %s' % variables)

        if len(list(uniq(variables))) != len(variables):
            raise ValueError('Variables are supposed to be unique symbols, got %s' % variables)

        if expr.is_Order:
            expr_vp = dict(expr.args[1:])
            new_vp = dict(expr_vp)
            vp = dict(zip(variables, point))
            for v, p in vp.items():
                if v in new_vp.keys():
                    if p != new_vp[v]:
                        raise NotImplementedError(
                            "Mixing Order at different points is not supported.")
                else:
                    new_vp[v] = p
            if set(expr_vp.keys()) == set(new_vp.keys()):
                return expr
            else:
                variables = list(new_vp.keys())
                point = [new_vp[v] for v in variables]

        if expr is S.NaN:
            return S.NaN

        if not all(p is S.Zero for p in point) and \
           not all(p is S.Infinity for p in point):
            raise NotImplementedError('Order at points other than 0 '
                'or oo not supported, got %s as a point.' % point)

        if variables:
            if len(variables) > 1:
                # XXX: better way?  We need this expand() to
                # workaround e.g: expr = x*(x + y).
                # (x*(x + y)).as_leading_term(x, y) currently returns
                # x*y (wrong order term!).  That's why we want to deal with
                # expand()'ed expr (handled in "if expr.is_Add" branch below).
                expr = expr.expand()

            if expr.is_Add:
                lst = expr.extract_leading_order(variables, point)
                expr = Add(*[f.expr for (e, f) in lst])

            elif expr:
                if point[0] == S.Zero:
                    expr = expr.as_leading_term(*variables)
                expr = expr.as_independent(*variables, as_Add=False)[1]

                expr = expand_power_base(expr)
                expr = expand_log(expr)

                if len(variables) == 1:
                    # The definition of O(f(x)) symbol explicitly stated that
                    # the argument of f(x) is irrelevant.  That's why we can
                    # combine some power exponents (only "on top" of the
                    # expression tree for f(x)), e.g.:
                    # x**p * (-x)**q -> x**(p+q) for real p, q.
                    x = variables[0]
                    margs = list(Mul.make_args(
                        expr.as_independent(x, as_Add=False)[1]))

                    for i, t in enumerate(margs):
                        if t.is_Pow:
                            b, q = t.args
                            if b in (x, -x) and q.is_real and not q.has(x):
                                margs[i] = x**q
                            elif b.is_Pow and not b.exp.has(x):
                                b, r = b.args
                                if b in (x, -x) and r.is_real:
                                    margs[i] = x**(r*q)
                            elif b.is_Mul and b.args[0] is S.NegativeOne:
                                b = -b
                                if b.is_Pow and not b.exp.has(x):
                                    b, r = b.args
                                    if b in (x, -x) and r.is_real:
                                        margs[i] = x**(r*q)

                    expr = Mul(*margs)

        if expr is S.Zero:
            return expr

        if expr.is_Order:
            expr = expr.expr

        if not expr.has(*variables):
            expr = S.One

        # create Order instance:
        variables.sort(key=default_sort_key)
        args = (expr,) + Tuple(*zip(variables, point))
        obj = Expr.__new__(cls, *args)
        return obj

    def _hashable_content(self):
        return self.args

    def oseries(self, order):
        return self

    def _eval_nseries(self, x, n, logx):
        return self

    @property
    def expr(self):
        return self.args[0]

    @property
    def variables(self):
        if self.args[1:]:
            return tuple(x[0] for x in self.args[1:])
        else:
            return ()

    @property
    def point(self):
        if self.args[1:]:
            return tuple(x[1] for x in self.args[1:])
        else:
            return ()

    @property
    def free_symbols(self):
        return self.expr.free_symbols | set(self.variables)

    def _eval_power(b, e):
        if e.is_Number and e.is_nonnegative:
            return b.func(b.expr ** e, *b.args[1:])
        return

    def as_expr_variables(self, order_symbols):
        if order_symbols is None:
            order_symbols = self.args[1:]
        else:
            if not all(o[1] == order_symbols[0][1] for o in order_symbols) and \
               not all(p == self.point[0] for p in self.point):
                raise NotImplementedError('Order at points other than 0 '
                    'or oo not supported, got %s as a point.' % point)
            if order_symbols[0][1] != self.point[0]:
                raise NotImplementedError(
                        "Multiplying Order at different points is not supported.")
            order_symbols = dict(order_symbols)
            for s, p in dict(self.args[1:]).items():
                if s not in order_symbols.keys():
                    order_symbols[s] = p
            order_symbols = sorted(order_symbols.items(), key=lambda x: default_sort_key(x[0]))
        return self.expr, tuple(order_symbols)

    def removeO(self):
        return S.Zero

    def getO(self):
        return self

    @cacheit
    def contains(self, expr):
        """
        Return True if expr belongs to Order(self.expr, \*self.variables).
        Return False if self belongs to expr.
        Return None if the inclusion relation cannot be determined
        (e.g. when self and expr have different symbols).
        """
        from sympy import powsimp, limit
        if expr is S.Zero:
            return True
        if expr is S.NaN:
            return False
        if expr.is_Order:
            if not all(p == expr.point[0] for p in expr.point) and \
               not all(p == self.point[0] for p in self.point):
                raise NotImplementedError('Order at points other than 0 '
                    'or oo not supported, got %s as a point.' % point)
            else:
                # self and/or expr is O(1):
                if any(not p for p in [expr.point, self.point]):
                    point = self.point + expr.point
                    if point:
                        point = point[0]
                    else:
                        point = S.Zero
                else:
                    point = self.point[0]
            if expr.expr == self.expr:
                # O(1) + O(1), O(1) + O(1, x), etc.
                return all([x in self.args[1:] for x in expr.args[1:]])
            if expr.expr.is_Add:
                return all([self.contains(x) for x in expr.expr.args])
            if self.expr.is_Add:
                return any([self.func(x, *self.args[1:]).contains(expr)
                            for x in self.expr.args])
            if self.variables and expr.variables:
                common_symbols = tuple(
                    [s for s in self.variables if s in expr.variables])
            elif self.variables:
                common_symbols = self.variables
            else:
                common_symbols = expr.variables
            if not common_symbols:
                return None
            r = None
            ratio = self.expr/expr.expr
            ratio = powsimp(ratio, deep=True, combine='exp')
            for s in common_symbols:
                l = limit(ratio, s, point) != 0
                if r is None:
                    r = l
                else:
                    if r != l:
                        return
            return r
        obj = self.func(expr, *self.args[1:])
        return self.contains(obj)

    def _eval_subs(self, old, new):
        if old.is_Symbol and old in self.variables:
            i = self.variables.index(old)
            newexpr = self.expr._subs(old, new)
            if isinstance(new, Symbol):
                newvars = list(self.variables)
                newvars[i] = new
                newpt = self.point
            else:
                newvars = tuple(newexpr.free_symbols) + \
                    self.variables[:i] + self.variables[i + 1:]
                newpt = self.point[0]**(new.as_numer_denom()[1].is_number*2 - 1)
                newpt = [newpt]*len(newvars)
            return Order(newexpr, *zip(newvars, newpt))
        return Order(self.expr._subs(old, new), *self.args[1:])

    def _eval_conjugate(self):
        expr = self.expr._eval_conjugate()
        if expr is not None:
            return self.func(expr, *self.args[1:])

    def _eval_derivative(self, x):
        return self.func(self.expr.diff(x), *self.args[1:]) or self

    def _eval_transpose(self):
        expr = self.expr._eval_transpose()
        if expr is not None:
            return self.func(expr, *self.args[1:])

    def _sage_(self):
        #XXX: SAGE doesn't have Order yet. Let's return 0 instead.
        return Rational(0)._sage_()

O = Order
<|endoftext|>


<commit_before>import errno

from tornado import gen
from tornado import iostream
from socks5 import GreetingRequest, Request
from socks5 import GreetingResponse, Response
from socks5 import VERSION as socks_version
from socks5 import RESP_STATUS, AUTH_TYPE
from socks5.connection import ServerConnection

from base import ProxyLayer

from microproxy.utils import get_logger
from microproxy.exception import SrcStreamClosedError, DestNotConnectedError

logger = get_logger(__name__)

class SocksLayer(ProxyLayer):
    def __init__(self, context):
        super(SocksLayer, self).__init__(context)
        self.socks_conn = ServerConnection()

    @gen.coroutine
    def process_and_return_context(self):
        self.socks_conn.initiate_connection()
        while True:
            data = yield self.context.src_stream.read_bytes(1024, partial=True)
            _event = self.socks_conn.receive(data)

            if isinstance(_event, GreetingRequest):
                yield self.handle_greeting_request(_event)

            elif isinstance(_event, Request):
                yield self.handle_request(_event)
                break
        raise gen.Return(self.context)

    @gen.coroutine
    def handle_greeting_request(self, event):
        if not AUTH_TYPE["NO_AUTH"] in event.methods:
            response_event = GreetingResponse(
                socks_version, AUTH_TYPE["NO_SUPPORT_AUTH_METHOD"])
        else:
            response_event = GreetingResponse(
                socks_version, AUTH_TYPE["NO_AUTH"])
        data = self.socks_conn.send(response_event)
        yield self.context.src_stream.write(data)

    @gen.coroutine
    def handle_request(self, event):
        host = event.addr
        port = event.port
        try:
            dest_stream = yield self.create_dest_stream((host, port))

        except gen.TimeoutError as e:
            logger.debug("connection timout {0}:{1}".format(
                host, port))
            response_event = Response(
                socks_version, RESP_STATUS["NETWORK_UNREACHABLE"],
                event.atyp, event.addr, event.port)

            data = self.socks_conn.send(response_event)
            yield self.context.src_stream.write(data)
            raise DestNotConnectedError(e)

        except iostream.StreamClosedError as e:
            if e.real_error:
                err_num = abs(e.real_error[0])
                logger.debug("connect to {0}:{1} with error code {2}".format(
                    host, port, errno.errorcode[err_num]))

                # NOTE: if we submit an incorrect address type,
                # the error code will be:
                # - ENOEXEC in macos.
                # - EBADF in linux.
                if err_num == errno.ENOEXEC or err_num == errno.EBADF:
                    response_event = Response(
                        socks_version,
                        RESP_STATUS["ADDRESS_TYPE_NOT_SUPPORTED"],
                        event.atyp, event.addr, event.port)

                elif err_num == errno.ETIMEDOUT:
                    response_event = Response(
                        socks_version, RESP_STATUS["NETWORK_UNREACHABLE"],
                        event.atyp, event.addr, event.port)

                else:
                    logger.error("unhandle error code {0} received".format(
                        errno.errorcode[err_num]))
                    response_event = Response(
                        socks_version, RESP_STATUS["GENRAL_FAILURE"],
                        event.atyp, event.addr, event.port)
                data = self.socks_conn.send(response_event)
                yield self.context.src_stream.write(data)
                raise DestNotConnectedError(e)
            else:
                # NOTE: if real_error is None, it imply the source stream is closed.
                dest_stream.close()
                raise SrcStreamClosedError(e)

        self.context.dest_stream = dest_stream
        self.context.host = host
        self.context.port = port

        response_event = Response(
            socks_version, RESP_STATUS["SUCCESS"],
            event.atyp, event.addr, event.port)
        data = self.socks_conn.send(response_event)
        yield self.context.src_stream.write(data)
<commit_msg>Slightly change socks layer and add missing checking for wrong command type.<commit_after>import errno

from tornado import gen
from tornado import iostream
from socks5 import GreetingRequest, Request
from socks5 import GreetingResponse, Response
from socks5 import VERSION as socks_version
from socks5 import RESP_STATUS, AUTH_TYPE, REQ_COMMAND
from socks5.connection import ServerConnection

from base import ProxyLayer

from microproxy.utils import get_logger
from microproxy.exception import SrcStreamClosedError, DestNotConnectedError, ProtocolError

logger = get_logger(__name__)

class SocksLayer(ProxyLayer):
    def __init__(self, context):
        super(SocksLayer, self).__init__(context)
        self.socks_conn = ServerConnection()

    @gen.coroutine
    def process_and_return_context(self):
        self.socks_conn.initiate_connection()
        while True:
            data = yield self.context.src_stream.read_bytes(1024, partial=True)
            _event = self.socks_conn.receive(data)

            if isinstance(_event, GreetingRequest):
                event = self.handle_greeting_request(_event)
                data = self.socks_conn.send(event)
                yield self.context.src_stream.write(data)

            elif isinstance(_event, Request):
                error, event = yield self.handle_request(_event)
                data = self.socks_conn.send(event)
                yield self.context.src_stream.write(data)

                if error:
                    raise error

                break
        raise gen.Return(self.context)

    def handle_greeting_request(self, event):
        if not AUTH_TYPE["NO_AUTH"] in event.methods:
            response_event = GreetingResponse(
                socks_version, AUTH_TYPE["NO_SUPPORT_AUTH_METHOD"])
        else:
            response_event = GreetingResponse(
                socks_version, AUTH_TYPE["NO_AUTH"])
        return response_event

    @gen.coroutine
    def handle_request(self, event):

        if event.cmd != REQ_COMMAND["CONNECT"]:
            logger.debug("Unsupport connect type")
            error = ProtocolError("Unsupport bind type")
            response_event = Response(
                socks_version, RESP_STATUS["COMMAND_NOT_SUPPORTED"],
                event.atyp, event.addr, event.port)
            raise gen.Return((error, response_event))

        host = event.addr
        port = event.port
        try:
            dest_stream = yield self.create_dest_stream((host, port))

        except gen.TimeoutError as e:
            logger.debug("connection timout {0}:{1}".format(
                host, port))
            response_event = Response(
                socks_version, RESP_STATUS["NETWORK_UNREACHABLE"],
                event.atyp, event.addr, event.port)

            error = DestNotConnectedError(e)
            raise gen.Return((error, response_event))

        except iostream.StreamClosedError as e:
            if e.real_error:
                err_num = abs(e.real_error[0])
                logger.debug("connect to {0}:{1} with error code {2}".format(
                    host, port, errno.errorcode[err_num]))

                # NOTE: if we submit an incorrect address type,
                # the error code will be:
                # - ENOEXEC in macos.
                # - EBADF in linux.
                if err_num == errno.ENOEXEC or err_num == errno.EBADF:
                    response_event = Response(
                        socks_version,
                        RESP_STATUS["ADDRESS_TYPE_NOT_SUPPORTED"],
                        event.atyp, event.addr, event.port)

                elif err_num == errno.ETIMEDOUT:
                    response_event = Response(
                        socks_version, RESP_STATUS["NETWORK_UNREACHABLE"],
                        event.atyp, event.addr, event.port)

                else:
                    logger.error("unhandle error code {0} received".format(
                        errno.errorcode[err_num]))
                    response_event = Response(
                        socks_version, RESP_STATUS["GENRAL_FAILURE"],
                        event.atyp, event.addr, event.port)
                error = DestNotConnectedError(e)
                raise gen.Return((error, response_event))
            else:
                # NOTE: if real_error is None, it imply the source stream is closed.
                dest_stream.close()
                raise SrcStreamClosedError(e)

        self.context.dest_stream = dest_stream
        self.context.host = host
        self.context.port = port

        error = None
        response_event = Response(
            socks_version, RESP_STATUS["SUCCESS"],
            event.atyp, event.addr, event.port)

        raise gen.Return((None, response_event))
<|endoftext|>