<commit_before>import logging
import os
import sys

def find_project_dir():
    """
        Go through the path, and look for manage.py
    """
    for path in sys.path:
        abs_path = os.path.join(os.path.abspath(path), "manage.py")
        if os.path.exists(abs_path):
            return os.path.dirname(abs_path)

    raise RuntimeError("Unable to locate manage.py on sys.path")

PROJECT_DIR = find_project_dir()
DATA_ROOT = os.path.join(PROJECT_DIR, '.gaedata')

# Overrides for os.environ.
env_ext = {}
if 'DJANGO_SETTINGS_MODULE' not in os.environ:
    env_ext['DJANGO_SETTINGS_MODULE'] = 'settings'

def setup_env():
    """Configures GAE environment for command-line apps."""

    # Try to import the appengine code from the system path.
    try:
        from google.appengine.api import apiproxy_stub_map
    except ImportError:
        for k in [k for k in sys.modules if k.startswith('google')]:
            del sys.modules[k]

        # Not on the system path. Build a list of alternative paths
        # where it may be. First look within the project for a local
        # copy, then look for where the Mac OS SDK installs it.
        paths = [os.path.join(PROJECT_DIR, 'google_appengine'),
                 os.environ.get('APP_ENGINE_SDK'),
                 '/usr/local/google_appengine',
                 '/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine']
        for path in os.environ.get('PATH', '').split(os.pathsep):
            path = path.rstrip(os.sep)
            if path.endswith('google_appengine'):
                paths.append(path)
        if os.name in ('nt', 'dos'):
            path = r'%(PROGRAMFILES)s\Google\google_appengine' % os.environ
            paths.append(path)

        # Loop through all possible paths and look for the SDK dir.
        sdk_path = None
        for path in paths:
            if not path:
                continue
            path = os.path.expanduser(path)
            path = os.path.realpath(path)
            if os.path.exists(path):
                sdk_path = path
                break

        # The SDK could not be found in any known location.
        if sdk_path is None:
            sys.stderr.write("The Google App Engine SDK could not be found!\n"
                             "Make sure it's accessible via your PATH "
                             "environment and called google_appengine.\n")
            sys.exit(1)

        # Add the SDK and the libraries within it to the system path.
        extra_paths = [sdk_path]
        lib = os.path.join(sdk_path, 'lib')
        # Automatically add all packages in the SDK's lib folder:
        for name in os.listdir(lib):
            root = os.path.join(lib, name)
            subdir = name
            # Package can be under 'lib/<pkg>/<pkg>/' or
            # 'lib/<pkg>/lib/<pkg>/'.
            detect = (os.path.join(root, subdir),
                      os.path.join(root, 'lib', subdir))
            for path in detect:
                if os.path.isdir(path):
                    extra_paths.append(os.path.dirname(path))
                    break
            else:
                if name == 'webapp2':
                    extra_paths.append(root)
                elif name == 'webob_0_9':
                    extra_paths.append(root)
        sys.path = extra_paths + sys.path
        from google.appengine.api import apiproxy_stub_map

    setup_project()
    from .utils import have_appserver
    if have_appserver:
        # App Engine's threading.local is broken.
        setup_threading()
    elif not os.path.exists(DATA_ROOT):
        os.mkdir(DATA_ROOT)
    setup_logging()

    if not have_appserver:
        # Patch Django to support loading management commands from zip
        # files.
        from django.core import management
        management.find_commands = find_commands

def find_commands(management_dir):
    """
    Given a path to a management directory, returns a list of all the
    command names that are available.
    This version works for django deployments which are file based or
    contained in a ZIP (in sys.path).

    Returns an empty list if no commands are defined.
    """
    import pkgutil
    return [modname for importer, modname, ispkg in pkgutil.iter_modules(
                [os.path.join(management_dir, 'commands')]) if not ispkg]

def setup_threading():
    if sys.version_info >= (2, 7):
        return
    # XXX: On Python 2.5 GAE's threading.local doesn't work correctly
    #      with subclassing.
    try:
        from django.utils._threading_local import local
        import threading
        threading.local = local
    except ImportError:
        pass

def setup_logging():
    # Fix Python 2.6 logging module.
    logging.logMultiprocessing = 0

    # Enable logging.
    level = logging.DEBUG
    from .utils import have_appserver
    if have_appserver:
        # We can't import settings at this point when running a normal
        # manage.py command because this module gets imported from
        # settings.py.
        from django.conf import settings
        if not settings.DEBUG:
            level = logging.INFO
    logging.getLogger().setLevel(level)

def setup_project():
    from .utils import have_appserver, on_production_server
    if have_appserver:
        # This fixes a pwd import bug for os.path.expanduser().
        env_ext['HOME'] = PROJECT_DIR

    # The dev_appserver creates a sandbox which restricts access to
    # certain modules and builtins in order to emulate the production
    # environment. Here we get the subprocess module back into the
    # dev_appserver sandbox.This module is just too important for
    # development. Also we add the compiler/parser module back and
    # enable https connections (seem to be broken on Windows because
    # the _ssl module is disallowed).
    if not have_appserver:
        from google.appengine.tools import dev_appserver
        try:
            # Backup os.environ. It gets overwritten by the
            # dev_appserver, but it's needed by the subprocess module.
            env = dev_appserver.DEFAULT_ENV
            dev_appserver.DEFAULT_ENV = os.environ.copy()
            dev_appserver.DEFAULT_ENV.update(env)
            # Backup the buffer() builtin. The subprocess in Python 2.5
            # on Linux and OS X uses needs it, but the dev_appserver
            # removes it.
            dev_appserver.buffer = buffer
        except AttributeError:
            logging.warn("Could not patch the default environment. "
                         "The subprocess module will not work correctly.")

        try:
            # Allow importing compiler/parser, _ssl (for https),
            # _io for Python 2.7 io support on OS X
            dev_appserver.HardenedModulesHook._WHITE_LIST_C_MODULES.extend(
                ('parser', '_ssl', '_io'))
        except AttributeError:
            logging.warn("Could not patch modules whitelist. the compiler "
                         "and parser modules will not work and SSL support "
                         "is disabled.")
    elif not on_production_server:
        try:
            # Restore the real subprocess module.
            from google.appengine.api.mail_stub import subprocess
            sys.modules['subprocess'] = subprocess
            # Re-inject the buffer() builtin into the subprocess module.
            from google.appengine.tools import dev_appserver
            subprocess.buffer = dev_appserver.buffer
        except Exception, e:
            logging.warn("Could not add the subprocess module to the "
                         "sandbox: %s" % e)

    os.environ.update(env_ext)

    extra_paths = [PROJECT_DIR, os.path.join(os.path.dirname(__file__), 'lib')]
    zip_packages_dir = os.path.join(PROJECT_DIR, 'zip-packages')

    # We support zipped packages in the common and project folders.
    if os.path.isdir(zip_packages_dir):
        for zip_package in os.listdir(zip_packages_dir):
            extra_paths.append(os.path.join(zip_packages_dir, zip_package))

    # App Engine causes main.py to be reloaded if an exception gets
    # raised on the first request of a main.py instance, so don't call
    # setup_project() multiple times. We ensure this indirectly by
    # checking if we've already modified sys.path, already.
    if len(sys.path) < len(extra_paths) or \
            sys.path[:len(extra_paths)] != extra_paths:
        for path in extra_paths:
            while path in sys.path:
                sys.path.remove(path)
        sys.path = extra_paths + sys.path
<commit_msg>Use fix_sys_path from the AppEngine SDK rather than try to find the libraries manually<commit_after>import logging
import os
import sys

def find_project_dir():
    """
        Go through the path, and look for manage.py
    """
    for path in sys.path:
        abs_path = os.path.join(os.path.abspath(path), "manage.py")
        if os.path.exists(abs_path):
            return os.path.dirname(abs_path)

    raise RuntimeError("Unable to locate manage.py on sys.path")

PROJECT_DIR = find_project_dir()
DATA_ROOT = os.path.join(PROJECT_DIR, '.gaedata')

# Overrides for os.environ.
env_ext = {}
if 'DJANGO_SETTINGS_MODULE' not in os.environ:
    env_ext['DJANGO_SETTINGS_MODULE'] = 'settings'

def setup_env():
    """Configures GAE environment for command-line apps."""

    # Try to import the appengine code from the system path.
    try:
        from google.appengine.api import apiproxy_stub_map
    except ImportError:
        for k in [k for k in sys.modules if k.startswith('google')]:
            del sys.modules[k]

        # Not on the system path. Build a list of alternative paths
        # where it may be. First look within the project for a local
        # copy, then look for where the Mac OS SDK installs it.
        paths = [os.path.join(PROJECT_DIR, 'google_appengine'),
                 os.environ.get('APP_ENGINE_SDK'),
                 '/usr/local/google_appengine',
                 '/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine']
        for path in os.environ.get('PATH', '').split(os.pathsep):
            path = path.rstrip(os.sep)
            if path.endswith('google_appengine'):
                paths.append(path)
        if os.name in ('nt', 'dos'):
            path = r'%(PROGRAMFILES)s\Google\google_appengine' % os.environ
            paths.append(path)

        # Loop through all possible paths and look for the SDK dir.
        sdk_path = None
        for path in paths:
            if not path:
                continue
            path = os.path.expanduser(path)
            path = os.path.realpath(path)
            if os.path.exists(path):
                sdk_path = path
                break

        # The SDK could not be found in any known location.
        if sdk_path is None:
            sys.stderr.write("The Google App Engine SDK could not be found!\n"
                             "Make sure it's accessible via your PATH "
                             "environment and called google_appengine.\n")
            sys.exit(1)

        #First add the found SDK to the path
        sys.path = [ sdk_path ] + sys.path

        #Then call fix_sys_path from the SDK
        from dev_appserver import fix_sys_path
        fix_sys_path()

    setup_project()
    from .utils import have_appserver
    if have_appserver:
        # App Engine's threading.local is broken.
        setup_threading()
    elif not os.path.exists(DATA_ROOT):
        os.mkdir(DATA_ROOT)
    setup_logging()

    if not have_appserver:
        # Patch Django to support loading management commands from zip
        # files.
        from django.core import management
        management.find_commands = find_commands

def find_commands(management_dir):
    """
    Given a path to a management directory, returns a list of all the
    command names that are available.
    This version works for django deployments which are file based or
    contained in a ZIP (in sys.path).

    Returns an empty list if no commands are defined.
    """
    import pkgutil
    return [modname for importer, modname, ispkg in pkgutil.iter_modules(
                [os.path.join(management_dir, 'commands')]) if not ispkg]

def setup_threading():
    if sys.version_info >= (2, 7):
        return
    # XXX: On Python 2.5 GAE's threading.local doesn't work correctly
    #      with subclassing.
    try:
        from django.utils._threading_local import local
        import threading
        threading.local = local
    except ImportError:
        pass

def setup_logging():
    # Fix Python 2.6 logging module.
    logging.logMultiprocessing = 0

    # Enable logging.
    level = logging.DEBUG
    from .utils import have_appserver
    if have_appserver:
        # We can't import settings at this point when running a normal
        # manage.py command because this module gets imported from
        # settings.py.
        from django.conf import settings
        if not settings.DEBUG:
            level = logging.INFO
    logging.getLogger().setLevel(level)

def setup_project():
    from .utils import have_appserver, on_production_server
    if have_appserver:
        # This fixes a pwd import bug for os.path.expanduser().
        env_ext['HOME'] = PROJECT_DIR

    # The dev_appserver creates a sandbox which restricts access to
    # certain modules and builtins in order to emulate the production
    # environment. Here we get the subprocess module back into the
    # dev_appserver sandbox.This module is just too important for
    # development. Also we add the compiler/parser module back and
    # enable https connections (seem to be broken on Windows because
    # the _ssl module is disallowed).
    if not have_appserver:
        from google.appengine.tools import dev_appserver
        try:
            # Backup os.environ. It gets overwritten by the
            # dev_appserver, but it's needed by the subprocess module.
            env = dev_appserver.DEFAULT_ENV
            dev_appserver.DEFAULT_ENV = os.environ.copy()
            dev_appserver.DEFAULT_ENV.update(env)
            # Backup the buffer() builtin. The subprocess in Python 2.5
            # on Linux and OS X uses needs it, but the dev_appserver
            # removes it.
            dev_appserver.buffer = buffer
        except AttributeError:
            logging.warn("Could not patch the default environment. "
                         "The subprocess module will not work correctly.")

        try:
            # Allow importing compiler/parser, _ssl (for https),
            # _io for Python 2.7 io support on OS X
            dev_appserver.HardenedModulesHook._WHITE_LIST_C_MODULES.extend(
                ('parser', '_ssl', '_io'))
        except AttributeError:
            logging.warn("Could not patch modules whitelist. the compiler "
                         "and parser modules will not work and SSL support "
                         "is disabled.")
    elif not on_production_server:
        try:
            # Restore the real subprocess module.
            from google.appengine.api.mail_stub import subprocess
            sys.modules['subprocess'] = subprocess
            # Re-inject the buffer() builtin into the subprocess module.
            from google.appengine.tools import dev_appserver
            subprocess.buffer = dev_appserver.buffer
        except Exception, e:
            logging.warn("Could not add the subprocess module to the "
                         "sandbox: %s" % e)

    os.environ.update(env_ext)

    extra_paths = [PROJECT_DIR, os.path.join(os.path.dirname(__file__), 'lib')]
    zip_packages_dir = os.path.join(PROJECT_DIR, 'zip-packages')

    # We support zipped packages in the common and project folders.
    if os.path.isdir(zip_packages_dir):
        for zip_package in os.listdir(zip_packages_dir):
            extra_paths.append(os.path.join(zip_packages_dir, zip_package))

    # App Engine causes main.py to be reloaded if an exception gets
    # raised on the first request of a main.py instance, so don't call
    # setup_project() multiple times. We ensure this indirectly by
    # checking if we've already modified sys.path, already.
    if len(sys.path) < len(extra_paths) or \
            sys.path[:len(extra_paths)] != extra_paths:
        for path in extra_paths:
            while path in sys.path:
                sys.path.remove(path)
        sys.path = extra_paths + sys.path
<|endoftext|>


<commit_before># Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import argparse

from telemetry.internal.util import command_line

class ArgParseCommand(command_line.Command):
  usage = ''

  @classmethod
  def CreateParser(cls):
    return argparse.ArgumentParser('%%prog %s %s' % (cls.Name(), cls.usage),
                                   description=cls.Description())

  @classmethod
  def AddCommandLineArgs(cls, parser, environment):
    # pylint: disable=arguments-differ
    pass

  @classmethod
  def ProcessCommandLineArgs(cls, parser, options, extra_args, environment):
    # pylint: disable=arguments-differ
    pass

  def Run(self, options, extra_args=None):
    # pylint: disable=arguments-differ
    raise NotImplementedError()

  @classmethod
  def main(cls, args=None):
    """Main method to run this command as a standalone script."""
    parser = cls.CreateParser()
    cls.AddCommandLineArgs(parser, None)
    options, extra_args = parser.parse_known_args(args=args)
    cls.ProcessCommandLineArgs(parser, options, extra_args, None)
    return min(cls().Run(options, extra_args), 255)
<commit_msg>[Telemetry] Remove unused ArgParseCommand<commit_after><|endoftext|>


<commit_before>#!/usr/bin/python

import requests
import json
import re
import random

import settings

from flask import Flask
from flask import request

app = Flask(__name__)

RE_LINK = re.compile('')
RE_HREF = re.compile('href=\"(.+)\"')
RE_TITLE = re.compile('>(.+)<')

def parse_post_html(base_url, html):
    link = base_url + RE_HREF.findall(html)[0]
    title = RE_TITLE.findall(html)[0]

    return (link, title)

def get_posts(blog_url = 'http://dangoldin.com'):
    r = requests.get(blog_url)
    c = r.content

    raw_posts = RE_LINK.findall(c)
    posts = [parse_post_html(blog_url, x) for x in raw_posts]

    return posts

@app.route('/danblogbot', methods=['POST'])
def dan_blog_bot():
    message = request.get_json()

    print message

    telegram_url = 'https://api.telegram.org/bot{0}/'.format(settings.TELEGRAM_TOKEN)

    posts = get_posts()

    if '/blogme' in message['message']['text']:
        post = random.choice(posts)
        chat_id = message['message']['chat']['id']
        text = '{1}'.format(post[0], post[1])
        parse_mode = 'HTML'
        r = requests.post(telegram_url + 'sendMessage', json={'chat_id': chat_id, 'text': text, 'parse_mode': parse_mode})
    else if '/help' in message['message']['text'] or '/start'  in message['message']['text']:
        chat_id = message['message']['chat']['id']
        text = 'Just reply with /blogme to get a random blog post'
        parse_mode = 'HTML'
        r = requests.post(telegram_url + 'sendMessage', json={'chat_id': chat_id, 'text': text, 'parse_mode': parse_mode})

    return 'Success'

def test():
    r = requests.get(telegram_url + 'getMe')
    print json.dumps(r.json(), indent=2)

    r = requests.get(telegram_url + 'getUpdates')
    print json.dumps(r.json(), indent=2)

    messages = r.json()['result']

    for message in messages:
        if '/blogme' in message['message']['text']:
            post = random.choice(posts)
            chat_id = message['message']['chat']['id']
            text = '{1}'.format(post[0], post[1])
            parse_mode = 'HTML'
            r = requests.post(telegram_url + 'sendMessage', json={'chat_id': chat_id, 'text': text, 'parse_mode': parse_mode})

if __name__ == '__main__':
    app.run(debug=True)
<commit_msg>Fix python<commit_after>#!/usr/bin/python

import requests
import json
import re
import random

import settings

from flask import Flask
from flask import request

app = Flask(__name__)

RE_LINK = re.compile('')
RE_HREF = re.compile('href=\"(.+)\"')
RE_TITLE = re.compile('>(.+)<')

def parse_post_html(base_url, html):
    link = base_url + RE_HREF.findall(html)[0]
    title = RE_TITLE.findall(html)[0]

    return (link, title)

def get_posts(blog_url = 'http://dangoldin.com'):
    r = requests.get(blog_url)
    c = r.content

    raw_posts = RE_LINK.findall(c)
    posts = [parse_post_html(blog_url, x) for x in raw_posts]

    return posts

@app.route('/danblogbot', methods=['POST'])
def dan_blog_bot():
    message = request.get_json()

    print message

    telegram_url = 'https://api.telegram.org/bot{0}/'.format(settings.TELEGRAM_TOKEN)

    posts = get_posts()

    if '/blogme' in message['message']['text']:
        post = random.choice(posts)
        chat_id = message['message']['chat']['id']
        text = '{1}'.format(post[0], post[1])
        parse_mode = 'HTML'
        r = requests.post(telegram_url + 'sendMessage', json={'chat_id': chat_id, 'text': text, 'parse_mode': parse_mode})
    elif '/help' in message['message']['text'] or '/start'  in message['message']['text']:
        chat_id = message['message']['chat']['id']
        text = 'Just reply with /blogme to get a random blog post'
        parse_mode = 'HTML'
        r = requests.post(telegram_url + 'sendMessage', json={'chat_id': chat_id, 'text': text, 'parse_mode': parse_mode})

    return 'Success'

def test():
    r = requests.get(telegram_url + 'getMe')
    print json.dumps(r.json(), indent=2)

    r = requests.get(telegram_url + 'getUpdates')
    print json.dumps(r.json(), indent=2)

    messages = r.json()['result']

    for message in messages:
        if '/blogme' in message['message']['text']:
            post = random.choice(posts)
            chat_id = message['message']['chat']['id']
            text = '{1}'.format(post[0], post[1])
            parse_mode = 'HTML'
            r = requests.post(telegram_url + 'sendMessage', json={'chat_id': chat_id, 'text': text, 'parse_mode': parse_mode})

if __name__ == '__main__':
    app.run(debug=True)
<|endoftext|>


<commit_before>#!/usr/bin/python
#------------------------------------------------------------
# Run me with (may need su privilege for logging):
#        gunicorn -w 4 -b 127.0.0.1:3031 pyxserver_wsgi:application
#  (remove the -w 4 for debugging--don't want 4 workers)
# gunicorn --preload -b 127.0.0.1:3031 --timeout=35 --pythonpath=. pyxserver_wsgi:application
#------------------------------------------------------------

import cgi    # for the escape() function
import json
import logging
import os
import os.path
from statsd import statsd
import sys
from time import localtime, strftime, time

import settings    # Not django, but do something similar

from sandbox import sandbox

# make sure we can find the grader files
sys.path.append(settings.GRADER_ROOT)
import grade

logging.config.dictConfig(settings.LOGGING)

log = logging.getLogger("xserver." + __name__)

results_template = u"""

Test results

    {status}

      {errors}
      {results}

"""

results_correct_template = u"""

{short-description}

{long-description}

Output:

{actual-output}

"""

results_incorrect_template = u"""

{short-description}

{long-description}

Your output:

{actual-output}

Correct output:

{expected-output}

"""

def format_errors(errors):
    esc = cgi.escape
    error_string = ''
    error_list = [esc(e) for e in errors or []]
    if error_list:
        items = u'\n'.join([u'

{0}

\n'.format(e) for e in error_list])
        error_string = u'
\n{0}
\n'.format(items)
        error_string = u'
{0}
'.format(error_string)
    return error_string

def to_dict(result):
    # long description may or may not be provided.  If not, don't display it.
    # TODO: replace with mako template
    esc = cgi.escape
    if result[1]:
        long_desc = u'
{0}
'.format(esc(result[1]))
    else:
        long_desc = u''
    return {'short-description': esc(result[0]),
            'long-description': long_desc,
            'correct': result[2],   # Boolean; don't escape.
            'expected-output': esc(result[3]),
            'actual-output': esc(result[4])
            }

def render_results(results):
    output = []
    test_results = [to_dict(r) for r in results['tests']]
    for result in test_results:
        if result['correct']:
            template = results_correct_template
        else:
            template = results_incorrect_template
        output += template.format(**result)

    errors = format_errors(results['errors'])

    status = 'INCORRECT'
    if errors:
        status = 'ERROR'
    elif results['correct']:
        status = 'CORRECT'

    return results_template.format(status=status,
                                   errors=errors,
                                   results=''.join(output))

def do_GET(data):
    return "Hey, the time is %s" % strftime("%a, %d %b %Y %H:%M:%S", localtime())

def do_POST(data):
    statsd.increment('xserver.post-requests')
    # This server expects jobs to be pushed to it from the queue
    xpackage = json.loads(data)
    body  = xpackage['xqueue_body']
    files = xpackage['xqueue_files']

    # Delivery from the lms
    body = json.loads(body)
    student_response = body['student_response']
    payload = body['grader_payload']
    try:
        grader_config = json.loads(payload)
    except ValueError as err:
        # If parsing json fails, erroring is fine--something is wrong in the content.
        # However, for debugging, still want to see what the problem is
        statsd.increment('xserver.grader_payload_error')

        log.debug("error parsing: '{0}' -- {1}".format(payload, err))
        raise

    log.debug("Processing submission, grader payload: {0}".format(payload))
    relative_grader_path = grader_config['grader']
    grader_path = os.path.join(settings.GRADER_ROOT, relative_grader_path)
    start = time()
    results = grade.grade(grader_path, student_response, sandbox)
    statsd.histogram('xserver.grading-time', time() - start)

    # Make valid JSON message
    reply = { 'correct': results['correct'],
              'score': results['score'],
              'msg': render_results(results) }

    statsd.increment('xserver.post-replies (non-exception)')

    return json.dumps(reply)

# Entry point
def application(env, start_response):

    log.info("Starting application")
    # Handle request
    method = env['REQUEST_METHOD']
    data = env['wsgi.input'].read()

    log.debug('-' * 60)
    log.debug(method)

    def post_wrapper(data):
        try:
            return do_POST(data)
        except:
            log.exception("Error processing request: {0}".format(data))
            return None

    handlers = {'GET': do_GET,
                 'POST': post_wrapper,
                 }
    if method in handlers.keys():
        reply = handlers[method](data)

        if reply is not None:
            log.debug(' [*] reply:\n%s\n' % reply)

            start_response('200 OK', [('Content-Type', 'text/html')])
            return reply

    # If we fell through to here, complain.
    start_response('404 Not Found', [('Content-Type', 'text/plain')])
    return ''
<commit_msg>pass the grader config through to the grader, if it's ready for it<commit_after>#!/usr/bin/python
#------------------------------------------------------------
# Run me with (may need su privilege for logging):
#        gunicorn -w 4 -b 127.0.0.1:3031 pyxserver_wsgi:application
#  (remove the -w 4 for debugging--don't want 4 workers)
# gunicorn --preload -b 127.0.0.1:3031 --timeout=35 --pythonpath=. pyxserver_wsgi:application
#------------------------------------------------------------

import cgi    # for the escape() function
import json
import logging
import os
import os.path
from statsd import statsd
import sys
from time import localtime, strftime, time

import settings    # Not django, but do something similar

from sandbox import sandbox

# make sure we can find the grader files
sys.path.append(settings.GRADER_ROOT)
import grade

logging.config.dictConfig(settings.LOGGING)

log = logging.getLogger("xserver." + __name__)

results_template = u"""

Test results

    {status}

      {errors}
      {results}

"""

results_correct_template = u"""

{short-description}

{long-description}

Output:

{actual-output}

"""

results_incorrect_template = u"""

{short-description}

{long-description}

Your output:

{actual-output}

Correct output:

{expected-output}

"""

def format_errors(errors):
    esc = cgi.escape
    error_string = ''
    error_list = [esc(e) for e in errors or []]
    if error_list:
        items = u'\n'.join([u'

{0}

\n'.format(e) for e in error_list])
        error_string = u'
\n{0}
\n'.format(items)
        error_string = u'
{0}
'.format(error_string)
    return error_string

def to_dict(result):
    # long description may or may not be provided.  If not, don't display it.
    # TODO: replace with mako template
    esc = cgi.escape
    if result[1]:
        long_desc = u'
{0}
'.format(esc(result[1]))
    else:
        long_desc = u''
    return {'short-description': esc(result[0]),
            'long-description': long_desc,
            'correct': result[2],   # Boolean; don't escape.
            'expected-output': esc(result[3]),
            'actual-output': esc(result[4])
            }

def render_results(results):
    output = []
    test_results = [to_dict(r) for r in results['tests']]
    for result in test_results:
        if result['correct']:
            template = results_correct_template
        else:
            template = results_incorrect_template
        output += template.format(**result)

    errors = format_errors(results['errors'])

    status = 'INCORRECT'
    if errors:
        status = 'ERROR'
    elif results['correct']:
        status = 'CORRECT'

    return results_template.format(status=status,
                                   errors=errors,
                                   results=''.join(output))

def do_GET(data):
    return "Hey, the time is %s" % strftime("%a, %d %b %Y %H:%M:%S", localtime())

def do_POST(data):
    statsd.increment('xserver.post-requests')
    # This server expects jobs to be pushed to it from the queue
    xpackage = json.loads(data)
    body  = xpackage['xqueue_body']
    files = xpackage['xqueue_files']

    # Delivery from the lms
    body = json.loads(body)
    student_response = body['student_response']
    payload = body['grader_payload']
    try:
        grader_config = json.loads(payload)
    except ValueError as err:
        # If parsing json fails, erroring is fine--something is wrong in the content.
        # However, for debugging, still want to see what the problem is
        statsd.increment('xserver.grader_payload_error')

        log.debug("error parsing: '{0}' -- {1}".format(payload, err))
        raise

    log.debug("Processing submission, grader payload: {0}".format(payload))
    relative_grader_path = grader_config['grader']
    grader_path = os.path.join(settings.GRADER_ROOT, relative_grader_path)
    start = time()
    # TODO: Temporary code, to make transition easier: once all deployed versions of 6.00 have code
    # that expects grade() to be passed the grader payload dict, will get rid of the conditional.
    if hasattr(grade, 'TEMPORARY_WANTS_CONFIG'):
        results = grade.grade(grader_path, student_response, sandbox, grader_config)
    else:
        # old version didn't take the config
        results = grade.grade(grader_path, student_response, sandbox)

    statsd.histogram('xserver.grading-time', time() - start)

    # Make valid JSON message
    reply = { 'correct': results['correct'],
              'score': results['score'],
              'msg': render_results(results) }

    statsd.increment('xserver.post-replies (non-exception)')

    return json.dumps(reply)

# Entry point
def application(env, start_response):

    log.info("Starting application")
    # Handle request
    method = env['REQUEST_METHOD']
    data = env['wsgi.input'].read()

    log.debug('-' * 60)
    log.debug(method)

    def post_wrapper(data):
        try:
            return do_POST(data)
        except:
            log.exception("Error processing request: {0}".format(data))
            return None

    handlers = {'GET': do_GET,
                 'POST': post_wrapper,
                 }
    if method in handlers.keys():
        reply = handlers[method](data)

        if reply is not None:
            log.debug(' [*] reply:\n%s\n' % reply)

            start_response('200 OK', [('Content-Type', 'text/html')])
            return reply

    # If we fell through to here, complain.
    start_response('404 Not Found', [('Content-Type', 'text/plain')])
    return ''
<|endoftext|>


<commit_before>import urllib.request
import json

def postillon_ticker():
    '''
    Get ticker data from the Postillon.
    TODO: abstract get_from_api() function?
    '''
    ret = urllib.request.urlopen('http://www.der-postillion.de/ticker/newsticker2.php')
    ticker = json.loads(ret.read().decode("utf-8-sig"))
    return ticker["tickers"]
<commit_msg>make ticker to single line string<commit_after>import urllib.request
import json

def postillon_ticker():
    '''
    Get ticker data from the Postillon.
    TODO: abstract get_from_api() function?
    '''
    ret = urllib.request.urlopen(
        'http://www.der-postillion.de/ticker/newsticker2.php')
    ticker = json.loads(ret.read().decode("utf-8-sig"))
    tickerstring = ''

    space = 6
    for line in ticker['tickers']:
        tickerstring += '+++{ticker}+++{space}'.format(
            ticker=line['text'], space=' ' * space)

    tickerstring = tickerstring[:-space]
    return tickerstring
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import sys
from collections import Counter, OrderedDict
try:
    from lxml.etree import Element
except ImportError:
    from xml.etree.cElementTree import Element

__author__ = 'S Anand'
__email__ = 'root.node@gmail.com'
__version__ = '0.1.8'

# Python 3: define unicode() as str()
if sys.version_info[0] == 3:
    unicode = str
    basestring = str

class XMLData(object):
    def __init__(self, xml_fromstring=True, xml_tostring=True, element=None, dict_type=None,
                 list_type=None, attr_prefix=None, text_content=None, simple_text=False):
        # xml_fromstring == False(y) => '1' -> '1'
        # xml_fromstring == True     => '1' -> 1
        # xml_fromstring == fn       => '1' -> fn(1)
        if callable(xml_fromstring):
            self._fromstring = xml_fromstring
        elif not xml_fromstring:
            self._fromstring = lambda v: v
        # custom conversion function to convert data string to XML string
        if callable(xml_tostring):
            self._tostring = xml_tostring
        # custom etree.Element to use
        self.element = Element if element is None else element
        # dict constructor (e.g. OrderedDict, defaultdict)
        self.dict = OrderedDict if dict_type is None else dict_type
        # list constructor (e.g. UserList)
        self.list = list if list_type is None else list_type
        # Prefix attributes with a string (e.g. '$')
        self.attr_prefix = attr_prefix
        # Key that stores text content (e.g. '$t')
        self.text_content = text_content
        # simple_text == False or None or 0 => '<x>a</x>' = {'x': {'a': {}}}
        # simple_text == True               => '<x>a</x>' = {'x': 'a'}
        self.simple_text = simple_text

    @staticmethod
    def _tostring(value):
        '''Convert value to XML compatible string'''
        if value is True:
            value = 'true'
        elif value is False:
            value = 'false'
        return unicode(value)       # noqa: convert to whatever native unicode repr

    @staticmethod
    def _fromstring(value):
        '''Convert XML string value to None, boolean, int or float'''
        if not value:
            return None
        std_value = value.strip().lower()
        if std_value == 'true':
            return True
        elif std_value == 'false':
            return False
        try:
            return int(std_value)
        except ValueError:
            pass
        try:
            return float(std_value)
        except ValueError:
            pass
        return value

    def etree(self, data, root=None):
        '''Convert data structure into a list of etree.Element'''
        result = self.list() if root is None else root
        if isinstance(data, (self.dict, dict)):
            for key, value in data.items():
                value_is_list = isinstance(value, (self.list, list))
                value_is_dict = isinstance(value, (self.dict, dict))
                # Add attributes and text to result (if root)
                if root is not None:
                    # Handle attribute prefixes (BadgerFish)
                    if self.attr_prefix is not None:
                        if key.startswith(self.attr_prefix):
                            key = key.lstrip(self.attr_prefix)
                            # @xmlns: {$: xxx, svg: yyy} becomes xmlns="xxx" xmlns:svg="yyy"
                            if value_is_dict:
                                raise ValueError('XML namespaces not yet supported')
                            else:
                                result.set(key, self._tostring(value))
                            continue
                    # Handle text content (BadgerFish, GData)
                    if self.text_content is not None:
                        if key == self.text_content:
                            result.text = self._tostring(value)
                            continue
                    # Treat scalars as text content, not children (GData)
                    if self.attr_prefix is None and self.text_content is not None:
                        if not value_is_dict and not value_is_list:
                            result.set(key, self._tostring(value))
                            continue
                # Add other keys as one or more children
                values = value if value_is_list else [value]
                for value in values:
                    elem = self.element(key)
                    result.append(elem)
                    # Treat scalars as text content, not children (Parker)
                    if not isinstance(value, (self.dict, dict, self.list, list)):
                        if self.text_content:
                            value = {self.text_content: value}
                    self.etree(value, root=elem)
        else:
            if self.text_content is None and root is not None:
                root.text = self._tostring(data)
            else:
                result.append(self.element(self._tostring(data)))
        return result

    def data(self, root):
        '''Convert etree.Element into a dictionary'''
        value = self.dict()
        children = [node for node in root if isinstance(node.tag, basestring)]
        for attr, attrval in root.attrib.items():
            attr = attr if self.attr_prefix is None else self.attr_prefix + attr
            value[attr] = self._fromstring(attrval)
        if root.text and self.text_content is not None:
            text = root.text.strip()
            if text:
                if self.simple_text and len(children) == len(root.attrib) == 0:
                    value = self._fromstring(text)
                else:
                    value[self.text_content] = self._fromstring(text)
        count = Counter(child.tag for child in children)
        for child in children:
            if count[child.tag] == 1:
                value.update(self.data(child))
            else:
                result = value.setdefault(child.tag, self.list())
                result += self.data(child).values()
        return self.dict([(root.tag, value)])

class BadgerFish(XMLData):
    '''Converts between XML and data using the BadgerFish convention'''
    def __init__(self, **kwargs):
        super(BadgerFish, self).__init__(attr_prefix='@', text_content='$', **kwargs)

class GData(XMLData):
    '''Converts between XML and data using the GData convention'''
    def __init__(self, **kwargs):
        super(GData, self).__init__(text_content='$t', **kwargs)

class Yahoo(XMLData):
    '''Converts between XML and data using the Yahoo convention'''
    def __init__(self, **kwargs):
        kwargs.setdefault('xml_fromstring', False)
        super(Yahoo, self).__init__(text_content='content', simple_text=True, **kwargs)

class Parker(XMLData):
    '''Converts between XML and data using the Parker convention'''
    def __init__(self, **kwargs):
        super(Parker, self).__init__(**kwargs)

    def data(self, root, preserve_root=False):
        'Convert etree.Element into a dictionary'
        # If preserve_root is False, return the root element. This is easiest
        # done by wrapping the XML in a dummy root element that will be ignored.
        if preserve_root:
            new_root = root.makeelement('dummy_root', {})
            new_root.insert(0, root)
            root = new_root

        # If no children, just return the text
        children = [node for node in root if isinstance(node.tag, basestring)]
        if len(children) == 0:
            return self._fromstring(root.text)

        # Element names become object properties
        count = Counter(child.tag for child in children)
        result = self.dict()
        for child in children:
            if count[child.tag] == 1:
                result[child.tag] = self.data(child)
            else:
                result.setdefault(child.tag, self.list()).append(self.data(child))

        return result

class Abdera(XMLData):
    '''Converts between XML and data using the Abdera convention'''
    def __init__(self, **kwargs):
        super(Abdera, self).__init__(simple_text=True, text_content=True, **kwargs)

    def data(self, root):
        '''Convert etree.Element into a dictionary'''

        value = self.dict()

        # Add attributes specific 'attributes' key
        if root.attrib:
            value[u'attributes'] = self.dict()
            for attr, attrval in root.attrib.items():
                value[u'attributes'][unicode(attr)] = self._fromstring(attrval)

        # Add children to specific 'children' key
        children_list = self.list()
        children = [node for node in root if isinstance(node.tag, basestring)]

        # Add root text
        if root.text and self.text_content is not None:
            text = root.text.strip()
            if text:
                if self.simple_text and len(children) == len(root.attrib) == 0:
                    value = self._fromstring(text)
                else:
                    children_list = [self._fromstring(text), ]

        count = Counter(child.tag for child in children)
        for child in children:
            child_data = self.data(child)
            if (count[child.tag] == 1
                    and len(children_list) > 1
                    and isinstance(children_list[-1], dict)):
                # Merge keys to existing dictionary
                children_list[-1].update(child_data)
            else:
                # Add additional text
                children_list.append(self.data(child))

        # Flatten children
        if len(root.attrib) == 0 and len(children_list) == 1:
            value = children_list[0]

        elif len(children_list) > 0:
            value[u'children'] = children_list

        return self.dict([(unicode(root.tag), value)])

# The difference between Cobra and Abdera is that Cobra _always_ has 'attributes' keys,
# 'children' key is remove when only one child and everything is a string.
# https://github.com/datacenter/cobra/blob/master/cobra/internal/codec/jsoncodec.py
class Cobra(XMLData):
    '''Converts between XML and data using the Cobra convention'''
    def __init__(self, **kwargs):
        super(Cobra, self).__init__(simple_text=True, text_content=True,
                                    xml_fromstring=False, **kwargs)

    def data(self, root):
        '''Convert etree.Element into a dictionary'''

        value = self.dict()

        # Add attributes to 'attributes' key (sorted!) even when empty
        value[u'attributes'] = self.dict()
        if root.attrib:
            for attr in sorted(root.attrib):
                value[u'attributes'][unicode(attr)] = root.attrib[attr]

        # Add children to specific 'children' key
        children_list = self.list()
        children = [node for node in root if isinstance(node.tag, basestring)]

        # Add root text
        if root.text and self.text_content is not None:
            text = root.text.strip()
            if text:
                if self.simple_text and len(children) == len(root.attrib) == 0:
                    value = self._fromstring(text)
                else:
                    children_list = [self._fromstring(text), ]

        count = Counter(child.tag for child in children)
        for child in children:
            child_data = self.data(child)
            if (count[child.tag] == 1
                    and len(children_list) > 1
                    and isinstance(children_list[-1], dict)):
                # Merge keys to existing dictionary
                children_list[-1].update(child_data)
            else:
                # Add additional text
                children_list.append(self.data(child))

        if len(children_list) > 0:
            value[u'children'] = children_list

        return self.dict([(unicode(root.tag), value)])

abdera = Abdera()
badgerfish = BadgerFish()
cobra = Cobra()
gdata = GData()
parker = Parker()
yahoo = Yahoo()
<commit_msg>resolved issue#21 of parent repository<commit_after># -*- coding: utf-8 -*-

import sys
from collections import Counter, OrderedDict
try:
    from lxml.etree import Element
except ImportError:
    from xml.etree.cElementTree import Element

__author__ = 'S Anand'
__email__ = 'root.node@gmail.com'
__version__ = '0.1.8'

# Python 3: define unicode() as str()
if sys.version_info[0] == 3:
    unicode = str
    basestring = str

class XMLData(object):
    def __init__(self, xml_fromstring=True, xml_tostring=True, element=None, dict_type=None,
                 list_type=None, attr_prefix=None, text_content=None, simple_text=False):
        # xml_fromstring == False(y) => '1' -> '1'
        # xml_fromstring == True     => '1' -> 1
        # xml_fromstring == fn       => '1' -> fn(1)
        if callable(xml_fromstring):
            self._fromstring = xml_fromstring
        elif not xml_fromstring:
            self._fromstring = lambda v: v
        # custom conversion function to convert data string to XML string
        if callable(xml_tostring):
            self._tostring = xml_tostring
        # custom etree.Element to use
        self.element = Element if element is None else element
        # dict constructor (e.g. OrderedDict, defaultdict)
        self.dict = OrderedDict if dict_type is None else dict_type
        # list constructor (e.g. UserList)
        self.list = list if list_type is None else list_type
        # Prefix attributes with a string (e.g. '$')
        self.attr_prefix = attr_prefix
        # Key that stores text content (e.g. '$t')
        self.text_content = text_content
        # simple_text == False or None or 0 => '<x>a</x>' = {'x': {'a': {}}}
        # simple_text == True               => '<x>a</x>' = {'x': 'a'}
        self.simple_text = simple_text

    @staticmethod
    def _tostring(value):
        '''Convert value to XML compatible string'''
        if value is True:
            value = 'true'
        elif value is False:
            value = 'false'
        return unicode(value)       # noqa: convert to whatever native unicode repr

    @staticmethod
    def _fromstring(value):
        '''Convert XML string value to None, boolean, int or float'''
        if not value:
            return None
        std_value = value.strip().lower()
        if std_value == 'true':
            return True
        elif std_value == 'false':
            return False
        try:
            return int(std_value)
        except ValueError:
            pass
        try:
            return float(std_value)
        except ValueError:
            pass
        return value

    def etree(self, data, root=None):
        '''Convert data structure into a list of etree.Element'''
        result = self.list() if root is None else root
        if isinstance(data, (self.dict, dict)):
            for key, value in data.items():
                value_is_list = isinstance(value, (self.list, list))
                value_is_dict = isinstance(value, (self.dict, dict))
                # Add attributes and text to result (if root)
                if root is not None:
                    # Handle attribute prefixes (BadgerFish)
                    if self.attr_prefix is not None:
                        if key.startswith(self.attr_prefix):
                            key = key.lstrip(self.attr_prefix)
                            # @xmlns: {$: xxx, svg: yyy} becomes xmlns="xxx" xmlns:svg="yyy"
                            if value_is_dict:
                                raise ValueError(
                                    'XML namespaces not yet supported')
                            else:
                                result.set(key, self._tostring(value))
                            continue
                    # Handle text content (BadgerFish, GData)
                    if self.text_content is not None:
                        if key == self.text_content:
                            result.text = self._tostring(value)
                            continue
                    # Treat scalars as text content, not children (GData)
                    if self.attr_prefix is None and self.text_content is not None:
                        if not value_is_dict and not value_is_list:
                            result.set(key, self._tostring(value))
                            continue
                # Add other keys as one or more children
                values = value if value_is_list else [value]
                for value in values:
                    elem = self.element(key)
                    result.append(elem)
                    # Treat scalars as text content, not children (Parker)
                    if not isinstance(value, (self.dict, dict, self.list, list)):
                        if self.text_content:
                            value = {self.text_content: value}
                    self.etree(value, root=elem)
        else:
            if self.text_content is None and root is not None:
                root.text = self._tostring(data)
            else:
                result.append(self.element(self._tostring(data)))
        return result

    def data(self, root):
        '''Convert etree.Element into a dictionary'''
        value = self.dict()
        children = [node for node in root if isinstance(node.tag, basestring)]
        for attr, attrval in root.attrib.items():
            attr = attr if self.attr_prefix is None else self.attr_prefix + attr
            value[attr] = self._fromstring(attrval)
        if root.text and self.text_content is not None:
            text = root.text.strip()
            if text:
                if self.simple_text and len(children) == len(root.attrib) == 0:
                    value = self._fromstring(text)
                else:
                    value[self.text_content] = self._fromstring(text)
        count = Counter(child.tag for child in children)
        for child in children:
            if count[child.tag] == 1:
                value.update(self.data(child))
            else:
                result = value.setdefault(child.tag, self.list())
                result += self.data(child).values()
        return self.dict([(root.tag, value)])

class BadgerFish(XMLData):
    '''Converts between XML and data using the BadgerFish convention'''

    def __init__(self, **kwargs):
        super(BadgerFish, self).__init__(
            attr_prefix='@', text_content='$', **kwargs)

class GData(XMLData):
    '''Converts between XML and data using the GData convention'''

    def __init__(self, **kwargs):
        super(GData, self).__init__(text_content='$t', **kwargs)

class Yahoo(XMLData):
    '''Converts between XML and data using the Yahoo convention'''

    def __init__(self, **kwargs):
        kwargs.setdefault('xml_fromstring', False)
        super(Yahoo, self).__init__(
            text_content='content', simple_text=True, **kwargs)

class Parker(XMLData):
    '''Converts between XML and data using the Parker convention'''

    def __init__(self, **kwargs):
        super(Parker, self).__init__(**kwargs)

    def data(self, root, preserve_root=False):
        'Convert etree.Element into a dictionary'
        # If preserve_root is False, return the root element. This is easiest
        # done by wrapping the XML in a dummy root element that will be
        # ignored.
        if preserve_root:
            new_root = root.makeelement('dummy_root', {})
            new_root.insert(0, root)
            root = new_root

        # If no children, just return the text
        children = [node for node in root if isinstance(node.tag, basestring)]
        if len(children) == 0:
            return self._fromstring(root.text)

        # Element names become object properties
        count = Counter(child.tag for child in children)
        result = self.dict()
        for child in children:
            if count[child.tag] == 1:
                result[child.tag] = self.data(child)
            else:
                result.setdefault(child.tag, self.list()
                                  ).append(self.data(child))

        return result

class Abdera(XMLData):
    '''Converts between XML and data using the Abdera convention'''

    def __init__(self, **kwargs):
        super(Abdera, self).__init__(
            simple_text=True, text_content=True, **kwargs)

    def data(self, root):
        '''Convert etree.Element into a dictionary'''

        value = self.dict()

        # Add attributes specific 'attributes' key
        if root.attrib:
            value[u'attributes'] = self.dict()
            for attr, attrval in root.attrib.items():
                value[u'attributes'][unicode(attr)] = self._fromstring(attrval)

        # Add children to specific 'children' key
        children_list = self.list()
        children = [node for node in root if isinstance(node.tag, basestring)]

        # Add root text
        if root.text and self.text_content is not None:
            text = root.text.strip()
            if text:
                if self.simple_text and len(children) == len(root.attrib) == 0:
                    value = self._fromstring(text)
                else:
                    children_list = [self._fromstring(text), ]
        count = Counter(child.tag for child in children)

        for child in children:
            child_data = self.data(child)
            if (count[child.tag] == 1 and
                    len(children_list) > 0 and
                    isinstance(children_list[-1], dict)):
                # Merge keys to existing dictionary
                children_list[-1].update(child_data)
            else:
                # Add additional text
                children_list.append(self.data(child))

        # Flatten children
        if len(root.attrib) == 0 and len(children_list) == 1:
            value = children_list[0]

        elif len(children_list) > 0:
            value[u'children'] = children_list

        return self.dict([(unicode(root.tag), value)])

# The difference between Cobra and Abdera is that Cobra _always_ has 'attributes' keys,
# 'children' key is remove when only one child and everything is a string.
# https://github.com/datacenter/cobra/blob/master/cobra/internal/codec/jsoncodec.py
class Cobra(XMLData):
    '''Converts between XML and data using the Cobra convention'''

    def __init__(self, **kwargs):
        super(Cobra, self).__init__(simple_text=True, text_content=True,
                                    xml_fromstring=False, **kwargs)

    def data(self, root):
        '''Convert etree.Element into a dictionary'''

        value = self.dict()

        # Add attributes to 'attributes' key (sorted!) even when empty
        value[u'attributes'] = self.dict()
        if root.attrib:
            for attr in sorted(root.attrib):
                value[u'attributes'][unicode(attr)] = root.attrib[attr]

        # Add children to specific 'children' key
        children_list = self.list()
        children = [node for node in root if isinstance(node.tag, basestring)]

        # Add root text
        if root.text and self.text_content is not None:
            text = root.text.strip()
            if text:
                if self.simple_text and len(children) == len(root.attrib) == 0:
                    value = self._fromstring(text)
                else:
                    children_list = [self._fromstring(text), ]

        count = Counter(child.tag for child in children)
        for child in children:
            child_data = self.data(child)
            if (count[child.tag] == 1 and
                    len(children_list) > 0 and
                    isinstance(children_list[-1], dict)):
                # Merge keys to existing dictionary
                children_list[-1].update(child_data)
            else:
                # Add additional text
                children_list.append(self.data(child))

        if len(children_list) > 0:
            value[u'children'] = children_list

        return self.dict([(unicode(root.tag), value)])

abdera = Abdera()
badgerfish = BadgerFish()
cobra = Cobra()
gdata = GData()
parker = Parker()
yahoo = Yahoo()
<|endoftext|>


<commit_before># ==============================================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Upgrade script to move from pre-release schema to new schema.

Usage examples:

bazel run tensorflow/lite/schema/upgrade_schema -- in.json out.json
bazel run tensorflow/lite/schema/upgrade_schema -- in.bin out.bin
bazel run tensorflow/lite/schema/upgrade_schema -- in.bin out.json
bazel run tensorflow/lite/schema/upgrade_schema -- in.json out.bin
bazel run tensorflow/lite/schema/upgrade_schema -- in.tflite out.tflite
"""
import argparse
import contextlib
import json
import os
import shutil
import subprocess
import sys
import tempfile

import tensorflow as tf
from tensorflow.python.platform import resource_loader

parser = argparse.ArgumentParser(
    description="Script to move TFLite models from pre-release schema to "
    "new schema.")
parser.add_argument(
    "input",
    type=str,
    help="Input TensorFlow lite file in `.json`, `.bin` or `.tflite` format.")
parser.add_argument(
    "output",
    type=str,
    help="Output json or bin TensorFlow lite model compliant with "
    "the new schema. Extension must be `.json`, `.bin` or `.tflite`.")

# RAII Temporary Directory, because flatc doesn't allow direct use of tempfiles.
@contextlib.contextmanager
def TemporaryDirectoryResource():
  temporary = tempfile.mkdtemp()
  try:
    yield temporary
  finally:
    shutil.rmtree(temporary)

class Converter(object):
  """Converts TensorFlow flatbuffer models from old to new version of schema.

  This can convert between any version to the latest version. It uses
  an incremental upgrade strategy to go from version to version.

  Usage:
    converter = Converter()
    converter.Convert("a.tflite", "a.json")
    converter.Convert("b.json", "b.tflite")
  """

  def __init__(self):
    # TODO(aselle): make this work in the open source version with better
    # path.
    paths_to_try = [
        "../../../../flatbuffers/flatc",  # not bazel
        "../../../../external/flatbuffers/flatc"  # bazel
    ]
    for p in paths_to_try:
      self._flatc_path = resource_loader.get_path_to_datafile(p)
      if os.path.exists(self._flatc_path): break

    def FindSchema(base_name):
      return resource_loader.get_path_to_datafile("%s" % base_name)

    # Supported schemas for upgrade.
    self._schemas = [
        (0, FindSchema("schema_v0.fbs"), True, self._Upgrade0To1),
        (1, FindSchema("schema_v1.fbs"), True, self._Upgrade1To2),
        (2, FindSchema("schema_v2.fbs"), True, self._Upgrade2To3),
        (3, FindSchema("schema_v3.fbs"), False, None)  # Non-callable by design.
    ]
    # Ensure schemas are sorted, and extract latest version and upgrade
    # dispatch function table.
    self._schemas.sort()
    self._new_version, self._new_schema = self._schemas[-1][:2]
    self._upgrade_dispatch = {
        version: dispatch
        for version, unused1, unused2, dispatch in self._schemas}

  def _Read(self, input_file, schema, raw_binary=False):
    """Read a tflite model assuming the given flatbuffer schema.

    If `input_file` is in bin, then we must use flatc to convert the schema
    from binary to json.

    Args:
      input_file: a binary (flatbuffer) or json file to read from. Extension
        must  be `.tflite`, `.bin`, or `.json` for FlatBuffer Binary or
        FlatBuffer JSON.
      schema: which schema to use for reading
      raw_binary: whether to assume raw_binary (versions previous to v3)
        that lacked file_identifier require this.

    Raises:
      RuntimeError: 1. When flatc cannot be invoked.
                    2. When json file does not exists.
      ValueError: When the extension is not json or bin.

    Returns:
      A dictionary representing the read tflite model.
    """
    raw_binary = ["--raw-binary"] if raw_binary else []
    with TemporaryDirectoryResource() as tempdir:
      basename = os.path.basename(input_file)
      basename_no_extension, extension = os.path.splitext(basename)
      if extension in [".bin", ".tflite"]:
        # Convert to json using flatc
        returncode = subprocess.call([
            self._flatc_path,
            "-t",
            "--strict-json",
            "--defaults-json",
        ] + raw_binary + ["-o", tempdir, schema, "--", input_file])
        if returncode != 0:
          raise RuntimeError("flatc failed to convert from binary to json.")
        json_file = os.path.join(tempdir, basename_no_extension + ".json")
        if not os.path.exists(json_file):
          raise RuntimeError("Could not find %r" % json_file)
      elif extension == ".json":
        json_file = input_file
      else:
        raise ValueError("Invalid extension on input file %r" % input_file)
      return json.load(open(json_file))

  def _Write(self, data, output_file):
    """Output a json or bin version of the flatbuffer model.

    Args:
      data: Dict representing the TensorFlow Lite model to write.
      output_file: filename to write the converted flatbuffer to. (json,
        tflite, or bin extension is required).
    Raises:
      ValueError: When the extension is not json or bin
      RuntimeError: When flatc fails to convert json data to binary.
    """
    _, extension = os.path.splitext(output_file)
    with TemporaryDirectoryResource() as tempdir:
      if extension == ".json":
        json.dump(data, open(output_file, "w"), sort_keys=True, indent=2)
      elif extension in [".tflite", ".bin"]:
        input_json = os.path.join(tempdir, "temp.json")
        with open(input_json, "w") as fp:
          json.dump(data, fp, sort_keys=True, indent=2)
        returncode = subprocess.call([
            self._flatc_path, "-b", "--defaults-json", "--strict-json", "-o",
            tempdir, self._new_schema, input_json
        ])
        if returncode != 0:
          raise RuntimeError("flatc failed to convert upgraded json to binary.")

        shutil.copy(os.path.join(tempdir, "temp.tflite"), output_file)
      else:
        raise ValueError("Invalid extension on output file %r" % output_file)

  def _Upgrade0To1(self, data):
    """Upgrade data from Version 0 to Version 1.

    Changes: Added subgraphs (which contains a subset of formally global
    entries).

    Args:
      data: Dictionary representing the TensorFlow lite data to be upgraded.
        This will be modified in-place to be an upgraded version.
    """
    subgraph = {}
    for key_to_promote in ["tensors", "operators", "inputs", "outputs"]:
      subgraph[key_to_promote] = data[key_to_promote]
      del data[key_to_promote]
    data["subgraphs"] = [subgraph]

  def _Upgrade1To2(self, data):
    """Upgrade data from Version 1 to Version 2.

    Changes: Rename operators to Conform to NN API.

    Args:
      data: Dictionary representing the TensorFlow lite data to be upgraded.
        This will be modified in-place to be an upgraded version.
    Raises:
      ValueError: Throws when model builtins are numeric rather than symbols.
    """

    def RemapOperator(opcode_name):
      """Go from old schema op name to new schema op name.

      Args:
        opcode_name: String representing the ops (see :schema.fbs).
      Returns:
        Converted opcode_name from V1 to V2.
      """
      old_name_to_new_name = {
          "CONVOLUTION": "CONV_2D",
          "DEPTHWISE_CONVOLUTION": "DEPTHWISE_CONV_2D",
          "AVERAGE_POOL": "AVERAGE_POOL_2D",
          "MAX_POOL": "MAX_POOL_2D",
          "L2_POOL": "L2_POOL_2D",
          "SIGMOID": "LOGISTIC",
          "L2NORM": "L2_NORMALIZATION",
          "LOCAL_RESPONSE_NORM": "LOCAL_RESPONSE_NORMALIZATION",
          "Basic_RNN": "RNN",
      }

      return (old_name_to_new_name[opcode_name]
              if opcode_name in old_name_to_new_name else opcode_name)

    def RemapOperatorType(operator_type):
      """Remap operator structs from old names to new names.

      Args:
        operator_type: String representing the builtin operator data type
          string.
        (see :schema.fbs).
      Raises:
        ValueError: When the model has consistency problems.
      Returns:
        Upgraded builtin operator data type as a string.
      """
      old_to_new = {
          "PoolOptions": "Pool2DOptions",
          "DepthwiseConvolutionOptions": "DepthwiseConv2DOptions",
          "ConvolutionOptions": "Conv2DOptions",
          "LocalResponseNormOptions": "LocalResponseNormalizationOptions",
          "BasicRNNOptions": "RNNOptions",
      }
      return (old_to_new[operator_type]
              if operator_type in old_to_new else operator_type)

    for subgraph in data["subgraphs"]:
      for ops in subgraph["operators"]:
        ops["builtin_options_type"] = RemapOperatorType(
            ops["builtin_options_type"])

    # Upgrade the operator codes
    for operator_code in data["operator_codes"]:
      # Check if builtin_code is the appropriate string type
      # use type("") instead of str or unicode. for py2and3
      if not isinstance(operator_code["builtin_code"], type(u"")):
        raise ValueError("builtin_code %r is non-string. this usually means "
                         "your model has consistency problems." %
                         (operator_code["builtin_code"]))
      operator_code["builtin_code"] = (RemapOperator(
          operator_code["builtin_code"]))

  def _Upgrade2To3(self, data):
    """Upgrade data from Version 2 to Version 3.

    Changed actual read-only tensor data to be in a buffers table instead
    of inline with the tensor.

    Args:
      data: Dictionary representing the TensorFlow lite data to be upgraded.
        This will be modified in-place to be an upgraded version.
    """
    buffers = [{"data": []}]  # Start with 1 empty buffer
    for subgraph in data["subgraphs"]:
      if "tensors" not in subgraph:
        continue
      for tensor in subgraph["tensors"]:
        if "data_buffer" not in tensor:
          tensor["buffer"] = 0
        else:
          if tensor["data_buffer"]:
            tensor[u"buffer"] = len(buffers)
            buffers.append({"data": tensor["data_buffer"]})
          else:
            tensor["buffer"] = 0
          del tensor["data_buffer"]
    data["buffers"] = buffers

  def _PerformUpgrade(self, data):
    """Manipulate the `data` (parsed JSON) based on changes in format.

    This incrementally will upgrade from version to version within data.

    Args:
      data: Dictionary representing the TensorFlow data. This will be upgraded
        in place.
    """
    while data["version"] < self._new_version:
      self._upgrade_dispatch[data["version"]](data)
      data["version"] += 1

  def Convert(self, input_file, output_file):
    """Perform schema conversion from input_file to output_file.

    Args:
      input_file: Filename of TensorFlow Lite data to convert from. Must
        be `.json` or `.bin` extension files for JSON or Binary forms of
        the TensorFlow FlatBuffer schema.
      output_file: Filename to write to. Extension also must be `.json`
        or `.bin`.

    Raises:
      RuntimeError: Generated when none of the upgrader supported schemas
        matche the `input_file` data.
    """
    # Read data in each schema (since they are incompatible). Version is
    # always present. Use the read data that matches the version of the
    # schema.
    for version, schema, raw_binary, _ in self._schemas:
      try:
        data_candidate = self._Read(input_file, schema, raw_binary)
      except RuntimeError:
        continue  # Skip and hope another schema works
      if "version" not in data_candidate:  # Assume version 1 if not present.
        data_candidate["version"] = 1
      elif data_candidate["version"] == 0:  # Version 0 doesn't exist in wild.
        data_candidate["version"] = 1

      if data_candidate["version"] == version:
        self._PerformUpgrade(data_candidate)
        self._Write(data_candidate, output_file)
        return
    raise RuntimeError("No schema that the converter understands worked with "
                       "the data file you provided.")

def main(argv):
  del argv
  Converter().Convert(FLAGS.input, FLAGS.output)

if __name__ == "__main__":
  FLAGS, unparsed = parser.parse_known_args()
  tf.compat.v1.app.run(main=main, argv=[sys.argv[0]] + unparsed)
<commit_msg>[tensorflow/lite/schema/upgrade_schema.py] Move `(see :schema.fbs).` up to `operator_type` arg descriptor<commit_after># ==============================================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Upgrade script to move from pre-release schema to new schema.

Usage examples:

bazel run tensorflow/lite/schema/upgrade_schema -- in.json out.json
bazel run tensorflow/lite/schema/upgrade_schema -- in.bin out.bin
bazel run tensorflow/lite/schema/upgrade_schema -- in.bin out.json
bazel run tensorflow/lite/schema/upgrade_schema -- in.json out.bin
bazel run tensorflow/lite/schema/upgrade_schema -- in.tflite out.tflite
"""
import argparse
import contextlib
import json
import os
import shutil
import subprocess
import sys
import tempfile

import tensorflow as tf
from tensorflow.python.platform import resource_loader

parser = argparse.ArgumentParser(
    description="Script to move TFLite models from pre-release schema to "
    "new schema.")
parser.add_argument(
    "input",
    type=str,
    help="Input TensorFlow lite file in `.json`, `.bin` or `.tflite` format.")
parser.add_argument(
    "output",
    type=str,
    help="Output json or bin TensorFlow lite model compliant with "
    "the new schema. Extension must be `.json`, `.bin` or `.tflite`.")

# RAII Temporary Directory, because flatc doesn't allow direct use of tempfiles.
@contextlib.contextmanager
def TemporaryDirectoryResource():
  temporary = tempfile.mkdtemp()
  try:
    yield temporary
  finally:
    shutil.rmtree(temporary)

class Converter(object):
  """Converts TensorFlow flatbuffer models from old to new version of schema.

  This can convert between any version to the latest version. It uses
  an incremental upgrade strategy to go from version to version.

  Usage:
    converter = Converter()
    converter.Convert("a.tflite", "a.json")
    converter.Convert("b.json", "b.tflite")
  """

  def __init__(self):
    # TODO(aselle): make this work in the open source version with better
    # path.
    paths_to_try = [
        "../../../../flatbuffers/flatc",  # not bazel
        "../../../../external/flatbuffers/flatc"  # bazel
    ]
    for p in paths_to_try:
      self._flatc_path = resource_loader.get_path_to_datafile(p)
      if os.path.exists(self._flatc_path): break

    def FindSchema(base_name):
      return resource_loader.get_path_to_datafile("%s" % base_name)

    # Supported schemas for upgrade.
    self._schemas = [
        (0, FindSchema("schema_v0.fbs"), True, self._Upgrade0To1),
        (1, FindSchema("schema_v1.fbs"), True, self._Upgrade1To2),
        (2, FindSchema("schema_v2.fbs"), True, self._Upgrade2To3),
        (3, FindSchema("schema_v3.fbs"), False, None)  # Non-callable by design.
    ]
    # Ensure schemas are sorted, and extract latest version and upgrade
    # dispatch function table.
    self._schemas.sort()
    self._new_version, self._new_schema = self._schemas[-1][:2]
    self._upgrade_dispatch = {
        version: dispatch
        for version, unused1, unused2, dispatch in self._schemas}

  def _Read(self, input_file, schema, raw_binary=False):
    """Read a tflite model assuming the given flatbuffer schema.

    If `input_file` is in bin, then we must use flatc to convert the schema
    from binary to json.

    Args:
      input_file: a binary (flatbuffer) or json file to read from. Extension
        must  be `.tflite`, `.bin`, or `.json` for FlatBuffer Binary or
        FlatBuffer JSON.
      schema: which schema to use for reading
      raw_binary: whether to assume raw_binary (versions previous to v3)
        that lacked file_identifier require this.

    Raises:
      RuntimeError: 1. When flatc cannot be invoked.
                    2. When json file does not exists.
      ValueError: When the extension is not json or bin.

    Returns:
      A dictionary representing the read tflite model.
    """
    raw_binary = ["--raw-binary"] if raw_binary else []
    with TemporaryDirectoryResource() as tempdir:
      basename = os.path.basename(input_file)
      basename_no_extension, extension = os.path.splitext(basename)
      if extension in [".bin", ".tflite"]:
        # Convert to json using flatc
        returncode = subprocess.call([
            self._flatc_path,
            "-t",
            "--strict-json",
            "--defaults-json",
        ] + raw_binary + ["-o", tempdir, schema, "--", input_file])
        if returncode != 0:
          raise RuntimeError("flatc failed to convert from binary to json.")
        json_file = os.path.join(tempdir, basename_no_extension + ".json")
        if not os.path.exists(json_file):
          raise RuntimeError("Could not find %r" % json_file)
      elif extension == ".json":
        json_file = input_file
      else:
        raise ValueError("Invalid extension on input file %r" % input_file)
      return json.load(open(json_file))

  def _Write(self, data, output_file):
    """Output a json or bin version of the flatbuffer model.

    Args:
      data: Dict representing the TensorFlow Lite model to write.
      output_file: filename to write the converted flatbuffer to. (json,
        tflite, or bin extension is required).
    Raises:
      ValueError: When the extension is not json or bin
      RuntimeError: When flatc fails to convert json data to binary.
    """
    _, extension = os.path.splitext(output_file)
    with TemporaryDirectoryResource() as tempdir:
      if extension == ".json":
        json.dump(data, open(output_file, "w"), sort_keys=True, indent=2)
      elif extension in [".tflite", ".bin"]:
        input_json = os.path.join(tempdir, "temp.json")
        with open(input_json, "w") as fp:
          json.dump(data, fp, sort_keys=True, indent=2)
        returncode = subprocess.call([
            self._flatc_path, "-b", "--defaults-json", "--strict-json", "-o",
            tempdir, self._new_schema, input_json
        ])
        if returncode != 0:
          raise RuntimeError("flatc failed to convert upgraded json to binary.")

        shutil.copy(os.path.join(tempdir, "temp.tflite"), output_file)
      else:
        raise ValueError("Invalid extension on output file %r" % output_file)

  def _Upgrade0To1(self, data):
    """Upgrade data from Version 0 to Version 1.

    Changes: Added subgraphs (which contains a subset of formally global
    entries).

    Args:
      data: Dictionary representing the TensorFlow lite data to be upgraded.
        This will be modified in-place to be an upgraded version.
    """
    subgraph = {}
    for key_to_promote in ["tensors", "operators", "inputs", "outputs"]:
      subgraph[key_to_promote] = data[key_to_promote]
      del data[key_to_promote]
    data["subgraphs"] = [subgraph]

  def _Upgrade1To2(self, data):
    """Upgrade data from Version 1 to Version 2.

    Changes: Rename operators to Conform to NN API.

    Args:
      data: Dictionary representing the TensorFlow lite data to be upgraded.
        This will be modified in-place to be an upgraded version.
    Raises:
      ValueError: Throws when model builtins are numeric rather than symbols.
    """

    def RemapOperator(opcode_name):
      """Go from old schema op name to new schema op name.

      Args:
        opcode_name: String representing the ops (see :schema.fbs).
      Returns:
        Converted opcode_name from V1 to V2.
      """
      old_name_to_new_name = {
          "CONVOLUTION": "CONV_2D",
          "DEPTHWISE_CONVOLUTION": "DEPTHWISE_CONV_2D",
          "AVERAGE_POOL": "AVERAGE_POOL_2D",
          "MAX_POOL": "MAX_POOL_2D",
          "L2_POOL": "L2_POOL_2D",
          "SIGMOID": "LOGISTIC",
          "L2NORM": "L2_NORMALIZATION",
          "LOCAL_RESPONSE_NORM": "LOCAL_RESPONSE_NORMALIZATION",
          "Basic_RNN": "RNN",
      }

      return (old_name_to_new_name[opcode_name]
              if opcode_name in old_name_to_new_name else opcode_name)

    def RemapOperatorType(operator_type):
      """Remap operator structs from old names to new names.

      Args:
        operator_type: String representing the builtin operator data type
          string. (see :schema.fbs).
      Raises:
        ValueError: When the model has consistency problems.
      Returns:
        Upgraded builtin operator data type as a string.
      """
      old_to_new = {
          "PoolOptions": "Pool2DOptions",
          "DepthwiseConvolutionOptions": "DepthwiseConv2DOptions",
          "ConvolutionOptions": "Conv2DOptions",
          "LocalResponseNormOptions": "LocalResponseNormalizationOptions",
          "BasicRNNOptions": "RNNOptions",
      }
      return (old_to_new[operator_type]
              if operator_type in old_to_new else operator_type)

    for subgraph in data["subgraphs"]:
      for ops in subgraph["operators"]:
        ops["builtin_options_type"] = RemapOperatorType(
            ops["builtin_options_type"])

    # Upgrade the operator codes
    for operator_code in data["operator_codes"]:
      # Check if builtin_code is the appropriate string type
      # use type("") instead of str or unicode. for py2and3
      if not isinstance(operator_code["builtin_code"], type(u"")):
        raise ValueError("builtin_code %r is non-string. this usually means "
                         "your model has consistency problems." %
                         (operator_code["builtin_code"]))
      operator_code["builtin_code"] = (RemapOperator(
          operator_code["builtin_code"]))

  def _Upgrade2To3(self, data):
    """Upgrade data from Version 2 to Version 3.

    Changed actual read-only tensor data to be in a buffers table instead
    of inline with the tensor.

    Args:
      data: Dictionary representing the TensorFlow lite data to be upgraded.
        This will be modified in-place to be an upgraded version.
    """
    buffers = [{"data": []}]  # Start with 1 empty buffer
    for subgraph in data["subgraphs"]:
      if "tensors" not in subgraph:
        continue
      for tensor in subgraph["tensors"]:
        if "data_buffer" not in tensor:
          tensor["buffer"] = 0
        else:
          if tensor["data_buffer"]:
            tensor[u"buffer"] = len(buffers)
            buffers.append({"data": tensor["data_buffer"]})
          else:
            tensor["buffer"] = 0
          del tensor["data_buffer"]
    data["buffers"] = buffers

  def _PerformUpgrade(self, data):
    """Manipulate the `data` (parsed JSON) based on changes in format.

    This incrementally will upgrade from version to version within data.

    Args:
      data: Dictionary representing the TensorFlow data. This will be upgraded
        in place.
    """
    while data["version"] < self._new_version:
      self._upgrade_dispatch[data["version"]](data)
      data["version"] += 1

  def Convert(self, input_file, output_file):
    """Perform schema conversion from input_file to output_file.

    Args:
      input_file: Filename of TensorFlow Lite data to convert from. Must
        be `.json` or `.bin` extension files for JSON or Binary forms of
        the TensorFlow FlatBuffer schema.
      output_file: Filename to write to. Extension also must be `.json`
        or `.bin`.

    Raises:
      RuntimeError: Generated when none of the upgrader supported schemas
        matche the `input_file` data.
    """
    # Read data in each schema (since they are incompatible). Version is
    # always present. Use the read data that matches the version of the
    # schema.
    for version, schema, raw_binary, _ in self._schemas:
      try:
        data_candidate = self._Read(input_file, schema, raw_binary)
      except RuntimeError:
        continue  # Skip and hope another schema works
      if "version" not in data_candidate:  # Assume version 1 if not present.
        data_candidate["version"] = 1
      elif data_candidate["version"] == 0:  # Version 0 doesn't exist in wild.
        data_candidate["version"] = 1

      if data_candidate["version"] == version:
        self._PerformUpgrade(data_candidate)
        self._Write(data_candidate, output_file)
        return
    raise RuntimeError("No schema that the converter understands worked with "
                       "the data file you provided.")

def main(argv):
  del argv
  Converter().Convert(FLAGS.input, FLAGS.output)

if __name__ == "__main__":
  FLAGS, unparsed = parser.parse_known_args()
  tf.compat.v1.app.run(main=main, argv=[sys.argv[0]] + unparsed)
<|endoftext|>