<commit_before>#!/usr/bin/python

import sys, string, os, getopt
import json
import re
import traceback
import numpy as np

class uuidDetails():
   def __init__(self, uuid):
       self.uuid            = uuid
       self.begin_timestamp = None
       self.end_timestamp   = None
       self.s3_read_dur     = None
       self.idx_read_dur    = None

   def __str__(self):
      return self.uuid + ": " + \
         str(self.begin_timestamp) + " / " + \
         str(self.end_timestamp) + " / " + \
         str(self.s3_read_dur) + " / " + \
         str(self.idx_read_dur)

def usage():
   print ("analyze-event-log.py [OPTIONS]")
   print ("  -e/--events_log    : Path to log file with CloudWatch events")
   print ("  -v/--verbose      : Verbose Output (Optional)")

eventsLog = None
verbose   = False

try:
   opts, args = getopt.getopt(sys.argv[1:],"he:v",["events_log","verbose"])
except getopt.GetoptError:
   usage()
   sys.exit(2)

for opt, arg in opts:
   if opt == '-h':
      usage()
      sys.exit()
   elif opt in ("-e", "--events_log"):
      eventsLog = arg
   elif opt in ("-v", "--verbose"):
      verbose = True

# Open the product configuration file
if not eventsLog:
   print("Events Log must be provided.")
   sys.exit(-1)

try:
   eventsData = json.load(open(eventsLog))['events']
except:
   print("Error parsing events log.\n" + traceback.format_exc())
   sys.exit(-1)

eventDetailsDict   = {}
getEventKByteItems = []

matchBegin = re.compile(".*begin_onearth_handle.*")
matchEnd   = re.compile(".*end_mod_mrf_handle.*")
matchS3    = re.compile(".*mod_mrf_s3_read.*")
matchIdx   = re.compile(".*mod_mrf_index_read.*")
matchGet   = re.compile(".*GET .mrf_endpoint.*")

# Parse event data into uuidDetails() objects
for event in eventsData:

   try :
      if matchBegin.search(event["message"]):
         m = re.match(".*timestamp=([0-9]*), uuid=(.*)", event["message"])
         if not m or len(m.groups()) != 2:
            print("Bogus log message: (" + str(event["timestamp"]) + "): " + event["message"])
         else:
            if m.group(2) not in eventDetailsDict:
               eventDetailsDict[m.group(2)] = uuidDetails(m.group(2))

            eventDetailsDict[m.group(2)].begin_timestamp = float(m.group(1)) / 1000

      elif matchEnd.search(event["message"]):
         m = re.match(".*timestamp=([0-9]*), uuid=(.*)", event["message"])
         if not m or len(m.groups()) != 2:
            print("Bogus log message: (" + str(event["timestamp"]) + "): " + event["message"])
         else:
            if m.group(2) not in eventDetailsDict:
               eventDetailsDict[m.group(2)] = uuidDetails(m.group(2))

            eventDetailsDict[m.group(2)].end_timestamp = float(m.group(1)) / 1000

      elif matchS3.search(event["message"]):
         m = re.match(".*duration=([0-9]*), uuid=(.*)", event["message"])
         if not m or len(m.groups()) != 2:
            print("Bogus log message: (" + str(event["timestamp"]) + "): " + event["message"])
         else:
            if m.group(2) not in eventDetailsDict:
               eventDetailsDict[m.group(2)] = uuidDetails(m.group(2))

            eventDetailsDict[m.group(2)].s3_read_dur = float(m.group(1)) / 1000

      elif matchIdx.search(event["message"]):
         m = re.match(".*duration=([0-9]*), uuid=(.*)", event["message"])
         if not m or len(m.groups()) != 2:
            print("Bogus log message: (" + str(event["timestamp"]) + "): " + event["message"])
         else:
            if m.group(2) not in eventDetailsDict:
               eventDetailsDict[m.group(2)] = uuidDetails(m.group(2))

            eventDetailsDict[m.group(2)].idx_read_dur = float(m.group(1)) / 1000

      elif matchGet.search(event["message"]):
         m = re.match(".*GET .*200 ([0-9]*) .*", event["message"])
         if not m or len(m.groups()) != 1:
            print("Bogus log message: (" + str(event["timestamp"]) + "): " + event["message"])
         else:
            getEventKByteItems.append(float(m.group(1)) / 1000)

      else:
         if verbose: print("Unknown log message: (" + str(event["timestamp"]) + "): " + event["message"])
   except:
      print("Error parsing message: (" + str(event["timestamp"]) + "): " + event["message"] + "\n" + traceback.format_exc())

# Look for invalid uuidDetails()
badUuids = []

for details in eventDetailsDict.values():

   valid = (details.begin_timestamp and details.end_timestamp and details.s3_read_dur and details.idx_read_dur) and \
           (details.end_timestamp - details.begin_timestamp > 0)

   if not valid:
      if verbose: print("Invalid uuid details: " + str(details))

      badUuids.append(details.uuid)

for uuid in badUuids:
   del(eventDetailsDict[uuid])

# Build arrays for analysis
durReadItems = []
s3ReadItems  = []
idxReadItems = []

for details in eventDetailsDict.values():
   durReadItems.append(details.end_timestamp - details.begin_timestamp)
   s3ReadItems.append(details.s3_read_dur)
   idxReadItems.append(details.idx_read_dur)

# Calculate and print statistics
durReadArr = np.array(durReadItems)

print("onearth-mod_mrf")
print("\tcount:           " + str(len(durReadItems)))
print("\tmean:            " + str(round(np.mean(durReadArr),2)) + " ms")
print("\tmax:             " + str(round(np.amax(durReadArr),2)) + " ms")
print("\tmin:             " + str(round(np.amin(durReadArr),2)) + " ms")
print("\t25th percentile: " + str(round(np.percentile(durReadArr, 25),2)) + " ms")
print("\t50th percentile: " + str(round(np.percentile(durReadArr, 50),2)) + " ms")
print("\t75th percentile: " + str(round(np.percentile(durReadArr, 75),2)) + " ms")
print("\t95th percentile: " + str(round(np.percentile(durReadArr, 95),2)) + " ms")
print("\t98th percentile: " + str(round(np.percentile(durReadArr, 98),2)) + " ms")
print("\t99th percentile: " + str(round(np.percentile(durReadArr, 99),2)) + " ms")

s3ReadArr = np.array(s3ReadItems)

print("mod_mrf_s3_read")
print("\tcount:           " + str(len(s3ReadItems)))
print("\tmean:            " + str(round(np.mean(s3ReadArr),2)) + " ms")
print("\tmax:             " + str(round(np.amax(s3ReadArr),2)) + " ms")
print("\tmin:             " + str(round(np.amin(s3ReadArr),2)) + " ms")
print("\t25th percentile: " + str(round(np.percentile(s3ReadArr, 25),2)) + " ms")
print("\t50th percentile: " + str(round(np.percentile(s3ReadArr, 50),2)) + " ms")
print("\t75th percentile: " + str(round(np.percentile(s3ReadArr, 75),2)) + " ms")
print("\t95th percentile: " + str(round(np.percentile(s3ReadArr, 95),2)) + " ms")
print("\t98th percentile: " + str(round(np.percentile(s3ReadArr, 98),2)) + " ms")
print("\t99th percentile: " + str(round(np.percentile(s3ReadArr, 99),2)) + " ms")

idxReadArr = np.array(idxReadItems)

print("mod_mrf_index_read")
print("\tcount:           " + str(len(idxReadItems)))
print("\tmean:            " + str(round(np.mean(idxReadArr),2)) + " ms")
print("\tmax:             " + str(round(np.amax(idxReadArr))) + " ms")
print("\tmin:             " + str(round(np.amin(idxReadArr))) + " ms")
print("\t25th percentile: " + str(round(np.percentile(idxReadArr, 25),2)) + " ms")
print("\t50th percentile: " + str(round(np.percentile(idxReadArr, 50),2)) + " ms")
print("\t75th percentile: " + str(round(np.percentile(idxReadArr, 75),2)) + " ms")
print("\t95th percentile: " + str(round(np.percentile(idxReadArr, 95),2)) + " ms")
print("\t98th percentile: " + str(round(np.percentile(idxReadArr, 98),2)) + " ms")
print("\t99th percentile: " + str(round(np.percentile(idxReadArr, 99),2)) + " ms")

getKBytesArr = np.array(getEventKByteItems)

print("apache_get_bytes")
print("\tcount:           " + str(len(getEventKByteItems)))
print("\tmean:            " + str(round(np.mean(getKBytesArr),2)) + " kb")
print("\tmax:             " + str(round(np.amax(getKBytesArr))) + " kb")
print("\tmin:             " + str(round(np.amin(getKBytesArr))) + " kb")
print("\t25th percentile: " + str(round(np.percentile(getKBytesArr, 25),2)) + " kb")
print("\t50th percentile: " + str(round(np.percentile(getKBytesArr, 50),2)) + " kb")
print("\t75th percentile: " + str(round(np.percentile(getKBytesArr, 75),2)) + " kb")
print("\t95th percentile: " + str(round(np.percentile(getKBytesArr, 95),2)) + " kb")
print("\t98th percentile: " + str(round(np.percentile(getKBytesArr, 98),2)) + " kb")
print("\t99th percentile: " + str(round(np.percentile(getKBytesArr, 99),2)) + " kb")
<commit_msg>Updated analyze-event-log.py to the latest version<commit_after>#!/usr/bin/python

import sys, string, os, getopt
import json
import re
import traceback
import numpy as np

logMetrics = [
               ["begin_onearth_handle","timestamp", True],
               ["end_onearth_handle","timestamp", True],
               ["begin_mod_mrf_handle","timestamp", False],
               ["end_mod_mrf_handle","timestamp", False],
               ["mod_mrf_index_read","duration", False],
               ["mod_mrf_s3_read","duration", False],
               ["begin_request_time_snap","timestamp", False],
               ["end_request_time_snap","timestamp", False],
               ["begin_mod_reproject_handle","timestamp", False],
               ["end_mod_reproject_handle","timestamp", False]
             ]

class uuidDetails():
   def __init__(self, uuid):
       self.uuid            = uuid

#   def __str__(self):
#      return self.uuid + ": " + \
#         str(self.begin_timestamp) + " / " + \
#         str(self.end_timestamp) + " / " + \
#         str(self.s3_read_dur) + " / " + \
#         str(self.idx_read_dur)

def usage():
   print ("analyze-event-log.py [OPTIONS]")
   print ("  -e/--events_log    : Path to log file with CloudWatch events")
   print ("  -v/--verbose      : Verbose Output (Optional)")

def calculateAndPrintStats(metric, values):
   npArray = np.array(values)

   print(metric)
   print("\tcount:           " + str(len(values)))
   print("\tmean:            " + str(round(np.mean(npArray),2)) + " ms")
   print("\tmax:             " + str(round(np.amax(npArray),2)) + " ms")
   print("\tmin:             " + str(round(np.amin(npArray),2)) + " ms")
   print("\t25th percentile: " + str(round(np.percentile(npArray, 25),2)) + " ms")
   print("\t50th percentile: " + str(round(np.percentile(npArray, 50),2)) + " ms")
   print("\t75th percentile: " + str(round(np.percentile(npArray, 75),2)) + " ms")
   print("\t95th percentile: " + str(round(np.percentile(npArray, 95),2)) + " ms")
   print("\t98th percentile: " + str(round(np.percentile(npArray, 98),2)) + " ms")
   print("\t99th percentile: " + str(round(np.percentile(npArray, 99),2)) + " ms")

eventsLog = None
verbose   = False

try:
   opts, args = getopt.getopt(sys.argv[1:],"he:v",["events_log","verbose"])
except getopt.GetoptError:
   usage()
   sys.exit(2)

for opt, arg in opts:
   if opt == '-h':
      usage()
      sys.exit()
   elif opt in ("-e", "--events_log"):
      eventsLog = arg
   elif opt in ("-v", "--verbose"):
      verbose = True

# Open the product configuration file
if not eventsLog:
   print("Events Log must be provided.")
   sys.exit(-1)

try:
   eventsData = json.load(open(eventsLog))['events']
except:
   print("Error parsing events log.\n" + traceback.format_exc())
   sys.exit(-1)

eventDetailsDict   = {}
getEventKByteItems = []
metricMatchers     = {}

for metric in logMetrics:
   messageMatcher = re.compile(".*" + metric[0] + ".*")
   messageParser  = re.compile((".*timestamp=([0-9]*)" if metric[1] == "timestamp"
                                 else ".*duration=([0-9]*)") + ", uuid=(.*)")
   metricMatchers[metric[0]] = [messageMatcher, messageParser]

#matchGet   = re.compile(".*GET .mrf_endpoint.*")

# Parse event data into uuidDetails() objects
for event in eventsData:

   try :
      for metric in metricMatchers:
         matched = False

         if metricMatchers[metric][0].search(event["message"]):
            matched = True

            m = metricMatchers[metric][1].match(event["message"])
            if not m or len(m.groups()) != 2:
               print("Bogus log message: (" + str(event["timestamp"]) + "): " + event["message"])
            else:
               if m.group(2) not in eventDetailsDict:
                  eventDetailsDict[m.group(2)] = uuidDetails(m.group(2))

               setattr(eventDetailsDict[m.group(2)], metric, float(m.group(1)) / 1000)

            break

      if not matched:
         if verbose: print("Unknown log message: (" + str(event["timestamp"]) + "): " + event["message"])
   except:
      print("Error parsing message: (" + str(event["timestamp"]) + "): " + event["message"] + "\n" + traceback.format_exc())

# Look for invalid uuidDetails()
badUuids = []

for details in eventDetailsDict.values():
   valid = True

   for metric in logMetrics:
      if metric[1] == "duration":
         # If required and missing...
         if metric[2] and metric[0] not in dir(details):
            valid = False
            if verbose: print("Missing metric (" + metric[0] + ") : " + details.uuid)

      elif metric[1] == "timestamp" and metric[0].startswith("begin"):
         # If required and 'begin' is missing...
         if metric[2] and metric[0] not in dir(details):
            valid = False
            if verbose: print("Missing metric (" + metric[0] + ") : " + details.uuid)

         #Elif the 'begin' was found, but 'end' was not
         elif metric[0] in dir(details) and metric[0].replace("begin","end") not in dir(details):
            valid = False
            if verbose: print("Missing metric (" + metric[0].replace("begin","end") + ") : " + details.uuid)

         #Elif
         elif dir(details) and metric[0].replace("begin","end") in dir(details) and \
               getattr(details, metric[0].replace("begin","end")) - getattr(details, metric[0]) < 0:
            if verbose: print("Invalid begin/end metric (" + metric[0].replace("begin_","") + ") : " + details.uuid)

   if not valid:
      badUuids.append(details.uuid)

for uuid in badUuids:
   del(eventDetailsDict[uuid])

# Calculate and Print Metrics
for metric in logMetrics:
   metricValues = []

   if metric[1] == "duration":
      for details in eventDetailsDict.values():
         if metric[0] in dir(details):
            metricValues.append(getattr(details, metric[0]))

   elif metric[1] == "timestamp" and metric[0].startswith("begin"):
      for details in eventDetailsDict.values():
         if metric[0] in dir(details):
            if getattr(details, metric[0].replace("begin","end")) - getattr(details, metric[0]) > 0:
                metricValues.append(getattr(details, metric[0].replace("begin","end")) - getattr(details, metric[0]))

   else:
      continue

   if len(metricValues) > 0:
      calculateAndPrintStats(metric[0], metricValues)

<|endoftext|>


<commit_before>from flask import Flask, session, request, send_from_directory, jsonify, \
    render_template, url_for, redirect
from helpers import make_json_response, parse_user_details
from flask_oauth import OAuth
from simplekv.fs import FilesystemStore
from flaskext.kvsession import KVSessionExtension
from flaskext.coffee import coffee
from random import choice
import geojson

import sys
from flask.ext.mongoengine import MongoEngine

try:
    import settings
except ImportError:
    sys.stderr.write("""There must be a settings.py file with a secret_key.
    Run bin/make_secret.py
    """)
    sys.exit(2)

# initialize server KV session store
store = FilesystemStore('./sessiondata')

# instantiate flask app
app = Flask(__name__)

# connect flask app to server KV session store
KVSessionExtension(store, app)

# Apps need a secret key
app.secret_key = settings.secret_key

# Coffeescript enable the app
coffee(app)

app.debug = True

# Adding MongoDB configs
app.config["MONGODB_SETTINGS"] = {'DB': "maproulette"}
db = MongoEngine(app)

#initialize osm oauth
# instantite OAuth object
oauth = OAuth()
osm = oauth.remote_app(
    'osm',
    base_url='http://openstreetmap.org/',
    request_token_url = 'http://www.openstreetmap.org/oauth/request_token',
    access_token_url = 'http://www.openstreetmap.org/oauth/access_token',
    authorize_url = 'http://www.openstreetmap.org/oauth/authorize',
    consumer_key = settings.consumer_key,
    consumer_secret = settings.consumer_secret
)

@osm.tokengetter
def get_osm_token(token=None):
    session.regenerate()
    return session.get('osm_token')

# By default, send out the standard client
@app.route('/')
def index():
    "Display the index.html"
    return render_template('index.html')

@app.route('/api/challenges')
def challenges_api():
    "Returns a list of challenges as json"
    pass

@app.route('/api/task')
def task():
    """Returns an appropriate task based on parameters"""
    # We need to find a task for the user to work on, based (as much
    # as possible)
    difficulty = request.args.get('difficulty', 'easy')
    near = request.args.get('near')
    if near:
        lat, lon = near.split(',')
    else:
        point = None
    pass

@app.route('/api/c/<challenge>/meta')
def challenge_meta(challenge):
    "Returns the metadata for a challenge"
    pass

@app.route('/api/c/<challenge>/stats')
def challenge_stats(challenge):
    "Returns stat data for a challenge"
    pass

@app.route('/api/c/<challenge>/task')
def challenge_task(challenge):
    "Returns a task for specified challenge"
    pass

@app.route('/api/c/<challenge>/task/<id>', methods = ['POST'])
def challenge_post(challenge, task_id):
    "Accepts data for completed task"
    pass

@app.route('/logout')
def logout():
    session.destroy()
    return redirect('/')

@app.route('/oauth/authorize')
def oauth_authorize():
    """Initiates OAuth authorization agains the OSM server"""
    return osm.authorize(callback=url_for('oauth_authorized',
      next=request.args.get('next') or request.referrer or None))

@app.route('/oauth/callback')
@osm.authorized_handler
def oauth_authorized(resp):
    """Receives the OAuth callback from OSM"""
    next_url = request.args.get('next') or url_for('index')
    if resp is None:
        return redirect(next_url)
    session['osm_token'] = (
      resp['oauth_token'],
      resp['oauth_token_secret']
    )
    print 'getting user data from osm'
    osmuserresp = osm.get('user/details')
    if osmuserresp.status == 200:
        session['user'] = get_user_attribs(osmuserresp.data)
    else:
        print 'not able to get osm user data'
    return redirect(next_url)

@app.route('/<path:path>')
def catch_all(path):
    "Returns static files based on path"
    return send_from_directory('static', path)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--port", type = int, help = "the port to bind to",
                        default = 8080)
    parser.add_argument("--host", help = "the host to bind to",
                        default = "localhost")
    args = parser.parse_args()
    app.run(port=args.port)
    app.run(host=args.host, port=args.port)
<commit_msg>Adding bogus OAuth settings for testing<commit_after>from flask import Flask, session, request, send_from_directory, jsonify, \
    render_template, url_for, redirect
from helpers import make_json_response, parse_user_details
from flask_oauth import OAuth
from simplekv.fs import FilesystemStore
from flaskext.kvsession import KVSessionExtension
from flaskext.coffee import coffee
from random import choice
import geojson

import sys
from flask.ext.mongoengine import MongoEngine

try:
    import settings
    settings_keys = dir(settings)
except ImportError:
    sys.stderr.write("""There must be a settings.py file with a secret_key.
    Run bin/make_secret.py
    """)
    sys.exit(2)

# initialize server KV session store
store = FilesystemStore('./sessiondata')

# instantiate flask app
app = Flask(__name__)

# connect flask app to server KV session store
KVSessionExtension(store, app)

# Apps need a secret key
app.secret_key = settings.secret_key

# Coffeescript enable the app
coffee(app)

app.debug = True

# Adding MongoDB configs
app.config["MONGODB_SETTINGS"] = {'DB': "maproulette"}
db = MongoEngine(app)

#initialize osm oauth
# instantite OAuth object
oauth = OAuth()
if 'consumer_key' in settings_keys:
    consumer_key = settings.consumer_key
else:
    consumer_key = "consumer_key"

if 'consumer_secret' in settings_keys:
    consumer_secret = settings.consumer_secret
else:
    consumer_secret = "111111"

osm = oauth.remote_app(
    'osm',
    base_url='http://openstreetmap.org/',
    request_token_url = 'http://www.openstreetmap.org/oauth/request_token',
    access_token_url = 'http://www.openstreetmap.org/oauth/access_token',
    authorize_url = 'http://www.openstreetmap.org/oauth/authorize',
    consumer_key = consumer_key,
    consumer_secret = consumer_secret
)

@osm.tokengetter
def get_osm_token(token=None):
    session.regenerate()
    return session.get('osm_token')

# By default, send out the standard client
@app.route('/')
def index():
    "Display the index.html"
    return render_template('index.html')

@app.route('/api/challenges')
def challenges_api():
    "Returns a list of challenges as json"
    pass

@app.route('/api/task')
def task():
    """Returns an appropriate task based on parameters"""
    # We need to find a task for the user to work on, based (as much
    # as possible)
    difficulty = request.args.get('difficulty', 'easy')
    near = request.args.get('near')
    if near:
        lat, lon = near.split(',')
    else:
        point = None
    pass

@app.route('/api/c/<challenge>/meta')
def challenge_meta(challenge):
    "Returns the metadata for a challenge"
    pass

@app.route('/api/c/<challenge>/stats')
def challenge_stats(challenge):
    "Returns stat data for a challenge"
    pass

@app.route('/api/c/<challenge>/task')
def challenge_task(challenge):
    "Returns a task for specified challenge"
    pass

@app.route('/api/c/<challenge>/task/<id>', methods = ['POST'])
def challenge_post(challenge, task_id):
    "Accepts data for completed task"
    pass

@app.route('/logout')
def logout():
    session.destroy()
    return redirect('/')

@app.route('/oauth/authorize')
def oauth_authorize():
    """Initiates OAuth authorization agains the OSM server"""
    return osm.authorize(callback=url_for('oauth_authorized',
      next=request.args.get('next') or request.referrer or None))

@app.route('/oauth/callback')
@osm.authorized_handler
def oauth_authorized(resp):
    """Receives the OAuth callback from OSM"""
    next_url = request.args.get('next') or url_for('index')
    if resp is None:
        return redirect(next_url)
    session['osm_token'] = (
      resp['oauth_token'],
      resp['oauth_token_secret']
    )
    print 'getting user data from osm'
    osmuserresp = osm.get('user/details')
    if osmuserresp.status == 200:
        session['user'] = get_user_attribs(osmuserresp.data)
    else:
        print 'not able to get osm user data'
    return redirect(next_url)

@app.route('/<path:path>')
def catch_all(path):
    "Returns static files based on path"
    return send_from_directory('static', path)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--port", type = int, help = "the port to bind to",
                        default = 8080)
    parser.add_argument("--host", help = "the host to bind to",
                        default = "localhost")
    args = parser.parse_args()
    app.run(port=args.port)
    app.run(host=args.host, port=args.port)
<|endoftext|>


<commit_before># Copyright 2011 OpenStack Foundation
# Copyright 2012-2013 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from distutils import log
import fnmatch
import os
import sys

try:
    import cStringIO
except ImportError:
    import io as cStringIO

try:
    import sphinx
    from sphinx import apidoc
    from sphinx import application
    from sphinx import setup_command
except Exception as e:
    # NOTE(dhellmann): During the installation of docutils, setuptools
    # tries to import pbr code to find the egg_info.writer hooks. That
    # imports this module, which imports sphinx, which imports
    # docutils, which is being installed. Because docutils uses 2to3
    # to convert its code during installation under python 3, the
    # import fails, but it fails with an error other than ImportError
    # (today it's a NameError on StandardError, an exception base
    # class). Convert the exception type here so it can be caught in
    # packaging.py where we try to determine if we can import and use
    # sphinx by importing this module.  See bug #1403510 for details.
    raise ImportError(str(e))
from pbr import git
from pbr import options
from pbr import version

_rst_template = """%(heading)s
%(underline)s

.. automodule:: %(module)s
  :members:
  :undoc-members:
  :show-inheritance:
"""

def _find_modules(arg, dirname, files):
    for filename in files:
        if filename.endswith('.py') and filename != '__init__.py':
            arg["%s.%s" % (dirname.replace('/', '.'),
                           filename[:-3])] = True

class LocalBuildDoc(setup_command.BuildDoc):

    builders = ['html']
    command_name = 'build_sphinx'
    sphinx_initialized = False

    def _get_source_dir(self):
        option_dict = self.distribution.get_option_dict('build_sphinx')
        pbr_option_dict = self.distribution.get_option_dict('pbr')
        _, api_doc_dir = pbr_option_dict.get('api_doc_dir', (None, 'api'))
        if 'source_dir' in option_dict:
            source_dir = os.path.join(option_dict['source_dir'][1],
                                      api_doc_dir)
        else:
            source_dir = 'doc/source/' + api_doc_dir
        if not os.path.exists(source_dir):
            os.makedirs(source_dir)
        return source_dir

    def generate_autoindex(self, excluded_modules=None):
        log.info("[pbr] Autodocumenting from %s"
                 % os.path.abspath(os.curdir))
        modules = {}
        source_dir = self._get_source_dir()
        for pkg in self.distribution.packages:
            if '.' not in pkg:
                for dirpath, dirnames, files in os.walk(pkg):
                    _find_modules(modules, dirpath, files)

        def include(module):
            return not any(fnmatch.fnmatch(module, pat)
                           for pat in excluded_modules)

        module_list = sorted(mod for mod in modules.keys() if include(mod))
        autoindex_filename = os.path.join(source_dir, 'autoindex.rst')
        with open(autoindex_filename, 'w') as autoindex:
            autoindex.write(""".. toctree::
   :maxdepth: 1

""")
            for module in module_list:
                output_filename = os.path.join(source_dir,
                                               "%s.rst" % module)
                heading = "The :mod:`%s` Module" % module
                underline = "=" * len(heading)
                values = dict(module=module, heading=heading,
                              underline=underline)

                log.info("[pbr] Generating %s"
                         % output_filename)
                with open(output_filename, 'w') as output_file:
                    output_file.write(_rst_template % values)
                autoindex.write("   %s.rst\n" % module)

    def _sphinx_tree(self):
            source_dir = self._get_source_dir()
            cmd = ['apidoc', '.', '-H', 'Modules', '-o', source_dir]
            apidoc.main(cmd + self.autodoc_tree_excludes)

    def _sphinx_run(self):
        if not self.verbose:
            status_stream = cStringIO.StringIO()
        else:
            status_stream = sys.stdout
        confoverrides = {}
        if self.project:
            confoverrides['project'] = self.project
        if self.version:
            confoverrides['version'] = self.version
        if self.release:
            confoverrides['release'] = self.release
        if self.today:
            confoverrides['today'] = self.today
        if self.sphinx_initialized:
            confoverrides['suppress_warnings'] = [
                'app.add_directive', 'app.add_role',
                'app.add_generic_role', 'app.add_node',
                'image.nonlocal_uri',
            ]
        app = application.Sphinx(
            self.source_dir, self.config_dir,
            self.builder_target_dir, self.doctree_dir,
            self.builder, confoverrides, status_stream,
            freshenv=self.fresh_env, warningiserror=self.warning_is_error)
        self.sphinx_initialized = True

        try:
            app.build(force_all=self.all_files)
        except Exception as err:
            from docutils import utils
            if isinstance(err, utils.SystemMessage):
                sys.stder.write('reST markup error:\n')
                sys.stderr.write(err.args[0].encode('ascii',
                                                    'backslashreplace'))
                sys.stderr.write('\n')
            else:
                raise

        if self.link_index:
            src = app.config.master_doc + app.builder.out_suffix
            dst = app.builder.get_outfilename('index')
            os.symlink(src, dst)

    def run(self):
        option_dict = self.distribution.get_option_dict('pbr')
        if git._git_is_installed():
            git.write_git_changelog(option_dict=option_dict)
            git.generate_authors(option_dict=option_dict)
        tree_index = options.get_boolean_option(option_dict,
                                                'autodoc_tree_index_modules',
                                                'AUTODOC_TREE_INDEX_MODULES')
        auto_index = options.get_boolean_option(option_dict,
                                                'autodoc_index_modules',
                                                'AUTODOC_INDEX_MODULES')
        if not os.getenv('SPHINX_DEBUG'):
            # NOTE(afazekas): These options can be used together,
            # but they do a very similar thing in a different way
            if tree_index:
                self._sphinx_tree()
            if auto_index:
                self.generate_autoindex(
                    set(option_dict.get(
                        "autodoc_exclude_modules",
                        [None, ""])[1].split()))

        self.finalize_options()

        is_multibuilder_sphinx = version.SemanticVersion.from_pip_string(
            sphinx.__version__) >= version.SemanticVersion(1, 6)

        # TODO(stephenfin): Remove support for Sphinx < 1.6 in 4.0
        if not is_multibuilder_sphinx:
            log.warning('[pbr] Support for Sphinx < 1.6 will be dropped in '
                        'pbr 4.0. Upgrade to Sphinx 1.6+')

        # TODO(stephenfin): Remove this at the next MAJOR version bump
        if self.builders != ['html']:
            log.warning("[pbr] Sphinx 1.6 added native support for "
                        "specifying multiple builders in the "
                        "'[sphinx_build] builder' configuration option, "
                        "found in 'setup.cfg'. As a result, the "
                        "'[sphinx_build] builders' option has been "
                        "deprecated and will be removed in pbr 4.0. Migrate "
                        "to the 'builder' configuration option.")
            if is_multibuilder_sphinx:
                self.builder = self.builders

        if is_multibuilder_sphinx:
            # Sphinx >= 1.6
            return setup_command.BuildDoc.run(self)

        # Sphinx < 1.6
        for builder in self.builder:
            self.builder = builder
            self.finalize_options()
            self._sphinx_run()

    def initialize_options(self):
        # Not a new style class, super keyword does not work.
        setup_command.BuildDoc.initialize_options(self)

        # NOTE(dstanek): exclude setup.py from the autodoc tree index
        # builds because all projects will have an issue with it
        self.autodoc_tree_excludes = ['setup.py']

    def finalize_options(self):
        # Not a new style class, super keyword does not work.
        setup_command.BuildDoc.finalize_options(self)

        # Handle builder option from command line - override cfg
        option_dict = self.distribution.get_option_dict('build_sphinx')
        if 'command line' in option_dict.get('builder', [[]])[0]:
            self.builders = option_dict['builder'][1]
        # Allow builders to be configurable - as a comma separated list.
        if not isinstance(self.builders, list) and self.builders:
            self.builders = self.builders.split(',')

        self.project = self.distribution.get_name()
        self.version = self.distribution.get_version()
        self.release = self.distribution.get_version()

        # NOTE(dstanek): check for autodoc tree exclusion overrides
        # in the setup.cfg
        opt = 'autodoc_tree_excludes'
        option_dict = self.distribution.get_option_dict('pbr')
        if opt in option_dict:
            self.autodoc_tree_excludes = option_dict[opt][1]
            self.ensure_string_list(opt)

        # handle Sphinx < 1.5.0
        if not hasattr(self, 'warning_is_error'):
            self.warning_is_error = False
<commit_msg>emit warning correctly<commit_after># Copyright 2011 OpenStack Foundation
# Copyright 2012-2013 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from distutils import log
import fnmatch
import os
import sys

try:
    import cStringIO
except ImportError:
    import io as cStringIO

try:
    import sphinx
    from sphinx import apidoc
    from sphinx import application
    from sphinx import setup_command
except Exception as e:
    # NOTE(dhellmann): During the installation of docutils, setuptools
    # tries to import pbr code to find the egg_info.writer hooks. That
    # imports this module, which imports sphinx, which imports
    # docutils, which is being installed. Because docutils uses 2to3
    # to convert its code during installation under python 3, the
    # import fails, but it fails with an error other than ImportError
    # (today it's a NameError on StandardError, an exception base
    # class). Convert the exception type here so it can be caught in
    # packaging.py where we try to determine if we can import and use
    # sphinx by importing this module.  See bug #1403510 for details.
    raise ImportError(str(e))
from pbr import git
from pbr import options
from pbr import version

_rst_template = """%(heading)s
%(underline)s

.. automodule:: %(module)s
  :members:
  :undoc-members:
  :show-inheritance:
"""

def _find_modules(arg, dirname, files):
    for filename in files:
        if filename.endswith('.py') and filename != '__init__.py':
            arg["%s.%s" % (dirname.replace('/', '.'),
                           filename[:-3])] = True

class LocalBuildDoc(setup_command.BuildDoc):

    builders = ['html']
    command_name = 'build_sphinx'
    sphinx_initialized = False

    def _get_source_dir(self):
        option_dict = self.distribution.get_option_dict('build_sphinx')
        pbr_option_dict = self.distribution.get_option_dict('pbr')
        _, api_doc_dir = pbr_option_dict.get('api_doc_dir', (None, 'api'))
        if 'source_dir' in option_dict:
            source_dir = os.path.join(option_dict['source_dir'][1],
                                      api_doc_dir)
        else:
            source_dir = 'doc/source/' + api_doc_dir
        if not os.path.exists(source_dir):
            os.makedirs(source_dir)
        return source_dir

    def generate_autoindex(self, excluded_modules=None):
        log.info("[pbr] Autodocumenting from %s"
                 % os.path.abspath(os.curdir))
        modules = {}
        source_dir = self._get_source_dir()
        for pkg in self.distribution.packages:
            if '.' not in pkg:
                for dirpath, dirnames, files in os.walk(pkg):
                    _find_modules(modules, dirpath, files)

        def include(module):
            return not any(fnmatch.fnmatch(module, pat)
                           for pat in excluded_modules)

        module_list = sorted(mod for mod in modules.keys() if include(mod))
        autoindex_filename = os.path.join(source_dir, 'autoindex.rst')
        with open(autoindex_filename, 'w') as autoindex:
            autoindex.write(""".. toctree::
   :maxdepth: 1

""")
            for module in module_list:
                output_filename = os.path.join(source_dir,
                                               "%s.rst" % module)
                heading = "The :mod:`%s` Module" % module
                underline = "=" * len(heading)
                values = dict(module=module, heading=heading,
                              underline=underline)

                log.info("[pbr] Generating %s"
                         % output_filename)
                with open(output_filename, 'w') as output_file:
                    output_file.write(_rst_template % values)
                autoindex.write("   %s.rst\n" % module)

    def _sphinx_tree(self):
            source_dir = self._get_source_dir()
            cmd = ['apidoc', '.', '-H', 'Modules', '-o', source_dir]
            apidoc.main(cmd + self.autodoc_tree_excludes)

    def _sphinx_run(self):
        if not self.verbose:
            status_stream = cStringIO.StringIO()
        else:
            status_stream = sys.stdout
        confoverrides = {}
        if self.project:
            confoverrides['project'] = self.project
        if self.version:
            confoverrides['version'] = self.version
        if self.release:
            confoverrides['release'] = self.release
        if self.today:
            confoverrides['today'] = self.today
        if self.sphinx_initialized:
            confoverrides['suppress_warnings'] = [
                'app.add_directive', 'app.add_role',
                'app.add_generic_role', 'app.add_node',
                'image.nonlocal_uri',
            ]
        app = application.Sphinx(
            self.source_dir, self.config_dir,
            self.builder_target_dir, self.doctree_dir,
            self.builder, confoverrides, status_stream,
            freshenv=self.fresh_env, warningiserror=self.warning_is_error)
        self.sphinx_initialized = True

        try:
            app.build(force_all=self.all_files)
        except Exception as err:
            from docutils import utils
            if isinstance(err, utils.SystemMessage):
                sys.stder.write('reST markup error:\n')
                sys.stderr.write(err.args[0].encode('ascii',
                                                    'backslashreplace'))
                sys.stderr.write('\n')
            else:
                raise

        if self.link_index:
            src = app.config.master_doc + app.builder.out_suffix
            dst = app.builder.get_outfilename('index')
            os.symlink(src, dst)

    def run(self):
        option_dict = self.distribution.get_option_dict('pbr')
        if git._git_is_installed():
            git.write_git_changelog(option_dict=option_dict)
            git.generate_authors(option_dict=option_dict)
        tree_index = options.get_boolean_option(option_dict,
                                                'autodoc_tree_index_modules',
                                                'AUTODOC_TREE_INDEX_MODULES')
        auto_index = options.get_boolean_option(option_dict,
                                                'autodoc_index_modules',
                                                'AUTODOC_INDEX_MODULES')
        if not os.getenv('SPHINX_DEBUG'):
            # NOTE(afazekas): These options can be used together,
            # but they do a very similar thing in a different way
            if tree_index:
                self._sphinx_tree()
            if auto_index:
                self.generate_autoindex(
                    set(option_dict.get(
                        "autodoc_exclude_modules",
                        [None, ""])[1].split()))

        self.finalize_options()

        is_multibuilder_sphinx = version.SemanticVersion.from_pip_string(
            sphinx.__version__) >= version.SemanticVersion(1, 6)

        # TODO(stephenfin): Remove support for Sphinx < 1.6 in 4.0
        if not is_multibuilder_sphinx:
            log.warn('[pbr] Support for Sphinx < 1.6 will be dropped in '
                     'pbr 4.0. Upgrade to Sphinx 1.6+')

        # TODO(stephenfin): Remove this at the next MAJOR version bump
        if self.builders != ['html']:
            log.warn("[pbr] Sphinx 1.6 added native support for "
                     "specifying multiple builders in the "
                     "'[sphinx_build] builder' configuration option, "
                     "found in 'setup.cfg'. As a result, the "
                     "'[sphinx_build] builders' option has been "
                     "deprecated and will be removed in pbr 4.0. Migrate "
                     "to the 'builder' configuration option.")
            if is_multibuilder_sphinx:
                self.builder = self.builders

        if is_multibuilder_sphinx:
            # Sphinx >= 1.6
            return setup_command.BuildDoc.run(self)

        # Sphinx < 1.6
        for builder in self.builder:
            self.builder = builder
            self.finalize_options()
            self._sphinx_run()

    def initialize_options(self):
        # Not a new style class, super keyword does not work.
        setup_command.BuildDoc.initialize_options(self)

        # NOTE(dstanek): exclude setup.py from the autodoc tree index
        # builds because all projects will have an issue with it
        self.autodoc_tree_excludes = ['setup.py']

    def finalize_options(self):
        # Not a new style class, super keyword does not work.
        setup_command.BuildDoc.finalize_options(self)

        # Handle builder option from command line - override cfg
        option_dict = self.distribution.get_option_dict('build_sphinx')
        if 'command line' in option_dict.get('builder', [[]])[0]:
            self.builders = option_dict['builder'][1]
        # Allow builders to be configurable - as a comma separated list.
        if not isinstance(self.builders, list) and self.builders:
            self.builders = self.builders.split(',')

        self.project = self.distribution.get_name()
        self.version = self.distribution.get_version()
        self.release = self.distribution.get_version()

        # NOTE(dstanek): check for autodoc tree exclusion overrides
        # in the setup.cfg
        opt = 'autodoc_tree_excludes'
        option_dict = self.distribution.get_option_dict('pbr')
        if opt in option_dict:
            self.autodoc_tree_excludes = option_dict[opt][1]
            self.ensure_string_list(opt)

        # handle Sphinx < 1.5.0
        if not hasattr(self, 'warning_is_error'):
            self.warning_is_error = False
<|endoftext|>


<commit_before>import hashlib
import os
import random
import re
from threading import Timer
import time
from twisted.application.service import Application
from twisted.python import log
from twisted.python.log import ILogObserver, FileLogObserver
from twisted.python.logfile import DailyLogFile
from coapthon import defines
from coapthon.client.coap_protocol import HelperClient
from coapthon.messages.message import Message
from coapthon.messages.request import Request
from coapthon.messages.response import Response
from coapthon.serializer import Serializer
from coapthon.server.coap_protocol import CoAP

__author__ = 'Giacomo Tanganelli'
__version__ = "2.0"

home = os.path.expanduser("~")
if not os.path.exists(home + "/.coapthon/"):
    os.makedirs(home + "/.coapthon/")

logfile = DailyLogFile("CoAPthon_forward_proxy.log", home + "/.coapthon/")
# Now add an observer that logs to a file
application = Application("CoAPthon_Forward_Proxy")
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)

class ProxyCoAP(CoAP):
    def __init__(self, server, client):
        """
        Initialize the CoAP protocol

        """
        CoAP.__init__(self)
        self.client = client
        self.server = server
        self._forward = {}
        self._forward_mid = {}
        self._token = random.randint(1, 1000)
        self.timer = None

    def startProtocol(self):
        """
        Called after protocol has started listening.
        """
        # Set the TTL>1 so multicast will cross router hops:
        # self.transport.setTTL(5)
        # Join a specific multicast group:
        # self.transport.joinGroup(defines.ALL_COAP_NODES)
        self.client.starting_mid = self._currentMID % (1 << 16)

    def datagramReceived(self, data, addr):
        """
        Handler for received dUDP datagram.

        :param data: the UDP datagram
        :param host: source host
        :param port: source port
        """
        try:
            host, port = addr
        except ValueError:
            host, port, tmp1, tmp2 = addr
        log.msg("Datagram received from " + str(host) + ":" + str(port))
        serializer = Serializer()
        message = serializer.deserialize(data, host, port)
        print "Message received from " + host + ":" + str(port)
        print "----------------------------------------"
        print message
        print "----------------------------------------"
        if isinstance(message, Request):
            log.msg("Received request")
            ret = self._request_layer.handle_request(message)
            if isinstance(ret, Request):
                self.forward_request(ret)
            else:
                return
        elif isinstance(message, Response):
            log.err("Received response")
            rst = Message.new_rst(message)
            rst = self._message_layer.matcher_response(rst)
            log.msg("Send RST")
            self.send(rst, host, port)
        elif isinstance(message, tuple):
            message, error = message
            response = Response()
            response.destination = (host, port)
            response.code = defines.responses[error]
            response = self.reliability_response(message, response)
            response = self._message_layer.matcher_response(response)
            log.msg("Send Error")
            self.send(response, host, port)
        elif message is not None:
            # ACK or RST
            log.msg("Received ACK or RST")
            self._message_layer.handle_message(message)

    def parse_path(self, path):
        return self.parse_path_ipv6(path)
        # m = re.match("([a-zA-Z]{4,5})://([a-zA-Z0-9.]*):([0-9]*)/(\S*)", path)
        # if m is None:
        #     m = re.match("([a-zA-Z]{4,5})://([a-zA-Z0-9.]*)/(\S*)", path)
        #     if m is None:
        #         m = re.match("([a-zA-Z]{4,5})://([a-zA-Z0-9.]*)", path)
        #         if m is None:
        #             ip, port, path = self.parse_path_ipv6(path)
        #         else:
        #             ip = m.group(2)
        #             port = 5683
        #             path = ""
        #     else:
        #         ip = m.group(2)
        #         port = 5683
        #         path = m.group(3)
        # else:
        #     ip = m.group(2)
        #     port = int(m.group(3))
        #     path = m.group(4)
        #
        # return ip, port, path

    @staticmethod
    def parse_path_ipv6(path):
        m = re.match("([a-zA-Z]{4,5})://\[([a-fA-F0-9:]*)\]:([0-9]*)/(\S*)", path)
        if m is None:
            m = re.match("([a-zA-Z]{4,5})://\[([a-fA-F0-9:]*)\]/(\S*)", path)
            if m is None:
                m = re.match("([a-zA-Z]{4,5})://\[([a-fA-F0-9:]*)\]", path)
                ip = m.group(2)
                port = 5683
                path = ""
            else:
                ip = m.group(2)
                port = 5683
                path = m.group(3)
        else:
            ip = m.group(2)
            port = int(m.group(3))
            path = m.group(4)

        return ip, port, path

    def forward_request(self, request):
        """
        Forward an incoming request to the specified server.

        :param request: the request to be forwarded
        :return: None if success, send an error otherwise
        """
        # print "FORWARD REQUEST\n"
        uri = request.proxy_uri
        response = Response()
        response.destination = request.source
        token = self.generate_token()
        if uri is None:
            return self.send_error(request, response, "BAD_REQUEST")
        host, port, path = self.parse_path(uri)

        request.uri_path = path
        self._currentMID += 1
        method = defines.codes[request.code]
        if method == 'GET':
            function = self.client.protocol.get
            req = Request()
            req.destination = (str(host), int(port))
            req.mid = self._currentMID % (1 << 16)
            req.token = str(token)
            req.uri_path = path
            req.type = defines.inv_types["CON"]
            req.code = defines.inv_codes['GET']
            args = (req,)
            kwargs = {}
            callback = self.result_forward
            err_callback = self.error
        elif method == 'POST':
            function = self.client.protocol.post
            req = Request()
            req.destination = (str(host), int(port))
            req.mid = self._currentMID % (1 << 16)
            req.token = str(token)
            req.uri_path = path
            req.payload = request.payload
            req.type = defines.inv_types["CON"]
            req.code = defines.inv_codes['POST']
            args = (req,)
            kwargs = {}
            callback = self.result_forward
            err_callback = self.error
        elif method == 'PUT':
            function = self.client.protocol.put
            req = Request()
            req.destination = (str(host), int(port))
            req.mid = self._currentMID % (1 << 16)
            req.token = str(token)
            req.uri_path = path
            req.payload = request.payload
            req.type = defines.inv_types["CON"]
            req.code = defines.inv_codes['PUT']
            args = (req,)
            kwargs = {}
            callback = self.result_forward
            err_callback = self.error
        elif method == 'DELETE':
            function = self.client.protocol.delete
            req = Request()
            req.destination = (str(host), int(port))
            req.mid = self._currentMID % (1 << 16)
            req.token = str(token)
            req.uri_path = path
            req.type = defines.inv_types["CON"]
            req.code = defines.inv_codes['DELETE']
            args = (req,)
            kwargs = {}
            callback = self.result_forward
            err_callback = self.error
        else:
            return self.send_error(request, response, "BAD_REQUEST")
        for option in request.options:
            if option.safe:
                kwargs[option.name] = option.value

        operations = [(function, args, kwargs, (callback, err_callback))]
        key = hash(str(host) + str(port) + str(token))
        self._forward[key] = request
        key = hash(str(host) + str(port) + str(self._currentMID % (1 << 16)))
        self._forward_mid[key] = request
        # print "************"
        # print str(host),  str(port), str(self._currentMID % (1 << 16))
        # print req
        # print "************"

        # self.sent[key] = (req, time.time())
        self.sent[str(self._currentMID % (1 << 16))] = (req, time.time())
        self.client.start(operations)
        # Render_GET
        # self.timer = Timer(defines.SEPARATE_TIMEOUT, self.send_ack, [request])
        # self.timer.start()
        return None

    def send_ack(self, list_request):
        """
        Send an ack to the client. Used mostly with separate.

        :param list_request: the request to be acknowledge.
        :type list_request: [Request] or Request
        """
        self.timer = None
        if isinstance(list_request, list):
            request = list_request[0]
        else:
            request = list_request
        host, port = request.source
        ack = Message.new_ack(request)
        self.send(ack, host, port)

    def result_forward(self, response, request=None):
        """
        Forward results to the client.

        :param response: the response sent by the server.
        """
        # print "RESULT FORWARD\n"
        skip_delete = False
        host, port = response.source
        key_mid = hash(str(host) + str(port) + str(response.mid))
        key = None
        if request is None:
            host, port = response.source
            key = hash(str(host) + str(port) + str(response.token))
            request = self._forward.get(key)
        else:
            skip_delete = True
        # print request
        if self.timer is not None:
            self.timer.cancel()
            response.type = defines.inv_types["ACK"]
            response.mid = request.mid
        elif skip_delete:
            response.type = defines.inv_types["ACK"]
            response.mid = request.mid
        else:
            if request.type == defines.inv_types["CON"]:
                response.type = defines.inv_types["CON"]
            else:
                response.type = defines.inv_types["NON"]

        if request is not None:
            response.destination = request.source
            response.token = request.token
            if not skip_delete:
                del self._forward[key]
                try:
                    del self._forward_mid[key_mid]
                except KeyError:
                    log.err("MID has not been deleted")
            host, port = request.source
            if response.mid is None:
                response.mid = self._currentMID
            self.send(response, host, port)

    def generate_token(self):
        """
        Generate tokens.

        :return: a token.
        """
        self._token += 1
        # md5 = hashlib.md5()
        # md5.update(str(self._token))
        # token = md5.digest()
        # return token[0:15]
        return str(self._token)

    def error(self, mid, host, port):
        """
        Handler of errors. Send an error message to the client.

        :param mid: the mid of the response that generates the error.
        :param host: the host of the server.
        :param port: the port of the server.
        """
        key = hash(str(host) + str(port) + str(mid))
        request = self._forward_mid.get(key)
        if request is not None:
            response = Response()
            host, port = request.source
            response.destination = request.source
            response.token = request.token
            response._mid = self._currentMID
            response.type = defines.inv_types["NON"]
            response.code = defines.responses["GATEWAY_TIMEOUT"]
            key = hash(str(host) + str(port) + str(response.token))
            del self._forward[key]
            key = hash(str(host) + str(port) + str(mid))
            try:
                del self._forward_mid[key]
            except KeyError:
                log.err("MID has not been deleted")
            self.send(response, host, port)<commit_msg>debug removed<commit_after>import hashlib
import os
import random
import re
from threading import Timer
import time
from twisted.application.service import Application
from twisted.python import log
from twisted.python.log import ILogObserver, FileLogObserver
from twisted.python.logfile import DailyLogFile
from coapthon import defines
from coapthon.client.coap_protocol import HelperClient
from coapthon.messages.message import Message
from coapthon.messages.request import Request
from coapthon.messages.response import Response
from coapthon.serializer import Serializer
from coapthon.server.coap_protocol import CoAP

__author__ = 'Giacomo Tanganelli'
__version__ = "2.0"

home = os.path.expanduser("~")
if not os.path.exists(home + "/.coapthon/"):
    os.makedirs(home + "/.coapthon/")

logfile = DailyLogFile("CoAPthon_forward_proxy.log", home + "/.coapthon/")
# Now add an observer that logs to a file
application = Application("CoAPthon_Forward_Proxy")
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)

class ProxyCoAP(CoAP):
    def __init__(self, server, client):
        """
        Initialize the CoAP protocol

        """
        CoAP.__init__(self)
        self.client = client
        self.server = server
        self._forward = {}
        self._forward_mid = {}
        self._token = random.randint(1, 1000)
        self.timer = None

    def startProtocol(self):
        """
        Called after protocol has started listening.
        """
        # Set the TTL>1 so multicast will cross router hops:
        # self.transport.setTTL(5)
        # Join a specific multicast group:
        # self.transport.joinGroup(defines.ALL_COAP_NODES)
        self.client.starting_mid = self._currentMID % (1 << 16)

    def datagramReceived(self, data, addr):
        """
        Handler for received dUDP datagram.

        :param data: the UDP datagram
        :param host: source host
        :param port: source port
        """
        try:
            host, port = addr
        except ValueError:
            host, port, tmp1, tmp2 = addr
        log.msg("Datagram received from " + str(host) + ":" + str(port))
        serializer = Serializer()
        message = serializer.deserialize(data, host, port)
        # print "Message received from " + host + ":" + str(port)
        # print "----------------------------------------"
        # print message
        # print "----------------------------------------"
        if isinstance(message, Request):
            log.msg("Received request")
            ret = self._request_layer.handle_request(message)
            if isinstance(ret, Request):
                self.forward_request(ret)
            else:
                return
        elif isinstance(message, Response):
            log.err("Received response")
            rst = Message.new_rst(message)
            rst = self._message_layer.matcher_response(rst)
            log.msg("Send RST")
            self.send(rst, host, port)
        elif isinstance(message, tuple):
            message, error = message
            response = Response()
            response.destination = (host, port)
            response.code = defines.responses[error]
            response = self.reliability_response(message, response)
            response = self._message_layer.matcher_response(response)
            log.msg("Send Error")
            self.send(response, host, port)
        elif message is not None:
            # ACK or RST
            log.msg("Received ACK or RST")
            self._message_layer.handle_message(message)

    def parse_path(self, path):
        return self.parse_path_ipv6(path)
        # m = re.match("([a-zA-Z]{4,5})://([a-zA-Z0-9.]*):([0-9]*)/(\S*)", path)
        # if m is None:
        #     m = re.match("([a-zA-Z]{4,5})://([a-zA-Z0-9.]*)/(\S*)", path)
        #     if m is None:
        #         m = re.match("([a-zA-Z]{4,5})://([a-zA-Z0-9.]*)", path)
        #         if m is None:
        #             ip, port, path = self.parse_path_ipv6(path)
        #         else:
        #             ip = m.group(2)
        #             port = 5683
        #             path = ""
        #     else:
        #         ip = m.group(2)
        #         port = 5683
        #         path = m.group(3)
        # else:
        #     ip = m.group(2)
        #     port = int(m.group(3))
        #     path = m.group(4)
        #
        # return ip, port, path

    @staticmethod
    def parse_path_ipv6(path):
        m = re.match("([a-zA-Z]{4,5})://\[([a-fA-F0-9:]*)\]:([0-9]*)/(\S*)", path)
        if m is None:
            m = re.match("([a-zA-Z]{4,5})://\[([a-fA-F0-9:]*)\]/(\S*)", path)
            if m is None:
                m = re.match("([a-zA-Z]{4,5})://\[([a-fA-F0-9:]*)\]", path)
                ip = m.group(2)
                port = 5683
                path = ""
            else:
                ip = m.group(2)
                port = 5683
                path = m.group(3)
        else:
            ip = m.group(2)
            port = int(m.group(3))
            path = m.group(4)

        return ip, port, path

    def forward_request(self, request):
        """
        Forward an incoming request to the specified server.

        :param request: the request to be forwarded
        :return: None if success, send an error otherwise
        """
        # print "FORWARD REQUEST\n"
        uri = request.proxy_uri
        response = Response()
        response.destination = request.source
        token = self.generate_token()
        if uri is None:
            return self.send_error(request, response, "BAD_REQUEST")
        host, port, path = self.parse_path(uri)

        request.uri_path = path
        self._currentMID += 1
        method = defines.codes[request.code]
        if method == 'GET':
            function = self.client.protocol.get
            req = Request()
            req.destination = (str(host), int(port))
            req.mid = self._currentMID % (1 << 16)
            req.token = str(token)
            req.uri_path = path
            req.type = defines.inv_types["CON"]
            req.code = defines.inv_codes['GET']
            args = (req,)
            kwargs = {}
            callback = self.result_forward
            err_callback = self.error
        elif method == 'POST':
            function = self.client.protocol.post
            req = Request()
            req.destination = (str(host), int(port))
            req.mid = self._currentMID % (1 << 16)
            req.token = str(token)
            req.uri_path = path
            req.payload = request.payload
            req.type = defines.inv_types["CON"]
            req.code = defines.inv_codes['POST']
            args = (req,)
            kwargs = {}
            callback = self.result_forward
            err_callback = self.error
        elif method == 'PUT':
            function = self.client.protocol.put
            req = Request()
            req.destination = (str(host), int(port))
            req.mid = self._currentMID % (1 << 16)
            req.token = str(token)
            req.uri_path = path
            req.payload = request.payload
            req.type = defines.inv_types["CON"]
            req.code = defines.inv_codes['PUT']
            args = (req,)
            kwargs = {}
            callback = self.result_forward
            err_callback = self.error
        elif method == 'DELETE':
            function = self.client.protocol.delete
            req = Request()
            req.destination = (str(host), int(port))
            req.mid = self._currentMID % (1 << 16)
            req.token = str(token)
            req.uri_path = path
            req.type = defines.inv_types["CON"]
            req.code = defines.inv_codes['DELETE']
            args = (req,)
            kwargs = {}
            callback = self.result_forward
            err_callback = self.error
        else:
            return self.send_error(request, response, "BAD_REQUEST")
        for option in request.options:
            if option.safe:
                kwargs[option.name] = option.value

        operations = [(function, args, kwargs, (callback, err_callback))]
        key = hash(str(host) + str(port) + str(token))
        self._forward[key] = request
        key = hash(str(host) + str(port) + str(self._currentMID % (1 << 16)))
        self._forward_mid[key] = request
        # print "************"
        # print str(host),  str(port), str(self._currentMID % (1 << 16))
        # print req
        # print "************"

        # self.sent[key] = (req, time.time())
        self.sent[str(self._currentMID % (1 << 16))] = (req, time.time())
        self.client.start(operations)
        # Render_GET
        # self.timer = Timer(defines.SEPARATE_TIMEOUT, self.send_ack, [request])
        # self.timer.start()
        return None

    def send_ack(self, list_request):
        """
        Send an ack to the client. Used mostly with separate.

        :param list_request: the request to be acknowledge.
        :type list_request: [Request] or Request
        """
        self.timer = None
        if isinstance(list_request, list):
            request = list_request[0]
        else:
            request = list_request
        host, port = request.source
        ack = Message.new_ack(request)
        self.send(ack, host, port)

    def result_forward(self, response, request=None):
        """
        Forward results to the client.

        :param response: the response sent by the server.
        """
        # print "RESULT FORWARD\n"
        skip_delete = False
        host, port = response.source
        key_mid = hash(str(host) + str(port) + str(response.mid))
        key = None
        if request is None:
            host, port = response.source
            key = hash(str(host) + str(port) + str(response.token))
            request = self._forward.get(key)
        else:
            skip_delete = True
        # print request
        if self.timer is not None:
            self.timer.cancel()
            response.type = defines.inv_types["ACK"]
            response.mid = request.mid
        elif skip_delete:
            response.type = defines.inv_types["ACK"]
            response.mid = request.mid
        else:
            if request.type == defines.inv_types["CON"]:
                response.type = defines.inv_types["CON"]
            else:
                response.type = defines.inv_types["NON"]

        if request is not None:
            response.destination = request.source
            response.token = request.token
            if not skip_delete:
                del self._forward[key]
                try:
                    del self._forward_mid[key_mid]
                except KeyError:
                    log.err("MID has not been deleted")
            host, port = request.source
            if response.mid is None:
                response.mid = self._currentMID
            self.send(response, host, port)

    def generate_token(self):
        """
        Generate tokens.

        :return: a token.
        """
        self._token += 1
        # md5 = hashlib.md5()
        # md5.update(str(self._token))
        # token = md5.digest()
        # return token[0:15]
        return str(self._token)

    def error(self, mid, host, port):
        """
        Handler of errors. Send an error message to the client.

        :param mid: the mid of the response that generates the error.
        :param host: the host of the server.
        :param port: the port of the server.
        """
        key = hash(str(host) + str(port) + str(mid))
        request = self._forward_mid.get(key)
        if request is not None:
            response = Response()
            host, port = request.source
            response.destination = request.source
            response.token = request.token
            response._mid = self._currentMID
            response.type = defines.inv_types["NON"]
            response.code = defines.responses["GATEWAY_TIMEOUT"]
            key = hash(str(host) + str(port) + str(response.token))
            del self._forward[key]
            key = hash(str(host) + str(port) + str(mid))
            try:
                del self._forward_mid[key]
            except KeyError:
                log.err("MID has not been deleted")
            self.send(response, host, port)<|endoftext|>


<commit_before>import os

source = '../src/Tween.js'
build = '../build/Tween.js'
header = '// Tween.js - http://github.com/sole/tween.js\n'

os.system( 'java -jar compiler/compiler.jar --language_in=ECMASCRIPT5 --js ' + source + ' --js_output_file ' + build )

file = open( build, 'r' )
contents = file.read();
file.close()

file = open( build, 'w' )
file.write( header + contents )
file.close()
<commit_msg>simplified builder.py<commit_after>import os

source = '../src/Tween.js'
output = '../build/tween.min.js'

os.system( 'java -jar compiler/compiler.jar --language_in=ECMASCRIPT5 --js ' + source + ' --js_output_file ' + output )

with open(output,'r') as f: text = f.read()
with open(output,'w') as f: f.write(("// tween.min.js - http://github.com/sole/tween.js\n" % os.path.basename(output)) + text)
<|endoftext|>


<commit_before>from itertools import cycle
from contextlib import ExitStack

from .. import lekvar
from ..errors import InternalError

from .util import *
from .state import State
from . import builtins

#
# class Object
#

@patch
def Object_eval(self):
    raise InternalError("Not Implemented")

@patch
def Object_evalContext(self):
    raise InternalError("Not Implemented")

@patch
def Object_evalCall(self, values):
    raise InternalError("Not Implemented")

@patch
def Object_evalAssign(self, value):
    raise InternalError("Not Implemented")

#
# class Type
#

@patch
def Type_evalSize(self):
    raise InternalError("Not Implemented")

@patch
def Type_evalNewValue(self):
    raise InternalError("Not Implemented")

#
# class Link
#

@patch
def Link_eval(self):
    return self.value.eval()

@patch
def Link_evalContext(self):
    return self.value.evalContext()

@patch
def Link_evalCall(self, values):
    return self.value.evalCall(values)

@patch
def Link_evalAssign(self, value):
    self.value.evalAssign(value)

@patch
def Link_evalSize(self):
    return self.value.evalSize()

@patch
def Link_evalNewValue(self):
    return self.value.evalNewValue()

#
# class Attribute
#

@patch
def Attribute_eval(self):
    with State.selfScope(self.object.eval()):
        return self.value.eval()

@patch
def Attribute_evalContext(self):
    return self.object.eval()

@patch
def Attribute_evalCall(self, values):
    with State.selfScope(self.object.eval()):
        return self.value.evalCall(values)

@patch
def Attribute_evalAssign(self, value):
    with State.selfScope(self.object.eval()):
        self.value.evalAssign(value)

#
# class ClosedTarget
#

@patch
def ClosedTarget_eval(self):
    with self.target():
        result = self.value.eval()

        if result == self.value:
            return self
        return result

@patch
def ClosedTarget_evalContext(self):
    with self.target():
        return self.value.evalContext()

@patch
def ClosedTarget_evalCall(self, values):
    with self.target():
        return self.value.evalCall(values)

@patch
def ClosedTarget_evalAssign(self, value):
    with self.target():
        self.value.evalAssign(value)

@patch
def ClosedTarget_evalNewValue(self):
    with self.target():
        return self.value.evalNewValue()

#
# class Reference
#

@patch
def Reference_eval(self):
    return self.value.eval()

@patch
def Reference_evalContext(self):
    return self.value.evalContext()

@patch
def Reference_evalCall(self, values):
    return self.value.evalCall(values)

@patch
def Reference_evalAssign(self, value):
    self.value.evalAssign(value)

@patch
def Reference_evalSize(self):
    return builtins.PTR_SIZE

#
# class Module
#

lekvar.Module.evaled = False

@patch
def Module_eval(self):
    if self.evaled: return self
    self.evaled = True

    for instr in self.main:
        instr.eval()

    for obj in self.context:
        obj.eval()

    return self

#
# class Variable
#

@patch
def Variable_eval(self):
    if isinstance(self.parent, lekvar.Class):
        assert State.self is not None

        if not isinstance(State.self, lekvar.Literal) or not isinstance(State.self.data, dict):
            return State.self
        return State.self.data[self.name]

    if self.value is not None:
        return self.value
    return self

@patch
def Variable_evalContext(self):
    return self.eval()

@patch
def Variable_evalAssign(self, value):
    if isinstance(self.parent, lekvar.Class):
        assert State.self is not None
        assert isinstance(State.self, lekvar.Literal)

        State.self.data[self.name] = value
    else:
        self.value = value

@patch
def Variable_evalSize(self):
    return self.value.evalSize()

@patch
def Variable_evalNewValue(self):
    return self.value.evalNewValue()

#
# class Assignment
#

@patch
def Assignment_eval(self):
    self.assigned.evalAssign(self.value.eval())

#
# class Method
#

@patch
def Method_eval(self):
    return self

@patch
def Method_evalContext(self):
    return None

@patch
def Method_evalCall(self, values):
    call = lekvar.FunctionType([value.resolveType() for value in values])
    call.verify()
    function = self.resolveCall(call)

    return function.evalCall(values)

#
# class MethodType
#

@patch
def MethodType_evalNewValue(self):
    return lekvar.Literal({}, self)

#
# class MethodInstance
#

@patch
def MethodInstance_eval(self):
    return State.self

@patch
def MethodInstance_evalCall(self, values):
    method = State.self

    call = lekvar.FunctionType([value.resolveType() for value in values])
    function = method.resolveCall(call)

    return function.evalCall(values)

#
# class Function
#

lekvar.Function.eval_returning = None
lekvar.Function.eval_returned = None

@patch
def Function_eval(self):
    return self

@patch
def Function_evalContext(self):
    #TODO: Implement closures
    assert len(self.closed_context) == 0

    return None

@patch
def Function_evalCall(self, values):
    self.eval_returned = False
    self.eval_returning = None

    previous_values = [arg.resolveValue().value for arg in self.arguments]
    for arg, val in zip(self.arguments, values):
        arg.evalAssign(val)

    for instr in self.instructions:
        instr.eval()

        if self.eval_returned:
            break

    for arg, val in zip(self.arguments, previous_values):
        arg.evalAssign(val)

    returning = self.eval_returning
    self.eval_returned = False
    self.eval_returning = None

    return returning

#
# class FunctionInstance
#

@patch
def FunctionInstance_eval(self):
    return self

@patch
def FunctionInstance_evalCall(self, values):
    return State.self.evalCall(values)

#
# class ExternalFunction
#

@patch
def ExternalFunction_eval(self):
    return self

@patch
def ExternalFunction_evalContext(self):
    return None

#
# class Class
#

@patch
def Class_eval(self):
    targets = [(value, value.eval()) for value in self.closed_context]

    if len(targets) == 0:
        return self
    return lekvar.ClosedTarget(self, targets)

@patch
def Class_evalCall(self, values):
    return self.constructor.evalCall(values)

@patch
def Class_evalContext(self):
    return None

@patch
def Class_evalSize(self):
    variables = [value for value in self.instance_context if isinstance(value, lekvar.Variable)]
    sizes = [value.resolveType().evalSize() for value in variables]

    return sum(sizes)

@patch
def Class_evalNewValue(self):
    values = {}

    for value in self.instance_context:
        if not isinstance(value, lekvar.Variable):
            continue

        values[value.name] = value.resolveType().evalNewValue()

    return lekvar.Literal(values, self)

#
# class Constructor
#

@patch
def Constructor_evalCall(self, values):
    self_value = self.constructing.evalNewValue()

    with State.selfScope(self_value):
        lekvar.Function.evalCall(self, values)

    return self_value

#
# class ForwardTarget
#

@patch
def ForwardTarget_eval(self):
    with self.target():
        result = self.value.eval()

    if result is self.value:
        return self
    return result

@patch
def ForwardTarget_evalContext(self):
    raise InternalError("Not Implemented")

@patch
def ForwardTarget_evalCall(self, values):
    with self.target():
        return self.value.evalCall(values)

@patch
def ForwardTarget_evalAssign(self, value):
    raise InternalError("Not Implemented")

@patch
def ForwardTarget_evalNewValue(self):
    with self.target():
        return self.value.evalNewValue()

#
# class ForwardObject
#

@patch
def ForwardObject_eval(self):
    assert self.target is not None
    value = self.target.eval()
    if value is self.target:
        return self
    return value

@patch
def ForwardObject_evalContext(self):
    return self.target.evalContext()

@patch
def ForwardObject_evalCall(self, values):
    return self.target.evalCall(values)

@patch
def ForwardObject_evalAssign(self, value):
    self.target.evalAssign(value)

@patch
def ForwardObject_evalSize(self):
    return self.target.evalSize()

@patch
def ForwardObject_evalNewValue(self):
    return self.target.evalNewValue()

#
# class Literal
#

@patch
def Literal_eval(self):
    return self

#
# class Call
#

@patch
def Call_eval(self):
    with State.selfScope(self.called.eval()):
        called = self.function.eval()

    # Hack, for now
    scope = ExitStack()
    if isinstance(self.function, lekvar.ForwardTarget):
        scope = self.function.target()

    with scope:
        values = [value.eval() for value in self.values]

        context = self.called.evalContext()
        with State.selfScope(context):
            return called.evalCall(values)

@patch
def Call_evalContext(self):
    return self.called.eval()

#
# class Return
#

@patch
def Return_eval(self):
    if self.value is not None:
        self.function.eval_returning = self.value.eval()

    self.function.eval_returned = True
    return None

#
# class Branch
#

@patch
def Branch_eval(self):
    if self.condition is not None:
        value = self.condition.eval()
        assert isinstance(value, lekvar.Literal)

        bool_value = value.data
        if isinstance(value.data, dict):
            bool_value = value.data["value"].data

        if bool_value is False:
            if self.next_branch is not None:
                self.next_branch.eval()
            return

    for instr in self.instructions:
        instr.eval()

#
# class Loop
#

lekvar.Loop.breaking = False

@patch
def Loop_eval(self):
    self.breaking = False

    for instr in cycle(self.instructions):
        instr.eval()

        if self.breaking: break

#
# class Break
#

@patch
def Break_eval(self):
    self.loop.breaking = True

#
# class SizeOf
#

@patch
def SizeOf_evalCall(self, values):
    assert len(values) == 1
    value = values[0]

    return lekvar.Literal(value.evalSize(), self.type.return_type)

#
# class VoidType
#

@patch
def VoidType_eval(self):
    return

@patch
def VoidType_evalNewValue(self):
    return lekvar.Literal(0, self)
<commit_msg>Fixed certain nested calls on instances failing due to wrong context evaluation for methods in interpreter<commit_after>from itertools import cycle
from contextlib import ExitStack

from .. import lekvar
from ..errors import InternalError

from .util import *
from .state import State
from . import builtins

#
# class Object
#

@patch
def Object_eval(self):
    raise InternalError("Not Implemented")

@patch
def Object_evalContext(self):
    raise InternalError("Not Implemented")

@patch
def Object_evalCall(self, values):
    raise InternalError("Not Implemented")

@patch
def Object_evalAssign(self, value):
    raise InternalError("Not Implemented")

#
# class Type
#

@patch
def Type_evalSize(self):
    raise InternalError("Not Implemented")

@patch
def Type_evalNewValue(self):
    raise InternalError("Not Implemented")

#
# class Link
#

@patch
def Link_eval(self):
    return self.value.eval()

@patch
def Link_evalContext(self):
    return self.value.evalContext()

@patch
def Link_evalCall(self, values):
    return self.value.evalCall(values)

@patch
def Link_evalAssign(self, value):
    self.value.evalAssign(value)

@patch
def Link_evalSize(self):
    return self.value.evalSize()

@patch
def Link_evalNewValue(self):
    return self.value.evalNewValue()

#
# class Attribute
#

@patch
def Attribute_eval(self):
    with State.selfScope(self.object.eval()):
        return self.value.eval()

@patch
def Attribute_evalContext(self):
    return self.object.eval()

@patch
def Attribute_evalCall(self, values):
    with State.selfScope(self.object.eval()):
        return self.value.evalCall(values)

@patch
def Attribute_evalAssign(self, value):
    with State.selfScope(self.object.eval()):
        self.value.evalAssign(value)

#
# class ClosedTarget
#

@patch
def ClosedTarget_eval(self):
    with self.target():
        result = self.value.eval()

        if result == self.value:
            return self
        return result

@patch
def ClosedTarget_evalContext(self):
    with self.target():
        return self.value.evalContext()

@patch
def ClosedTarget_evalCall(self, values):
    with self.target():
        return self.value.evalCall(values)

@patch
def ClosedTarget_evalAssign(self, value):
    with self.target():
        self.value.evalAssign(value)

@patch
def ClosedTarget_evalNewValue(self):
    with self.target():
        return self.value.evalNewValue()

#
# class Reference
#

@patch
def Reference_eval(self):
    return self.value.eval()

@patch
def Reference_evalContext(self):
    return self.value.evalContext()

@patch
def Reference_evalCall(self, values):
    return self.value.evalCall(values)

@patch
def Reference_evalAssign(self, value):
    self.value.evalAssign(value)

@patch
def Reference_evalSize(self):
    return builtins.PTR_SIZE

#
# class Module
#

lekvar.Module.evaled = False

@patch
def Module_eval(self):
    if self.evaled: return self
    self.evaled = True

    for instr in self.main:
        instr.eval()

    for obj in self.context:
        obj.eval()

    return self

#
# class Variable
#

@patch
def Variable_eval(self):
    if isinstance(self.parent, lekvar.Class):
        assert State.self is not None

        if not isinstance(State.self, lekvar.Literal) or not isinstance(State.self.data, dict):
            return State.self
        return State.self.data[self.name]

    if self.value is not None:
        return self.value
    return self

@patch
def Variable_evalContext(self):
    return self.eval()

@patch
def Variable_evalAssign(self, value):
    if isinstance(self.parent, lekvar.Class):
        assert State.self is not None
        assert isinstance(State.self, lekvar.Literal)

        State.self.data[self.name] = value
    else:
        self.value = value

@patch
def Variable_evalSize(self):
    return self.value.evalSize()

@patch
def Variable_evalNewValue(self):
    return self.value.evalNewValue()

#
# class Assignment
#

@patch
def Assignment_eval(self):
    self.assigned.evalAssign(self.value.eval())

#
# class Method
#

@patch
def Method_eval(self):
    return self

@patch
def Method_evalContext(self):
    return State.self

@patch
def Method_evalCall(self, values):
    call = lekvar.FunctionType([value.resolveType() for value in values])
    call.verify()
    function = self.resolveCall(call)

    return function.evalCall(values)

#
# class MethodType
#

@patch
def MethodType_evalNewValue(self):
    return lekvar.Literal({}, self)

#
# class MethodInstance
#

@patch
def MethodInstance_eval(self):
    return State.self

@patch
def MethodInstance_evalCall(self, values):
    method = State.self

    call = lekvar.FunctionType([value.resolveType() for value in values])
    function = method.resolveCall(call)

    return function.evalCall(values)

#
# class Function
#

lekvar.Function.eval_returning = None
lekvar.Function.eval_returned = None

@patch
def Function_eval(self):
    return self

@patch
def Function_evalContext(self):
    #TODO: Implement closures
    assert len(self.closed_context) == 0

    return None

@patch
def Function_evalCall(self, values):
    self.eval_returned = False
    self.eval_returning = None

    previous_values = [arg.resolveValue().value for arg in self.arguments]
    for arg, val in zip(self.arguments, values):
        arg.evalAssign(val)

    for instr in self.instructions:
        instr.eval()

        if self.eval_returned:
            break

    for arg, val in zip(self.arguments, previous_values):
        arg.evalAssign(val)

    returning = self.eval_returning
    self.eval_returned = False
    self.eval_returning = None

    return returning

#
# class FunctionInstance
#

@patch
def FunctionInstance_eval(self):
    return self

@patch
def FunctionInstance_evalCall(self, values):
    return State.self.evalCall(values)

#
# class ExternalFunction
#

@patch
def ExternalFunction_eval(self):
    return self

@patch
def ExternalFunction_evalContext(self):
    return None

#
# class Class
#

@patch
def Class_eval(self):
    targets = [(value, value.eval()) for value in self.closed_context]

    if len(targets) == 0:
        return self
    return lekvar.ClosedTarget(self, targets)

@patch
def Class_evalCall(self, values):
    return self.constructor.evalCall(values)

@patch
def Class_evalContext(self):
    return None

@patch
def Class_evalSize(self):
    variables = [value for value in self.instance_context if isinstance(value, lekvar.Variable)]
    sizes = [value.resolveType().evalSize() for value in variables]

    return sum(sizes)

@patch
def Class_evalNewValue(self):
    values = {}

    for value in self.instance_context:
        if not isinstance(value, lekvar.Variable):
            continue

        values[value.name] = value.resolveType().evalNewValue()

    return lekvar.Literal(values, self)

#
# class Constructor
#

@patch
def Constructor_evalCall(self, values):
    self_value = self.constructing.evalNewValue()

    with State.selfScope(self_value):
        lekvar.Function.evalCall(self, values)

    return self_value

#
# class ForwardTarget
#

@patch
def ForwardTarget_eval(self):
    with self.target():
        result = self.value.eval()

    if result is self.value:
        return self
    return result

@patch
def ForwardTarget_evalContext(self):
    raise InternalError("Not Implemented")

@patch
def ForwardTarget_evalCall(self, values):
    with self.target():
        return self.value.evalCall(values)

@patch
def ForwardTarget_evalAssign(self, value):
    raise InternalError("Not Implemented")

@patch
def ForwardTarget_evalNewValue(self):
    with self.target():
        return self.value.evalNewValue()

#
# class ForwardObject
#

@patch
def ForwardObject_eval(self):
    assert self.target is not None
    value = self.target.eval()
    if value is self.target:
        return self
    return value

@patch
def ForwardObject_evalContext(self):
    return self.target.evalContext()

@patch
def ForwardObject_evalCall(self, values):
    return self.target.evalCall(values)

@patch
def ForwardObject_evalAssign(self, value):
    self.target.evalAssign(value)

@patch
def ForwardObject_evalSize(self):
    return self.target.evalSize()

@patch
def ForwardObject_evalNewValue(self):
    return self.target.evalNewValue()

#
# class Literal
#

@patch
def Literal_eval(self):
    return self

#
# class Call
#

@patch
def Call_eval(self):
    with State.selfScope(self.called.eval()):
        called = self.function.eval()

    # Hack, for now
    scope = ExitStack()
    if isinstance(self.function, lekvar.ForwardTarget):
        scope = self.function.target()

    with scope:
        values = [value.eval() for value in self.values]

        context = self.called.evalContext()
        with State.selfScope(context):
            return called.evalCall(values)

@patch
def Call_evalContext(self):
    return self.called.eval()

#
# class Return
#

@patch
def Return_eval(self):
    if self.value is not None:
        self.function.eval_returning = self.value.eval()

    self.function.eval_returned = True
    return None

#
# class Branch
#

@patch
def Branch_eval(self):
    if self.condition is not None:
        value = self.condition.eval()
        assert isinstance(value, lekvar.Literal)

        bool_value = value.data
        if isinstance(value.data, dict):
            bool_value = value.data["value"].data

        if bool_value is False:
            if self.next_branch is not None:
                self.next_branch.eval()
            return

    for instr in self.instructions:
        instr.eval()

#
# class Loop
#

lekvar.Loop.breaking = False

@patch
def Loop_eval(self):
    self.breaking = False

    for instr in cycle(self.instructions):
        instr.eval()

        if self.breaking: break

#
# class Break
#

@patch
def Break_eval(self):
    self.loop.breaking = True

#
# class SizeOf
#

@patch
def SizeOf_evalCall(self, values):
    assert len(values) == 1
    value = values[0]

    return lekvar.Literal(value.evalSize(), self.type.return_type)

#
# class VoidType
#

@patch
def VoidType_eval(self):
    return

@patch
def VoidType_evalNewValue(self):
    return lekvar.Literal(0, self)
<|endoftext|>