<commit_before>
import csv
with open('../Data/Test_Data//RECS2009/recs2009_public.csv') as csvfile:
    readCSV = csv.reader(csvfile, delimiter=',')

    totrooms = []
    playsta1 = []

    for row in readCSV:
        room_total = row[34]
        playstation = row[246]

        totrooms.append(room_total)
        playsta1.append(playstation)

    print(totrooms)
    print(playsta1)
<commit_msg>Converted the lists to np.array and found averages<commit_after>
import csv
import numpy as np

with open("/home/adam/GitHub/RedRocksCommunityCollege/Clair-Global-Collab/Data/Test_Data/RECS2009/recs2009_public.csv") as csvfile:
    readCSV = csv.reader(csvfile, delimiter=',')

    totrooms = []
    playsta1 = []

    for row in readCSV:
        room_total = row[34]
        playstation = row[246]

        totrooms.append(room_total)
        playsta1.append(playstation)

    print(totrooms)
    print(playsta1)

#Check the types
type(totrooms)
type(playsta1)

#Convert to a numpy array
roomarray = np.delete(np.asarray(totrooms),0,0) #np.delete will remove the name since it is the first entry.
plarray = np.delete(np.asarray(playsta1),0,0)

print(roomarray)
roomarray[0] #Pull the first element of the arr
np.shape(roomarray)

print(plarray)
playsta1[0]
np.shape(plarray)

room = np.array([])

for i in range(0,np.shape(roomarray)[0]):
    d = int(roomarray[i])
    room = np.append(room,d)

np.sum(room)
np.sum(room)/np.shape(roomarray)[0]

games = np.array([])
for i in range(0,np.shape(plarray)[0]):
    d = int(plarray[i])
    games = np.append(games,d)

np.sum(games)
np.sum(games)/np.shape(plarray)[0]
<|endoftext|>


<commit_before>""" A stylish alternative for caching your map tiles.

TileStache is a Python-based server application that can serve up map tiles
based on rendered geographic data. You might be familiar with TileCache
(http://tilecache.org), the venerable open source WMS server from MetaCarta.
TileStache is similar, but we hope simpler and better-suited to the needs of
designers and cartographers.

Documentation available at http://tilestache.org/doc/
"""
__version__ = 'N.N.N'

import re

from sys import stdout
try:
    from urlparse import parse_qs
except ImportError:
    from cgi import parse_qs
from StringIO import StringIO
from os.path import dirname, join as pathjoin, realpath
from datetime import datetime, timedelta
from urlparse import urljoin, urlparse
from urllib import urlopen
from os import getcwd
from time import time

try:
    from json import load as json_load
except ImportError:
    from simplejson import load as json_load

from ModestMaps.Core import Coordinate

import Core
import Config

# dictionary of configuration objects for requestLayer().
_previous_configs = {}

# dictionary of tiles seen recently in this process, when ignore_cached is on.
_recent_tiles = {}

# regular expression for PATH_INFO
_pathinfo_pat = re.compile(r'^/?(?P<l>\w.+)/(?P<z>\d+)/(?P<x>-?\d+)/(?P<y>-?\d+)\.(?P<e>\w+)$')
_preview_pat = re.compile(r'^/?(?P<l>\w.+)/(preview\.html)?$')

def getTile(layer, coord, extension, ignore_cached=False):
    """ Get a type string and tile binary for a given request layer tile.

        Arguments:
        - layer: instance of Core.Layer to render.
        - coord: one ModestMaps.Core.Coordinate corresponding to a single tile.
        - extension: filename extension to choose response type, e.g. "png" or "jpg".
        - ignore_cached: always re-render the tile, whether it's in the cache or not.

        This is the main entry point, after site configuration has been loaded
        and individual tiles need to be rendered.
    """
    mimetype, format = layer.getTypeByExtension(extension)
    cache = layer.config.cache

    # key in _recent_tiles
    _tile = (layer, coord, extension)

    if not ignore_cached:
        # Start by checking for a tile in the cache.
        body = cache.read(layer, coord, format)

    elif _tile in _recent_tiles:
        # Then look in the bag of recent tiles.
        body, use_by = _recent_tiles[_tile]
        if time() > use_by:
            del _recent_tiles[_tile]
            body = None

    else:
        # Bypass the cache
        body = None

    # If no tile was found, dig deeper
    if body is None:
        try:
            lockCoord = None

            if layer.write_cache:
                # this is the coordinate that actually gets locked.
                lockCoord = layer.metatile.firstCoord(coord)

                # We may need to write a new tile, so acquire a lock.
                cache.lock(layer, lockCoord, format)

            if not ignore_cached:
                # There's a chance that some other process has
                # written the tile while the lock was being acquired.
                body = cache.read(layer, coord, format)

            if body is None:
                # No one else wrote the tile, do it here.
                buff = StringIO()

                try:
                    tile = layer.render(coord, format)
                    save = True
                except Core.NoTileLeftBehind, e:
                    tile = e.tile
                    save = False

                if not layer.write_cache:
                    save = False

                if format.lower() == 'jpeg':
                    save_kwargs = layer.jpeg_options
                elif format.lower() == 'png':
                    save_kwargs = layer.png_options
                else:
                    save_kwargs = {}

                tile.save(buff, format, **save_kwargs)
                body = buff.getvalue()

                if save:
                    cache.save(body, layer, coord, format)

        finally:
            if lockCoord:
                # Always clean up a lock when it's no longer being used.
                cache.unlock(layer, lockCoord, format)

    if ignore_cached:
        _recent_tiles[_tile] = body, time() + 300

    return mimetype, body

def getPreview(layer):
    """ Get a type string and dynamic map viewer HTML for a given layer.
    """
    return 'text/html', Core._preview(layer)

def parseConfigfile(configpath):
    """ Parse a configuration file and return a Configuration object.

        Configuration file is formatted as JSON with two sections, "cache" and "layers":

          {
            "cache": { ... },
            "layers": {
              "layer-1": { ... },
              "layer-2": { ... },
              ...
            }
          }

        The full path to the file is significant, used to
        resolve any relative paths found in the configuration.

        See the Caches module for more information on the "caches" section,
        and the Core and Providers modules for more information on the
        "layers" section.
    """
    config_dict = json_load(urlopen(configpath))

    scheme, host, path, p, q, f = urlparse(configpath)

    if scheme == '':
        scheme = 'file'
        path = realpath(path)

    dirpath = '%s://%s%s' % (scheme, host, dirname(path).rstrip('/') + '/')

    return Config.buildConfiguration(config_dict, dirpath)

def splitPathInfo(pathinfo):
    """ Converts a PATH_INFO string to layer name, coordinate, and extension parts.

        Example: "/layer/0/0/0.png", leading "/" optional.
    """
    if _pathinfo_pat.match(pathinfo):
        path = _pathinfo_pat.match(pathinfo)
        layer, row, column, zoom, extension = [path.group(p) for p in 'lyxze']
        coord = Coordinate(int(row), int(column), int(zoom))

    elif _preview_pat.match(pathinfo):
        path = _preview_pat.match(pathinfo)
        layer, extension = path.group('l'), 'html'
        coord = None

    else:
        raise Core.KnownUnknown('Bad path: "%s". I was expecting something more like "/example/0/0/0.png"' % pathinfo)

    return layer, coord, extension

def requestLayer(config, path_info):
    """ Return a Layer.

        Requires a configuration and PATH_INFO (e.g. "/example/0/0/0.png").

        Config parameter can be a file path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.
    """
    if type(config) in (str, unicode):
        #
        # Should be a path to a configuration file we can load;
        # build a tuple key into previously-seen config objects.
        #
        key = hasattr(config, '__hash__') and (config, getcwd())

        if key in _previous_configs:
            config = _previous_configs[key]

        else:
            config = parseConfigfile(config)

            if key:
                _previous_configs[key] = config

    else:
        assert hasattr(config, 'cache'), 'Configuration object must have a cache.'
        assert hasattr(config, 'layers'), 'Configuration object must have layers.'
        assert hasattr(config, 'dirpath'), 'Configuration object must have a dirpath.'

    layername = splitPathInfo(path_info)[0]

    if layername not in config.layers:
        raise Core.KnownUnknown('"%s" is not a layer I know about. Here are some that I do know about: %s.' % (layername, ', '.join(sorted(config.layers.keys()))))

    return config.layers[layername]

def requestHandler(config, path_info, query_string):
    """ Generate a mime-type and response body for a given request.

        Requires a configuration and PATH_INFO (e.g. "/example/0/0/0.png").

        Config parameter can be a file path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.

        Query string is optional, currently used for JSON callbacks.

        Calls getTile() to render actual tiles, and getPreview() to render preview.html.
    """
    try:
        if path_info is None:
            raise Core.KnownUnknown('Missing path_info in requestHandler().')

        layer = requestLayer(config, path_info)
        query = parse_qs(query_string or '')
        try:
            callback = query['callback'][0]
        except KeyError:
            callback = None

        coord, extension = splitPathInfo(path_info)[1:]

        if extension == 'html' and coord is None:
            mimetype, content = getPreview(layer)

        else:
            mimetype, content = getTile(layer, coord, extension)

        if callback and 'json' in mimetype:
            mimetype, content = 'application/javascript', '%s(%s)' % (callback, content)

    except Core.KnownUnknown, e:
        out = StringIO()

        print >> out, 'Known unknown!'
        print >> out, e
        print >> out, ''
        print >> out, '\n'.join(Core._rummy())

        mimetype, content = 'text/plain', out.getvalue()

    return mimetype, content

def cgiHandler(environ, config='./tilestache.cfg', debug=False):
    """ Read environment PATH_INFO, load up configuration, talk to stdout by CGI.

        Calls requestHandler().

        Config parameter can be a file path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.
    """
    if debug:
        import cgitb
        cgitb.enable()

    path_info = environ.get('PATH_INFO', None)
    query_string = environ.get('QUERY_STRING', None)

    mimetype, content = requestHandler(config, path_info, query_string)
    layer = requestLayer(config, path_info)

    if layer.allowed_origin:
        print >> stdout, 'Access-Control-Allow-Origin:', layer.allowed_origin

    if layer.max_cache_age is not None:
        expires = datetime.utcnow() + timedelta(seconds=layer.max_cache_age)
        print >> stdout, 'Expires:', expires.strftime('%a %d %b %Y %H:%M:%S GMT')
        print >> stdout, 'Cache-Control: public, max-age=%d' % layer.max_cache_age

    print >> stdout, 'Content-Length: %d' % len(content)
    print >> stdout, 'Content-Type: %s\n' % mimetype
    print >> stdout, content

class WSGITileServer:
    """ Create a WSGI application that can handle requests from any server that talks WSGI.

        The WSGI application is an instance of this class. Example:

          app = WSGITileServer('/path/to/tilestache.cfg')
          werkzeug.serving.run_simple('localhost', 8080, app)
    """

    def __init__(self, config, autoreload=False):
        """ Initialize a callable WSGI instance.

            Config parameter can be a file path string for a JSON configuration
            file or a configuration object with 'cache', 'layers', and
            'dirpath' properties.

            Optional autoreload boolean parameter causes config to be re-read
            on each request, applicable only when config is a JSON file.
        """

        if type(config) in (str, unicode):
            self.autoreload = autoreload
            self.config_path = config

            try:
                self.config = parseConfigfile(config)
            except Exception, e:
                raise Core.KnownUnknown("Error loading Tilestache config file:\n%s" % str(e))

        else:
            assert hasattr(config, 'cache'), 'Configuration object must have a cache.'
            assert hasattr(config, 'layers'), 'Configuration object must have layers.'
            assert hasattr(config, 'dirpath'), 'Configuration object must have a dirpath.'

            self.autoreload = False
            self.config_path = None
            self.config = config

    def __call__(self, environ, start_response):
        """
        """
        if self.autoreload: # re-parse the config file on every request
            try:
                self.config = parseConfigfile(self.config_path)
            except Exception, e:
                raise Core.KnownUnknown("Error loading Tilestache config file:\n%s" % str(e))

        try:
            layer, coord, ext = splitPathInfo(environ['PATH_INFO'])
        except Core.KnownUnknown, e:
            return self._response(start_response, '400 Bad Request', str(e))

        if layer not in self.config.layers:
            return self._response(start_response, '404 Not Found')

        mimetype, content = requestHandler(self.config, environ['PATH_INFO'], environ['QUERY_STRING'])
        request_layer = requestLayer(self.config, environ['PATH_INFO'])
        allowed_origin = request_layer.allowed_origin
        max_cache_age = request_layer.max_cache_age
        return self._response(start_response, '200 OK', str(content), mimetype, allowed_origin, max_cache_age)

    def _response(self, start_response, code, content='', mimetype='text/plain', allowed_origin='', max_cache_age=None):
        """
        """
        headers = [('Content-Type', mimetype), ('Content-Length', str(len(content)))]

        if allowed_origin:
            headers.append(('Access-Control-Allow-Origin', allowed_origin))

        if max_cache_age is not None:
            expires = datetime.utcnow() + timedelta(seconds=max_cache_age)
            headers.append(('Expires', expires.strftime('%a %d %b %Y %H:%M:%S GMT')))
            headers.append(('Cache-Control', 'public, max-age=%d' % max_cache_age))

        start_response(code, headers)
        return [content]

def modpythonHandler(request):
    """ Handle a mod_python request.

        Calls requestHandler().

        Example Apache configuration for TileStache:

        <Directory /home/migurski/public_html/TileStache>
            AddHandler mod_python .py
            PythonHandler TileStache::modpythonHandler
            PythonOption config /etc/tilestache.cfg
        </Directory>

        Configuration options, using PythonOption directive:
        - config: path to configuration file, defaults to "tilestache.cfg",
            using request.filename as the current working directory.
    """
    from mod_python import apache

    config_path = request.get_options().get('config', 'tilestache.cfg')
    config_path = realpath(pathjoin(dirname(request.filename), config_path))

    path_info = request.path_info
    query_string = request.args

    mimetype, content = requestHandler(config_path, path_info, query_string)

    request.status = apache.HTTP_OK
    request.content_type = mimetype
    request.set_content_length(len(content))
    request.send_http_header()

    request.write(content)

    return apache.OK
<commit_msg>Added support for basic index page instead of throwing errors<commit_after>""" A stylish alternative for caching your map tiles.

TileStache is a Python-based server application that can serve up map tiles
based on rendered geographic data. You might be familiar with TileCache
(http://tilecache.org), the venerable open source WMS server from MetaCarta.
TileStache is similar, but we hope simpler and better-suited to the needs of
designers and cartographers.

Documentation available at http://tilestache.org/doc/
"""
__version__ = 'N.N.N'

import re

from sys import stdout
try:
    from urlparse import parse_qs
except ImportError:
    from cgi import parse_qs
from StringIO import StringIO
from os.path import dirname, join as pathjoin, realpath
from datetime import datetime, timedelta
from urlparse import urljoin, urlparse
from urllib import urlopen
from os import getcwd
from time import time

try:
    from json import load as json_load
except ImportError:
    from simplejson import load as json_load

from ModestMaps.Core import Coordinate

import Core
import Config

# dictionary of configuration objects for requestLayer().
_previous_configs = {}

# dictionary of tiles seen recently in this process, when ignore_cached is on.
_recent_tiles = {}

# regular expression for PATH_INFO
_pathinfo_pat = re.compile(r'^/?(?P<l>\w.+)/(?P<z>\d+)/(?P<x>-?\d+)/(?P<y>-?\d+)\.(?P<e>\w+)$')
_preview_pat = re.compile(r'^/?(?P<l>\w.+)/(preview\.html)?$')

def getTile(layer, coord, extension, ignore_cached=False):
    """ Get a type string and tile binary for a given request layer tile.

        Arguments:
        - layer: instance of Core.Layer to render.
        - coord: one ModestMaps.Core.Coordinate corresponding to a single tile.
        - extension: filename extension to choose response type, e.g. "png" or "jpg".
        - ignore_cached: always re-render the tile, whether it's in the cache or not.

        This is the main entry point, after site configuration has been loaded
        and individual tiles need to be rendered.
    """
    mimetype, format = layer.getTypeByExtension(extension)
    cache = layer.config.cache

    # key in _recent_tiles
    _tile = (layer, coord, extension)

    if not ignore_cached:
        # Start by checking for a tile in the cache.
        body = cache.read(layer, coord, format)

    elif _tile in _recent_tiles:
        # Then look in the bag of recent tiles.
        body, use_by = _recent_tiles[_tile]
        if time() > use_by:
            del _recent_tiles[_tile]
            body = None

    else:
        # Bypass the cache
        body = None

    # If no tile was found, dig deeper
    if body is None:
        try:
            lockCoord = None

            if layer.write_cache:
                # this is the coordinate that actually gets locked.
                lockCoord = layer.metatile.firstCoord(coord)

                # We may need to write a new tile, so acquire a lock.
                cache.lock(layer, lockCoord, format)

            if not ignore_cached:
                # There's a chance that some other process has
                # written the tile while the lock was being acquired.
                body = cache.read(layer, coord, format)

            if body is None:
                # No one else wrote the tile, do it here.
                buff = StringIO()

                try:
                    tile = layer.render(coord, format)
                    save = True
                except Core.NoTileLeftBehind, e:
                    tile = e.tile
                    save = False

                if not layer.write_cache:
                    save = False

                if format.lower() == 'jpeg':
                    save_kwargs = layer.jpeg_options
                elif format.lower() == 'png':
                    save_kwargs = layer.png_options
                else:
                    save_kwargs = {}

                tile.save(buff, format, **save_kwargs)
                body = buff.getvalue()

                if save:
                    cache.save(body, layer, coord, format)

        finally:
            if lockCoord:
                # Always clean up a lock when it's no longer being used.
                cache.unlock(layer, lockCoord, format)

    if ignore_cached:
        _recent_tiles[_tile] = body, time() + 300

    return mimetype, body

def getPreview(layer):
    """ Get a type string and dynamic map viewer HTML for a given layer.
    """
    return 'text/html', Core._preview(layer)

def parseConfigfile(configpath):
    """ Parse a configuration file and return a Configuration object.

        Configuration file is formatted as JSON with two sections, "cache" and "layers":

          {
            "cache": { ... },
            "layers": {
              "layer-1": { ... },
              "layer-2": { ... },
              ...
            }
          }

        The full path to the file is significant, used to
        resolve any relative paths found in the configuration.

        See the Caches module for more information on the "caches" section,
        and the Core and Providers modules for more information on the
        "layers" section.
    """
    config_dict = json_load(urlopen(configpath))

    scheme, host, path, p, q, f = urlparse(configpath)

    if scheme == '':
        scheme = 'file'
        path = realpath(path)

    dirpath = '%s://%s%s' % (scheme, host, dirname(path).rstrip('/') + '/')

    return Config.buildConfiguration(config_dict, dirpath)

def splitPathInfo(pathinfo):
    """ Converts a PATH_INFO string to layer name, coordinate, and extension parts.

        Example: "/layer/0/0/0.png", leading "/" optional.
    """
    if _pathinfo_pat.match(pathinfo or ''):
        path = _pathinfo_pat.match(pathinfo)
        layer, row, column, zoom, extension = [path.group(p) for p in 'lyxze']
        coord = Coordinate(int(row), int(column), int(zoom))

    elif _preview_pat.match(pathinfo or ''):
        path = _preview_pat.match(pathinfo)
        layer, extension = path.group('l'), 'html'
        coord = None

    else:
        raise Core.KnownUnknown('Bad path: "%s". I was expecting something more like "/example/0/0/0.png"' % pathinfo)

    return layer, coord, extension

def requestLayer(config, path_info):
    """ Return a Layer.

        Requires a configuration and PATH_INFO (e.g. "/example/0/0/0.png").

        Config parameter can be a file path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.
    """
    if type(config) in (str, unicode):
        #
        # Should be a path to a configuration file we can load;
        # build a tuple key into previously-seen config objects.
        #
        key = hasattr(config, '__hash__') and (config, getcwd())

        if key in _previous_configs:
            config = _previous_configs[key]

        else:
            config = parseConfigfile(config)

            if key:
                _previous_configs[key] = config

    else:
        assert hasattr(config, 'cache'), 'Configuration object must have a cache.'
        assert hasattr(config, 'layers'), 'Configuration object must have layers.'
        assert hasattr(config, 'dirpath'), 'Configuration object must have a dirpath.'

    # ensure that path_info is at least a single "/"
    path_info = '/' + (path_info or '').lstrip('/')

    if path_info == '/':
        return Core.Layer(config, None, None)

    layername = splitPathInfo(path_info)[0]

    if layername not in config.layers:
        raise Core.KnownUnknown('"%s" is not a layer I know about. Here are some that I do know about: %s.' % (layername, ', '.join(sorted(config.layers.keys()))))

    return config.layers[layername]

def requestHandler(config, path_info, query_string):
    """ Generate a mime-type and response body for a given request.

        Requires a configuration and PATH_INFO (e.g. "/example/0/0/0.png").

        Config parameter can be a file path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.

        Query string is optional, currently used for JSON callbacks.

        Calls getTile() to render actual tiles, and getPreview() to render preview.html.
    """
    try:
        # ensure that path_info is at least a single "/"
        path_info = '/' + (path_info or '').lstrip('/')

        #
        # Special case for index page.
        #
        if path_info == '/':
            return 'text/html', 'TileStache says hello.'

        layer = requestLayer(config, path_info)
        query = parse_qs(query_string or '')
        try:
            callback = query['callback'][0]
        except KeyError:
            callback = None

        coord, extension = splitPathInfo(path_info)[1:]

        if path_info == '/':
            raise Exception(path_info)

        elif extension == 'html' and coord is None:
            mimetype, content = getPreview(layer)

        else:
            mimetype, content = getTile(layer, coord, extension)

        if callback and 'json' in mimetype:
            mimetype, content = 'application/javascript', '%s(%s)' % (callback, content)

    except Core.KnownUnknown, e:
        out = StringIO()

        print >> out, 'Known unknown!'
        print >> out, e
        print >> out, ''
        print >> out, '\n'.join(Core._rummy())

        mimetype, content = 'text/plain', out.getvalue()

    return mimetype, content

def cgiHandler(environ, config='./tilestache.cfg', debug=False):
    """ Read environment PATH_INFO, load up configuration, talk to stdout by CGI.

        Calls requestHandler().

        Config parameter can be a file path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.
    """
    if debug:
        import cgitb
        cgitb.enable()

    path_info = environ.get('PATH_INFO', None)
    query_string = environ.get('QUERY_STRING', None)

    mimetype, content = requestHandler(config, path_info, query_string)
    layer = requestLayer(config, path_info)

    if layer.allowed_origin:
        print >> stdout, 'Access-Control-Allow-Origin:', layer.allowed_origin

    if layer.max_cache_age is not None:
        expires = datetime.utcnow() + timedelta(seconds=layer.max_cache_age)
        print >> stdout, 'Expires:', expires.strftime('%a %d %b %Y %H:%M:%S GMT')
        print >> stdout, 'Cache-Control: public, max-age=%d' % layer.max_cache_age

    print >> stdout, 'Content-Length: %d' % len(content)
    print >> stdout, 'Content-Type: %s\n' % mimetype
    print >> stdout, content

class WSGITileServer:
    """ Create a WSGI application that can handle requests from any server that talks WSGI.

        The WSGI application is an instance of this class. Example:

          app = WSGITileServer('/path/to/tilestache.cfg')
          werkzeug.serving.run_simple('localhost', 8080, app)
    """

    def __init__(self, config, autoreload=False):
        """ Initialize a callable WSGI instance.

            Config parameter can be a file path string for a JSON configuration
            file or a configuration object with 'cache', 'layers', and
            'dirpath' properties.

            Optional autoreload boolean parameter causes config to be re-read
            on each request, applicable only when config is a JSON file.
        """

        if type(config) in (str, unicode):
            self.autoreload = autoreload
            self.config_path = config

            try:
                self.config = parseConfigfile(config)
            except Exception, e:
                raise Core.KnownUnknown("Error loading Tilestache config file:\n%s" % str(e))

        else:
            assert hasattr(config, 'cache'), 'Configuration object must have a cache.'
            assert hasattr(config, 'layers'), 'Configuration object must have layers.'
            assert hasattr(config, 'dirpath'), 'Configuration object must have a dirpath.'

            self.autoreload = False
            self.config_path = None
            self.config = config

    def __call__(self, environ, start_response):
        """
        """
        if self.autoreload: # re-parse the config file on every request
            try:
                self.config = parseConfigfile(self.config_path)
            except Exception, e:
                raise Core.KnownUnknown("Error loading Tilestache config file:\n%s" % str(e))

        try:
            layer, coord, ext = splitPathInfo(environ['PATH_INFO'])
        except Core.KnownUnknown, e:
            return self._response(start_response, '400 Bad Request', str(e))

        if layer not in self.config.layers:
            return self._response(start_response, '404 Not Found')

        mimetype, content = requestHandler(self.config, environ['PATH_INFO'], environ['QUERY_STRING'])
        request_layer = requestLayer(self.config, environ['PATH_INFO'])
        allowed_origin = request_layer.allowed_origin
        max_cache_age = request_layer.max_cache_age
        return self._response(start_response, '200 OK', str(content), mimetype, allowed_origin, max_cache_age)

    def _response(self, start_response, code, content='', mimetype='text/plain', allowed_origin='', max_cache_age=None):
        """
        """
        headers = [('Content-Type', mimetype), ('Content-Length', str(len(content)))]

        if allowed_origin:
            headers.append(('Access-Control-Allow-Origin', allowed_origin))

        if max_cache_age is not None:
            expires = datetime.utcnow() + timedelta(seconds=max_cache_age)
            headers.append(('Expires', expires.strftime('%a %d %b %Y %H:%M:%S GMT')))
            headers.append(('Cache-Control', 'public, max-age=%d' % max_cache_age))

        start_response(code, headers)
        return [content]

def modpythonHandler(request):
    """ Handle a mod_python request.

        Calls requestHandler().

        Example Apache configuration for TileStache:

        <Directory /home/migurski/public_html/TileStache>
            AddHandler mod_python .py
            PythonHandler TileStache::modpythonHandler
            PythonOption config /etc/tilestache.cfg
        </Directory>

        Configuration options, using PythonOption directive:
        - config: path to configuration file, defaults to "tilestache.cfg",
            using request.filename as the current working directory.
    """
    from mod_python import apache

    config_path = request.get_options().get('config', 'tilestache.cfg')
    config_path = realpath(pathjoin(dirname(request.filename), config_path))

    path_info = request.path_info
    query_string = request.args

    mimetype, content = requestHandler(config_path, path_info, query_string)

    request.status = apache.HTTP_OK
    request.content_type = mimetype
    request.set_content_length(len(content))
    request.send_http_header()

    request.write(content)

    return apache.OK
<|endoftext|>


<commit_before>from System import Action
import clr
clr.AddReference('Vsync') # The profile of the dll file.
import Vsync
from SimpleXMLRPCServer import SimpleXMLRPCServer
from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler
import json
import os.path

#from threading import Thread

Vsync.VsyncSystem.Start()

# define RequestHandler
class RequestHandler(SimpleXMLRPCRequestHandler):
    rpc_paths = ('/RPC2',)

# define the server for later use
server = SimpleXMLRPCServer(("localhost", 8000),requestHandler=RequestHandler)
server.register_introspection_functions()

group = Vsync.Group("Master")

print "Created Master group"
users = {}
schedules = {}
messages = {}
cache = {}
counter = 0
masterId = -1
myRank = -1

'''
global
'''
def backup():
    '''
        backup data.
    '''
    global counter
    return

    counter = counter + 1
    if counter % 15 == 1:
	counter = 1
    	print "###backing#####"
    	print "user length"+str(len(users.keys()))
    	print "schedule length"+str(len(schedules.keys()))
    	print "message length"+str(len(messages.keys()))
    	print "cache length"+str(len(cache.keys()))

    	with open('data.json', 'w') as outfile:
    	    outfile.write(json.dumps({"user":users,"schedule":schedules,"message":messages,"cache":cache}))

def loadBK():
    '''
        load backup.
    '''
    if not os.path.isfile('data.json'):
	return
    global users, schedules, cache, messages

    data = ''
    with open('data.json') as data_file:
	data = data_file.read()

    dataset = json.loads(data)
    users = dataset["user"]
    schedules = dataset["schedule"]
    messages = dataset["message"]
    cache = dataset["cache"]

    del dataset

"""
    User DHT
"""
def postUserData_api(id, profile):
	"""
            Forward user POST to vsync
            @param {string} id - user id
            @param {string} profile - stringified user profile
	"""
        # delegate to vsync
	group.Send(0, id, profile)

	return "profile" # just for flask

def postUserData(id, profile):
        '''
            add a profile with id to the users DHT.
            @param {string} id - user id
            @param {string} profile - stringified user profile
        '''
        # for debugging
        if profile == "-1":
            print "Delete the user with id :"+id
            del users[id]
        else:
            print "Add the user with id :"+id
	    users[id] = profile
        backup()

def getUserData_api(id):
	"""
            Forward user GET to vsync
            @param {string} id - user id
            @return {string} - either return the profile content or -1 for not found / deleted
	"""
        try:
	    if id in users:
	    	return users[id]
	    else:
	    	res = []
	    	nr = group.Query(Vsync.Group.ALL, 1, id, Vsync.EOLMarker(), res)
	    	for ele in res:
	    		if ele != "-1":
	    			return ele
	    return "-1"
        except:
            return "-1"

def getUserData(id):
        '''
            get a user profile from the user DHT with the given id.
            @param {string} id - user id
        '''
	if id in users:
		group.Reply(users[id])
	else:
		group.Reply(-1)
        print("Get user with id :" + id)

def removeUserData_api(id):
	"""
            Forward user DELETE to vsync
            @param {string} id - user id
	"""
        # delegate to vsync post
	group.Send(0, id, '-1')
	return "profile" # just for flask

"""
Message DHT
"""
def postMessageData_api(id, message):
	"""
            Forward a message POST to vsync.
            @param {string} id - message id
            @param {string} message - message content
	"""
        # delegate to vsync
	group.Send(3, id, message)
	return "message" # just for flask

def postMessageData(id, message):
        '''
            Add a message to message DHT with a id.
            @param {string} id - message id
            @param {string} message - message content
        '''

        # for debugging
        if message == "-1":
            print "Delete the message with id :"+id
            del messages[id]
        else:
            print "Add the message with id :"+id
	    messages[id] = message
        backup()

def getMessageData_api(id):
	"""
            Forward message GET to vsync.
            @param {string} id - message id
            @return {string} - either return the message content or -1 for not found / deleted
	"""
        try:
	    if id in messages:
	    	return messages[id]
	    else:
	    	res = []
	    	nr = group.Query(Vsync.Group.ALL, 4, id, Vsync.EOLMarker(), res)
	    	print "result from group getting message"
	    	print res
	    	for ele in res:
	    		if ele != "-1":
	    			return ele
	    return "-1"
        except:
            return "-1"

def getMessageData(id):
        '''
            get a message from the message DHT with the given id.
            @param {string} id - message id
        '''
	if id in messages:
		group.Reply(messages[id])
	else:
		group.Reply('-1')
	print("Vsync server getMessageData with id=" + id.ToString())

def removeMessageData_api(id):
	"""
            Forward message DELETE to user.
            @param {string} id - message id
	"""
	group.Send(3, id, '-1')
	return "message" # just for flask

"""
    Schedule DHT
"""
def postScheduleData_api(id, schedule):
	"""
            Forward schedule POST to vsync
            @param {string} id - schedule id
            @param {string} schedule - schedule content
	"""
	group.Send(6, id, schedule)
	return "schedule" # just for flask

def postScheduleData(id, schedule):
        '''
            add a schedule with id to the schedule DHT.
            @param {string} id - schedule id
            @param {string} schedule - schedule content
        '''
        # for debugging
        if schedule == "-1":
            print "Delete the schedule with id :"+id
            del schedules[id]
        else:
            print "Add the schedule with id :"+id
	    schedules[id] = schedule

        backup()

def getScheduleData_api(id):
	"""
            Forward schedule GET to vsync
            @param {string} id - schedule id
            @return {string} - either return schedule content or -1 if not found / deleted
	"""
        try:
	    if id in schedules and schedules[id] != "-1":
	    	return schedules[id]
	    else:
	    	res = []
	    	nr = group.Query(Vsync.Group.ALL, 7, id, Vsync.EOLMarker(), res)
	    	print "result from group getting"
	    	print res
	    	for ele in res:
	    		if ele != "-1":
	    			return ele
	    return "-1"
        except:
            return "-1"

def getScheduleData(id):
        '''
            get a schedule from the schedule DHT with the given id.
            @param {string} id - schedule id
        '''
	if id in schedules:
		group.Reply(schedules[id])
	else:
		group.Reply('-1')
	print("Vsync server getScheduleData with id=" + id.ToString())

def removeScheduleData_api(id):
	"""
            Forward schedule DELETE to vsync POST with content as -1.
            @param {string} id - schedule id
	"""
	group.Send(6, id, '-1')
	return "profile"

"""
    Cache DHT - Define cache collection (cache_id - '{time:"1231231212.123",id_list:"[]"}'
"""
def postCacheData_api(id, cache_content):
	"""
            Forward cache POST to vsync
            @param {string} id - cache id
            @param {string} cache - cache content
	"""
        # delegate to vsync
	group.Send(9, id, cache_content)
        # just for flask
	return "cache"

def postCacheData(id, cache_content):
        '''
            add a cache to Cache DHT with given id.
            @param {string} id - cache id
            @param {string} cache - cache content
        '''
        # for debugging
        if cache_content == "-1":
            print "Delete the user with id :"+id
            del cache[id]
        else:
            cache[id] = cache_content
            print "Add the user with id :"+id
        backup()

def getCacheData_api(id):
	"""
            Forward cache GET to vsync.
            @param {string} id - cache id
            @return {string} either return the found cache content or -1 represent not found / deleted
	"""
        try:
	    if id in cache:
	    	return cache[id]
	    else:
	    	res = []
	    	nr = group.Query(Vsync.Group.ALL, 10, id, Vsync.EOLMarker(), res)
	    	print "result from group getting cache"
	    	print res
	    	for ele in res:
	    		if ele != "-1":
	    			return ele
	    return "-1"
        except:
            return "-1"

def getCacheData(id):
        '''
            get a schedule from the schedule DHT with the given id.
            @param {string} id - cache id
        '''
	if id in cache:
		group.Reply(cache[id])
	else:
		group.Reply('-1')
	print("Vsync server get Cache with id=" + id.ToString())

def removeCacheData_api(id):
	"""
            Forward cache DELETE to vsync, remove the cache = set the value for this key(id) as [], empty list.
            @param {string} id - cache id
            @return {string} useless for now
	"""
        # delegate to vsync
	group.Send(9, id, '-1')
	return "cache"

# for retrieving all user / message / schedule
def getAllUsers_api():
	"""
            Retrieve all users, including all keys and values.
            @return {string} - return stringify content of users
	"""
        return json.dumps(users)

def getAllMessages_api():
	"""
            Retrieve all messages, including all keys and values.
            @return {string} - return stringify content of users
	"""
        return json.dumps(messages)

def getAllCache_api():
	"""
            Retrieve all cache, including all keys and values.
            @return {string} - return stringify content of users
	"""
        return json.dumps(cache)

def getAllSchedules_api():
	"""
            Retrieve all schedules including all keys and values.
            @return {string} - return stringify content of users
	"""
        return json.dumps(schedules)

def giveMeAll(action):
    print action

### Vsycn register
def myViewFunc(v):
    global myRank,masterId
    print('New view: ' + v.ToString())
    print('My rank = ' + v.GetMyRank().ToString())
    myRank = v.GetMyRank()
    for a in v.joiners:
        print('  Joining: ' + a.ToString() + ', isMyAddress='+a.isMyAddress().ToString())
    for a in v.leavers:
        print('  Leaving: ' + a.ToString() + ', isMyAddress='+a.isMyAddress().ToString())
    if (v.GetMyRank().ToString() == "2"):
        # get data from other nodes
        print "#"
        group.Send(15, "givemeall")
    elif:
        print "#load from recovery data"
        loadBK()
    return

# All functions registered by rpc-server

# user related functions
server.register_function(postUserData_api,'postUserData')
server.register_function(getUserData_api, 'getUserData')
server.register_function(removeUserData_api,'removeUserData')
# schedule related functions
server.register_function(postScheduleData_api,'postScheduleData')
server.register_function(getScheduleData_api,'getScheduleData')
server.register_function(removeScheduleData_api,'removeScheduleData')
# message related functions
server.register_function(postMessageData_api,'postMessageData')
server.register_function(getMessageData_api,'getMessageData')
server.register_function(removeMessageData_api,'removeMessageData')
# cache related functions
server.register_function(postCacheData_api,'postCacheData')
server.register_function(getCacheData_api,'getCacheData')
server.register_function(removeCacheData_api,'removeCacheData')
# retrieve all
server.register_function(getAllUsers_api,'getAllUsers')
server.register_function(getAllSchedules_api,'getAllSchedules')
server.register_function(getAllMessages_api,'getAllMessages')
server.register_function(getAllCache_api,'getAllCache')
# view
server.register_function(myViewFunc,'myViewFunc')

# All functions registered by vsync

# user related functions
group.RegisterHandler(0, Action[str, str](postUserData))
group.RegisterHandler(1, Action[str](getUserData))
# group.RegisterHandler(2, Action[str](removeUserData))

# message related functions
group.RegisterHandler(3, Action[str, str](postMessageData))
group.RegisterHandler(4, Action[str](getMessageData))
# group.RegisterHandler(5, Action[str](removeMessageData))

# schedule related functions
group.RegisterHandler(6, Action[str, str](postScheduleData))
group.RegisterHandler(7, Action[str](getScheduleData))
# group.RegisterHandler(8, Action[str](removeScheduleData))

# cache related functions
group.RegisterHandler(9, Action[str, str](postCacheData))
group.RegisterHandler(10, Action[str](getCacheData))
# group.RegisterHandler(11, Action[str](removeCacheData))

# get all data
group.RegisterHandler(15, Action[str](giveMeAll))

# view
group.RegisterViewHandler(Vsync.ViewHandler(myViewFunc))

group.Join()

### run
server.serve_forever()
Vsync.VsyncSystem.WaitForever()
<commit_msg>hotfix<commit_after>from System import Action
import clr
clr.AddReference('Vsync') # The profile of the dll file.
import Vsync
from SimpleXMLRPCServer import SimpleXMLRPCServer
from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler
import json
import os.path

#from threading import Thread

Vsync.VsyncSystem.Start()

# define RequestHandler
class RequestHandler(SimpleXMLRPCRequestHandler):
    rpc_paths = ('/RPC2',)

# define the server for later use
server = SimpleXMLRPCServer(("localhost", 8000),requestHandler=RequestHandler)
server.register_introspection_functions()

group = Vsync.Group("Master")

print "Created Master group"
users = {}
schedules = {}
messages = {}
cache = {}
counter = 0
masterId = -1
myRank = -1

'''
global
'''
def backup():
    '''
        backup data.
    '''
    global counter
    return

    counter = counter + 1
    if counter % 15 == 1:
	counter = 1
    	print "###backing#####"
    	print "user length"+str(len(users.keys()))
    	print "schedule length"+str(len(schedules.keys()))
    	print "message length"+str(len(messages.keys()))
    	print "cache length"+str(len(cache.keys()))

    	with open('data.json', 'w') as outfile:
    	    outfile.write(json.dumps({"user":users,"schedule":schedules,"message":messages,"cache":cache}))

def loadBK():
    '''
        load backup.
    '''
    if not os.path.isfile('data.json'):
	return
    global users, schedules, cache, messages

    data = ''
    with open('data.json') as data_file:
	data = data_file.read()

    dataset = json.loads(data)
    users = dataset["user"]
    schedules = dataset["schedule"]
    messages = dataset["message"]
    cache = dataset["cache"]

    del dataset

"""
    User DHT
"""
def postUserData_api(id, profile):
	"""
            Forward user POST to vsync
            @param {string} id - user id
            @param {string} profile - stringified user profile
	"""
        # delegate to vsync
	group.Send(0, id, profile)

	return "profile" # just for flask

def postUserData(id, profile):
        '''
            add a profile with id to the users DHT.
            @param {string} id - user id
            @param {string} profile - stringified user profile
        '''
        # for debugging
        if profile == "-1":
            print "Delete the user with id :"+id
            del users[id]
        else:
            print "Add the user with id :"+id
	    users[id] = profile
        backup()

def getUserData_api(id):
	"""
            Forward user GET to vsync
            @param {string} id - user id
            @return {string} - either return the profile content or -1 for not found / deleted
	"""
        try:
	    if id in users:
	    	return users[id]
	    else:
	    	res = []
	    	nr = group.Query(Vsync.Group.ALL, 1, id, Vsync.EOLMarker(), res)
	    	for ele in res:
	    		if ele != "-1":
	    			return ele
	    return "-1"
        except:
            return "-1"

def getUserData(id):
        '''
            get a user profile from the user DHT with the given id.
            @param {string} id - user id
        '''
	if id in users:
		group.Reply(users[id])
	else:
		group.Reply(-1)
        print("Get user with id :" + id)

def removeUserData_api(id):
	"""
            Forward user DELETE to vsync
            @param {string} id - user id
	"""
        # delegate to vsync post
	group.Send(0, id, '-1')
	return "profile" # just for flask

"""
Message DHT
"""
def postMessageData_api(id, message):
	"""
            Forward a message POST to vsync.
            @param {string} id - message id
            @param {string} message - message content
	"""
        # delegate to vsync
	group.Send(3, id, message)
	return "message" # just for flask

def postMessageData(id, message):
        '''
            Add a message to message DHT with a id.
            @param {string} id - message id
            @param {string} message - message content
        '''

        # for debugging
        if message == "-1":
            print "Delete the message with id :"+id
            del messages[id]
        else:
            print "Add the message with id :"+id
	    messages[id] = message
        backup()

def getMessageData_api(id):
	"""
            Forward message GET to vsync.
            @param {string} id - message id
            @return {string} - either return the message content or -1 for not found / deleted
	"""
        try:
	    if id in messages:
	    	return messages[id]
	    else:
	    	res = []
	    	nr = group.Query(Vsync.Group.ALL, 4, id, Vsync.EOLMarker(), res)
	    	print "result from group getting message"
	    	print res
	    	for ele in res:
	    		if ele != "-1":
	    			return ele
	    return "-1"
        except:
            return "-1"

def getMessageData(id):
        '''
            get a message from the message DHT with the given id.
            @param {string} id - message id
        '''
	if id in messages:
		group.Reply(messages[id])
	else:
		group.Reply('-1')
	print("Vsync server getMessageData with id=" + id.ToString())

def removeMessageData_api(id):
	"""
            Forward message DELETE to user.
            @param {string} id - message id
	"""
	group.Send(3, id, '-1')
	return "message" # just for flask

"""
    Schedule DHT
"""
def postScheduleData_api(id, schedule):
	"""
            Forward schedule POST to vsync
            @param {string} id - schedule id
            @param {string} schedule - schedule content
	"""
	group.Send(6, id, schedule)
	return "schedule" # just for flask

def postScheduleData(id, schedule):
        '''
            add a schedule with id to the schedule DHT.
            @param {string} id - schedule id
            @param {string} schedule - schedule content
        '''
        # for debugging
        if schedule == "-1":
            print "Delete the schedule with id :"+id
            del schedules[id]
        else:
            print "Add the schedule with id :"+id
	    schedules[id] = schedule

        backup()

def getScheduleData_api(id):
	"""
            Forward schedule GET to vsync
            @param {string} id - schedule id
            @return {string} - either return schedule content or -1 if not found / deleted
	"""
        try:
	    if id in schedules and schedules[id] != "-1":
	    	return schedules[id]
	    else:
	    	res = []
	    	nr = group.Query(Vsync.Group.ALL, 7, id, Vsync.EOLMarker(), res)
	    	print "result from group getting"
	    	print res
	    	for ele in res:
	    		if ele != "-1":
	    			return ele
	    return "-1"
        except:
            return "-1"

def getScheduleData(id):
        '''
            get a schedule from the schedule DHT with the given id.
            @param {string} id - schedule id
        '''
	if id in schedules:
		group.Reply(schedules[id])
	else:
		group.Reply('-1')
	print("Vsync server getScheduleData with id=" + id.ToString())

def removeScheduleData_api(id):
	"""
            Forward schedule DELETE to vsync POST with content as -1.
            @param {string} id - schedule id
	"""
	group.Send(6, id, '-1')
	return "profile"

"""
    Cache DHT - Define cache collection (cache_id - '{time:"1231231212.123",id_list:"[]"}'
"""
def postCacheData_api(id, cache_content):
	"""
            Forward cache POST to vsync
            @param {string} id - cache id
            @param {string} cache - cache content
	"""
        # delegate to vsync
	group.Send(9, id, cache_content)
        # just for flask
	return "cache"

def postCacheData(id, cache_content):
        '''
            add a cache to Cache DHT with given id.
            @param {string} id - cache id
            @param {string} cache - cache content
        '''
        # for debugging
        if cache_content == "-1":
            print "Delete the user with id :"+id
            del cache[id]
        else:
            cache[id] = cache_content
            print "Add the user with id :"+id
        backup()

def getCacheData_api(id):
	"""
            Forward cache GET to vsync.
            @param {string} id - cache id
            @return {string} either return the found cache content or -1 represent not found / deleted
	"""
        try:
	    if id in cache:
	    	return cache[id]
	    else:
	    	res = []
	    	nr = group.Query(Vsync.Group.ALL, 10, id, Vsync.EOLMarker(), res)
	    	print "result from group getting cache"
	    	print res
	    	for ele in res:
	    		if ele != "-1":
	    			return ele
	    return "-1"
        except:
            return "-1"

def getCacheData(id):
        '''
            get a schedule from the schedule DHT with the given id.
            @param {string} id - cache id
        '''
	if id in cache:
		group.Reply(cache[id])
	else:
		group.Reply('-1')
	print("Vsync server get Cache with id=" + id.ToString())

def removeCacheData_api(id):
	"""
            Forward cache DELETE to vsync, remove the cache = set the value for this key(id) as [], empty list.
            @param {string} id - cache id
            @return {string} useless for now
	"""
        # delegate to vsync
	group.Send(9, id, '-1')
	return "cache"

# for retrieving all user / message / schedule
def getAllUsers_api():
	"""
            Retrieve all users, including all keys and values.
            @return {string} - return stringify content of users
	"""
        return json.dumps(users)

def getAllMessages_api():
	"""
            Retrieve all messages, including all keys and values.
            @return {string} - return stringify content of users
	"""
        return json.dumps(messages)

def getAllCache_api():
	"""
            Retrieve all cache, including all keys and values.
            @return {string} - return stringify content of users
	"""
        return json.dumps(cache)

def getAllSchedules_api():
	"""
            Retrieve all schedules including all keys and values.
            @return {string} - return stringify content of users
	"""
        return json.dumps(schedules)

def giveMeAll(action):
    print action

### Vsycn register
def myViewFunc(v):
    global myRank,masterId
    print('New view: ' + v.ToString())
    print('My rank = ' + v.GetMyRank().ToString())
    myRank = v.GetMyRank()
    for a in v.joiners:
        print('  Joining: ' + a.ToString() + ', isMyAddress='+a.isMyAddress().ToString())
    for a in v.leavers:
        print('  Leaving: ' + a.ToString() + ', isMyAddress='+a.isMyAddress().ToString())
    if (v.GetMyRank().ToString() == "2"):
        # get data from other nodes
        print "#"
        group.Send(15, "givemeall")
    else:
        print "#load from recovery data"
        loadBK()
    return

# All functions registered by rpc-server

# user related functions
server.register_function(postUserData_api,'postUserData')
server.register_function(getUserData_api, 'getUserData')
server.register_function(removeUserData_api,'removeUserData')
# schedule related functions
server.register_function(postScheduleData_api,'postScheduleData')
server.register_function(getScheduleData_api,'getScheduleData')
server.register_function(removeScheduleData_api,'removeScheduleData')
# message related functions
server.register_function(postMessageData_api,'postMessageData')
server.register_function(getMessageData_api,'getMessageData')
server.register_function(removeMessageData_api,'removeMessageData')
# cache related functions
server.register_function(postCacheData_api,'postCacheData')
server.register_function(getCacheData_api,'getCacheData')
server.register_function(removeCacheData_api,'removeCacheData')
# retrieve all
server.register_function(getAllUsers_api,'getAllUsers')
server.register_function(getAllSchedules_api,'getAllSchedules')
server.register_function(getAllMessages_api,'getAllMessages')
server.register_function(getAllCache_api,'getAllCache')
# view
server.register_function(myViewFunc,'myViewFunc')

# All functions registered by vsync

# user related functions
group.RegisterHandler(0, Action[str, str](postUserData))
group.RegisterHandler(1, Action[str](getUserData))
# group.RegisterHandler(2, Action[str](removeUserData))

# message related functions
group.RegisterHandler(3, Action[str, str](postMessageData))
group.RegisterHandler(4, Action[str](getMessageData))
# group.RegisterHandler(5, Action[str](removeMessageData))

# schedule related functions
group.RegisterHandler(6, Action[str, str](postScheduleData))
group.RegisterHandler(7, Action[str](getScheduleData))
# group.RegisterHandler(8, Action[str](removeScheduleData))

# cache related functions
group.RegisterHandler(9, Action[str, str](postCacheData))
group.RegisterHandler(10, Action[str](getCacheData))
# group.RegisterHandler(11, Action[str](removeCacheData))

# get all data
group.RegisterHandler(15, Action[str](giveMeAll))

# view
group.RegisterViewHandler(Vsync.ViewHandler(myViewFunc))

group.Join()

### run
server.serve_forever()
Vsync.VsyncSystem.WaitForever()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import os
import logging
import traceback
import yaml
from datetime import timedelta
from datetime import datetime
from dateutil.parser import parse as parse_datetime
from pytimeparse.timeparse import timeparse
from functools import wraps
from threading import Lock
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.interval import IntervalTrigger
from apscheduler.triggers.date import DateTrigger
from f8a_worker.conf import get_postgres_connection_string
import f8a_jobs.handlers as handlers
from f8a_jobs.utils import is_failed_job_handler_name

class ScheduleJobError(Exception):
    """An exception raised on job creation error"""
    pass

class Scheduler(object):
    """Scheduler wrapper to ensure that we have a single scheduler per process"""
    _scheduler = None
    _scheduler_creation_lock = Lock()
    scheduler_lock = Lock()
    log = logging.getLogger(__name__)
    _SCHEDULER_CONF = {
        'apscheduler.jobstores.default': {
            'type': 'sqlalchemy',
            'url': get_postgres_connection_string()
        },
        'apscheduler.executors.default': {
            'class': 'apscheduler.executors.pool:ThreadPoolExecutor',
            'max_workers': '20'
        },
        'apscheduler.executors.processpool': {
            'type': 'processpool',
            'max_workers': '5'
        },
        'apscheduler.job_defaults.coalesce': 'true',
        'apscheduler.job_defaults.max_instances': '1',
        'apscheduler.timezone': 'UTC'
    }

    def __init__(self):
        raise NotImplementedError()

    @classmethod
    def get_scheduler(cls):
        """Get scheduler instance

        :return: scheduler instance
        """
        if cls._scheduler is None:
            with cls._scheduler_creation_lock:
                if cls._scheduler is None:
                    cls._scheduler = BackgroundScheduler(cls._SCHEDULER_CONF)
                    cls._scheduler.start(paused=bool(os.environ.get('JOB_SERVICE_PAUSED')))

        return cls._scheduler

    @classmethod
    def get_paused_scheduler(cls):
        """Return paused scheduler for feeding jobs

        :return: scheduler instance
        """
        # As we would like to feed default jobs, not to run any job, make sure we start scheduler in paused mode.
        if cls._scheduler is not None:
            return cls._scheduler
        else:
            scheduler = BackgroundScheduler(cls._SCHEDULER_CONF)
            scheduler.start(paused=True)
            return scheduler

    @classmethod
    def schedule_job(cls, scheduler, handler_name,
                     job_id=None, when=None, periodically=None, misfire_grace_time=None, state=None,
                     modify_existing_job=False, **kwargs):
        """Schedule a job

        :param scheduler: scheduler that should be used to schedule a job
        :param handler_name: name of handler that is used to handle the given job
        :param job_id: unique job id, if None, job_id is generated
        :param when: date and time (string representation) when the given job should be fired
        :param periodically: string representation of the periodical execution (None = job will be executed only once)
        :param misfire_grace_time: time after which the given job should be thrown away because of misfire
        :param state: a string ('paused'/'running') representation of job state
        :param modify_existing_job: if True, existing job will be modified according to new job spec
        :param kwargs: handler kwargs
        :return: scheduled apscheduler.Job instance
        """
        if state not in (None, "running", "paused"):
            raise ValueError("Unknown state '%s' provided, could be 'running' or 'paused'")

        # just to be sure that the handler actually exists
        if not hasattr(handlers, handler_name):
            raise ValueError("Unknown handler '%s'" % handler_name)

        if when:
            try:
                when = parse_datetime(when)
            except ValueError as exc:
                raise ScheduleJobError("Unable to parse datetime format for 'when': '%s'" % when)\
                    from exc
            if when < datetime.now():
                raise ScheduleJobError("Cannot schedule event at '%s' to past" % str(when))

        if misfire_grace_time:
            seconds = timeparse(misfire_grace_time)

            if seconds is None:
                raise ScheduleJobError("Unable to parse format for 'misfire_grace_time': '%s'" % misfire_grace_time)

            misfire_grace_time = seconds

        if periodically:
            seconds = timeparse(periodically)

            if seconds is None:
                raise ScheduleJobError("Unable to parse format for 'periodically': '%s'" % periodically)

            trigger = 'interval'
            trigger_kwargs = {
                'seconds': seconds,
                'start_date': when
            }
        else:
            # One time job
            trigger = 'date'
            trigger_kwargs = {
                'run_date': when
            }

        if state == 'paused':
            trigger_kwargs['next_run_time'] = None

        try:
            job = scheduler.get_job(job_id)
            if modify_existing_job and job is not None:
                cls.log.info("Performing job change (if any) on already existing job '%s'", job_id)
                job.modify(
                    kwargs=kwargs or {},
                    misfire_grace_time=misfire_grace_time,
                )
                # Check for trigger configuration changes
                old_trigger = job.trigger
                trigger_change = (trigger == 'date' and not isinstance(old_trigger, DateTrigger)) or \
                                 (trigger == 'interval' and not isinstance(old_trigger, IntervalTrigger))
                if not trigger_change:
                    if isinstance(old_trigger, DateTrigger) \
                            and old_trigger.run_date != when:
                        cls.log.info("Job execution has changed from %s to %s, modifying entry...",
                                     old_trigger.run_date, when)
                        job.reschedule(trigger=trigger, **trigger_kwargs)
                    elif isinstance(old_trigger, IntervalTrigger) \
                            and old_trigger.interval != timedelta(seconds=seconds):
                        cls.log.info("Job periodic interval has changed from %s to %s, modifying entry...",
                                     old_trigger.interval, timedelta(seconds=seconds))
                        job.reschedule(trigger=trigger, **trigger_kwargs)
                    else:
                        cls.log.info("Job execution time was left untouched - no change performed")
                else:
                    cls.log.info("Trigger type has changed, force rescheduling job")
                    job.reschedule(trigger, **trigger_kwargs)
            else:
                # force replace existing one
                job = scheduler.add_job(
                    job_execute,
                    args=(handler_name, job_id),
                    kwargs=kwargs or {},
                    id=job_id,
                    replace_existing=True,
                    trigger=trigger,
                    misfire_grace_time=misfire_grace_time,
                    **trigger_kwargs
                )
        except Exception as e:
            cls.log.exception(str(e))
            raise ScheduleJobError("Unable to schedule job: '%s'" % str(e)) from e

        return job

    @classmethod
    def register_default_jobs(cls, job_dir):
        """Register default jobs as stated in configuration files

        :param job_dir: directory in which the default jobs sit (YAML files)
        """
        cls.log.info("Registering default jobs")
        scheduler = cls.get_paused_scheduler()

        for job_file_basename in os.listdir(job_dir):
            cls.log.info("Processing file '%s'" % job_file_basename)
            job_file = os.path.join(job_dir, job_file_basename)

            if not os.path.isfile(job_file) or job_file_basename.startswith("."):
                cls.log.warning("Skipping file '%s'", job_file)

            with open(job_file, 'r') as f:
                job_info = yaml.load(f, Loader=yaml.SafeLoader)

            if 'handler' not in job_info:
                raise ValueError("Expected handler name under 'handler' key in file '%s'", job_file)

            if 'job_id' not in job_info:
                raise ValueError("Expected job ID under 'job_id' key in file '%s'", job_file)

            cls.log.info("Registering default job '%s'", job_info['job_id'])

            try:
                job = cls.schedule_job(scheduler, job_info.pop('handler'), modify_existing_job=True, **job_info)
                cls.log.info("Job '%s' from file '%s' successfully added", job_file, job.id)
            except ScheduleJobError as exc:
                cls.log.exception("Failed to register job from file '%s': %s", job_file, str(exc))

def job_execute(handler_name, job_id, **handler_kwargs):
    """ Instantiate and run the handler

    :param handler_name: name of the handler that should be run
    :param job_id: id of the handler that should be run
    :param handler_kwargs: handler keyword arguments
    """
    # This has to ba a function as apscheduler does not handler classmethods transparently
    handler = getattr(handlers, handler_name)
    instance = handler(job_id)

    try:
        instance.execute(**handler_kwargs)
    except Exception as exc:
        logging.exception("Job '%s' failed, registering ErrorHandler: %s", job_id, str(exc))
        if not is_failed_job_handler_name(handler_name):
            exc_traceback = traceback.format_exc()
            error_handler_kwargs = {
                'exc_str': str(exc),
                'exc_traceback': exc_traceback,
                'failed_job_id': job_id,
                'failed_job_handler': handler_name,
                'failed_handler_kwargs': handler_kwargs
            }
            Scheduler.schedule_job(Scheduler.get_paused_scheduler(), handlers.ErrorHandler.__name__,
                                   state='paused', kwargs=error_handler_kwargs)
        else:
            Scheduler.log.exception("Failed to record job failure job")
    else:
        logging.info("Job '%s' successfully finished", job_id)

def uses_scheduler(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        # possible bottleneck here - I'm not sure if apscheduler implementation is thread safe - suppose not
        with Scheduler.scheduler_lock:
            return func(Scheduler.get_scheduler(), *args, **kwargs)
    return wrapper
<commit_msg>Fixes in scheduler.py<commit_after>#!/usr/bin/env python3
import os
import logging
import traceback
import yaml
from datetime import timedelta
from datetime import datetime
from dateutil.parser import parse as parse_datetime
from pytimeparse.timeparse import timeparse
from functools import wraps
from threading import Lock
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.interval import IntervalTrigger
from apscheduler.triggers.date import DateTrigger
from f8a_worker.conf import get_postgres_connection_string
import f8a_jobs.handlers as handlers
from f8a_jobs.utils import is_failed_job_handler_name

class ScheduleJobError(Exception):
    """An exception raised on job creation error"""
    pass

class Scheduler(object):
    """Scheduler wrapper to ensure that we have a single scheduler per process"""
    _scheduler = None
    _scheduler_creation_lock = Lock()
    scheduler_lock = Lock()
    log = logging.getLogger(__name__)
    _SCHEDULER_CONF = {
        'apscheduler.jobstores.default': {
            'type': 'sqlalchemy',
            'url': get_postgres_connection_string()
        },
        'apscheduler.executors.default': {
            'class': 'apscheduler.executors.pool:ThreadPoolExecutor',
            'max_workers': '20'
        },
        'apscheduler.executors.processpool': {
            'type': 'processpool',
            'max_workers': '5'
        },
        'apscheduler.job_defaults.coalesce': 'true',
        'apscheduler.job_defaults.max_instances': '1',
        'apscheduler.timezone': 'UTC'
    }

    def __init__(self):
        raise NotImplementedError()

    @classmethod
    def get_scheduler(cls):
        """Get scheduler instance

        :return: scheduler instance
        """
        if cls._scheduler is None:
            with cls._scheduler_creation_lock:
                if cls._scheduler is None:
                    cls._scheduler = BackgroundScheduler(cls._SCHEDULER_CONF)
                    cls._scheduler.start(paused=bool(os.environ.get('JOB_SERVICE_PAUSED')))

        return cls._scheduler

    @classmethod
    def get_paused_scheduler(cls):
        """Return paused scheduler for feeding jobs

        :return: scheduler instance
        """
        # As we would like to feed default jobs, not to run any job, make sure
        # we start scheduler in paused mode.
        if cls._scheduler is not None:
            return cls._scheduler
        else:
            scheduler = BackgroundScheduler(cls._SCHEDULER_CONF)
            scheduler.start(paused=True)
            return scheduler

    @classmethod
    def schedule_job(cls, scheduler, handler_name,
                     job_id=None, when=None, periodically=None, misfire_grace_time=None,
                     state=None, modify_existing_job=False, **kwargs):
        """Schedule a job

        :param scheduler: scheduler that should be used to schedule a job
        :param handler_name: name of handler that is used to handle the given job
        :param job_id: unique job id, if None, job_id is generated
        :param when: date and time (string representation) when the given job should be fired
        :param periodically: string representation of the periodical execution
        (None = job will be executed only once)
        :param misfire_grace_time: time after which the given job should be
        thrown away because of misfire
        :param state: a string ('paused'/'running') representation of job state
        :param modify_existing_job: if True, existing job will be modified
        according to new job spec
        :param kwargs: handler kwargs
        :return: scheduled apscheduler.Job instance
        """
        if state not in (None, "running", "paused"):
            raise ValueError("Unknown state '%s' provided, could be 'running' or 'paused'")

        # just to be sure that the handler actually exists
        if not hasattr(handlers, handler_name):
            raise ValueError("Unknown handler '%s'" % handler_name)

        if when:
            try:
                when = parse_datetime(when)
            except ValueError as exc:
                raise ScheduleJobError("Unable to parse datetime format for 'when': '%s'" % when)\
                    from exc
            if when < datetime.now():
                raise ScheduleJobError("Cannot schedule event at '%s' to past" % str(when))

        if misfire_grace_time:
            seconds = timeparse(misfire_grace_time)

            if seconds is None:
                raise ScheduleJobError("Unable to parse format for 'misfire_grace_time': '%s'" %
                                       misfire_grace_time)

            misfire_grace_time = seconds

        if periodically:
            seconds = timeparse(periodically)

            if seconds is None:
                raise ScheduleJobError("Unable to parse format for 'periodically': '%s'" %
                                       periodically)

            trigger = 'interval'
            trigger_kwargs = {
                'seconds': seconds,
                'start_date': when
            }
        else:
            # One time job
            trigger = 'date'
            trigger_kwargs = {
                'run_date': when
            }

        if state == 'paused':
            trigger_kwargs['next_run_time'] = None

        try:
            job = scheduler.get_job(job_id)
            if modify_existing_job and job is not None:
                cls.log.info("Performing job change (if any) on already existing job '%s'", job_id)
                job.modify(
                    kwargs=kwargs or {},
                    misfire_grace_time=misfire_grace_time,
                )
                # Check for trigger configuration changes
                old_trigger = job.trigger
                trigger_change = (
                    trigger == 'date' and not isinstance(old_trigger, DateTrigger)) or \
                    (trigger == 'interval' and not isinstance(old_trigger, IntervalTrigger))
                if not trigger_change:
                    if isinstance(old_trigger, DateTrigger) \
                            and old_trigger.run_date != when:
                        cls.log.info("Job execution has changed from %s to %s, modifying entry...",
                                     old_trigger.run_date, when)
                        job.reschedule(trigger=trigger, **trigger_kwargs)
                    elif isinstance(old_trigger, IntervalTrigger) \
                            and old_trigger.interval != timedelta(seconds=seconds):
                        cls.log.info("Job periodic interval has changed from %s to %s, "
                                     "modifying entry...",
                                     old_trigger.interval, timedelta(seconds=seconds))
                        job.reschedule(trigger=trigger, **trigger_kwargs)
                    else:
                        cls.log.info("Job execution time was left untouched - no change performed")
                else:
                    cls.log.info("Trigger type has changed, force rescheduling job")
                    job.reschedule(trigger, **trigger_kwargs)
            else:
                # force replace existing one
                job = scheduler.add_job(
                    job_execute,
                    args=(handler_name, job_id),
                    kwargs=kwargs or {},
                    id=job_id,
                    replace_existing=True,
                    trigger=trigger,
                    misfire_grace_time=misfire_grace_time,
                    **trigger_kwargs
                )
        except Exception as e:
            cls.log.exception(str(e))
            raise ScheduleJobError("Unable to schedule job: '%s'" % str(e)) from e

        return job

    @classmethod
    def register_default_jobs(cls, job_dir):
        """Register default jobs as stated in configuration files

        :param job_dir: directory in which the default jobs sit (YAML files)
        """
        cls.log.info("Registering default jobs")
        scheduler = cls.get_paused_scheduler()

        for job_file_basename in os.listdir(job_dir):
            cls.log.info("Processing file '%s'" % job_file_basename)
            job_file = os.path.join(job_dir, job_file_basename)

            if not os.path.isfile(job_file) or job_file_basename.startswith("."):
                cls.log.warning("Skipping file '%s'", job_file)

            with open(job_file, 'r') as f:
                job_info = yaml.load(f, Loader=yaml.SafeLoader)

            if 'handler' not in job_info:
                raise ValueError("Expected handler name under 'handler' key in file '%s'", job_file)

            if 'job_id' not in job_info:
                raise ValueError("Expected job ID under 'job_id' key in file '%s'", job_file)

            cls.log.info("Registering default job '%s'", job_info['job_id'])

            try:
                job = cls.schedule_job(scheduler, job_info.pop('handler'),
                                       modify_existing_job=True, **job_info)
                cls.log.info("Job '%s' from file '%s' successfully added", job_file, job.id)
            except ScheduleJobError as exc:
                cls.log.exception("Failed to register job from file '%s': %s", job_file, str(exc))

def job_execute(handler_name, job_id, **handler_kwargs):
    """ Instantiate and run the handler

    :param handler_name: name of the handler that should be run
    :param job_id: id of the handler that should be run
    :param handler_kwargs: handler keyword arguments
    """
    # This has to ba a function as apscheduler does not handler classmethods transparently
    handler = getattr(handlers, handler_name)
    instance = handler(job_id)

    try:
        instance.execute(**handler_kwargs)
    except Exception as exc:
        logging.exception("Job '%s' failed, registering ErrorHandler: %s", job_id, str(exc))
        if not is_failed_job_handler_name(handler_name):
            exc_traceback = traceback.format_exc()
            error_handler_kwargs = {
                'exc_str': str(exc),
                'exc_traceback': exc_traceback,
                'failed_job_id': job_id,
                'failed_job_handler': handler_name,
                'failed_handler_kwargs': handler_kwargs
            }
            Scheduler.schedule_job(Scheduler.get_paused_scheduler(), handlers.ErrorHandler.__name__,
                                   state='paused', kwargs=error_handler_kwargs)
        else:
            Scheduler.log.exception("Failed to record job failure job")
    else:
        logging.info("Job '%s' successfully finished", job_id)

def uses_scheduler(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        # possible bottleneck here - I'm not sure if apscheduler implementation
        # is thread safe - suppose not
        with Scheduler.scheduler_lock:
            return func(Scheduler.get_scheduler(), *args, **kwargs)
    return wrapper
<|endoftext|>