<commit_before>""" A stylish alternative for caching your map tiles.

TileStache is a Python-based server application that can serve up map tiles
based on rendered geographic data. You might be familiar with TileCache
(http://tilecache.org), the venerable open source WMS server from MetaCarta.
TileStache is similar, but we hope simpler and better-suited to the needs of
designers and cartographers.

Documentation available at http://tilestache.org/doc/
"""
__version__ = 'N.N.N'

import re

from sys import stdout
try:
    from urlparse import parse_qs
except ImportError:
    from cgi import parse_qs
from StringIO import StringIO
from os.path import dirname, join as pathjoin, realpath
from datetime import datetime, timedelta
from urlparse import urljoin, urlparse
from wsgiref.headers import Headers
from urllib import urlopen
from os import getcwd
from time import time

import httplib
import logging

try:
    from json import load as json_load
except ImportError:
    from simplejson import load as json_load

from ModestMaps.Core import Coordinate

# dictionary of configuration objects for requestLayer().
_previous_configs = {}

import Core
import Config

# regular expression for PATH_INFO
_pathinfo_pat = re.compile(r'^/?(?P<l>\w.+)/(?P<z>\d+)/(?P<x>-?\d+)/(?P<y>-?\d+)\.(?P<e>\w+)$')
_preview_pat = re.compile(r'^/?(?P<l>\w.+)/(preview\.html)?$')

def getTile(layer, coord, extension, ignore_cached=False):
    ''' Get a type string and tile binary for a given request layer tile.

        This function is documented as part of TileStache's public API:
            http://tilestache.org/doc/#tilestache-gettile

        Arguments:
        - layer: instance of Core.Layer to render.
        - coord: one ModestMaps.Core.Coordinate corresponding to a single tile.
        - extension: filename extension to choose response type, e.g. "png" or "jpg".
        - ignore_cached: always re-render the tile, whether it's in the cache or not.

        This is the main entry point, after site configuration has been loaded
        and individual tiles need to be rendered.
    '''
    status_code, headers, body = layer.getTileResponse(coord, extension, ignore_cached)
    mime = headers.get('Content-Type')

    return mime, body

def getPreview(layer):
    """ Get a type string and dynamic map viewer HTML for a given layer.
    """
    return 200, Headers([('Content-Type', 'text/html')]), Core._preview(layer)

def parseConfigfile(configpath):
    """ Parse a configuration file and return a Configuration object.

        Configuration file is formatted as JSON with two sections, "cache" and "layers":

          {
            "cache": { ... },
            "layers": {
              "layer-1": { ... },
              "layer-2": { ... },
              ...
            }
          }

        The full path to the file is significant, used to
        resolve any relative paths found in the configuration.

        See the Caches module for more information on the "caches" section,
        and the Core and Providers modules for more information on the
        "layers" section.
    """
    config_dict = json_load(urlopen(configpath))

    scheme, host, path, p, q, f = urlparse(configpath)

    if scheme == '':
        scheme = 'file'
        path = realpath(path)

    dirpath = '%s://%s%s' % (scheme, host, dirname(path).rstrip('/') + '/')

    return Config.buildConfiguration(config_dict, dirpath)

def splitPathInfo(pathinfo):
    """ Converts a PATH_INFO string to layer name, coordinate, and extension parts.

        Example: "/layer/0/0/0.png", leading "/" optional.
    """
    if pathinfo == '/':
        return None, None, None

    if _pathinfo_pat.match(pathinfo or ''):
        path = _pathinfo_pat.match(pathinfo)
        layer, row, column, zoom, extension = [path.group(p) for p in 'lyxze']
        coord = Coordinate(int(row), int(column), int(zoom))

    elif _preview_pat.match(pathinfo or ''):
        path = _preview_pat.match(pathinfo)
        layer, extension = path.group('l'), 'html'
        coord = None

    else:
        raise Core.KnownUnknown('Bad path: "%s". I was expecting something more like "/example/0/0/0.png"' % pathinfo)

    return layer, coord, extension

def mergePathInfo(layer, coord, extension):
    """ Converts layer name, coordinate and extension back to a PATH_INFO string.

        See also splitPathInfo().
    """
    z = coord.zoom
    x = coord.column
    y = coord.row

    return '/%(layer)s/%(z)d/%(x)d/%(y)d.%(extension)s' % locals()

def requestLayer(config, path_info):
    """ Return a Layer.

        Requires a configuration and PATH_INFO (e.g. "/example/0/0/0.png").

        Config parameter can be a file path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.
    """
    if type(config) in (str, unicode):
        #
        # Should be a path to a configuration file we can load;
        # build a tuple key into previously-seen config objects.
        #
        key = hasattr(config, '__hash__') and (config, getcwd())

        if key in _previous_configs:
            config = _previous_configs[key]

        else:
            config = parseConfigfile(config)

            if key:
                _previous_configs[key] = config

    else:
        assert hasattr(config, 'cache'), 'Configuration object must have a cache.'
        assert hasattr(config, 'layers'), 'Configuration object must have layers.'
        assert hasattr(config, 'dirpath'), 'Configuration object must have a dirpath.'

    # ensure that path_info is at least a single "/"
    path_info = '/' + (path_info or '').lstrip('/')

    if path_info == '/':
        return Core.Layer(config, None, None)

    layername = splitPathInfo(path_info)[0]

    if layername not in config.layers:
        raise Core.KnownUnknown('"%s" is not a layer I know about. Here are some that I do know about: %s.' % (layername, ', '.join(sorted(config.layers.keys()))))

    return config.layers[layername]

def requestHandler(config_hint, path_info, query_string):
    """ Generate a mime-type and response body for a given request.

        This function is documented as part of TileStache's public API:
            http://tilestache.org/doc/#tilestache-requesthandler

        TODO: replace this with requestHandler2() in TileStache 2.0.0.

        Calls requestHandler2().
    """
    status_code, headers, content = requestHandler2(config_hint, path_info, query_string)
    mimetype = headers.get('Content-Type')

    return mimetype, content

def requestHandler2(config_hint, path_info, query_string, script_name=''):
    """ Generate a set of headers and response body for a given request.

        TODO: Replace requestHandler() with this function in TileStache 2.0.0.

        Requires a configuration and PATH_INFO (e.g. "/example/0/0/0.png").

        Config_hint parameter can be a path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.

        Query string is optional, currently used for JSON callbacks.

        Calls Layer.getTileResponse() to render actual tiles, and getPreview() to render preview.html.
    """
    headers = Headers([])

    try:
        # ensure that path_info is at least a single "/"
        path_info = '/' + (path_info or '').lstrip('/')

        layer = requestLayer(config_hint, path_info)
        query = parse_qs(query_string or '')
        try:
            callback = query['callback'][0]
        except KeyError:
            callback = None

        #
        # Special case for index page.
        #
        if path_info == '/':
            mimetype, content = getattr(layer.config, 'index', ('text/plain', 'TileStache says hello.'))
            return 200, Headers([('Content-Type', mimetype)]), content

        coord, extension = splitPathInfo(path_info)[1:]

        if extension == 'html' and coord is None:
            status_code, headers, content = getPreview(layer)

        elif extension.lower() in layer.redirects:
            other_extension = layer.redirects[extension.lower()]

            redirect_uri = script_name
            redirect_uri += mergePathInfo(layer.name(), coord, other_extension)

            if query_string:
                redirect_uri += '?' + query_string

            headers['Location'] = redirect_uri
            headers['Content-Type'] = 'text/plain'

            return 302, headers, 'You are being redirected to %s\n' % redirect_uri

        else:
            status_code, headers, content = layer.getTileResponse(coord, extension)

        if callback and 'json' in headers['Content-Type']:
            headers['Content-Type'] = 'application/javascript; charset=utf-8'
            content = '%s(%s)' % (callback, content)

        allowed_origin = layer.allowed_origin
        max_cache_age = layer.max_cache_age

        if allowed_origin:
            headers.setdefault('Access-Control-Allow-Origin', allowed_origin)

        if max_cache_age is not None:
            expires = datetime.utcnow() + timedelta(seconds=max_cache_age)
            headers.setdefault('Expires', expires.strftime('%a %d %b %Y %H:%M:%S GMT'))
            headers.setdefault('Cache-Control', 'public, max-age=%d' % max_cache_age)

    except Core.KnownUnknown, e:
        out = StringIO()

        print >> out, 'Known unknown!'
        print >> out, e
        print >> out, ''
        print >> out, '\n'.join(Core._rummy())

        headers['Content-Type'] = 'text/plain'
        status_code, content = 500, out.getvalue()

    return status_code, headers, content

def cgiHandler(environ, config='./tilestache.cfg', debug=False):
    """ Read environment PATH_INFO, load up configuration, talk to stdout by CGI.

        This function is documented as part of TileStache's public API:
            http://tilestache.org/doc/#cgi

        Calls requestHandler().

        Config parameter can be a file path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.
    """
    if debug:
        import cgitb
        cgitb.enable()

    path_info = environ.get('PATH_INFO', None)
    query_string = environ.get('QUERY_STRING', None)
    script_name = environ.get('SCRIPT_NAME', None)

    status_code, headers, content = requestHandler2(config, path_info, query_string, script_name)

    headers.setdefault('Content-Length', str(len(content)))

    # output the status code as a header
    stdout.write('Status: %d\n' % status_code)

    # output gathered headers
    for k, v in headers.items():
        stdout.write('%s: %s\n' % (k, v))

    stdout.write('\n')
    stdout.write(content)

class WSGITileServer:
    """ Create a WSGI application that can handle requests from any server that talks WSGI.

        This class is documented as part of TileStache's public API:
            http://tilestache.org/doc/#wsgi

        The WSGI application is an instance of this class. Example:

          app = WSGITileServer('/path/to/tilestache.cfg')
          werkzeug.serving.run_simple('localhost', 8080, app)
    """

    def __init__(self, config, autoreload=False):
        """ Initialize a callable WSGI instance.

            Config parameter can be a file path string for a JSON configuration
            file or a configuration object with 'cache', 'layers', and
            'dirpath' properties.

            Optional autoreload boolean parameter causes config to be re-read
            on each request, applicable only when config is a JSON file.
        """

        if type(config) in (str, unicode):
            self.autoreload = autoreload
            self.config_path = config

            try:
                self.config = parseConfigfile(config)
            except:
                print "Error loading Tilestache config:"
                raise

        else:
            assert hasattr(config, 'cache'), 'Configuration object must have a cache.'
            assert hasattr(config, 'layers'), 'Configuration object must have layers.'
            assert hasattr(config, 'dirpath'), 'Configuration object must have a dirpath.'

            self.autoreload = False
            self.config_path = None
            self.config = config

    def __call__(self, environ, start_response):
        """
        """
        if self.autoreload: # re-parse the config file on every request
            try:
                self.config = parseConfigfile(self.config_path)
            except Exception, e:
                raise Core.KnownUnknown("Error loading Tilestache config file:\n%s" % str(e))

        try:
            layer, coord, ext = splitPathInfo(environ['PATH_INFO'])
        except Core.KnownUnknown, e:
            return self._response(start_response, 400, str(e))

        #
        # WSGI behavior is different from CGI behavior, because we may not want
        # to return a chatty rummy for likely-deployed WSGI vs. testing CGI.
        #
        if layer and layer not in self.config.layers:
            return self._response(start_response, 404)

        path_info = environ.get('PATH_INFO', None)
        query_string = environ.get('QUERY_STRING', None)
        script_name = environ.get('SCRIPT_NAME', None)

        status_code, headers, content = requestHandler2(self.config, path_info, query_string, script_name)

        return self._response(start_response, status_code, str(content), headers)

    def _response(self, start_response, code, content='', headers=None):
        """
        """
        headers = headers or Headers([])

        if content:
            headers.setdefault('Content-Length', str(len(content)))

        start_response('%d %s' % (code, httplib.responses[code]), headers.items())
        return [content]

def modpythonHandler(request):
    """ Handle a mod_python request.

        TODO: Upgrade to new requestHandler() so this can return non-200 HTTP.

        Calls requestHandler().

        Example Apache configuration for TileStache:

        <Directory /home/migurski/public_html/TileStache>
            AddHandler mod_python .py
            PythonHandler TileStache::modpythonHandler
            PythonOption config /etc/tilestache.cfg
        </Directory>

        Configuration options, using PythonOption directive:
        - config: path to configuration file, defaults to "tilestache.cfg",
            using request.filename as the current working directory.
    """
    from mod_python import apache

    config_path = request.get_options().get('config', 'tilestache.cfg')
    config_path = realpath(pathjoin(dirname(request.filename), config_path))

    path_info = request.path_info
    query_string = request.args

    mimetype, content = requestHandler(config_path, path_info, query_string)

    request.status = apache.HTTP_OK
    request.content_type = mimetype
    request.set_content_length(len(content))
    request.send_http_header()

    request.write(content)

    return apache.OK
<commit_msg>Made Rummy CORS-compliant, closes #132<commit_after>""" A stylish alternative for caching your map tiles.

TileStache is a Python-based server application that can serve up map tiles
based on rendered geographic data. You might be familiar with TileCache
(http://tilecache.org), the venerable open source WMS server from MetaCarta.
TileStache is similar, but we hope simpler and better-suited to the needs of
designers and cartographers.

Documentation available at http://tilestache.org/doc/
"""
__version__ = 'N.N.N'

import re

from sys import stdout
try:
    from urlparse import parse_qs
except ImportError:
    from cgi import parse_qs
from StringIO import StringIO
from os.path import dirname, join as pathjoin, realpath
from datetime import datetime, timedelta
from urlparse import urljoin, urlparse
from wsgiref.headers import Headers
from urllib import urlopen
from os import getcwd
from time import time

import httplib
import logging

try:
    from json import load as json_load
except ImportError:
    from simplejson import load as json_load

from ModestMaps.Core import Coordinate

# dictionary of configuration objects for requestLayer().
_previous_configs = {}

import Core
import Config

# regular expression for PATH_INFO
_pathinfo_pat = re.compile(r'^/?(?P<l>\w.+)/(?P<z>\d+)/(?P<x>-?\d+)/(?P<y>-?\d+)\.(?P<e>\w+)$')
_preview_pat = re.compile(r'^/?(?P<l>\w.+)/(preview\.html)?$')

def getTile(layer, coord, extension, ignore_cached=False):
    ''' Get a type string and tile binary for a given request layer tile.

        This function is documented as part of TileStache's public API:
            http://tilestache.org/doc/#tilestache-gettile

        Arguments:
        - layer: instance of Core.Layer to render.
        - coord: one ModestMaps.Core.Coordinate corresponding to a single tile.
        - extension: filename extension to choose response type, e.g. "png" or "jpg".
        - ignore_cached: always re-render the tile, whether it's in the cache or not.

        This is the main entry point, after site configuration has been loaded
        and individual tiles need to be rendered.
    '''
    status_code, headers, body = layer.getTileResponse(coord, extension, ignore_cached)
    mime = headers.get('Content-Type')

    return mime, body

def getPreview(layer):
    """ Get a type string and dynamic map viewer HTML for a given layer.
    """
    return 200, Headers([('Content-Type', 'text/html')]), Core._preview(layer)

def parseConfigfile(configpath):
    """ Parse a configuration file and return a Configuration object.

        Configuration file is formatted as JSON with two sections, "cache" and "layers":

          {
            "cache": { ... },
            "layers": {
              "layer-1": { ... },
              "layer-2": { ... },
              ...
            }
          }

        The full path to the file is significant, used to
        resolve any relative paths found in the configuration.

        See the Caches module for more information on the "caches" section,
        and the Core and Providers modules for more information on the
        "layers" section.
    """
    config_dict = json_load(urlopen(configpath))

    scheme, host, path, p, q, f = urlparse(configpath)

    if scheme == '':
        scheme = 'file'
        path = realpath(path)

    dirpath = '%s://%s%s' % (scheme, host, dirname(path).rstrip('/') + '/')

    return Config.buildConfiguration(config_dict, dirpath)

def splitPathInfo(pathinfo):
    """ Converts a PATH_INFO string to layer name, coordinate, and extension parts.

        Example: "/layer/0/0/0.png", leading "/" optional.
    """
    if pathinfo == '/':
        return None, None, None

    if _pathinfo_pat.match(pathinfo or ''):
        path = _pathinfo_pat.match(pathinfo)
        layer, row, column, zoom, extension = [path.group(p) for p in 'lyxze']
        coord = Coordinate(int(row), int(column), int(zoom))

    elif _preview_pat.match(pathinfo or ''):
        path = _preview_pat.match(pathinfo)
        layer, extension = path.group('l'), 'html'
        coord = None

    else:
        raise Core.KnownUnknown('Bad path: "%s". I was expecting something more like "/example/0/0/0.png"' % pathinfo)

    return layer, coord, extension

def mergePathInfo(layer, coord, extension):
    """ Converts layer name, coordinate and extension back to a PATH_INFO string.

        See also splitPathInfo().
    """
    z = coord.zoom
    x = coord.column
    y = coord.row

    return '/%(layer)s/%(z)d/%(x)d/%(y)d.%(extension)s' % locals()

def requestLayer(config, path_info):
    """ Return a Layer.

        Requires a configuration and PATH_INFO (e.g. "/example/0/0/0.png").

        Config parameter can be a file path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.
    """
    if type(config) in (str, unicode):
        #
        # Should be a path to a configuration file we can load;
        # build a tuple key into previously-seen config objects.
        #
        key = hasattr(config, '__hash__') and (config, getcwd())

        if key in _previous_configs:
            config = _previous_configs[key]

        else:
            config = parseConfigfile(config)

            if key:
                _previous_configs[key] = config

    else:
        assert hasattr(config, 'cache'), 'Configuration object must have a cache.'
        assert hasattr(config, 'layers'), 'Configuration object must have layers.'
        assert hasattr(config, 'dirpath'), 'Configuration object must have a dirpath.'

    # ensure that path_info is at least a single "/"
    path_info = '/' + (path_info or '').lstrip('/')

    if path_info == '/':
        return Core.Layer(config, None, None)

    layername = splitPathInfo(path_info)[0]

    if layername not in config.layers:
        raise Core.KnownUnknown('"%s" is not a layer I know about. Here are some that I do know about: %s.' % (layername, ', '.join(sorted(config.layers.keys()))))

    return config.layers[layername]

def requestHandler(config_hint, path_info, query_string):
    """ Generate a mime-type and response body for a given request.

        This function is documented as part of TileStache's public API:
            http://tilestache.org/doc/#tilestache-requesthandler

        TODO: replace this with requestHandler2() in TileStache 2.0.0.

        Calls requestHandler2().
    """
    status_code, headers, content = requestHandler2(config_hint, path_info, query_string)
    mimetype = headers.get('Content-Type')

    return mimetype, content

def requestHandler2(config_hint, path_info, query_string, script_name=''):
    """ Generate a set of headers and response body for a given request.

        TODO: Replace requestHandler() with this function in TileStache 2.0.0.

        Requires a configuration and PATH_INFO (e.g. "/example/0/0/0.png").

        Config_hint parameter can be a path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.

        Query string is optional, currently used for JSON callbacks.

        Calls Layer.getTileResponse() to render actual tiles, and getPreview() to render preview.html.
    """
    headers = Headers([])

    try:
        # ensure that path_info is at least a single "/"
        path_info = '/' + (path_info or '').lstrip('/')

        layer = requestLayer(config_hint, path_info)
        query = parse_qs(query_string or '')
        try:
            callback = query['callback'][0]
        except KeyError:
            callback = None

        if layer.allowed_origin:
            headers.setdefault('Access-Control-Allow-Origin', layer.allowed_origin)

        #
        # Special case for index page.
        #
        if path_info == '/':
            mimetype, content = getattr(layer.config, 'index', ('text/plain', 'TileStache says hello.'))
            return 200, Headers([('Content-Type', mimetype)]), content

        coord, extension = splitPathInfo(path_info)[1:]

        if extension == 'html' and coord is None:
            status_code, headers, content = getPreview(layer)

        elif extension.lower() in layer.redirects:
            other_extension = layer.redirects[extension.lower()]

            redirect_uri = script_name
            redirect_uri += mergePathInfo(layer.name(), coord, other_extension)

            if query_string:
                redirect_uri += '?' + query_string

            headers['Location'] = redirect_uri
            headers['Content-Type'] = 'text/plain'

            return 302, headers, 'You are being redirected to %s\n' % redirect_uri

        else:
            status_code, headers, content = layer.getTileResponse(coord, extension)

        if callback and 'json' in headers['Content-Type']:
            headers['Content-Type'] = 'application/javascript; charset=utf-8'
            content = '%s(%s)' % (callback, content)

        if layer.max_cache_age is not None:
            expires = datetime.utcnow() + timedelta(seconds=layer.max_cache_age)
            headers.setdefault('Expires', expires.strftime('%a %d %b %Y %H:%M:%S GMT'))
            headers.setdefault('Cache-Control', 'public, max-age=%d' % layer.max_cache_age)

    except Core.KnownUnknown, e:
        out = StringIO()

        print >> out, 'Known unknown!'
        print >> out, e
        print >> out, ''
        print >> out, '\n'.join(Core._rummy())

        headers['Content-Type'] = 'text/plain'
        status_code, content = 500, out.getvalue()

    return status_code, headers, content

def cgiHandler(environ, config='./tilestache.cfg', debug=False):
    """ Read environment PATH_INFO, load up configuration, talk to stdout by CGI.

        This function is documented as part of TileStache's public API:
            http://tilestache.org/doc/#cgi

        Calls requestHandler().

        Config parameter can be a file path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.
    """
    if debug:
        import cgitb
        cgitb.enable()

    path_info = environ.get('PATH_INFO', None)
    query_string = environ.get('QUERY_STRING', None)
    script_name = environ.get('SCRIPT_NAME', None)

    status_code, headers, content = requestHandler2(config, path_info, query_string, script_name)

    headers.setdefault('Content-Length', str(len(content)))

    # output the status code as a header
    stdout.write('Status: %d\n' % status_code)

    # output gathered headers
    for k, v in headers.items():
        stdout.write('%s: %s\n' % (k, v))

    stdout.write('\n')
    stdout.write(content)

class WSGITileServer:
    """ Create a WSGI application that can handle requests from any server that talks WSGI.

        This class is documented as part of TileStache's public API:
            http://tilestache.org/doc/#wsgi

        The WSGI application is an instance of this class. Example:

          app = WSGITileServer('/path/to/tilestache.cfg')
          werkzeug.serving.run_simple('localhost', 8080, app)
    """

    def __init__(self, config, autoreload=False):
        """ Initialize a callable WSGI instance.

            Config parameter can be a file path string for a JSON configuration
            file or a configuration object with 'cache', 'layers', and
            'dirpath' properties.

            Optional autoreload boolean parameter causes config to be re-read
            on each request, applicable only when config is a JSON file.
        """

        if type(config) in (str, unicode):
            self.autoreload = autoreload
            self.config_path = config

            try:
                self.config = parseConfigfile(config)
            except:
                print "Error loading Tilestache config:"
                raise

        else:
            assert hasattr(config, 'cache'), 'Configuration object must have a cache.'
            assert hasattr(config, 'layers'), 'Configuration object must have layers.'
            assert hasattr(config, 'dirpath'), 'Configuration object must have a dirpath.'

            self.autoreload = False
            self.config_path = None
            self.config = config

    def __call__(self, environ, start_response):
        """
        """
        if self.autoreload: # re-parse the config file on every request
            try:
                self.config = parseConfigfile(self.config_path)
            except Exception, e:
                raise Core.KnownUnknown("Error loading Tilestache config file:\n%s" % str(e))

        try:
            layer, coord, ext = splitPathInfo(environ['PATH_INFO'])
        except Core.KnownUnknown, e:
            return self._response(start_response, 400, str(e))

        #
        # WSGI behavior is different from CGI behavior, because we may not want
        # to return a chatty rummy for likely-deployed WSGI vs. testing CGI.
        #
        if layer and layer not in self.config.layers:
            return self._response(start_response, 404)

        path_info = environ.get('PATH_INFO', None)
        query_string = environ.get('QUERY_STRING', None)
        script_name = environ.get('SCRIPT_NAME', None)

        status_code, headers, content = requestHandler2(self.config, path_info, query_string, script_name)

        return self._response(start_response, status_code, str(content), headers)

    def _response(self, start_response, code, content='', headers=None):
        """
        """
        headers = headers or Headers([])

        if content:
            headers.setdefault('Content-Length', str(len(content)))

        start_response('%d %s' % (code, httplib.responses[code]), headers.items())
        return [content]

def modpythonHandler(request):
    """ Handle a mod_python request.

        TODO: Upgrade to new requestHandler() so this can return non-200 HTTP.

        Calls requestHandler().

        Example Apache configuration for TileStache:

        <Directory /home/migurski/public_html/TileStache>
            AddHandler mod_python .py
            PythonHandler TileStache::modpythonHandler
            PythonOption config /etc/tilestache.cfg
        </Directory>

        Configuration options, using PythonOption directive:
        - config: path to configuration file, defaults to "tilestache.cfg",
            using request.filename as the current working directory.
    """
    from mod_python import apache

    config_path = request.get_options().get('config', 'tilestache.cfg')
    config_path = realpath(pathjoin(dirname(request.filename), config_path))

    path_info = request.path_info
    query_string = request.args

    mimetype, content = requestHandler(config_path, path_info, query_string)

    request.status = apache.HTTP_OK
    request.content_type = mimetype
    request.set_content_length(len(content))
    request.send_http_header()

    request.write(content)

    return apache.OK
<|endoftext|>


<commit_before><commit_msg>Arch: Prevent exporting materials as IFC objects<commit_after><|endoftext|>


<commit_before><commit_msg>Draft: Added shortcut to toggle near snap during drawing<commit_after><|endoftext|>


<commit_before><commit_msg>[Draft] DraftGui Bugfix on selecting inputfields<commit_after><|endoftext|>


<commit_before>#! /usr/bin/python
# -*- coding: utf-8 -*-
"""Effortlessly read a text file using counting numbers.

Created 2013-2017 Caleb Ely
<https://CodeTri.net>

Licensed under The MIT License
<http://opensource.org/licenses/MIT/>

"""

import locale

__all__ = ("LineScan")

class LineScan(object):
    """Main linescan class.

    Instance using linescan.LineScan()
    Primary entry point is scan() function.
    """

    def __init__(self):
        """Initialize private and public variables."""
        self.__enable_exceptions = False

        self.__file_name = ""
        self.__start_line = None
        self.__end_line = None
        self.__encoding = None

    # ------- Private Methods ------- #
    def __set_details(self, file_name, start_line, end_line, encoding):
        """Store scan details.

        @private
        """
        self.__file_name = file_name
        self.__start_line = start_line
        self.__end_line = end_line
        self.__encoding = encoding

    def __raise_exception(self, exc, generic=True):
        """Handle errors per exception option.

        @private
        """
        # Raise an exception if they are enabled
        if self.__enable_exceptions:
            # A "generic" exception should be raised
            if generic:
                raise exc

            # A rescaning error should be raised instead
            raise FileNotFoundError(exc)

        # If exceptions have not been enabled, simply return False
        return False

    def __scanner(self):
        """Perform the actual scan.

        @private
        """
        try:
            # Since line numbers start at 0, get the starting line number.
            # No need to do the same for the ending line, as it is inclusive.
            start_line = self.__start_line - 1

            with open(self.__file_name, "rt", encoding=self.__encoding) as f:
                # The user wants to scan until the end of the file
                if self.__end_line == "end":
                    lines = f.readlines()[start_line:]

                # The user wants to scan one or more lines
                else:
                    lines = f.readlines()[start_line:self.__end_line]
            return "".join(lines)

        except Exception as exc:
            return self.__raise_exception(exc)

    # ------- Public Methods ------- #
    def show_errors(self, enable=False):
        """Enable exception raising instead of returning False on error.

        @param {Boolean} [enable=False] - Passing a value of True
                                          enables exception raising.
        """
        self.__enable_exceptions = bool(enable)

    def scan(self, file_name, start_line, end_line=None, encoding=None):
        """Scan lines with the option of using a different file encoding.

        filename (String): The desired file to scan.
        lineno (Integer): The line you wish to scan.
        endline (Optional, Integer, String): The last line to want to scan.
        Specify when scanning multiple lines. Specifying "end" will scan
        the file from lineno to the end of the file.
        encoding (Optional, String): Specify a file encoding to use.
        Defaults to default system encoding.
        @returns {String} The line(s) scanned.
        """
        # Use the system default encoding if one is not specified.
        if encoding is None:
            encoding = locale.getpreferredencoding(False)

        # If no ending line is given, default to a single line
        # TODO Write a test for this condition
        if end_line is None:
            end_line = start_line

        # Set the scan details for use in other methods.
        self.__set_details(file_name, start_line, end_line, encoding)

        # Perform the scan and store it only if it is valid.
        the_scan = self.__scanner()
        return the_scan
<commit_msg>Remove internal _setDetails method<commit_after>#! /usr/bin/python
# -*- coding: utf-8 -*-
"""Effortlessly read a text file using counting numbers.

Created 2013-2017 Caleb Ely
<https://CodeTri.net>

Licensed under The MIT License
<http://opensource.org/licenses/MIT/>

"""

import locale

__all__ = ("LineScan")

class LineScan(object):
    """Main linescan class.

    Instance using linescan.LineScan()
    Primary entry point is scan() function.
    """

    def __init__(self):
        """Initialize private and public variables."""
        self.__enable_exceptions = False

        self.__file_name = ""
        self.__start_line = None
        self.__end_line = None
        self.__encoding = None

    # ------- Private Methods ------- #
    def __raise_exception(self, exc, generic=True):
        """Handle errors per exception option.

        @private
        """
        # Raise an exception if they are enabled
        if self.__enable_exceptions:
            # A "generic" exception should be raised
            if generic:
                raise exc

            # A rescaning error should be raised instead
            raise FileNotFoundError(exc)

        # If exceptions have not been enabled, simply return False
        return False

    def __scanner(self, file_name, start_line, end_line, _encoding):
        """Perform the actual scan.

        @private
        """
        try:
            # Since line numbers start at 0, get the starting line number.
            # No need to do the same for the ending line, as it is inclusive.
            start_line -= 1

            with open(file_name, "rt", encoding=_encoding) as f:
                # The user wants to scan until the end of the file
                if end_line == "end":
                    lines = f.readlines()[start_line:]

                # The user wants to scan one or more lines
                else:
                    lines = f.readlines()[start_line:end_line]
            return "".join(lines)

        except Exception as exc:
            return self.__raise_exception(exc)

    # ------- Public Methods ------- #
    def show_errors(self, enable=False):
        """Enable exception raising instead of returning False on error.

        @param {Boolean} [enable=False] - Passing a value of True
                                          enables exception raising.
        """
        self.__enable_exceptions = bool(enable)

    def scan(self, file_name, start_line, end_line=None, encoding=None):
        """Scan lines with the option of using a different file encoding.

        filename (String): The desired file to scan.
        lineno (Integer): The line you wish to scan.
        endline (Optional, Integer, String): The last line to want to scan.
        Specify when scanning multiple lines. Specifying "end" will scan
        the file from lineno to the end of the file.
        encoding (Optional, String): Specify a file encoding to use.
        Defaults to default system encoding.
        @returns {String} The line(s) scanned.
        """
        # Use the system default encoding if one is not specified.
        if encoding is None:
            encoding = locale.getpreferredencoding(False)

        # If no ending line is given, default to a single line
        # TODO Write a test for this condition
        if end_line is None:
            end_line = start_line

        # Perform the scan and store it only if it is valid.
        the_scan = self.__scanner(file_name, start_line, end_line, encoding)
        return the_scan
<|endoftext|>


<commit_before>import pickle, os, numpy as np
from sklearn import linear_model

def train(features, codes, penalty='l1', C=1):
    """
    Train logistic regression classifiers,
    independently for each code
    :param features: input matrix
    :param codes: output matrix
    :param penalty: type of regularisation (l1 or l2)
    :param C: inverse of regularisation strength
    :return: list of classifiers
    """
    classifiers = []
    # Iterate throgh each code (i.e. each column of codes matrix)
    for code_i in codes.transpose():
        # Define a logistic regression model for each code
        model = linear_model.LogisticRegression(
            penalty=penalty,
            C=C)
        # (Could use class_weight to make positive instances more important...)

        if code_i.sum() > 1:  # Make sure there is more than one training example
            # Train the model
            model.fit(features, code_i)

            classifiers.append(model)

        else:
            classifiers.append(None)

    return classifiers

def train_on_file(input_name, output_suffix=None, directory='../data', **kwargs):
    """
    Train logistic regression classifiers on a file
    :param input_name: name of input file (without .pkl file extension)
    :param output_suffix: string to append to name of output file
    (if none is given, no file is saved)
    :param directory: directory of data files (default ../data)
    :param **kwargs: additional keyward arguments will be passed to train
    :return: features, codes, classifiers
    """
    # Load features and codes
    with open(os.path.join(directory, input_name+'.pkl'), 'rb') as f:
        features, codes = pickle.load(f)
    # Train model
    classifiers = train(features, codes, **kwargs)
    # Save model
    if output_suffix:
        with open(os.path.join(directory, '{}_{}.pkl'.format(input_name, output_suffix)), 'wb') as f:
            pickle.dump(classifiers, f)
    return features, codes, classifiers

def predict(classifiers, messages):
    """
    Apply a number of classifiers to a number of messages,
    returning the most likely result for each classfier ond message
    :param classifiers: classifier or list of classifiers
    :param messages: feature vectors (as a matrix)
    :return: array of predictions
    """
    # If more than one classifier is given, apply each
    if isinstance(classifiers, list):
        # Get the predictions from each classifier, giving zeros when a classifier is None
        predictions = [c.predict(messages) if c is not None else np.zeros(len(messages)) for c in classifiers]
        # Transpose so that the shape is (n_datapoints, n_classifiers)
        return np.array(predictions, dtype='bool').transpose()
    else:
        return classifiers.predict(messages)

def predict_prob(classifiers, messages):
    """
    Apply a number of classifiers to a number of messages,
    returning the probability of predicting each code for each message
    :param classifiers: classifier or list of classifiers
    :param messages: feature vectors (as a matrix)
    :return: array of probabilities
    """
    # If more than one classifier is given, apply each
    if isinstance(classifiers, list):
        # Get the prediction probabilities from each classifier
        # c.predict_proba returns probabilities for [False, True]
        # taking [:,1] will just give us probability of True
        prob = [c.predict_proba(messages)[:,1] if c is not None else np.zeros(len(messages)) for c in classifiers]
        # Transpose so that the shape is (n_datapoints, n_classifiers)
        return np.array(prob).transpose()
    else:
        return classifiers.predict_proba(messages)[:,1]

def evaluate(pred, gold, verbose=True):
    """
    Calculate the accuracy, precision, recall, and F1 score,
    for a set of predictions, compared to a gold standard
    :param pred: predictions of classifiers
    :param gold: gold standard annotations
    :param verbose: whether to print results (default True)
    :return: accuracy, precision, recall, F1 (each as an array)
    """
    # Check which predictions are correct
    correct = (pred == gold)
    n_correct = correct.sum(0)

    # Calculate different types of mistake
    n_true_correct = (correct * gold).sum(0)
    n_false_correct = (correct * (1-gold)).sum(0)
    n_true_wrong = ((1-correct) * gold).sum(0)
    n_false_wrong = ((1-correct) * (1-gold)).sum(0)

    # Accuracy: proportion correct, out of all messages
    accuracy = n_correct / pred.shape[0]
    # Precision: proportion correct, out of those predicted to have a code
    precision = n_true_correct / (n_true_correct + n_false_wrong)
    # Recall: proportion correct, out of those annotated with a code
    recall = n_true_correct / (n_true_correct + n_true_wrong)
    # F1: harmonic mean of precision and recall
    f1 = 2 * precision * recall / (precision + recall)

    if verbose:
        # Print results
        print('Accuracy:', accuracy)
        print('Precision:', precision)
        print('Recall:', recall)
        print('F1:', f1)

    return accuracy, precision, recall, f1

if __name__ == "__main__":
    'example use:'
    #features, codes, classifiers = train_on_file('delivery', 'C1', C=1)
    #predictions = predict(classifiers, features)
    #evaluate(predictions, codes)
<commit_msg>Add support for keywords, add options for weighting classes<commit_after>import pickle, os, numpy as np
from sklearn import linear_model

def train(features, codes, penalty='l1', C=1, keywords=None, keyword_strength=1, keyword_weight=1, weight_option='balanced', smoothing=0):
    """
    Train logistic regression classifiers,
    independently for each code
    :param features: input matrix, of shape [num_messages, num_features]
    :param codes: output matrix, of shape [num_messages, num_codes]
    :param penalty: type of regularisation ('l1' or 'l2')
    :param C: inverse of regularisation strength
    :param keywords: (optional) matrix of shape [num_codes, num_features],
        with a nonzero value when a feature should be considered a keyword for that code.
        The value should be the importance of this keyword, relative to a single message
    :param keyword_strength: (default 1) value to give to the vector for each keyword feature
    :param keyword_weight: (default 1) weight to assign a keyword vector (relative to a normal message)
    :param weight_option: determine the total weight for each code across all messages. Options are:
        - 'balanced': total weight is the same for true and false
        - 'smoothed': total weight is the observed number, plus smoothing
    :param smoothing: (default 0, i.e. no smoothing) constant to add,
        to reweight frequencies of each code
    :return: list of classifiers
    """
    N, F = features.shape
    classifiers = []
    # Iterate through each code (i.e. each column of codes matrix)
    for i, code_i in enumerate(codes.transpose()):
        # If there are no training examples, return None for this code
        N_pos = code_i.sum()
        if N_pos == 0:
            classifiers.append(None)
            continue

        # Check if keywords were given
        if keywords is None:
            # If no keywords, leave matrices the same
            feat_mat = features
            code_vec = code_i
            N_key = 0
        else:
            # If given, add the keywords as additional messages
            indices = keywords[i].nonzero()[0]
            N_key = len(indices)
            # Treat each keyword feature as a separate message
            key_features = np.zeros((N_key, F))
            for j, j_ind in enumerate(indices):
                # Set the strength of the feature as asked for
                key_features[j, j_ind] = keyword_strength
            key_codes = np.ones(N_key, dtype='bool')
            # Extend the feature and code arrays
            feat_mat = np.concatenate((features, key_features))
            code_vec = np.concatenate((code_i, key_codes))

        # Weight classes as asked for
        if weight_option == 'balanced':
            class_weight = 'balanced'
        elif weight_option == 'smoothed':
            N_neg = N - N_pos
            class_weight = {True: (N_pos + smoothing) / (N_pos + N_key*keyword_weight),
                            False: (N_neg + smoothing) / N_neg}
        else:
            raise ValueError('weight option not recognised')

        # Weight keyword examples as asked for
        if keyword_weight == 1:
            sample_weight = None
        else:
            sample_weight = np.ones(N+N_key)
            sample_weight[N:] = keyword_weight

        # Initialise a logistic regression model
        model = linear_model.LogisticRegression(penalty=penalty, C=C, class_weight=class_weight)

        # Train the model
        model.fit(feat_mat, code_vec, sample_weight=sample_weight)

        classifiers.append(model)

    return classifiers

def train_on_file(input_name, output_suffix=None, directory='../data', **kwargs):
    """
    Train logistic regression classifiers on a file
    :param input_name: name of input file (without .pkl file extension)
    :param output_suffix: string to append to name of output file
    (if none is given, no file is saved)
    :param directory: directory of data files (default ../data)
    :param **kwargs: additional keyward arguments will be passed to train
    :return: features, codes, classifiers
    """
    # Load features and codes
    with open(os.path.join(directory, input_name+'.pkl'), 'rb') as f:
        features, codes = pickle.load(f)
    # Train model
    classifiers = train(features, codes, **kwargs)
    # Save model
    if output_suffix:
        with open(os.path.join(directory, '{}_{}.pkl'.format(input_name, output_suffix)), 'wb') as f:
            pickle.dump(classifiers, f)
    return features, codes, classifiers

def predict(classifiers, messages):
    """
    Apply a number of classifiers to a number of messages,
    returning the most likely result for each classfier ond message
    :param classifiers: classifier or list of classifiers
    :param messages: feature vectors (as a matrix)
    :return: array of predictions
    """
    # If more than one classifier is given, apply each
    if isinstance(classifiers, list):
        # Get the predictions from each classifier, giving zeros when a classifier is None
        predictions = [c.predict(messages) if c is not None else np.zeros(len(messages)) for c in classifiers]
        # Transpose so that the shape is (n_datapoints, n_classifiers)
        return np.array(predictions, dtype='bool').transpose()
    else:
        return classifiers.predict(messages)

def predict_prob(classifiers, messages):
    """
    Apply a number of classifiers to a number of messages,
    returning the probability of predicting each code for each message
    :param classifiers: classifier or list of classifiers
    :param messages: feature vectors (as a matrix)
    :return: array of probabilities
    """
    # If more than one classifier is given, apply each
    if isinstance(classifiers, list):
        # Get the prediction probabilities from each classifier
        # c.predict_proba returns probabilities for [False, True]
        # taking [:,1] will just give us probability of True
        prob = [c.predict_proba(messages)[:,1] if c is not None else np.zeros(len(messages)) for c in classifiers]
        # Transpose so that the shape is (n_datapoints, n_classifiers)
        return np.array(prob).transpose()
    else:
        return classifiers.predict_proba(messages)[:,1]

def evaluate(pred, gold, verbose=True):
    """
    Calculate the accuracy, precision, recall, and F1 score,
    for a set of predictions, compared to a gold standard
    :param pred: predictions of classifiers
    :param gold: gold standard annotations
    :param verbose: whether to print results (default True)
    :return: accuracy, precision, recall, F1 (each as an array)
    """
    # Check which predictions are correct
    correct = (pred == gold)
    n_correct = correct.sum(0)

    # Calculate different types of mistake
    n_true_correct = (correct * gold).sum(0)
    n_false_correct = (correct * (1-gold)).sum(0)
    n_true_wrong = ((1-correct) * gold).sum(0)
    n_false_wrong = ((1-correct) * (1-gold)).sum(0)

    # Accuracy: proportion correct, out of all messages
    accuracy = n_correct / pred.shape[0]
    # Precision: proportion correct, out of those predicted to have a code
    precision = n_true_correct / (n_true_correct + n_false_wrong)
    # Recall: proportion correct, out of those annotated with a code
    recall = n_true_correct / (n_true_correct + n_true_wrong)
    # F1: harmonic mean of precision and recall
    f1 = 2 * precision * recall / (precision + recall)

    if verbose:
        # Print results
        print('Accuracy:', accuracy)
        print('Precision:', precision)
        print('Recall:', recall)
        print('F1:', f1)

    return accuracy, precision, recall, f1

if __name__ == "__main__":
    'example use:'
    #features, codes, classifiers = train_on_file('delivery', 'C1', C=1)
    #predictions = predict(classifiers, features)
    #evaluate(predictions, codes)
<|endoftext|>


<commit_before>"""
This script adversarially trains a model using iterative attacks on multiple
GPUs.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import logging

from tensorflow.python.platform import app
from tensorflow.python.platform import flags

from trainer_multigpu import TrainerMultiGPU
from trainer_singlegpu import TrainerSingleGPU

FLAGS = flags.FLAGS

def main(argv=None):
    logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)

    hparams = flags.FLAGS

    if hparams.ngpu > 1:
        logging.info('Multi GPU Trainer.')
        trainer = TrainerMultiGPU(hparams)
    else:
        logging.info('Single GPU Trainer.')
        trainer = TrainerSingleGPU(hparams)
    trainer.model_train()
    trainer.evaluate(inc_epoch=False)

    trainer.finish()

if __name__ == '__main__':
    flags.DEFINE_integer('nb_epochs', 6, 'Number of epochs to train model')
    flags.DEFINE_integer('batch_size', 128, 'Size of training batches')
    flags.DEFINE_float('learning_rate', 0.001, 'Learning rate for training')
    flags.DEFINE_boolean('adv_train', False,
                         'Whether to do adversarial training')
    flags.DEFINE_boolean('save', True,
                         'Whether to save from a checkpoint')
    flags.DEFINE_string('save_dir', 'runs/X',
                        'Location to store logs/model.')
    flags.DEFINE_string('model_type', 'madry',
                        'Model type: madry|resnet_tf')
    flags.DEFINE_string('attack_type_train', 'MadryEtAl_y_multigpu',
                        'Attack type for adversarial training:\
                        FGSM|MadryEtAl{,_y,_y_multigpu}')
    flags.DEFINE_string('attack_type_test', 'FGSM',
                        'Attack type for test: FGSM|MadryEtAl{,_y}')
    flags.DEFINE_string('dataset', 'mnist', 'Dataset mnist|cifar10')
    flags.DEFINE_boolean('only_adv_train', False,
                         'Do not train with clean examples when adv training.')
    flags.DEFINE_integer('save_steps', 50, 'Save model per X steps.')
    flags.DEFINE_integer('attack_nb_iter_train', None,
                         'Number of iterations of training attack')
    flags.DEFINE_integer('eval_iters', 1, 'Evaluate every X steps')
    flags.DEFINE_integer('lrn_step', 30000, 'Step to decrease learning rate')
    flags.DEFINE_float('resnet_lrn', .1, 'Initial learning rate for resnet')
    flags.DEFINE_string('optimizer', 'mom', 'Optimizer for resnet')
    flags.DEFINE_integer('ngpu', 1, 'Number of gpus')
    flags.DEFINE_integer('sync_step', 1, 'Sync params frequency')
    flags.DEFINE_boolean('fast_tests', False, 'Fast tests against attacks')
    flags.DEFINE_boolean('debug_graph', False,
                         'Saves the graph to Tensorobard.')
    flags.DEFINE_boolean('data_path', './datasets/', 'Path to datasets.'
                         'Each dataset should be in a subdirectory.')

    app.run()
<commit_msg>fix data_path<commit_after>"""
This script adversarially trains a model using iterative attacks on multiple
GPUs.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import logging

from tensorflow.python.platform import app
from tensorflow.python.platform import flags

from trainer_multigpu import TrainerMultiGPU
from trainer_singlegpu import TrainerSingleGPU

FLAGS = flags.FLAGS

def main(argv=None):
    logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)

    hparams = flags.FLAGS

    if hparams.ngpu > 1:
        logging.info('Multi GPU Trainer.')
        trainer = TrainerMultiGPU(hparams)
    else:
        logging.info('Single GPU Trainer.')
        trainer = TrainerSingleGPU(hparams)
    trainer.model_train()
    trainer.evaluate(inc_epoch=False)

    trainer.finish()

if __name__ == '__main__':
    flags.DEFINE_integer('nb_epochs', 6, 'Number of epochs to train model')
    flags.DEFINE_integer('batch_size', 128, 'Size of training batches')
    flags.DEFINE_float('learning_rate', 0.001, 'Learning rate for training')
    flags.DEFINE_boolean('adv_train', False,
                         'Whether to do adversarial training')
    flags.DEFINE_boolean('save', True,
                         'Whether to save from a checkpoint')
    flags.DEFINE_string('save_dir', 'runs/X',
                        'Location to store logs/model.')
    flags.DEFINE_string('model_type', 'madry',
                        'Model type: madry|resnet_tf')
    flags.DEFINE_string('attack_type_train', 'MadryEtAl_y_multigpu',
                        'Attack type for adversarial training:\
                        FGSM|MadryEtAl{,_y,_y_multigpu}')
    flags.DEFINE_string('attack_type_test', 'FGSM',
                        'Attack type for test: FGSM|MadryEtAl{,_y}')
    flags.DEFINE_string('dataset', 'mnist', 'Dataset mnist|cifar10')
    flags.DEFINE_boolean('only_adv_train', False,
                         'Do not train with clean examples when adv training.')
    flags.DEFINE_integer('save_steps', 50, 'Save model per X steps.')
    flags.DEFINE_integer('attack_nb_iter_train', None,
                         'Number of iterations of training attack')
    flags.DEFINE_integer('eval_iters', 1, 'Evaluate every X steps')
    flags.DEFINE_integer('lrn_step', 30000, 'Step to decrease learning rate')
    flags.DEFINE_float('resnet_lrn', .1, 'Initial learning rate for resnet')
    flags.DEFINE_string('optimizer', 'mom', 'Optimizer for resnet')
    flags.DEFINE_integer('ngpu', 1, 'Number of gpus')
    flags.DEFINE_integer('sync_step', 1, 'Sync params frequency')
    flags.DEFINE_boolean('fast_tests', False, 'Fast tests against attacks')
    flags.DEFINE_boolean('debug_graph', False,
                         'Saves the graph to Tensorobard.')
    flags.DEFINE_string('data_path', './datasets/', 'Path to datasets.'
                        'Each dataset should be in a subdirectory.')

    app.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8
# vim: ts=4 sw=4 et
"""
=================
Live Event Dumper
=================

Recieves triggered events from the detector and dumps them to a file.

"""
# Author: Tamas Gal <tgal@km3net.de>
# License: MIT
"""Read & dump events through the CH Pump.
"""
from km3pipe import Pipeline, Module
from km3pipe.io.ch import CHPump
from km3pipe.io.daq import DAQProcessor

class Dumper(Module):
    def configure(self):
        self.counter = 0

    def process(self, blob):
	tag = str(blob['CHPrefix'].tag)
	data = blob['CHData']
	self.dump(data, tag)
	print(blob["Hits"])
        return blob

    def dump(self, data, tag):
        with open('{0}-{1:06}.dat'.format(tag, self.counter), 'bw') as f:
            self.counter += 1
            f.write(data)

pipe = Pipeline()
pipe.attach(CHPump, host='xxx.xxx.xxx.xxx',
            port=5553,
            tags='IO_EVT',
            timeout=60*60*24,
            max_queue=42)
pipe.attach(DAQProcessor)
pipe.attach(Dumper)
pipe.drain()
<commit_msg>Fix indentationg<commit_after>#!/usr/bin/env python
# coding=utf-8
# vim: ts=4 sw=4 et
"""
=================
Live Event Dumper
=================

Recieves triggered events from the detector and dumps them to a file.

"""
# Author: Tamas Gal <tgal@km3net.de>
# License: MIT
"""Read & dump events through the CH Pump.
"""
from km3pipe import Pipeline, Module
from km3pipe.io.ch import CHPump
from km3pipe.io.daq import DAQProcessor

class Dumper(Module):
    def configure(self):
        self.counter = 0

    def process(self, blob):
        tag = str(blob['CHPrefix'].tag)
        data = blob['CHData']
        self.dump(data, tag)
        print(blob["Hits"])
        return blob

    def dump(self, data, tag):
        with open('{0}-{1:06}.dat'.format(tag, self.counter), 'bw') as f:
            self.counter += 1
            f.write(data)

pipe = Pipeline()
pipe.attach(CHPump, host='xxx.xxx.xxx.xxx',
            port=5553,
            tags='IO_EVT',
            timeout=60*60*24,
            max_queue=42)
pipe.attach(DAQProcessor)
pipe.attach(Dumper)
pipe.drain()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# ======================================================================
#
# hc.py
# ------
# Simple, pure Python model of the HC stream cipher with
# support for 128 and 256 bit keys.
#
# Note: Currently the model is only implementing HC-256.
#
#
# Author: Joachim Strömbergson
# Copyright (c) 2017, Assured AB
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# ======================================================================

# ------------------------------------------------------------------
# Python module imports.
# ------------------------------------------------------------------
import sys

# -------------------------------------------------------------------
# Defines.
# -------------------------------------------------------------------
MAX_W32 = 0xffffffff
HC256_TSIZE = 1024
HC128_TSIZE = 512

# -------------------------------------------------------------------
# HC
# The HC stream cipher class.
# ------------------------------------------------------------------
class HC():
    def __init__(self, verbose=False):
        # Allocate the internal state variables.
        self.verbose = verbose
        self.W = [0] * 2560
        self.P = [0] * 1024
        self.Q = [0] * 1024
        self.i = 0

    # Initialize the internal state given the key and iv.
    def init(self, key, iv):
        for i in range(8):
            self.W[i] = key[i]
            self.W[(i + 8)] = iv[i]

        for i in range(16, 2560):
            self.W[i] = (self.f2(self.W[(i - 2)]) + self.W[(i - 7)] +
                         self.f1(self.W[(i - 15)]) + self.W[(i - 15)] + i) & MAX_W32

        # We shoudl the cipher update for 4096 iterations too!

        for i in range(1024):
            self.P[i] = self.W[(i + 512)]
            self.Q[i] = self.W[(i + 1536)]

    # Update internal state and return the next word.
    def next(self):
        j = self.i % 1024
        if (self.i % 2048) < 1024:
            self.P[j] = (self.P[j] + self.P[self.subm(j, 10)]
                             + self.g1(self.P[self.subm(j, 3)], self.P[self.subm(j, 1023)])) & MAX_W32
            s = self.h1(self.P[self.subm(j, 12)]) ^ self.P[j]

        else:
            self.Q[j] = (self.Q[j] + self.Q[self.subm(j, 10)]
                             + self.g2(Q[self.subm(j, 3)], Q[self.subm(j, 1023)])) & MAX_W32
            s = self.h2(self.Q[self.subm(j, 12)]) ^ self.Q[j]

        self.i += 1
        return s

    # HC internal functions.
    def f1(self, x):
        return self.rotr(x, 7) ^ self.rotr(x, 18) ^ self.rotr(x, 3)

    def f2(self, x):
        return self.rotr(x, 17) ^ self.rotr(x, 19) ^ self.rotr(x, 10)

    def g1(self, x, y):
        return (self.rotr(x, 10) ^ self.rotr(x, 23) + self.Q[((x ^ y) % 1024)]) & MAX_W32

    def g2(self, x, y):
        return (self.rotr(x, 10) ^ self.rotr(x, 23) + self.P[((x ^ y) % 1024)]) & MAX_W32

    def h1(self, x):
        (x0, x1, x2, x3) = self.w2b(x)
        return (self.Q[x0] + self.Q[(x1 + 256)] +
                    self.Q[(x2 + 512)] + self.Q[(x3 + 768)]) & MAX_W32

    def h2(self, x):
        (x0, x1, x2, x3) = self.w2b(x)
        return (self.P[x0] + self.P[(x1 + 256)] +
                    self.P[(x2 + 512)] + self.P[(x3 + 768)]) & MAX_W32

    # Helper functions neeed to implement the HC functions.
    def rotr(self, w, b):
        return ((w >> b) | (w << (32 - b))) & 0xffffffff

    def w2b(self, x):
        x0 = x >> 24
        x1 = x >> 16 & 0xff
        x2 = x >> 8 & 0xff
        x3 = x & 0xff
        return (x0, x1, x2, x3)

    def subm(self, x, y):
        return (x - y) % 1024

    def dump_w(self):
        for i in range(0, 128, 4):
            print("W[%04d..%04d]: 0x%08x 0x%08x 0x%08x 0x%08x" %
                      (i, i+3, self.W[i], self.W[i+1], self.W[i+2], self.W[i+3]))

# ------------------------------------------------------------------
# test_rotr()
# Test the hc help function rotr()
# ------------------------------------------------------------------
def test_rotr():
    my_hc = HC()
    print("Testing rotr help function.")
    for i in range(33):
        print("rotr %02d steps: 0x%08x" % (i, my_hc.rotr(0x00000001, i)))
    print("")

# ------------------------------------------------------------------
# test_hc()
# Test the HC implementation with 128 and 256 bit keys.
# ------------------------------------------------------------------
def test_hc():
    my_key = [0] * 8
    my_iv = [0] * 8

    my_hc = HC()
    my_hc.init(my_key, my_iv)
    my_hc.dump_w()

    for i in range(16):
        print("keystream %02d = 0x%08x" % (i, my_hc.next()))

# ------------------------------------------------------------------
# main()
# If executed tests the ChaCha class using known test vectors.
# ------------------------------------------------------------------
def main():
    print("Testing the HC stream cipher model")
    print("==================================")
    print

#    test_rotr()
    test_hc()

# ------------------------------------------------------------------
# __name__
# Python thingy which allows the file to be run standalone as
# well as parsed from within a Python interpreter.
# ------------------------------------------------------------------
if __name__ == "__main__":
    # Run the main function.
    sys.exit(main())

# ======================================================================
# EOF hc.py
# ======================================================================
<commit_msg>making dumping of data a class behavior controlled using verbosity.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# ======================================================================
#
# hc.py
# ------
# Simple, pure Python model of the HC stream cipher with
# support for 128 and 256 bit keys.
#
# Note: Currently the model is only implementing HC-256.
#
#
# Author: Joachim Strömbergson
# Copyright (c) 2017, Assured AB
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# ======================================================================

# ------------------------------------------------------------------
# Python module imports.
# ------------------------------------------------------------------
import sys

# -------------------------------------------------------------------
# Defines.
# -------------------------------------------------------------------
MAX_W32 = 0xffffffff
HC256_TSIZE = 1024
HC128_TSIZE = 512
DUMP_W_ELEMENTS = 128

# -------------------------------------------------------------------
# HC
# The HC stream cipher class.
# ------------------------------------------------------------------
class HC():
    def __init__(self, verbose=False):
        # Allocate the internal state variables.
        self.verbose = verbose
        self.W = [0] * 2560
        self.P = [0] * 1024
        self.Q = [0] * 1024
        self.i = 0

    # Initialize the internal state given the key and iv.
    def init(self, key, iv):
        for i in range(8):
            self.W[i] = key[i]
            self.W[(i + 8)] = iv[i]

        for i in range(16, 2560):
            self.W[i] = (self.f2(self.W[(i - 2)]) + self.W[(i - 7)] +
                         self.f1(self.W[(i - 15)]) + self.W[(i - 15)] + i) & MAX_W32

        if self.verbose:
            self.dump_w()

        for i in range(1024):
            self.P[i] = self.W[(i + 512)]
            self.Q[i] = self.W[(i + 1536)]

        # We should run the cipher for 4096 iterations too.

    # Update internal state and return the next word.
    def next(self):
        j = self.i % 1024
        if (self.i % 2048) < 1024:
            self.P[j] = (self.P[j] + self.P[self.subm(j, 10)]
                             + self.g1(self.P[self.subm(j, 3)], self.P[self.subm(j, 1023)])) & MAX_W32
            s = self.h1(self.P[self.subm(j, 12)]) ^ self.P[j]

        else:
            self.Q[j] = (self.Q[j] + self.Q[self.subm(j, 10)]
                             + self.g2(Q[self.subm(j, 3)], Q[self.subm(j, 1023)])) & MAX_W32
            s = self.h2(self.Q[self.subm(j, 12)]) ^ self.Q[j]

        self.i += 1
        return s

    # Internal HC functions.
    def f1(self, x):
        return self.rotr(x, 7) ^ self.rotr(x, 18) ^ self.rotr(x, 3)

    def f2(self, x):
        return self.rotr(x, 17) ^ self.rotr(x, 19) ^ self.rotr(x, 10)

    def g1(self, x, y):
        return (self.rotr(x, 10) ^ self.rotr(x, 23) + self.Q[((x ^ y) % 1024)]) & MAX_W32

    def g2(self, x, y):
        return (self.rotr(x, 10) ^ self.rotr(x, 23) + self.P[((x ^ y) % 1024)]) & MAX_W32

    def h1(self, x):
        (x0, x1, x2, x3) = self.w2b(x)
        return (self.Q[x0] + self.Q[(x1 + 256)] +
                    self.Q[(x2 + 512)] + self.Q[(x3 + 768)]) & MAX_W32

    def h2(self, x):
        (x0, x1, x2, x3) = self.w2b(x)
        return (self.P[x0] + self.P[(x1 + 256)] +
                    self.P[(x2 + 512)] + self.P[(x3 + 768)]) & MAX_W32

    # Helper functions needed to implement the HC functions.
    def rotr(self, w, b):
        return ((w >> b) | (w << (32 - b))) & 0xffffffff

    def w2b(self, x):
        x0 = x >> 24
        x1 = x >> 16 & 0xff
        x2 = x >> 8 & 0xff
        x3 = x & 0xff
        return (x0, x1, x2, x3)

    def subm(self, x, y):
        return (x - y) % 1024

    def dump_w(self):
        print("State of the W array:")
        for i in range(0, DUMP_W_ELEMENTS, 4):
            print("W[%04d..%04d]: 0x%08x 0x%08x 0x%08x 0x%08x" %
                      (i, i+3, self.W[i], self.W[i+1], self.W[i+2], self.W[i+3]))
        print("")

# ------------------------------------------------------------------
# test_rotr()
# Test the hc help function rotr()
# ------------------------------------------------------------------
def test_rotr():
    my_hc = HC()
    print("Testing rotr help function.")
    for i in range(33):
        print("rotr %02d steps: 0x%08x" % (i, my_hc.rotr(0x00000001, i)))
    print("")

# ------------------------------------------------------------------
# test_hc()
# Test the HC implementation with 128 and 256 bit keys.
# ------------------------------------------------------------------
def test_hc():
    my_key = [0] * 8
    my_iv = [0] * 8

    my_hc = HC(verbose=True)
    my_hc.init(my_key, my_iv)

    for i in range(16):
        print("keystream %02d = 0x%08x" % (i, my_hc.next()))

# ------------------------------------------------------------------
# main()
# If executed tests the ChaCha class using known test vectors.
# ------------------------------------------------------------------
def main():
    print("Testing the HC stream cipher model")
    print("==================================")
    print

#    test_rotr()
    test_hc()

# ------------------------------------------------------------------
# __name__
# Python thingy which allows the file to be run standalone as
# well as parsed from within a Python interpreter.
# ------------------------------------------------------------------
if __name__ == "__main__":
    # Run the main function.
    sys.exit(main())

# ======================================================================
# EOF hc.py
# ======================================================================
<|endoftext|>


<commit_before># Main network and testnet3 definitions

# params = {
#     'bitcoin_main': {
#         'pubkey_address': 0,
#         'script_address': 5,
#         'genesis_hash': '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f'
#     },
#     'bitcoin_test': {
#         'pubkey_address': 111,
#         'script_address': 196,
#         'genesis_hash': '000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943'
#     }
# }

params = {
    'vcoin_main': {
        'pubkey_address': 70,
        'script_address': 30,
        'genesis_hash': '00000b7e804f0de87e7752550ff04d7686a4599509897feefd7f03904eb45633'
    },
    'vcoin_test': {
        'pubkey_address': 130,
        'script_address': 30,
        'genesis_hash': '13d8d31dde96874006da503dd2b63fa68c698dc823334359e417aa3a92f80433'
    }
}
<commit_msg>Update networks.py<commit_after># Main network and testnet3 definitions

# params = {
#     'bitcoin_main': {
#         'pubkey_address': 0,
#         'script_address': 5,
#         'genesis_hash': '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f'
#     },
#     'bitcoin_test': {
#         'pubkey_address': 111,
#         'script_address': 196,
#         'genesis_hash': '000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943'
#     }
# }

params = {
    'vcoin_main': {
        'pubkey_address': 70,
        'script_address': 30,
        'genesis_hash': '00000b7e804f0de87e7752550ff04d7686a4599509897feefd7f03904eb45633'
    },
    'vcoin_test': {
        'pubkey_address': 130,
        'script_address': 30,
        'genesis_hash': '00000b62077c6e53a2c7ada6e14feb5dfdeecd6001cc83d3a281145572112b17'
    }
}
<|endoftext|>


<commit_before># Natural Language Toolkit: Sets
#
# Copyright (C) 2001 University of Pennsylvania
# Author: Edward Loper <edloper@gradient.cis.upenn.edu>
# URL: <http://nltk.sf.net>
# For license information, see LICENSE.TXT
#
# $Id$

from chktype import chktype as _chktype
from chktype import chkclass as _chkclass

class Set:
    """
    An unordered container class that contains no duplicate elements.
    In particular, a set contains no elements M{e1} and M{e2} such
    that M{e1=e2}.  Currently, the C{Set} class is given a fairly
    minimal implementation.  However, more members (e.g., to iterate
    over a set) may be defined in the future.

    Although the C{Set} class attempts to ensure that it
    contains no duplicate elements, it can only do so under the
    following circumstances:

         - For all elements M{ei}, M{ej} added to the C{Set},
           M{ei=ej} if and only if M{ej=ei}.  This should always be
           the case as long as the elements in the C{Set} use
           well-defined comparison functions.  An example where it
           would not be the case would be if M{ei} defined
           C{__eq__}() to always return 0, and M{ej} defined
           C{__eq__}() to always return 1.

         - Mutable elements inserted in the C{Set} are not
           modified after they are inserted.

    If these circumstances are not met, the C{Set} will
    continue to function, but it will no longer guarantee that it
    contains no duplicate elements.
    """
    def __init__(self, *lst):
        """
        Construct a new Set, containing the elements in
        C{lst}.  If C{lst} contains any duplicate
        elements, only one of the elements will be included.  Example
        usage::

          set = Set('apple', 'orange', 'pear')

        @param lst: The elements that will be contained by the new Set.
        @type lst: Any
        """
        self._dict = {}
        for elt in lst:
            self._dict[elt] = 1

    def insert(self, elt):
        """
        Adds the specified element to this Set if it is not already
        present.  Formally, add C{elt} to this Set if and
        only if this set contains no element M{ei} such that
        C{elt}=M{e}.

        @param elt: The element to insert into the Set.
        @type elt: Any
        @rtype: C{boolean}
        @return: C{true} if the C{insert} operation added an element
            to the set; false if C{elt} was already present in the
            set.
        """
        if self._dict.has_key(elt): return 0
        self._dict[elt] = 1
        return 1

    def union(self, other):
        """
        Return the union of this Set and another Set.  Formally,
        construct and return a new Set containing an element M{e}
        if and only if either C{self} or C{other}
        contain M{e}.

        @param other: The Set with which this Set will be unioned.
        @type other: Set
        @return: The union of C{self} and C{other}.
        @rtype: Set
        """
        _chktype("union", 1, other, (Set,))
        newSet = self.copy()
        newSet._dict.update(other._dict)
        return newSet

    def intersection(self, other):
        """
        Return the intersection of this Set and another Set.
        Formally, construct and return a new Set containing an
        element M{e} if and only if both C{self} and
        C{other} contain M{e}.

        @param other: The Set with which this Set will be intersected.
        @type other: Set
        @return: The intersection of C{self} and
                C{other}.
        @rtype: Set
        """
        _chktype("intersection", 1, other, (Set,))
        newSet = self.copy()
        for elt in self._dict.keys():
            if not other._dict.has_key(elt):
                del newSet._dict[elt]
        return newSet

    def difference(self, other):
        """
        Return the difference between this Set and another Set.
        Formally, construct and return a new Set containing an element
        M{e} if and only if C{self} contains M{e} and C{other} does
        not contain M{e}.

        @param other: The set to subtract from this set.
        @type other: Set
        @return: The difference between this Set and another Set.
        @rtype: Set
        """
        _chktype("difference", 1, other, (Set,))
        newSet = self.copy()
        for elt in self._dict.keys():
            if other._dict.has_key(elt):
                del newSet._dict[elt]
        return newSet

    # set1 & set2
    def __and__(self, other):
        """
        Return the intersection of this Set and another Set.
        Formally, construct and return a new Set containing an
        element M{e} if and only if both C{self} and
        C{other} contain M{e}.

        This method is invoked for expressions of the form::

          set1 & set2

        @param other: The Set with which this Set will be intersected.
        @type other: Set
        @return: The intersection of C{self} and
                C{other}.
        @rtype: Set
        """
        return self.intersection(other)

    # set1 | set2
    def __or__(self, other):
        """
        Return the union of this Set and another Set.  Formally,
        construct and return a new Set containing an element M{e}
        if and only if either C{self} or C{other}
        contain M{e}.

        This method is invoked for expressions of the form::

          set1 | set2

        @param other: The Set with which this Set will be unioned.
        @type other: Set
        @return: The union of C{self} and C{other}.
        @rtype: Set
        """
        return self.union(other)

    def __minus__(self, other):
        """
        Return the difference between this Set and another Set.
        Formally, construct and return a new Set containing an element
        M{e} if and only if C{self} contains M{e} and C{other} does
        not contain M{e}.

        This method is invoked for expressions of the form::

          set1 - set2

        @param other: The set to subtract from this set.
        @type other: Set
        @return: The difference between this Set and another Set.
        @rtype: Set
        """
        return self.difference(other)

    def contains(self, elt):
        """
        Return true if this set contains the given element.
        Formally, return true if and only if this Set contains an
        element M{e} such that C{elt}=M{e}.

        @param elt: The element whose presence in the set is to be
               tested.
        @type elt: any
        @return: True if this set contains the given element.
        @rtype: boolean
        """
        return self._dict.has_key(elt)

    # elt in set
    def __contains__(self, elt):
        """
        Return true if this set contains the given element.
        Formally, return true if and only if this Set contains an
        element M{e} such that C{elt}=M{e}.

        This method is invoked for expressions of the form::

          elt in set

        @param elt: The element whose presence in the set is to be
               tested.
        @type elt: any
        @return: True if this set contains the given element.
        @rtype: boolean
        """
        return self.contains(elt)

    def copy(self):
        """
        Return a copy of this set.  Changes to the copy will not be
        reflected in the original, and vice versa.  However, changes
        to the copy's elements M{will} be reflected in the
        original, and vice versa.

        Currently, Sets are immutable, so this method has little use.
        However, the Set class may eventually be extended to make Sets
        mutable.

        @return: A copy of this set.
        @rtype: Set.
        """
        s=Set()
        s._dict.update(self._dict)
        return s

    def __repr__(self):
        """
        Return the string representation of this Set.  Sets
        are represented by strings of the form::

          {elt1, elt2, ..., eltn}

        For example, the string representation of
        C{Set('apple', 'orange', 'pear')} is::

          {'apple', 'orange', 'pear'}

        @return: The string representation of this Set.
        @rtype: string
        """
        return '{'+str(self._dict.keys())[1:-1]+'}'

    def __len__(self):
        """
        @return: The number of elements contained in this Set.
        @rtype: int
        """
        return len(self._dict)

    def count(self):
        """
        @return: The number of elements contained in this Set.
        @rtype: int
        """
        return len(self._dict)

    def __eq__(self, other):
        """
        Return 1 if the given object is equal to this Set.  In
        particular, return 1 if and only if C{other} is a
        Set, every member of this set is contained in
        C{other}, and every member of C{other} is
        contained in this Set.  Otherwise, return 0.

        @raise TypeError: if C{other} is not a C{Set}.
        @param other: The object to compare this Set to.
        @type other: any
        @return: 1 if the given object is equal to this Set.
        @rtype: int
        """
        _chkclass(self, other)
        return self._dict == other._dict

    def elements(self):
        """
        Return a C{list} of the elements in this
        C{Set}.  Changes to this C{list} will not
        be reflected in the C{Set}, and changes in the
        C{Set} will not be reflected in this
        C{list}.  This function is intended to allow
        iteration over a Set.

        @rtype: list
        @return: A C{list} of the elements in this
            C{Set}.
        """
        # We have to make a copy of the list.
        return self._dict.keys()

    def __hash__(self):
        """
        Return the hash value for this Set.  If two Sets are equal,
        they are guaranteed to have the same hash value.  However, two
        Sets may have the same hash value and still not be equal.

        @raise TypeError: if some element of the set is not a hashable
               type.
        @return: The hash value for this Set.
        @rtype: int
        """
        return hash(self._dict)
<commit_msg>    - Added an elements() method     - Renamed __minus__ to __sub__ (the proper name)     - Fixed some comments     - removed hash() method (sets are mutable)<commit_after># Natural Language Toolkit: Sets
#
# Copyright (C) 2001 University of Pennsylvania
# Author: Edward Loper <edloper@gradient.cis.upenn.edu>
# URL: <http://nltk.sf.net>
# For license information, see LICENSE.TXT
#
# $Id$

# To do:
#    - Add more mutation methods.

from chktype import chktype as _chktype
from chktype import chkclass as _chkclass

class Set:
    """
    An unordered container class that contains no duplicate elements.
    In particular, a set contains no elements M{e1} and M{e2} such
    that M{e1=e2}.  Currently, the C{Set} class is given a fairly
    minimal implementation.  However, more members (e.g., to iterate
    over a set) may be defined in the future.

    Although the C{Set} class attempts to ensure that it
    contains no duplicate elements, it can only do so under the
    following circumstances:

         - For all elements M{ei}, M{ej} added to the C{Set},
           M{ei=ej} if and only if M{ej=ei}.  This should always be
           the case as long as the elements in the C{Set} use
           well-defined comparison functions.  An example where it
           would not be the case would be if M{ei} defined
           C{__eq__}() to always return 0, and M{ej} defined
           C{__eq__}() to always return 1.

         - Mutable elements inserted in the C{Set} are not
           modified after they are inserted.

    If these circumstances are not met, the C{Set} will
    continue to function, but it will no longer guarantee that it
    contains no duplicate elements.
    """
    def __init__(self, *lst):
        """
        Construct a new Set, containing the elements in
        C{lst}.  If C{lst} contains any duplicate
        elements, only one of the elements will be included.  Example
        usage::

          set = Set('apple', 'orange', 'pear')

        @param lst: The elements that will be contained by the new Set.
        @type lst: Any
        """
        self._dict = {}
        for elt in lst:
            self._dict[elt] = 1

    def elements(self):
        """
        @return: a list of the elements contained in this Set.  The
            elements will be listed in arbitrary order, with each
            element appearing exactly once.
        @rtype: C{list}
        """
        return self._dict.keys()

    def insert(self, elt):
        """
        Adds the specified element to this Set if it is not already
        present.  Formally, add C{elt} to this Set if and
        only if this set contains no element M{ei} such that
        C{elt}=M{e}.

        @param elt: The element to insert into the Set.
        @type elt: Any
        @rtype: C{boolean}
        @return: C{true} if the C{insert} operation added an element
            to the set; false if C{elt} was already present in the
            set.
        """
        if self._dict.has_key(elt): return 0
        self._dict[elt] = 1
        return 1

    def union(self, other):
        """
        Return the union of this Set and another Set.  Formally,
        construct and return a new Set containing an element M{e}
        if and only if either C{self} or C{other}
        contain M{e}.

        @param other: The Set with which this Set will be unioned.
        @type other: Set
        @return: The union of C{self} and C{other}.
        @rtype: Set
        """
        _chktype("union", 1, other, (Set,))
        newSet = self.copy()
        newSet._dict.update(other._dict)
        return newSet

    def intersection(self, other):
        """
        Return the intersection of this Set and another Set.
        Formally, construct and return a new Set containing an
        element M{e} if and only if both C{self} and
        C{other} contain M{e}.

        @param other: The Set with which this Set will be intersected.
        @type other: Set
        @return: The intersection of C{self} and
                C{other}.
        @rtype: Set
        """
        _chktype("intersection", 1, other, (Set,))
        newSet = self.copy()
        for elt in self._dict.keys():
            if not other._dict.has_key(elt):
                del newSet._dict[elt]
        return newSet

    def difference(self, other):
        """
        Return the difference between this Set and another Set.
        Formally, construct and return a new Set containing an element
        M{e} if and only if C{self} contains M{e} and C{other} does
        not contain M{e}.

        @param other: The set to subtract from this set.
        @type other: Set
        @return: The difference between this Set and another Set.
        @rtype: Set
        """
        _chktype("difference", 1, other, (Set,))
        newSet = self.copy()
        for elt in self._dict.keys():
            if other._dict.has_key(elt):
                del newSet._dict[elt]
        return newSet

    # set1 & set2
    def __and__(self, other):
        """
        Return the intersection of this Set and another Set.
        Formally, construct and return a new Set containing an
        element M{e} if and only if both C{self} and
        C{other} contain M{e}.

        This method is invoked for expressions of the form::

          set1 & set2

        @param other: The Set with which this Set will be intersected.
        @type other: Set
        @return: The intersection of C{self} and
                C{other}.
        @rtype: Set
        """
        return self.intersection(other)

    # set1 | set2
    def __or__(self, other):
        """
        Return the union of this Set and another Set.  Formally,
        construct and return a new Set containing an element M{e}
        if and only if either C{self} or C{other}
        contain M{e}.

        This method is invoked for expressions of the form::

          set1 | set2

        @param other: The Set with which this Set will be unioned.
        @type other: Set
        @return: The union of C{self} and C{other}.
        @rtype: Set
        """
        return self.union(other)

    def __sub__(self, other):
        """
        Return the difference between this Set and another Set.
        Formally, construct and return a new Set containing an element
        M{e} if and only if C{self} contains M{e} and C{other} does
        not contain M{e}.

        This method is invoked for expressions of the form::

          set1 - set2

        @param other: The set to subtract from this set.
        @type other: Set
        @return: The difference between this Set and another Set.
        @rtype: Set
        """
        return self.difference(other)

    def contains(self, elt):
        """
        Return true if this set contains the given element.
        Formally, return true if and only if this Set contains an
        element M{e} such that C{elt}=M{e}.

        @param elt: The element whose presence in the set is to be
               tested.
        @type elt: any
        @return: True if this set contains the given element.
        @rtype: boolean
        """
        return self._dict.has_key(elt)

    # elt in set
    def __contains__(self, elt):
        """
        Return true if this set contains the given element.
        Formally, return true if and only if this Set contains an
        element M{e} such that C{elt}=M{e}.

        This method is invoked for expressions of the form::

          elt in set

        @param elt: The element whose presence in the set is to be
               tested.
        @type elt: any
        @return: True if this set contains the given element.
        @rtype: boolean
        """
        return self.contains(elt)

    def copy(self):
        """
        Return a copy of this set.  Changes to the copy will not be
        reflected in the original, and vice versa.  However, changes
        to the copy's elements M{will} be reflected in the
        original, and vice versa.

        @return: A copy of this set.
        @rtype: Set.
        """
        s=Set()
        s._dict.update(self._dict)
        return s

    def __repr__(self):
        """
        Return the string representation of this Set.  Sets
        are represented by strings of the form::

          {elt1, elt2, ..., eltn}

        For example, the string representation of
        C{Set('apple', 'orange', 'pear')} is::

          {'apple', 'orange', 'pear'}

        @return: The string representation of this Set.
        @rtype: string
        """
        return '{'+str(self._dict.keys())[1:-1]+'}'

    def __len__(self):
        """
        @return: The number of elements contained in this Set.
        @rtype: int
        """
        return len(self._dict)

    def count(self):
        """
        @return: The number of elements contained in this Set.
        @rtype: int
        """
        return len(self._dict)

    def __eq__(self, other):
        """
        Return 1 if the given object is equal to this Set.  In
        particular, return 1 if and only if C{other} is a
        Set, every member of this set is contained in
        C{other}, and every member of C{other} is
        contained in this Set.  Otherwise, return 0.

        @raise TypeError: if C{other} is not a C{Set}.
        @param other: The object to compare this Set to.
        @type other: any
        @return: 1 if the given object is equal to this Set.
        @rtype: int
        """
        _chkclass(self, other)
        return self._dict == other._dict

    def elements(self):
        """
        Return a C{list} of the elements in this
        C{Set}.  Changes to this C{list} will not
        be reflected in the C{Set}, and changes in the
        C{Set} will not be reflected in this
        C{list}.  This function is intended to allow
        iteration over a Set.

        @rtype: list
        @return: A C{list} of the elements in this
            C{Set}.
        """
        # We have to make a copy of the list.
        return self._dict.keys()

#     ## IS THIS A GOOD THING!?!??!?
#     def __hash__(self):
#         """
#         Return the hash value for this Set.  If two Sets are equal,
#         they are guaranteed to have the same hash value.  However, two
#         Sets may have the same hash value and still not be equal.
#
#         @raise TypeError: if some element of the set is not a hashable
#         type.
#         @return: The hash value for this Set.
#         @rtype: int
#         """
#         return hash(tuple(self._dict.keys()))
<|endoftext|>