<commit_before># baddies.py

from random import randint
from sprite import *
class baddies(pygame.sprite.Group):
	def __init__(self):
		pygame.sprite.Group.__init__(self)

	def generateBaddies(self, char, map):
		noBaddies = map.copy()
		playerSprite = pygame.sprite.Sprite()

		x, y = char.sprite.get_position()
		no_zone = pygame.rect.Rect((x,y),(100,100))
		playerSprite.rect = no_zone
		noBaddies.add(playerSprite)

		# baddies = pygame.sprite.Group()

		i = 1
		while i <= 10:
			b = Sprite('../resources/baddie.png')
			posx, posy = randint(0,800-b.rect.width), randint(0,600-b.rect.height)
			b.set_position(posx, posy)
			b.group = 0
			if pygame.sprite.spritecollideany(b, noBaddies) == None:
				self.add(b)
				i += 1

		return baddies

	def update(self, windowSurfaceObj):
		self.draw(windowSurfaceObj)<commit_msg>modded baddies<commit_after># baddies.py

from random import randint
from sprite import *
class baddies(pygame.sprite.Group):
	def __init__(self):
		pygame.sprite.Group.__init__(self)

	def generateBaddies(self, char, map):
		noBaddies = map.copy()
		playerSprite = pygame.sprite.Sprite()

		x, y = char.sprite.get_position()
		no_zone = pygame.rect.Rect((x,y),(100,100))
		playerSprite.rect = no_zone
		noBaddies.add(playerSprite)

		# baddies = pygame.sprite.Group()

		i = 1
		while i <= 10:
			b = baddy('../resources/baddie.png')
			posx, posy = randint(0,800-b.rect.width), randint(0,600-b.rect.height)
			b.set_position(posx, posy)
			b.group = 0
			if pygame.sprite.spritecollideany(b, noBaddies) == None:
				self.add(b)
				i += 1

		return baddies

	def update(self, windowSurfaceObj):
		self.draw(windowSurfaceObj)<|endoftext|>


<commit_before>import logging

from django.db import models
from django.contrib.auth.models import User

from django.dispatch import receiver
from django.db.models.signals import post_save
from django.utils.translation import ugettext_noop
from student.models import CourseEnrollment

from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule_django.models import CourseKeyField, NoneToEmptyManager

FORUM_ROLE_ADMINISTRATOR = ugettext_noop('Administrator')
FORUM_ROLE_MODERATOR = ugettext_noop('Moderator')
FORUM_ROLE_COMMUNITY_TA = ugettext_noop('Community TA')
FORUM_ROLE_STUDENT = ugettext_noop('Student')

@receiver(post_save, sender=CourseEnrollment)
def assign_default_role_on_enrollment(sender, instance, **kwargs):
    """
    Assign forum default role 'Student'
    """
    # The code below would remove all forum Roles from a user when they unenroll
    # from a course. Concerns were raised that it should apply only to students,
    # or that even the history of student roles is important for research
    # purposes. Since this was new functionality being added in this release,
    # I'm just going to comment it out for now and let the forums team deal with
    # implementing the right behavior.
    #
    # # We've unenrolled the student, so remove all roles for this course
    # if not instance.is_active:
    #     course_roles = list(Role.objects.filter(course_id=instance.course_id))
    #     instance.user.roles.remove(*course_roles)
    #     return

    # We've enrolled the student, so make sure they have the Student role
    assign_default_role(instance.course_id, instance.user)

def assign_default_role(course_id, user):
    """
    Assign forum default role 'Student' to user
    """
    role, __ = Role.objects.get_or_create(course_id=course_id, name=FORUM_ROLE_STUDENT)
    user.roles.add(role)

class Role(models.Model):

    objects = NoneToEmptyManager()

    name = models.CharField(max_length=30, null=False, blank=False)
    users = models.ManyToManyField(User, related_name="roles")
    course_id = CourseKeyField(max_length=255, blank=True, db_index=True)

    class Meta:
        # use existing table that was originally created from django_comment_client app
        db_table = 'django_comment_client_role'

    def __unicode__(self):
        # pylint: disable=no-member
        return self.name + " for " + (self.course_id.to_deprecated_string() if self.course_id else "all courses")

    def inherit_permissions(self, role):   # TODO the name of this method is a little bit confusing,
                                         # since it's one-off and doesn't handle inheritance later
        if role.course_id and role.course_id != self.course_id:
            logging.warning("%s cannot inherit permissions from %s due to course_id inconsistency", \
                            self, role)
        for per in role.permissions.all():
            self.add_permission(per)

    def add_permission(self, permission):
        self.permissions.add(Permission.objects.get_or_create(name=permission)[0])

    def has_permission(self, permission):
        course = modulestore().get_course(self.course_id)
        if course is None:
            raise ItemNotFoundError(self.course_id)
        if self.name == FORUM_ROLE_STUDENT and \
           (permission.startswith('edit') or permission.startswith('update') or permission.startswith('create')) and \
           (not course.forum_posts_allowed):
            return False

        return self.permissions.filter(name=permission).exists()

class Permission(models.Model):
    name = models.CharField(max_length=30, null=False, blank=False, primary_key=True)
    roles = models.ManyToManyField(Role, related_name="permissions")

    class Meta:
        # use existing table that was originally created from django_comment_client app
        db_table = 'django_comment_client_permission'

    def __unicode__(self):
        return self.name
<commit_msg>Fix PEP8: E502 the backslash is redundant<commit_after>import logging

from django.db import models
from django.contrib.auth.models import User

from django.dispatch import receiver
from django.db.models.signals import post_save
from django.utils.translation import ugettext_noop
from student.models import CourseEnrollment

from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule_django.models import CourseKeyField, NoneToEmptyManager

FORUM_ROLE_ADMINISTRATOR = ugettext_noop('Administrator')
FORUM_ROLE_MODERATOR = ugettext_noop('Moderator')
FORUM_ROLE_COMMUNITY_TA = ugettext_noop('Community TA')
FORUM_ROLE_STUDENT = ugettext_noop('Student')

@receiver(post_save, sender=CourseEnrollment)
def assign_default_role_on_enrollment(sender, instance, **kwargs):
    """
    Assign forum default role 'Student'
    """
    # The code below would remove all forum Roles from a user when they unenroll
    # from a course. Concerns were raised that it should apply only to students,
    # or that even the history of student roles is important for research
    # purposes. Since this was new functionality being added in this release,
    # I'm just going to comment it out for now and let the forums team deal with
    # implementing the right behavior.
    #
    # # We've unenrolled the student, so remove all roles for this course
    # if not instance.is_active:
    #     course_roles = list(Role.objects.filter(course_id=instance.course_id))
    #     instance.user.roles.remove(*course_roles)
    #     return

    # We've enrolled the student, so make sure they have the Student role
    assign_default_role(instance.course_id, instance.user)

def assign_default_role(course_id, user):
    """
    Assign forum default role 'Student' to user
    """
    role, __ = Role.objects.get_or_create(course_id=course_id, name=FORUM_ROLE_STUDENT)
    user.roles.add(role)

class Role(models.Model):

    objects = NoneToEmptyManager()

    name = models.CharField(max_length=30, null=False, blank=False)
    users = models.ManyToManyField(User, related_name="roles")
    course_id = CourseKeyField(max_length=255, blank=True, db_index=True)

    class Meta:
        # use existing table that was originally created from django_comment_client app
        db_table = 'django_comment_client_role'

    def __unicode__(self):
        # pylint: disable=no-member
        return self.name + " for " + (self.course_id.to_deprecated_string() if self.course_id else "all courses")

    def inherit_permissions(self, role):   # TODO the name of this method is a little bit confusing,
                                         # since it's one-off and doesn't handle inheritance later
        if role.course_id and role.course_id != self.course_id:
            logging.warning("%s cannot inherit permissions from %s due to course_id inconsistency",
                            self, role)
        for per in role.permissions.all():
            self.add_permission(per)

    def add_permission(self, permission):
        self.permissions.add(Permission.objects.get_or_create(name=permission)[0])

    def has_permission(self, permission):
        course = modulestore().get_course(self.course_id)
        if course is None:
            raise ItemNotFoundError(self.course_id)
        if self.name == FORUM_ROLE_STUDENT and \
           (permission.startswith('edit') or permission.startswith('update') or permission.startswith('create')) and \
           (not course.forum_posts_allowed):
            return False

        return self.permissions.filter(name=permission).exists()

class Permission(models.Model):
    name = models.CharField(max_length=30, null=False, blank=False, primary_key=True)
    roles = models.ManyToManyField(Role, related_name="permissions")

    class Meta:
        # use existing table that was originally created from django_comment_client app
        db_table = 'django_comment_client_permission'

    def __unicode__(self):
        return self.name
<|endoftext|>


<commit_before>"""Flask-RAML (REST API Markup Language) API server with parameter conversion, response encoding, and examples."""

__all__ = 'MimeEncoders API Loader Converter Content ApiError RequestError ParameterError AuthError'.split()

__version__ = '0.2.1'

from operator import itemgetter
from functools import wraps

from flask import abort, request, has_request_context, Response
from flask.app import HTTPException
from werkzeug.datastructures import MultiDict

import flask.ext.mime_encoders
import flask.ext.mime_encoders.json

import raml
from raml import Content, ApiError, RequestError, ParameterError, AuthError
   # Export raml module properties.

class MimeEncoders(flask.ext.mime_encoders.MimeEncoders):
    default = flask.ext.mime_encoders.MimeEncoders.json

class Converter(raml.Converter):
    log = True

    def convert_params(self, specification, params):
        if isinstance(params, MultiDict):
            params, multidict = {}, params
            for key, values in multidict.iteritems():
                params[key] = values[0] if len(values) == 1 else values

        return super(Converter, self).convert_params(specification, params)

class Loader(raml.Loader):
    log = True

    spec_param_template = '{{{name}}}'
    flask_param_template = '<{flask_type}:{name}>'

    flask_types = {
        'integer': 'int',
        'number': 'float',
        'string': 'string',
        'boolean': 'bool',
        'date': 'date',
    }

    def get_resource_uri(self, resource):
        uri = resource['relativeUri']
        if 'uriParameters' in resource:
            spec_format, flask_format = self.spec_param_template.format, self.flask_param_template.format
            for name, param in resource['uriParameters'].items():
                param['name'] = name
                param['flask_type'] = self.flask_types[param['type']]
                uri = uri.replace(spec_format(**param), flask_format(**param))
            resource['allUriParameters'].update(resource['uriParameters'])
        return uri

class API(raml.API):
    """Flask API.
    """
    plugins = dict(raml.API.plugins, loader=Loader, encoders=MimeEncoders, converter=Converter)

    auth = None
    logger_name = '{app}:api'
    decode_request = True
    encode_response = True
    convert_query_params = True
    convert_uri_params = True
    endpoint_template = '{api}{resource}_{methods}'
    requested_response_status_header = 'X-Test-Response-Status'
    default_error_status = 500
    default_error_message = 'internal server error'

    config_exclude = raml.API.config_exclude.union('unhandled_uris unhandled_methods'.split())

    def __init__(self, app, path, uri=None, id=None, log=None, **options):
        self.app = app
        self.views = {}

        if log is None or isinstance(log, basestring):
            log = app.logger.manager.getLogger(log or options.get('logger_name', self.logger_name).format(app=app.name))

        super(API, self).__init__(path, uri, id, log, **options)

        self.default_mimetype = self.encoders.default.mimetype

        if self.auth and getattr(self.auth, 'log', None) is True:
            self.auth.log = log

        if log:
            log.debug(repr(self))

    @property
    def unhandled_uris(self):
        return [uri for uri in self.api if uri not in self.views]

    @property
    def unhandled_methods(self):
        result = []
        for uri, resource in self.api.iteritems():
            methods = self.views.get(uri, ())
            result.extend((uri, method) for method in resource['methodsByName'] if method.upper() not in methods)
        return result

    def abort(self, status, error=None, encoder=True):
        if error:
            return abort(status, response = self.encoders[encoder].make_response(
                dict(status=status, error=error), status=status))
        else:
            return abort(status)

    def add_route(self, resource, view, methods=None, endpoint=None, **options):
        return self.route(resource, methods, endpoint, **options)(view)

    def route(self, resource, methods=None, endpoint=None, **options):
        resource = self.get_resource(resource)
        uri = resource['uri']
        config = dict(self.config, **options) if options else self.config
        methods = self.get_resource_methods(resource, methods)

        if endpoint is None:
            endpoint = self.get_endpoint(resource, methods, self.endpoint_template)

        auth = config['auth']
        decorate = config.get('decorate', None)
        decode_request = self.encoders[config['decode_request']]
        encode_response = self.encoders[config['encode_response']]
        convert_uri_params = config['convert_uri_params']
        convert_query_params = config['convert_query_params']

        def decorator(view):
            self.log.debug('map %s %s %s', self.id, '/'.join(sorted(methods)), uri)

            @wraps(view)
            def decorated_view(**uri_params):
                try:
                    self.log.info('%s %s %s [%s|%s|%s]', request.method, uri, uri_params,
                        len(uri_params) or '-', len(request.args) or '-', len(request.data) or '-')

                    if auth:
                        auth.authorize(uri_params, request)

                    method = self.get_method_spec(resource, request.method)

                    if convert_uri_params:
                        uri_params = self.converter.convert_params(resource['allUriParameters'], uri_params)

                    if convert_query_params:
                        if 'queryParameters' in method:
                            uri_params.update(self.converter.convert_params(method['queryParameters'], request.args))
                        elif request.args:
                            self.abort(400, 'resource does not accept query parameters')

                    if decode_request:
                        self.log.debug('%s %s << %s [%s]', request.method, uri, decode_request.name,
                            len(request.data))

                        uri_params.update(decode_request.get_request_data())

                    self.log.debug('%s %s %s [%s]', request.method, resource['uri'], uri_params,
                        len(uri_params) or '-')

                    response = view(**uri_params)

                    if encode_response and not isinstance(response, (Response, basestring)):
                        response = encode_response.make_response(response)

                        self.log.debug('%s %s >> %s [%s:%s] (%s)', request.method, uri, encode_response.name,
                            type(response.response), len(response.response), response.status)

                    return response

                except HTTPException:
                    raise
                except ApiError as error:
                    self.abort(error.status, error.message)
                except Exception as error:
                    msg =  str(error) if self.app.debug else self.default_error_message
                    self.log.exception('%r: %s', self.default_error_status, msg);
                    self.abort(self.default_error_status, msg)

            if decorate:
                decorated_view = decorate(decorated_view)

            self.app.add_url_rule(uri, endpoint, decorated_view, methods=methods)

            for method in methods:
                self.views.setdefault(uri, {})[method] = decorated_view

            return decorated_view

        return decorator

    def serve(self, view, *args, **kwargs):
        try:
            return view(*args, **kwargs)
        except ApiError as error:
            self.abort(error.status, error.message)

    def get_endpoint(self, resource, methods=None, template=None):
        return (template or self.endpoint_template).format(
            api=self.id,
            resource=resource['uniqueId'],
            methods='+'.join(methods) if methods else 'any',
            )

    def get_response_mimetype(self, response, accept=None, request=request):
        if accept is None:
            if request and has_request_context():
                accept = map(itemgetter(0), request.accept_mimetypes)
        return super(API, self).get_response_mimetype(response, accept)

    def get_default_status(self, status=None, request=request):
        try:
            return request.headers[self.requested_response_status_header]
        except (KeyError, RuntimeError):
            return super(API, self).get_default_status()

    def serve_examples(self, **options):
        for uri, method in self.unhandled_methods:
            self.serve_example(uri, method)

    def serve_example(self, resource, methods=None, **options):
        resource = self.get_resource(resource)

        for method in self.get_resource_methods(resource, methods):
            method_spec = self.get_method_spec(resource, method)
            self.route(resource, method, **options)(self.create_example_view(method_spec))

    def create_example_view(self, method_spec):
        def view(**params):
            return self.serve(self.get_example, method_spec)
        return view

    def get_example(self, method_spec, status=None, mimetype=None):
        response = self.get_response(method_spec, status)
        body = self.get_example_body(response, mimetype)
        headers = self.get_example_headers(response)

        self.log.info('%s %s: %s %s (%d bytes, %d headers)', method_spec['method'].upper(), method_spec['uri'],
            response['status'], body.mimetype, len(body), len(headers))

        return Response(body.content, status=response['status'], headers=headers, mimetype=body.mimetype)
<commit_msg>feat: create encoded response for default http exceptions, add error logging<commit_after>"""Flask-RAML (REST API Markup Language) API server with parameter conversion, response encoding, and examples."""

__all__ = 'MimeEncoders API Loader Converter Content ApiError RequestError ParameterError AuthError'.split()

__version__ = '0.2.1'

from sys import exc_info
from operator import itemgetter
from functools import wraps

from flask import abort, request, has_request_context, Response
from flask.app import HTTPException
from werkzeug.http import HTTP_STATUS_CODES
from werkzeug.datastructures import MultiDict

import flask.ext.mime_encoders
import flask.ext.mime_encoders.json

import raml
from raml import Content, ApiError, RequestError, ParameterError, AuthError
   # Export raml module properties.

class MimeEncoders(flask.ext.mime_encoders.MimeEncoders):
    default = flask.ext.mime_encoders.MimeEncoders.json

class Converter(raml.Converter):
    log = True

    def convert_params(self, specification, params):
        if isinstance(params, MultiDict):
            params, multidict = {}, params
            for key, values in multidict.iteritems():
                params[key] = values[0] if len(values) == 1 else values

        return super(Converter, self).convert_params(specification, params)

class Loader(raml.Loader):
    log = True

    spec_param_template = '{{{name}}}'
    flask_param_template = '<{flask_type}:{name}>'

    flask_types = {
        'integer': 'int',
        'number': 'float',
        'string': 'string',
        'boolean': 'bool',
        'date': 'date',
    }

    def get_resource_uri(self, resource):
        uri = resource['relativeUri']
        if 'uriParameters' in resource:
            spec_format, flask_format = self.spec_param_template.format, self.flask_param_template.format
            for name, param in resource['uriParameters'].items():
                param['name'] = name
                param['flask_type'] = self.flask_types[param['type']]
                uri = uri.replace(spec_format(**param), flask_format(**param))
            resource['allUriParameters'].update(resource['uriParameters'])
        return uri

class API(raml.API):
    """Flask API.
    """
    plugins = dict(raml.API.plugins, loader=Loader, encoders=MimeEncoders, converter=Converter)

    auth = None
    logger_name = '{app}:api'
    decode_request = True
    encode_response = True
    convert_query_params = True
    convert_uri_params = True
    endpoint_template = '{api}{resource}_{methods}'
    requested_response_status_header = 'X-Test-Response-Status'
    default_error_status = 500
    default_error_message = 'internal server error'

    config_exclude = raml.API.config_exclude.union('unhandled_uris unhandled_methods'.split())

    def __init__(self, app, path, uri=None, id=None, log=None, **options):
        self.app = app
        self.views = {}

        if log is None or isinstance(log, basestring):
            log = app.logger.manager.getLogger(log or options.get('logger_name', self.logger_name).format(app=app.name))

        super(API, self).__init__(path, uri, id, log, **options)

        self.default_mimetype = self.encoders.default.mimetype

        if self.auth and getattr(self.auth, 'log', None) is True:
            self.auth.log = log

        if log:
            log.debug(repr(self))

    @property
    def unhandled_uris(self):
        return [uri for uri in self.api if uri not in self.views]

    @property
    def unhandled_methods(self):
        result = []
        for uri, resource in self.api.iteritems():
            methods = self.views.get(uri, ())
            result.extend((uri, method) for method in resource['methodsByName'] if method.upper() not in methods)
        return result

    def abort(self, status, error=None, encoder=True):
        (self.log.exception if self.app.debug and exc_info()[0] else self.log.error)(
             '%r %s %s >> %s', status, request.method, request.path,
             error or HTTP_STATUS_CODES.get(status, 'Unknown Error'))

        if error:
            return abort(status, description=error, response = self.encoders[encoder].make_response(
                dict(status=status, error=error), status=status))
        else:
            return abort(status)

    def add_route(self, resource, view, methods=None, endpoint=None, **options):
        return self.route(resource, methods, endpoint, **options)(view)

    def route(self, resource, methods=None, endpoint=None, **options):
        resource = self.get_resource(resource)
        uri = resource['uri']
        config = dict(self.config, **options) if options else self.config
        methods = self.get_resource_methods(resource, methods)

        if endpoint is None:
            endpoint = self.get_endpoint(resource, methods, self.endpoint_template)

        auth = config['auth']
        decorate = config.get('decorate', None)
        decode_request = self.encoders[config['decode_request']]
        encode_response = self.encoders[config['encode_response']]
        convert_uri_params = config['convert_uri_params']
        convert_query_params = config['convert_query_params']

        def decorator(view):
            self.log.debug('map %s %s %s', self.id, '/'.join(sorted(methods)), uri)

            @wraps(view)
            def decorated_view(**uri_params):
                try:
                    url = request.path

                    self.log.info('%s %s << %s [%s|%s|%s]', request.method, url,
                        uri_params if self.app.debug or not uri_params else '{...}',
                        len(uri_params) or '-', len(request.args) or '-', len(request.data) or '-')

                    if auth:
                        auth.authorize(uri_params, request)

                    method = self.get_method_spec(resource, request.method)

                    if convert_uri_params:
                        uri_params = self.converter.convert_params(resource['allUriParameters'], uri_params)

                    if convert_query_params:
                        if 'queryParameters' in method:
                            uri_params.update(self.converter.convert_params(method['queryParameters'], request.args))
                        elif request.args:
                            self.abort(400, 'resource does not accept query parameters')

                    if uri_params:
                        self.log.debug('%s %s << args: %s [%s]', request.method, url, uri_params,
                            len(uri_params) or '-')

                    if decode_request:
                        self.log.debug('%s %s << data: %s [%s]', request.method, url, decode_request.name,
                            len(request.data))

                        uri_params.update(decode_request.get_request_data())

                    response = view(**uri_params)

                    if encode_response and not isinstance(response, (Response, basestring)):
                        response = encode_response.make_response(response)

                        self.log.debug('%s %s >> %s [%s:%s] (%s)', request.method, url, encode_response.name,
                            type(response.response), len(response.response), response.status)

                    return response

                except HTTPException as error:
                    if error.response:
                        # Use exception response if it was already created, either by API.abort(), or custom way.
                        raise
                    else:
                        # Otherwise, create a custom response via API.abort().
                        self.abort(error.code, error.description)
                except ApiError as error:
                    self.abort(error.status, error.message)
                except Exception as error:
                    msg =  str(error) if self.app.debug else self.default_error_message
                    self.abort(self.default_error_status, msg)

            if decorate:
                decorated_view = decorate(decorated_view)

            self.app.add_url_rule(uri, endpoint, decorated_view, methods=methods)

            for method in methods:
                self.views.setdefault(uri, {})[method] = decorated_view

            return decorated_view

        return decorator

    def serve(self, view, *args, **kwargs):
        try:
            return view(*args, **kwargs)
        except ApiError as error:
            self.abort(error.status, error.message)

    def get_endpoint(self, resource, methods=None, template=None):
        return (template or self.endpoint_template).format(
            api=self.id,
            resource=resource['uniqueId'],
            methods='+'.join(methods) if methods else 'any',
            )

    def get_response_mimetype(self, response, accept=None, request=request):
        if accept is None:
            if request and has_request_context():
                accept = map(itemgetter(0), request.accept_mimetypes)
        return super(API, self).get_response_mimetype(response, accept)

    def get_default_status(self, status=None, request=request):
        try:
            return request.headers[self.requested_response_status_header]
        except (KeyError, RuntimeError):
            return super(API, self).get_default_status()

    def serve_examples(self, **options):
        for uri, method in self.unhandled_methods:
            self.serve_example(uri, method)

    def serve_example(self, resource, methods=None, **options):
        resource = self.get_resource(resource)

        for method in self.get_resource_methods(resource, methods):
            method_spec = self.get_method_spec(resource, method)
            self.route(resource, method, **options)(self.create_example_view(method_spec))

    def create_example_view(self, method_spec):
        def view(**params):
            return self.serve(self.get_example, method_spec)
        return view

    def get_example(self, method_spec, status=None, mimetype=None):
        response = self.get_response(method_spec, status)
        body = self.get_example_body(response, mimetype)
        headers = self.get_example_headers(response)

        self.log.info('%s %s: %s %s (%d bytes, %d headers)', method_spec['method'].upper(), method_spec['uri'],
            response['status'], body.mimetype, len(body), len(headers))

        return Response(body.content, status=response['status'], headers=headers, mimetype=body.mimetype)
<|endoftext|>


<commit_before>##   transports.py
##
##   Copyright (C) 2003-2004 Alexey "Snake" Nezhdanov
##
##   This program is free software; you can redistribute it and/or modify
##   it under the terms of the GNU General Public License as published by
##   the Free Software Foundation; either version 2, or (at your option)
##   any later version.
##
##   This program is distributed in the hope that it will be useful,
##   but WITHOUT ANY WARRANTY; without even the implied warranty of
##   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
##   GNU General Public License for more details.

# $Id: transports.py,v 1.28 2006/01/26 13:09:35 snakeru Exp $

"""
This module contains the low-level implementations of xmpppy connect methods or
(in other words) transports for xmpp-stanzas.
Currently here is three transports:
direct TCP connect - TCPsocket class
proxied TCP connect - HTTPPROXYsocket class (CONNECT proxies)
TLS connection - TLS class. Can be used for SSL connections also.

Transports are stackable so you - f.e. TLS use HTPPROXYsocket or TCPsocket as more low-level transport.

Also exception 'error' is defined to allow capture of this module specific exceptions.
"""

import socket,select,base64,dispatcher,sys
from simplexml import ustr
from client import PlugIn
from protocol import *

# determine which DNS resolution library is available
HAVE_DNSPYTHON = False
HAVE_PYDNS = False

try:
    import dns.resolver # http://dnspython.org/
    HAVE_DNSPYTHON = True
except ImportError:
    try:
        import DNS # http://pydns.sf.net/
        HAVE_PYDNS = True
    except ImportError:
        #TODO: use self.DEBUG()
        sys.stderr.write("Could not load one of the supported DNS libraries (dnspython or pydns). SRV records will not be queried and you may need to set custom hostname/port for some servers to be accessible.\n")

DATA_RECEIVED='DATA RECEIVED'
DATA_SENT='DATA SENT'

class error:
    """An exception to be raised in case of low-level errors in methods of 'transports' module."""
    def __init__(self,comment):
        """Cache the descriptive string"""
        self._comment=comment

    def __str__(self):
        """Serialise exception into pre-cached descriptive string."""
        return self._comment

BUFLEN=1024
class TCPsocket(PlugIn):
    """ This class defines direct TCP connection method. """
    def __init__(self, server=None, use_srv=True):
        """ Cache connection point 'server'. 'server' is the tuple of (host, port)
            absolutely the same as standard tcp socket uses. """
        PlugIn.__init__(self)
        self.DBG_LINE='socket'
        self._exported_methods=[self.send,self.disconnect]

        # SRV resolver
        if use_srv and (HAVE_DNSPYTHON or HAVE_PYDNS):
            host, port = server
            possible_queries = ['_xmpp-client._tcp.' + host]

            for query in possible_queries:
                try:
                    if HAVE_DNSPYTHON:
                        answers = [x for x in dns.resolver.query(query, 'SRV')]
                        if answers:
                            host = str(answers[0].target)
                            port = int(answers[0].port)
                            break
                    elif HAVE_PYDNS:
                        # ensure we haven't cached an old configuration
                        DNS.ParseResolvConf()
                        response = DNS.Request().req(query, qtype='SRV')
                        answers = response.answers
                        if len(answers) > 0:
                            # ignore the priority and weight for now
                            _, _, port, host = answers[0]['data']
                            del _
                            port = int(port)
                            break
                except:
                    #TODO: use self.DEBUG()
                    print 'An error occurred while looking up %s' % query
            server = (host, port)
        # end of SRV resolver

        self._server = server

    def plugin(self, owner):
        """ Fire up connection. Return non-empty string on success.
            Also registers self.disconnected method in the owner's dispatcher.
            Called internally. """
        if not self._server: self._server=(self._owner.Server,5222)
        if not self.connect(self._server): return
        self._owner.Connection=self
        self._owner.RegisterDisconnectHandler(self.disconnected)
        return 'ok'

    def getHost(self):
        """ Return the 'host' value that is connection is [will be] made to."""
        return self._server[0]
    def getPort(self):
        """ Return the 'port' value that is connection is [will be] made to."""
        return self._server[1]

    def connect(self,server=None):
        """ Try to connect. Returns non-empty string on success. """
        try:
            if not server: server=self._server
            self._sock=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self._sock.connect(server)
            self._send=self._sock.sendall
            self._recv=self._sock.recv
            self.DEBUG("Successfully connected to remote host %s"%`server`,'start')
            return 'ok'
        except socket.error, (errno, strerror):
            self.DEBUG("Failed to connect to remote host %s: %s (%s)"%(`server`, strerror, errno),'error')
        except: pass

    def plugout(self):
        """ Disconnect from the remote server and unregister self.disconnected method from
            the owner's dispatcher. """
        self._sock.close()
        if self._owner.__dict__.has_key('Connection'):
            del self._owner.Connection
            self._owner.UnregisterDisconnectHandler(self.disconnected)

    def receive(self):
        """ Reads all pending incoming data.
            In case of disconnection calls owner's disconnected() method and then raises IOError exception."""
        try: received = self._recv(BUFLEN)
        except socket.sslerror,e:
            self._seen_data=0
            if e[0]==2: return ''
            self.DEBUG('Socket error while receiving data','warn')
            self._owner.disconnected()
            raise IOError("Disconnected from server")
        except: received = ''

        while self.pending_data(0):
            try: add = self._recv(BUFLEN)
            except: add=''
            received +=add
            if not add: break

        if len(received): # length of 0 means disconnect
            self._seen_data=1
            self.DEBUG(received,'got')
            if hasattr(self._owner, 'Dispatcher'):
                self._owner.Dispatcher.Event('', DATA_RECEIVED, received)
        else:
            self.DEBUG('Socket error while receiving data','warn')
            self._owner.disconnected()
            raise IOError("Disconnected from server")
        return received

    def send(self,raw_data):
        """ Writes raw outgoing data. Blocks until done.
            If supplied data is unicode string, encodes it to utf-8 before send."""
        if type(raw_data)==type(u''): raw_data = raw_data.encode('utf-8')
        elif type(raw_data)<>type(''): raw_data = ustr(raw_data).encode('utf-8')
        try:
            self._send(raw_data)
            # Avoid printing messages that are empty keepalive packets.
            if raw_data.strip():
                self.DEBUG(raw_data,'sent')
                self._owner.Dispatcher.Event('', DATA_SENT, raw_data)
        except:
            self.DEBUG("Socket error while sending data",'error')
            self._owner.disconnected()

    def pending_data(self,timeout=0):
        """ Returns true if there is a data ready to be read. """
        return select.select([self._sock],[],[],timeout)[0]

    def disconnect(self):
        """ Closes the socket. """
        self.DEBUG("Closing socket",'stop')
        self._sock.close()

    def disconnected(self):
        """ Called when a Network Error or disconnection occurs.
            Designed to be overidden. """
        self.DEBUG("Socket operation failed",'error')

DBG_CONNECT_PROXY='CONNECTproxy'
class HTTPPROXYsocket(TCPsocket):
    """ HTTP (CONNECT) proxy connection class. Uses TCPsocket as the base class
        redefines only connect method. Allows to use HTTP proxies like squid with
        (optionally) simple authentication (using login and password). """
    def __init__(self,proxy,server,use_srv=True):
        """ Caches proxy and target addresses.
            'proxy' argument is a dictionary with mandatory keys 'host' and 'port' (proxy address)
            and optional keys 'user' and 'password' to use for authentication.
            'server' argument is a tuple of host and port - just like TCPsocket uses. """
        TCPsocket.__init__(self,server,use_srv)
        self.DBG_LINE=DBG_CONNECT_PROXY
        self._proxy=proxy

    def plugin(self, owner):
        """ Starts connection. Used interally. Returns non-empty string on success."""
        owner.debug_flags.append(DBG_CONNECT_PROXY)
        return TCPsocket.plugin(self,owner)

    def connect(self,dupe=None):
        """ Starts connection. Connects to proxy, supplies login and password to it
            (if were specified while creating instance). Instructs proxy to make
            connection to the target server. Returns non-empty sting on success. """
        if not TCPsocket.connect(self,(self._proxy['host'],self._proxy['port'])): return
        self.DEBUG("Proxy server contacted, performing authentification",'start')
        connector = ['CONNECT %s:%s HTTP/1.0'%self._server,
            'Proxy-Connection: Keep-Alive',
            'Pragma: no-cache',
            'Host: %s:%s'%self._server,
            'User-Agent: HTTPPROXYsocket/v0.1']
        if self._proxy.has_key('user') and self._proxy.has_key('password'):
            credentials = '%s:%s'%(self._proxy['user'],self._proxy['password'])
            credentials = base64.encodestring(credentials).strip()
            connector.append('Proxy-Authorization: Basic '+credentials)
        connector.append('\r\n')
        self.send('\r\n'.join(connector))
        try: reply = self.receive().replace('\r','')
        except IOError:
            self.DEBUG('Proxy suddenly disconnected','error')
            self._owner.disconnected()
            return
        try: proto,code,desc=reply.split('\n')[0].split(' ',2)
        except: raise error('Invalid proxy reply')
        if code<>'200':
            self.DEBUG('Invalid proxy reply: %s %s %s'%(proto,code,desc),'error')
            self._owner.disconnected()
            return
        while reply.find('\n\n') == -1:
            try: reply += self.receive().replace('\r','')
            except IOError:
                self.DEBUG('Proxy suddenly disconnected','error')
                self._owner.disconnected()
                return
        self.DEBUG("Authentification successfull. Jabber server contacted.",'ok')
        return 'ok'

    def DEBUG(self,text,severity):
        """Overwrites DEBUG tag to allow debug output be presented as "CONNECTproxy"."""
        return self._owner.DEBUG(DBG_CONNECT_PROXY,text,severity)

class TLS(PlugIn):
    """ TLS connection used to encrypts already estabilished tcp connection."""
    def PlugIn(self,owner,now=0):
        """ If the 'now' argument is true then starts using encryption immidiatedly.
            If 'now' in false then starts encryption as soon as TLS feature is
            declared by the server (if it were already declared - it is ok).
        """
        if owner.__dict__.has_key('TLS'): return  # Already enabled.
        PlugIn.PlugIn(self,owner)
        DBG_LINE='TLS'
        if now: return self._startSSL()
        if self._owner.Dispatcher.Stream.features:
            try: self.FeaturesHandler(self._owner.Dispatcher,self._owner.Dispatcher.Stream.features)
            except NodeProcessed: pass
        else: self._owner.RegisterHandlerOnce('features',self.FeaturesHandler,xmlns=NS_STREAMS)
        self.starttls=None

    def plugout(self,now=0):
        """ Unregisters TLS handler's from owner's dispatcher. Take note that encription
            can not be stopped once started. You can only break the connection and start over."""
        self._owner.UnregisterHandler('features',self.FeaturesHandler,xmlns=NS_STREAMS)
	#self._owner.UnregisterHandlerOnce('proceed',self.StartTLSHandler,xmlns=NS_TLS)
	self._owner.UnregisterHandler('proceed',self.StartTLSHandler,xmlns=NS_TLS)
        #self._owner.UnregisterHandlerOnce('failure',self.StartTLSHandler,xmlns=NS_TLS)
        self._owner.UnregisterHandler('failure',self.StartTLSHandler,xmlns=NS_TLS)

    def FeaturesHandler(self, conn, feats):
        """ Used to analyse server <features/> tag for TLS support.
            If TLS is supported starts the encryption negotiation. Used internally"""
        if not feats.getTag('starttls',namespace=NS_TLS):
            self.DEBUG("TLS unsupported by remote server.",'warn')
            return
        self.DEBUG("TLS supported by remote server. Requesting TLS start.",'ok')
        self._owner.RegisterHandlerOnce('proceed',self.StartTLSHandler,xmlns=NS_TLS)
        self._owner.RegisterHandlerOnce('failure',self.StartTLSHandler,xmlns=NS_TLS)
        self._owner.Connection.send('<starttls xmlns="%s"/>'%NS_TLS)
        raise NodeProcessed

    def pending_data(self,timeout=0):
        """ Returns true if there possible is a data ready to be read. """
        return self._tcpsock._seen_data or select.select([self._tcpsock._sock],[],[],timeout)[0]

    def _startSSL(self):
        """ Immidiatedly switch socket to TLS mode. Used internally."""
        """ Here we should switch pending_data to hint mode."""
        tcpsock=self._owner.Connection
        tcpsock._sslObj    = socket.ssl(tcpsock._sock, None, None)
        tcpsock._sslIssuer = tcpsock._sslObj.issuer()
        tcpsock._sslServer = tcpsock._sslObj.server()
        tcpsock._recv = tcpsock._sslObj.read
        tcpsock._send = tcpsock._sslObj.write

        tcpsock._seen_data=1
        self._tcpsock=tcpsock
        tcpsock.pending_data=self.pending_data
        tcpsock._sock.setblocking(0)

        self.starttls='success'

    def StartTLSHandler(self, conn, starttls):
        """ Handle server reply if TLS is allowed to process. Behaves accordingly.
            Used internally."""
        if starttls.getNamespace()<>NS_TLS: return
        self.starttls=starttls.getName()
        if self.starttls=='failure':
            self.DEBUG("Got starttls response: "+self.starttls,'error')
            return
        self.DEBUG("Got starttls proceed response. Switching to TLS/SSL...",'ok')
        self._startSSL()
        self._owner.Dispatcher.PlugOut()
        dispatcher.Dispatcher().PlugIn(self._owner)
<commit_msg>Commented the dns import error. Tried to import dnspython and did not work, so we choose to hide the error.<commit_after>##   transports.py
##
##   Copyright (C) 2003-2004 Alexey "Snake" Nezhdanov
##
##   This program is free software; you can redistribute it and/or modify
##   it under the terms of the GNU General Public License as published by
##   the Free Software Foundation; either version 2, or (at your option)
##   any later version.
##
##   This program is distributed in the hope that it will be useful,
##   but WITHOUT ANY WARRANTY; without even the implied warranty of
##   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
##   GNU General Public License for more details.

# $Id: transports.py,v 1.28 2006/01/26 13:09:35 snakeru Exp $

"""
This module contains the low-level implementations of xmpppy connect methods or
(in other words) transports for xmpp-stanzas.
Currently here is three transports:
direct TCP connect - TCPsocket class
proxied TCP connect - HTTPPROXYsocket class (CONNECT proxies)
TLS connection - TLS class. Can be used for SSL connections also.

Transports are stackable so you - f.e. TLS use HTPPROXYsocket or TCPsocket as more low-level transport.

Also exception 'error' is defined to allow capture of this module specific exceptions.
"""

import socket,select,base64,dispatcher,sys
from simplexml import ustr
from client import PlugIn
from protocol import *

# determine which DNS resolution library is available
HAVE_DNSPYTHON = False
HAVE_PYDNS = False

try:
    import dns.resolver # http://dnspython.org/
    HAVE_DNSPYTHON = True
except ImportError:
    try:
        import DNS # http://pydns.sf.net/
        HAVE_PYDNS = True
    except ImportError: pass
        #TODO: use self.DEBUG()
        #sys.stderr.write("Could not load one of the supported DNS libraries (dnspython or pydns). SRV records will not be queried and you may need to set custom hostname/port for some servers to be accessible.\n")

DATA_RECEIVED='DATA RECEIVED'
DATA_SENT='DATA SENT'

class error:
    """An exception to be raised in case of low-level errors in methods of 'transports' module."""
    def __init__(self,comment):
        """Cache the descriptive string"""
        self._comment=comment

    def __str__(self):
        """Serialise exception into pre-cached descriptive string."""
        return self._comment

BUFLEN=1024
class TCPsocket(PlugIn):
    """ This class defines direct TCP connection method. """
    def __init__(self, server=None, use_srv=True):
        """ Cache connection point 'server'. 'server' is the tuple of (host, port)
            absolutely the same as standard tcp socket uses. """
        PlugIn.__init__(self)
        self.DBG_LINE='socket'
        self._exported_methods=[self.send,self.disconnect]

        # SRV resolver
        if use_srv and (HAVE_DNSPYTHON or HAVE_PYDNS):
            host, port = server
            possible_queries = ['_xmpp-client._tcp.' + host]

            for query in possible_queries:
                try:
                    if HAVE_DNSPYTHON:
                        answers = [x for x in dns.resolver.query(query, 'SRV')]
                        if answers:
                            host = str(answers[0].target)
                            port = int(answers[0].port)
                            break
                    elif HAVE_PYDNS:
                        # ensure we haven't cached an old configuration
                        DNS.ParseResolvConf()
                        response = DNS.Request().req(query, qtype='SRV')
                        answers = response.answers
                        if len(answers) > 0:
                            # ignore the priority and weight for now
                            _, _, port, host = answers[0]['data']
                            del _
                            port = int(port)
                            break
                except:
                    #TODO: use self.DEBUG()
                    print 'An error occurred while looking up %s' % query
            server = (host, port)
        # end of SRV resolver

        self._server = server

    def plugin(self, owner):
        """ Fire up connection. Return non-empty string on success.
            Also registers self.disconnected method in the owner's dispatcher.
            Called internally. """
        if not self._server: self._server=(self._owner.Server,5222)
        if not self.connect(self._server): return
        self._owner.Connection=self
        self._owner.RegisterDisconnectHandler(self.disconnected)
        return 'ok'

    def getHost(self):
        """ Return the 'host' value that is connection is [will be] made to."""
        return self._server[0]
    def getPort(self):
        """ Return the 'port' value that is connection is [will be] made to."""
        return self._server[1]

    def connect(self,server=None):
        """ Try to connect. Returns non-empty string on success. """
        try:
            if not server: server=self._server
            self._sock=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self._sock.connect(server)
            self._send=self._sock.sendall
            self._recv=self._sock.recv
            self.DEBUG("Successfully connected to remote host %s"%`server`,'start')
            return 'ok'
        except socket.error, (errno, strerror):
            self.DEBUG("Failed to connect to remote host %s: %s (%s)"%(`server`, strerror, errno),'error')
        except: pass

    def plugout(self):
        """ Disconnect from the remote server and unregister self.disconnected method from
            the owner's dispatcher. """
        self._sock.close()
        if self._owner.__dict__.has_key('Connection'):
            del self._owner.Connection
            self._owner.UnregisterDisconnectHandler(self.disconnected)

    def receive(self):
        """ Reads all pending incoming data.
            In case of disconnection calls owner's disconnected() method and then raises IOError exception."""
        try: received = self._recv(BUFLEN)
        except socket.sslerror,e:
            self._seen_data=0
            if e[0]==2: return ''
            self.DEBUG('Socket error while receiving data','warn')
            self._owner.disconnected()
            raise IOError("Disconnected from server")
        except: received = ''

        while self.pending_data(0):
            try: add = self._recv(BUFLEN)
            except: add=''
            received +=add
            if not add: break

        if len(received): # length of 0 means disconnect
            self._seen_data=1
            self.DEBUG(received,'got')
            if hasattr(self._owner, 'Dispatcher'):
                self._owner.Dispatcher.Event('', DATA_RECEIVED, received)
        else:
            self.DEBUG('Socket error while receiving data','warn')
            self._owner.disconnected()
            raise IOError("Disconnected from server")
        return received

    def send(self,raw_data):
        """ Writes raw outgoing data. Blocks until done.
            If supplied data is unicode string, encodes it to utf-8 before send."""
        if type(raw_data)==type(u''): raw_data = raw_data.encode('utf-8')
        elif type(raw_data)<>type(''): raw_data = ustr(raw_data).encode('utf-8')
        try:
            self._send(raw_data)
            # Avoid printing messages that are empty keepalive packets.
            if raw_data.strip():
                self.DEBUG(raw_data,'sent')
                self._owner.Dispatcher.Event('', DATA_SENT, raw_data)
        except:
            self.DEBUG("Socket error while sending data",'error')
            self._owner.disconnected()

    def pending_data(self,timeout=0):
        """ Returns true if there is a data ready to be read. """
        return select.select([self._sock],[],[],timeout)[0]

    def disconnect(self):
        """ Closes the socket. """
        self.DEBUG("Closing socket",'stop')
        self._sock.close()

    def disconnected(self):
        """ Called when a Network Error or disconnection occurs.
            Designed to be overidden. """
        self.DEBUG("Socket operation failed",'error')

DBG_CONNECT_PROXY='CONNECTproxy'
class HTTPPROXYsocket(TCPsocket):
    """ HTTP (CONNECT) proxy connection class. Uses TCPsocket as the base class
        redefines only connect method. Allows to use HTTP proxies like squid with
        (optionally) simple authentication (using login and password). """
    def __init__(self,proxy,server,use_srv=True):
        """ Caches proxy and target addresses.
            'proxy' argument is a dictionary with mandatory keys 'host' and 'port' (proxy address)
            and optional keys 'user' and 'password' to use for authentication.
            'server' argument is a tuple of host and port - just like TCPsocket uses. """
        TCPsocket.__init__(self,server,use_srv)
        self.DBG_LINE=DBG_CONNECT_PROXY
        self._proxy=proxy

    def plugin(self, owner):
        """ Starts connection. Used interally. Returns non-empty string on success."""
        owner.debug_flags.append(DBG_CONNECT_PROXY)
        return TCPsocket.plugin(self,owner)

    def connect(self,dupe=None):
        """ Starts connection. Connects to proxy, supplies login and password to it
            (if were specified while creating instance). Instructs proxy to make
            connection to the target server. Returns non-empty sting on success. """
        if not TCPsocket.connect(self,(self._proxy['host'],self._proxy['port'])): return
        self.DEBUG("Proxy server contacted, performing authentification",'start')
        connector = ['CONNECT %s:%s HTTP/1.0'%self._server,
            'Proxy-Connection: Keep-Alive',
            'Pragma: no-cache',
            'Host: %s:%s'%self._server,
            'User-Agent: HTTPPROXYsocket/v0.1']
        if self._proxy.has_key('user') and self._proxy.has_key('password'):
            credentials = '%s:%s'%(self._proxy['user'],self._proxy['password'])
            credentials = base64.encodestring(credentials).strip()
            connector.append('Proxy-Authorization: Basic '+credentials)
        connector.append('\r\n')
        self.send('\r\n'.join(connector))
        try: reply = self.receive().replace('\r','')
        except IOError:
            self.DEBUG('Proxy suddenly disconnected','error')
            self._owner.disconnected()
            return
        try: proto,code,desc=reply.split('\n')[0].split(' ',2)
        except: raise error('Invalid proxy reply')
        if code<>'200':
            self.DEBUG('Invalid proxy reply: %s %s %s'%(proto,code,desc),'error')
            self._owner.disconnected()
            return
        while reply.find('\n\n') == -1:
            try: reply += self.receive().replace('\r','')
            except IOError:
                self.DEBUG('Proxy suddenly disconnected','error')
                self._owner.disconnected()
                return
        self.DEBUG("Authentification successfull. Jabber server contacted.",'ok')
        return 'ok'

    def DEBUG(self,text,severity):
        """Overwrites DEBUG tag to allow debug output be presented as "CONNECTproxy"."""
        return self._owner.DEBUG(DBG_CONNECT_PROXY,text,severity)

class TLS(PlugIn):
    """ TLS connection used to encrypts already estabilished tcp connection."""
    def PlugIn(self,owner,now=0):
        """ If the 'now' argument is true then starts using encryption immidiatedly.
            If 'now' in false then starts encryption as soon as TLS feature is
            declared by the server (if it were already declared - it is ok).
        """
        if owner.__dict__.has_key('TLS'): return  # Already enabled.
        PlugIn.PlugIn(self,owner)
        DBG_LINE='TLS'
        if now: return self._startSSL()
        if self._owner.Dispatcher.Stream.features:
            try: self.FeaturesHandler(self._owner.Dispatcher,self._owner.Dispatcher.Stream.features)
            except NodeProcessed: pass
        else: self._owner.RegisterHandlerOnce('features',self.FeaturesHandler,xmlns=NS_STREAMS)
        self.starttls=None

    def plugout(self,now=0):
        """ Unregisters TLS handler's from owner's dispatcher. Take note that encription
            can not be stopped once started. You can only break the connection and start over."""
        self._owner.UnregisterHandler('features',self.FeaturesHandler,xmlns=NS_STREAMS)
	#self._owner.UnregisterHandlerOnce('proceed',self.StartTLSHandler,xmlns=NS_TLS)
	self._owner.UnregisterHandler('proceed',self.StartTLSHandler,xmlns=NS_TLS)
        #self._owner.UnregisterHandlerOnce('failure',self.StartTLSHandler,xmlns=NS_TLS)
        self._owner.UnregisterHandler('failure',self.StartTLSHandler,xmlns=NS_TLS)

    def FeaturesHandler(self, conn, feats):
        """ Used to analyse server <features/> tag for TLS support.
            If TLS is supported starts the encryption negotiation. Used internally"""
        if not feats.getTag('starttls',namespace=NS_TLS):
            self.DEBUG("TLS unsupported by remote server.",'warn')
            return
        self.DEBUG("TLS supported by remote server. Requesting TLS start.",'ok')
        self._owner.RegisterHandlerOnce('proceed',self.StartTLSHandler,xmlns=NS_TLS)
        self._owner.RegisterHandlerOnce('failure',self.StartTLSHandler,xmlns=NS_TLS)
        self._owner.Connection.send('<starttls xmlns="%s"/>'%NS_TLS)
        raise NodeProcessed

    def pending_data(self,timeout=0):
        """ Returns true if there possible is a data ready to be read. """
        return self._tcpsock._seen_data or select.select([self._tcpsock._sock],[],[],timeout)[0]

    def _startSSL(self):
        """ Immidiatedly switch socket to TLS mode. Used internally."""
        """ Here we should switch pending_data to hint mode."""
        tcpsock=self._owner.Connection
        tcpsock._sslObj    = socket.ssl(tcpsock._sock, None, None)
        tcpsock._sslIssuer = tcpsock._sslObj.issuer()
        tcpsock._sslServer = tcpsock._sslObj.server()
        tcpsock._recv = tcpsock._sslObj.read
        tcpsock._send = tcpsock._sslObj.write

        tcpsock._seen_data=1
        self._tcpsock=tcpsock
        tcpsock.pending_data=self.pending_data
        tcpsock._sock.setblocking(0)

        self.starttls='success'

    def StartTLSHandler(self, conn, starttls):
        """ Handle server reply if TLS is allowed to process. Behaves accordingly.
            Used internally."""
        if starttls.getNamespace()<>NS_TLS: return
        self.starttls=starttls.getName()
        if self.starttls=='failure':
            self.DEBUG("Got starttls response: "+self.starttls,'error')
            return
        self.DEBUG("Got starttls proceed response. Switching to TLS/SSL...",'ok')
        self._startSSL()
        self._owner.Dispatcher.PlugOut()
        dispatcher.Dispatcher().PlugIn(self._owner)
<|endoftext|>


<commit_before>"""Word cloud is ungraded xblock used by students to
generate and view word cloud.

On the client side we show:
If student does not yet answered - `num_inputs` numbers of text inputs.
If student have answered - words he entered and cloud.
"""

import json
import logging

from pkg_resources import resource_string
from xmodule.raw_module import EmptyDataRawDescriptor
from xmodule.editing_module import MetadataOnlyEditingDescriptor
from xmodule.x_module import XModule

from xblock.fields import Scope, Dict, Boolean, List, Integer, String

log = logging.getLogger(__name__)

def _(s):
    return s;

def pretty_bool(value):
    """Check value for possible `True` value.

    Using this function we can manage different type of Boolean value
    in xml files.
    """
    bool_dict = [True, "True", "true", "T", "t", "1"]
    return value in bool_dict

class MasterClassFields(object):
    """XFields for word cloud."""
    display_name = String(
        display_name=_("Display Name"),
        help=_("Display name for this module"),
        scope=Scope.settings,
        default="Master Class"
    )
    total_places = Integer(
        display_name=_("Max places"),
        help=_("Number of text boxes available for students to input words/sentences."),
        scope=Scope.settings,
        default=30,
        values={"min": 1}
    )
    autopass_score = Integer(
        display_name=_("Autopass score"),
        help=_("Maximum number of words to be displayed in generated word cloud."),
        scope=Scope.settings,
        default=250,
        values={"min": 1}
    )

    # Fields for descriptor.
    submitted = Boolean(
        help=_("Whether this student has been register for this master class."),
        scope=Scope.user_state,
        default=False
    )
    all_registrations = List(
        help=_("All registrations from all students."),
        scope=Scope.user_state_summary
    )
    passed_registrations = List(
        help=_("Passed registrations."),
        scope=Scope.user_state_summary
    )

class MasterClassModule(MasterClassFields, XModule):
    """MasterClass Xmodule"""
    js = {
        'coffee': [resource_string(__name__, 'js/src/javascript_loader.coffee')],
        'js': [resource_string(__name__, 'js/src/word_cloud/logme.js'),
        resource_string(__name__, 'js/src/master_class/master_class.js'),
        resource_string(__name__, 'js/src/master_class/master_class_main.js')]
    }
    css = {'scss': [resource_string(__name__, 'css/master_class/display.scss')]}
    js_module_name = "MasterClass"

    def get_state(self):
        """Return success json answer for client."""
        total_register = len(self.passed_registrations)
        message = ""
        if self.runtime.anonymous_student_id in self.passed_registrations:
            message = _("You have been registered for this master class. We will provide addition information soon.")
        else:
            message = _("You are pending for registration for this master class. Please visit this page later for result.")
        if (total_register is None):
            total_register = 0
        additional_data = {}
        if self.runtime.user_is_staff:
            additional_data['all_registrations'] = self.all_registrations
            additional_data['passed_registrations'] = self.passed_registrations
            additional_data['is_staff'] = self.runtime.user_is_staff
        if self.submitted:
            data = {
                'status': 'success',
                'submitted': True,
                'total_places': self.total_places,
                'total_register': total_register,
                'message': message
            }
            data.update(additional_data)
            return json.dumps(data)
        else:
            data = {
                'status': 'success',
                'submitted': False,
                'total_places': self.total_places,
                'total_register': total_register,
            }
            data.update(additional_data)
            return json.dumps(data)

    def handle_ajax(self, dispatch, data):
        """Ajax handler.

        Args:
            dispatch: string request slug
            data: dict request get parameters

        Returns:
            json string
        """
        if dispatch == 'submit':
            if self.submitted:
                return json.dumps({
                    'status': 'fail',
                    'error': 'You have already posted your data.'
                })

            # Student words from client.
            # FIXME: we must use raw JSON, not a post data (multipart/form-data)
            master_class = data.getlist('master_class')[0]

            self.all_registrations.append(self.runtime.user.email)

            self.submitted = True

            return self.get_state()
        elif dispatch == 'get_state':
            return self.get_state()
        elif dispatch == 'register':
            logging.error(data)
            if self.runtime.user_is_staff:
                for email in data.getlist('emails[]'):
                    if (len(self.passed_registrations) < self.total_places):
                        if (self.all_registrations.count(email) > 0):
                            self.passed_registrations.append(email)
                            self.all_registrations.remove(email)
                    else:
                        break
                return self.get_state()
            else:
                return json.dumps({
                    'status': 'fail',
                    'error': 'Unknown Command!'
                })
        else:
            return json.dumps({
                'status': 'fail',
                'error': 'Unknown Command!'
            })

    def get_html(self):
        """Template rendering."""
        logging.info(type(self.location))
        logging.info(self.get_progress())
        logging.info(self.runtime.seed)
        logging.info(self.runtime.anonymous_student_id)
        logging.info(self.runtime)
        context = {
            'display_name': self.display_name,
            'element_id': self.location.html_id(),
            'element_class': self.location.category,
            'ajax_url': self.system.ajax_url,
            'submitted': self.submitted,
            'is_staff': self.runtime.user_is_staff,
            'all_registrations': self.all_registrations,
            'passed_registrations': self.passed_registrations
        }
        self.content = self.system.render_template('master_class.html', context)
        return self.content

class MasterClassDescriptor(MasterClassFields, MetadataOnlyEditingDescriptor, EmptyDataRawDescriptor):
    """Descriptor for MasterClass Xmodule."""
    module_class = MasterClassModule
    template_dir_name = 'master_class'
<commit_msg>fixes<commit_after>"""Word cloud is ungraded xblock used by students to
generate and view word cloud.

On the client side we show:
If student does not yet answered - `num_inputs` numbers of text inputs.
If student have answered - words he entered and cloud.
"""

import json
import logging

from pkg_resources import resource_string
from xmodule.raw_module import EmptyDataRawDescriptor
from xmodule.editing_module import MetadataOnlyEditingDescriptor
from xmodule.x_module import XModule

from xblock.fields import Scope, Dict, Boolean, List, Integer, String

log = logging.getLogger(__name__)

def _(s):
    return s;

def pretty_bool(value):
    """Check value for possible `True` value.

    Using this function we can manage different type of Boolean value
    in xml files.
    """
    bool_dict = [True, "True", "true", "T", "t", "1"]
    return value in bool_dict

class MasterClassFields(object):
    """XFields for word cloud."""
    display_name = String(
        display_name=_("Display Name"),
        help=_("Display name for this module"),
        scope=Scope.settings,
        default="Master Class"
    )
    total_places = Integer(
        display_name=_("Max places"),
        help=_("Number of text boxes available for students to input words/sentences."),
        scope=Scope.settings,
        default=30,
        values={"min": 1}
    )
    autopass_score = Integer(
        display_name=_("Autopass score"),
        help=_("Maximum number of words to be displayed in generated word cloud."),
        scope=Scope.settings,
        default=250,
        values={"min": 1}
    )

    # Fields for descriptor.
    submitted = Boolean(
        help=_("Whether this student has been register for this master class."),
        scope=Scope.user_state,
        default=False
    )
    all_registrations = List(
        help=_("All registrations from all students."),
        scope=Scope.user_state_summary
    )
    passed_registrations = List(
        help=_("Passed registrations."),
        scope=Scope.user_state_summary
    )

class MasterClassModule(MasterClassFields, XModule):
    """MasterClass Xmodule"""
    js = {
        'coffee': [resource_string(__name__, 'js/src/javascript_loader.coffee')],
        'js': [resource_string(__name__, 'js/src/word_cloud/logme.js'),
        resource_string(__name__, 'js/src/master_class/master_class.js'),
        resource_string(__name__, 'js/src/master_class/master_class_main.js')]
    }
    css = {'scss': [resource_string(__name__, 'css/master_class/display.scss')]}
    js_module_name = "MasterClass"

    def get_state(self):
        """Return success json answer for client."""
        total_register = len(self.passed_registrations)
        message = ""
        if self.runtime.user.email in self.passed_registrations:
            message = _("You have been registered for this master class. We will provide addition information soon.")
        else:
            message = _("You are pending for registration for this master class. Please visit this page later for result.")
        if (total_register is None):
            total_register = 0
        additional_data = {}
        if self.runtime.user_is_staff:
            additional_data['all_registrations'] = self.all_registrations
            additional_data['passed_registrations'] = self.passed_registrations
            additional_data['is_staff'] = self.runtime.user_is_staff
        if self.submitted:
            data = {
                'status': 'success',
                'submitted': True,
                'total_places': self.total_places,
                'total_register': total_register,
                'message': message
            }
            data.update(additional_data)
            return json.dumps(data)
        else:
            data = {
                'status': 'success',
                'submitted': False,
                'total_places': self.total_places,
                'total_register': total_register,
            }
            data.update(additional_data)
            return json.dumps(data)

    def handle_ajax(self, dispatch, data):
        """Ajax handler.

        Args:
            dispatch: string request slug
            data: dict request get parameters

        Returns:
            json string
        """
        if dispatch == 'submit':
            if self.submitted:
                return json.dumps({
                    'status': 'fail',
                    'error': 'You have already posted your data.'
                })

            # Student words from client.
            # FIXME: we must use raw JSON, not a post data (multipart/form-data)
            master_class = data.getlist('master_class')[0]

            self.all_registrations.append(self.runtime.user.email)

            self.submitted = True

            return self.get_state()
        elif dispatch == 'get_state':
            return self.get_state()
        elif dispatch == 'register':
            logging.error(data)
            if self.runtime.user_is_staff:
                for email in data.getlist('emails[]'):
                    if (len(self.passed_registrations) < self.total_places):
                        if (self.all_registrations.count(email) > 0):
                            self.passed_registrations.append(email)
                            self.all_registrations.remove(email)
                    else:
                        break
                return self.get_state()
            else:
                return json.dumps({
                    'status': 'fail',
                    'error': 'Unknown Command!'
                })
        else:
            return json.dumps({
                'status': 'fail',
                'error': 'Unknown Command!'
            })

    def get_html(self):
        """Template rendering."""
        logging.info(type(self.location))
        logging.info(self.get_progress())
        logging.info(self.runtime.seed)
        logging.info(self.runtime.anonymous_student_id)
        logging.info(self.runtime)
        context = {
            'display_name': self.display_name,
            'element_id': self.location.html_id(),
            'element_class': self.location.category,
            'ajax_url': self.system.ajax_url,
            'submitted': self.submitted,
            'is_staff': self.runtime.user_is_staff,
            'all_registrations': self.all_registrations,
            'passed_registrations': self.passed_registrations
        }
        self.content = self.system.render_template('master_class.html', context)
        return self.content

class MasterClassDescriptor(MasterClassFields, MetadataOnlyEditingDescriptor, EmptyDataRawDescriptor):
    """Descriptor for MasterClass Xmodule."""
    module_class = MasterClassModule
    template_dir_name = 'master_class'
<|endoftext|>


<commit_before><commit_msg>Use only one forcefield object in setup<commit_after><|endoftext|>


<commit_before>from django.core.exceptions import ObjectDoesNotExist
from tastypie import fields
from tastypie.authorization import DjangoAuthorization
from tastypie.constants import ALL, ALL_WITH_RELATIONS
from tastypie.exceptions import Unauthorized
from tastypie.resources import ModelResource
from django_images.models import Thumbnail

from .models import Pin, Image
from users.models import User

class PinryAuthorization(DjangoAuthorization):
    """
    Pinry-specific Authorization backend with object-level permission checking.
    """
    def update_detail(self, object_list, bundle):
        klass = self.base_checks(bundle.request, bundle.obj.__class__)

        if klass is False:
            raise Unauthorized("You are not allowed to access that resource.")

        permission = '%s.change_%s' % (klass._meta.app_label, klass._meta.model_name)

        if not bundle.request.user.has_perm(permission, bundle.obj):
            raise Unauthorized("You are not allowed to access that resource.")

        return True

    def delete_detail(self, object_list, bundle):
        klass = self.base_checks(bundle.request, bundle.obj.__class__)

        if klass is False:
            raise Unauthorized("You are not allowed to access that resource.")

        permission = '%s.delete_%s' % (klass._meta.app_label, klass._meta.model_name)

        if not bundle.request.user.has_perm(permission, bundle.obj):
            raise Unauthorized("You are not allowed to access that resource.")

        return True

class UserResource(ModelResource):
    gravatar = fields.CharField(readonly=True)

    def dehydrate_gravatar(self, bundle):
        return bundle.obj.gravatar

    class Meta:
        list_allowed_methods = ['get']
        filtering = {
            'username': ALL
        }
        queryset = User.objects.all()
        resource_name = 'user'
        fields = ['username']
        include_resource_uri = False

def filter_generator_for(size):
    def wrapped_func(bundle, **kwargs):
        if hasattr(bundle.obj, '_prefetched_objects_cache') and 'thumbnail' in bundle.obj._prefetched_objects_cache:
            for thumbnail in bundle.obj._prefetched_objects_cache['thumbnail']:
                if thumbnail.size == size:
                    return thumbnail
            raise ObjectDoesNotExist()
        else:
            return bundle.obj.get_by_size(size)
    return wrapped_func

class ThumbnailResource(ModelResource):
    class Meta:
        list_allowed_methods = ['get']
        fields = ['image', 'width', 'height']
        queryset = Thumbnail.objects.all()
        resource_name = 'thumbnail'
        include_resource_uri = False

class ImageResource(ModelResource):
    standard = fields.ToOneField(ThumbnailResource, full=True,
                                 attribute=lambda bundle: filter_generator_for('standard')(bundle))
    thumbnail = fields.ToOneField(ThumbnailResource, full=True,
                                  attribute=lambda bundle: filter_generator_for('thumbnail')(bundle))
    square = fields.ToOneField(ThumbnailResource, full=True,
                               attribute=lambda bundle: filter_generator_for('square')(bundle))

    class Meta:
        fields = ['image', 'width', 'height']
        include_resource_uri = False
        resource_name = 'image'
        queryset = Image.objects.all()
        authorization = DjangoAuthorization()

class PinResource(ModelResource):
    submitter = fields.ToOneField(UserResource, 'submitter', full=True)
    image = fields.ToOneField(ImageResource, 'image', full=True)
    tags = fields.ListField()

    def hydrate_image(self, bundle):
        url = bundle.data.get('url', None)
        if url:
            image = Image.objects.create_for_url(url)
            bundle.data['image'] = '/api/v1/image/{}/'.format(image.pk)
        return bundle

    def hydrate(self, bundle):
        """Run some early/generic processing

        Make sure that user is authorized to create Pins first, before
        we hydrate the Image resource, creating the Image object in process
        """
        submitter = bundle.data.get('submitter', None)
        if not submitter:
            bundle.data['submitter'] = '/api/v1/user/{}/'.format(bundle.request.user.pk)
        else:
            if not '/api/v1/user/{}/'.format(bundle.request.user.pk) == submitter:
                raise Unauthorized("You are not authorized to create Pins for other users")
        return bundle

    def dehydrate_tags(self, bundle):
        return map(str, bundle.obj.tags.all())

    def build_filters(self, filters=None):
        orm_filters = super(PinResource, self).build_filters(filters)
        if filters and 'tag' in filters:
            orm_filters['tags__name__in'] = filters['tag'].split(',')
        return orm_filters

    def save_m2m(self, bundle):
        tags = bundle.data.get('tags', None)
        if tags:
            bundle.obj.tags.set(*tags)
        return super(PinResource, self).save_m2m(bundle)

    class Meta:
        fields = ['id', 'url', 'origin', 'description']
        ordering = ['id']
        filtering = {
            'submitter': ALL_WITH_RELATIONS
        }
        queryset = Pin.objects.all().select_related('submitter', 'image'). \
            prefetch_related('image__thumbnail_set', 'tags')
        resource_name = 'pin'
        include_resource_uri = False
        always_return_data = True
        authorization = PinryAuthorization()
<commit_msg>Fix tags being serialized as stringified map objects<commit_after>from django.core.exceptions import ObjectDoesNotExist
from tastypie import fields
from tastypie.authorization import DjangoAuthorization
from tastypie.constants import ALL, ALL_WITH_RELATIONS
from tastypie.exceptions import Unauthorized
from tastypie.resources import ModelResource
from django_images.models import Thumbnail

from .models import Pin, Image
from users.models import User

class PinryAuthorization(DjangoAuthorization):
    """
    Pinry-specific Authorization backend with object-level permission checking.
    """
    def update_detail(self, object_list, bundle):
        klass = self.base_checks(bundle.request, bundle.obj.__class__)

        if klass is False:
            raise Unauthorized("You are not allowed to access that resource.")

        permission = '%s.change_%s' % (klass._meta.app_label, klass._meta.model_name)

        if not bundle.request.user.has_perm(permission, bundle.obj):
            raise Unauthorized("You are not allowed to access that resource.")

        return True

    def delete_detail(self, object_list, bundle):
        klass = self.base_checks(bundle.request, bundle.obj.__class__)

        if klass is False:
            raise Unauthorized("You are not allowed to access that resource.")

        permission = '%s.delete_%s' % (klass._meta.app_label, klass._meta.model_name)

        if not bundle.request.user.has_perm(permission, bundle.obj):
            raise Unauthorized("You are not allowed to access that resource.")

        return True

class UserResource(ModelResource):
    gravatar = fields.CharField(readonly=True)

    def dehydrate_gravatar(self, bundle):
        return bundle.obj.gravatar

    class Meta:
        list_allowed_methods = ['get']
        filtering = {
            'username': ALL
        }
        queryset = User.objects.all()
        resource_name = 'user'
        fields = ['username']
        include_resource_uri = False

def filter_generator_for(size):
    def wrapped_func(bundle, **kwargs):
        if hasattr(bundle.obj, '_prefetched_objects_cache') and 'thumbnail' in bundle.obj._prefetched_objects_cache:
            for thumbnail in bundle.obj._prefetched_objects_cache['thumbnail']:
                if thumbnail.size == size:
                    return thumbnail
            raise ObjectDoesNotExist()
        else:
            return bundle.obj.get_by_size(size)
    return wrapped_func

class ThumbnailResource(ModelResource):
    class Meta:
        list_allowed_methods = ['get']
        fields = ['image', 'width', 'height']
        queryset = Thumbnail.objects.all()
        resource_name = 'thumbnail'
        include_resource_uri = False

class ImageResource(ModelResource):
    standard = fields.ToOneField(ThumbnailResource, full=True,
                                 attribute=lambda bundle: filter_generator_for('standard')(bundle))
    thumbnail = fields.ToOneField(ThumbnailResource, full=True,
                                  attribute=lambda bundle: filter_generator_for('thumbnail')(bundle))
    square = fields.ToOneField(ThumbnailResource, full=True,
                               attribute=lambda bundle: filter_generator_for('square')(bundle))

    class Meta:
        fields = ['image', 'width', 'height']
        include_resource_uri = False
        resource_name = 'image'
        queryset = Image.objects.all()
        authorization = DjangoAuthorization()

class PinResource(ModelResource):
    submitter = fields.ToOneField(UserResource, 'submitter', full=True)
    image = fields.ToOneField(ImageResource, 'image', full=True)
    tags = fields.ListField()

    def hydrate_image(self, bundle):
        url = bundle.data.get('url', None)
        if url:
            image = Image.objects.create_for_url(url)
            bundle.data['image'] = '/api/v1/image/{}/'.format(image.pk)
        return bundle

    def hydrate(self, bundle):
        """Run some early/generic processing

        Make sure that user is authorized to create Pins first, before
        we hydrate the Image resource, creating the Image object in process
        """
        submitter = bundle.data.get('submitter', None)
        if not submitter:
            bundle.data['submitter'] = '/api/v1/user/{}/'.format(bundle.request.user.pk)
        else:
            if not '/api/v1/user/{}/'.format(bundle.request.user.pk) == submitter:
                raise Unauthorized("You are not authorized to create Pins for other users")
        return bundle

    def dehydrate_tags(self, bundle):
        return list(map(str, bundle.obj.tags.all()))

    def build_filters(self, filters=None):
        orm_filters = super(PinResource, self).build_filters(filters)
        if filters and 'tag' in filters:
            orm_filters['tags__name__in'] = filters['tag'].split(',')
        return orm_filters

    def save_m2m(self, bundle):
        tags = bundle.data.get('tags', None)
        if tags:
            bundle.obj.tags.set(*tags)
        return super(PinResource, self).save_m2m(bundle)

    class Meta:
        fields = ['id', 'url', 'origin', 'description']
        ordering = ['id']
        filtering = {
            'submitter': ALL_WITH_RELATIONS
        }
        queryset = Pin.objects.all().select_related('submitter', 'image'). \
            prefetch_related('image__thumbnail_set', 'tags')
        resource_name = 'pin'
        include_resource_uri = False
        always_return_data = True
        authorization = PinryAuthorization()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import datetime
import logging
import os
import re
from urllib.parse import urljoin

from utils import utils, inspector

# http://www.cftc.gov/About/OfficeoftheInspectorGeneral/index.htm
archive = 2000

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
# - Add published dates for all reports in REPORT_PUBLISHED_MAPPING

REPORT_PUBLISHED_MAPPING = {
  "oig_mgmtuseitresources": datetime.datetime(2014, 10, 1),
  "oig_auditreportp05": datetime.datetime(2014, 7, 17),
  "oigocoaudit2014": datetime.datetime(2014, 5, 1),
  "oigcommentletter042214": datetime.datetime(2014, 4, 22),
  "oig_auditruleenforcementreview": datetime.datetime(2015, 8, 5),
  "oig_pensionawardsaudit": datetime.datetime(2015, 9, 24),
  "oigfinancialstatementalert": datetime.datetime(2016, 1, 15),
  "2015finstatementaudit": datetime.datetime(2016, 1, 15),
  "2009par": datetime.datetime(2009, 11, 13),
  "2004finstatementaudit": datetime.datetime(2004, 11, 16),
  "oigmgmtchall2015": datetime.datetime(2015, 10, 26),
  "oigmgmtchall2014": datetime.datetime(2014, 11, 17),
  "oigmgmtchall2013": datetime.datetime(2013, 12, 13),
  "oigmgmtchall2012": datetime.datetime(2012, 11, 14),
  "oigmgmtchall2011": datetime.datetime(2011, 11, 10),
  "oigmgmtchall2010": datetime.datetime(2010, 10, 10),
  "oigmgmtchall2009": datetime.datetime(2009, 11, 16),
  "oigmgmtchall2008": datetime.datetime(2008, 11, 14),
  "oigmgmtchall2007": datetime.datetime(2007, 11, 15),
  "oigmgmtchall2006": datetime.datetime(2006, 10, 31),
  "oigmgmtchall2005": datetime.datetime(2005, 10, 10),
  "oigmgmtchall2004": datetime.datetime(2004, 11, 12),
  "FOIAaudit20122011": datetime.datetime(2013, 1, 28),
  "cpfreport2015": datetime.datetime(2015, 10, 30),
  "cpfreport2014": datetime.datetime(2014, 10, 30),
  "cpfreport2013": datetime.datetime(2013, 10, 31),
  "cpfreport2012": datetime.datetime(2012, 10, 25),
  "cpfreport2011": datetime.datetime(2011, 10, 25),
  "fisma2014": datetime.datetime(2014, 2, 20),
}

REPORTS_URL = "http://www.cftc.gov/About/OfficeoftheInspectorGeneral/index.htm"

def run(options):
  year_range = inspector.year_range(options, archive)

  # Pull the audit reports
  doc = utils.beautifulsoup_from_url(REPORTS_URL)
  results = doc.select("ul.text > ul > li")
  if not results:
    raise inspector.NoReportsFoundError("CFTC audit reports")
  for result in results:
    report = report_from(result, year_range)
    if report:
      inspector.save_report(report)

  # Pull the semiannual reports
  results = doc.select("ul.text td a")
  if not results:
    raise inspector.NoReportsFoundError("CFTC semiannual reports")
  for result in results:
    report = report_from(result, year_range, report_type="semiannual_report")
    if report:
      inspector.save_report(report)

def extract_report_type(text):
  if 'Peer Review' in text:
    return "peer_review"
  elif 'Audit' in text:
    return "audit"
  elif 'Investigation' in text:
    return 'investigation'
  elif 'Inspection' in text:
    return "inspection"
  elif 'Management Challenges' in text:
    return "management_challenges"

def report_from(result, year_range, report_type=None):
  if result.name == 'a':
    link = result
  else:
    link = result.select("a")[-1]

  report_url = urljoin(REPORTS_URL, link.get('href'))
  report_filename = report_url.split("/")[-1]
  report_id, _ = os.path.splitext(report_filename)

  title = link.text

  published_on = None
  if report_id in REPORT_PUBLISHED_MAPPING:
    published_on = REPORT_PUBLISHED_MAPPING[report_id]
  if not published_on:
    try:
      published_on_text = "/".join(re.search("(\w+) (\d+), (\d+)", title).groups())
      published_on = datetime.datetime.strptime(published_on_text, '%B/%d/%Y')
    except AttributeError:
      pass
  if not published_on:
    try:
      published_on_text = "/".join(re.search("(\w+) (\d+), (\d+)", str(link.next_sibling)).groups())
      published_on = datetime.datetime.strptime(published_on_text, '%B/%d/%Y')
    except AttributeError:
      pass
  if not published_on:
    raise inspector.NoDateFoundError(report_id, title)

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  if not report_type:
    report_type = extract_report_type(title)
  if not report_type:
    report_type = extract_report_type(result.find_previous("p").text)
  if not report_type:
    report_type = "other"

  report = {
    'inspector': 'cftc',
    'inspector_url': 'http://www.cftc.gov/About/OfficeoftheInspectorGeneral/index.htm',
    'agency': 'cftc',
    'agency_name': 'Commodity Futures Trading Commission',
    'file_type': 'pdf',
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'type': report_type,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  return report

utils.run(run) if (__name__ == "__main__") else None
<commit_msg>[cftc] Add date<commit_after>#!/usr/bin/env python

import datetime
import logging
import os
import re
from urllib.parse import urljoin

from utils import utils, inspector

# http://www.cftc.gov/About/OfficeoftheInspectorGeneral/index.htm
archive = 2000

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
# - Add published dates for all reports in REPORT_PUBLISHED_MAPPING

REPORT_PUBLISHED_MAPPING = {
  "oig_mgmtuseitresources": datetime.datetime(2014, 10, 1),
  "oig_auditreportp05": datetime.datetime(2014, 7, 17),
  "oigocoaudit2014": datetime.datetime(2014, 5, 1),
  "oigcommentletter042214": datetime.datetime(2014, 4, 22),
  "oig_auditruleenforcementreview": datetime.datetime(2015, 8, 5),
  "oig_pensionawardsaudit": datetime.datetime(2015, 9, 24),
  "oigfinancialstatementalert": datetime.datetime(2016, 1, 15),
  "2015finstatementaudit": datetime.datetime(2016, 1, 15),
  "2009par": datetime.datetime(2009, 11, 13),
  "2004finstatementaudit": datetime.datetime(2004, 11, 16),
  "oigmgmtchall2015": datetime.datetime(2015, 10, 26),
  "oigmgmtchall2014": datetime.datetime(2014, 11, 17),
  "oigmgmtchall2013": datetime.datetime(2013, 12, 13),
  "oigmgmtchall2012": datetime.datetime(2012, 11, 14),
  "oigmgmtchall2011": datetime.datetime(2011, 11, 10),
  "oigmgmtchall2010": datetime.datetime(2010, 10, 10),
  "oigmgmtchall2009": datetime.datetime(2009, 11, 16),
  "oigmgmtchall2008": datetime.datetime(2008, 11, 14),
  "oigmgmtchall2007": datetime.datetime(2007, 11, 15),
  "oigmgmtchall2006": datetime.datetime(2006, 10, 31),
  "oigmgmtchall2005": datetime.datetime(2005, 10, 10),
  "oigmgmtchall2004": datetime.datetime(2004, 11, 12),
  "FOIAaudit20122011": datetime.datetime(2013, 1, 28),
  "cpfreport2015": datetime.datetime(2015, 10, 30),
  "cpfreport2014": datetime.datetime(2014, 10, 30),
  "cpfreport2013": datetime.datetime(2013, 10, 31),
  "cpfreport2012": datetime.datetime(2012, 10, 25),
  "cpfreport2011": datetime.datetime(2011, 10, 25),
  "fisma2014": datetime.datetime(2014, 2, 20),
  "oig_riskassess2015": datetime.datetime(2016, 2, 19),
}

REPORTS_URL = "http://www.cftc.gov/About/OfficeoftheInspectorGeneral/index.htm"

def run(options):
  year_range = inspector.year_range(options, archive)

  # Pull the audit reports
  doc = utils.beautifulsoup_from_url(REPORTS_URL)
  results = doc.select("ul.text > ul > li")
  if not results:
    raise inspector.NoReportsFoundError("CFTC audit reports")
  for result in results:
    report = report_from(result, year_range)
    if report:
      inspector.save_report(report)

  # Pull the semiannual reports
  results = doc.select("ul.text td a")
  if not results:
    raise inspector.NoReportsFoundError("CFTC semiannual reports")
  for result in results:
    report = report_from(result, year_range, report_type="semiannual_report")
    if report:
      inspector.save_report(report)

def extract_report_type(text):
  if 'Peer Review' in text:
    return "peer_review"
  elif 'Audit' in text:
    return "audit"
  elif 'Investigation' in text:
    return 'investigation'
  elif 'Inspection' in text:
    return "inspection"
  elif 'Management Challenges' in text:
    return "management_challenges"

def report_from(result, year_range, report_type=None):
  if result.name == 'a':
    link = result
  else:
    link = result.select("a")[-1]

  report_url = urljoin(REPORTS_URL, link.get('href'))
  report_filename = report_url.split("/")[-1]
  report_id, _ = os.path.splitext(report_filename)

  title = link.text

  published_on = None
  if report_id in REPORT_PUBLISHED_MAPPING:
    published_on = REPORT_PUBLISHED_MAPPING[report_id]
  if not published_on:
    try:
      published_on_text = "/".join(re.search("(\w+) (\d+), (\d+)", title).groups())
      published_on = datetime.datetime.strptime(published_on_text, '%B/%d/%Y')
    except AttributeError:
      pass
  if not published_on:
    try:
      published_on_text = "/".join(re.search("(\w+) (\d+), (\d+)", str(link.next_sibling)).groups())
      published_on = datetime.datetime.strptime(published_on_text, '%B/%d/%Y')
    except AttributeError:
      pass
  if not published_on:
    raise inspector.NoDateFoundError(report_id, title)

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  if not report_type:
    report_type = extract_report_type(title)
  if not report_type:
    report_type = extract_report_type(result.find_previous("p").text)
  if not report_type:
    report_type = "other"

  report = {
    'inspector': 'cftc',
    'inspector_url': 'http://www.cftc.gov/About/OfficeoftheInspectorGeneral/index.htm',
    'agency': 'cftc',
    'agency_name': 'Commodity Futures Trading Commission',
    'file_type': 'pdf',
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'type': report_type,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  return report

utils.run(run) if (__name__ == "__main__") else None
<|endoftext|>


<commit_before>import re
class sql_utility:
	def __init__(self):
		self.stop_chars=['\n', ' ', '\t']
		self.statements=[]
		self.tokens=[]

	def tokenise(self, sql_string):
		token_list=[]

		self.statements=sql_string.split(';')
		for statement in self.statements:
			token_list=re.sub("[^\w][(][)]", " ",  statement).split()
			self.tokens.append(token_list)
		print self.tokens
<commit_msg>added comments to the class<commit_after>import re
class sql_utility:
	"""
	Class sql_utility. Tokenise the sql statements captured by the mysql replication.
	Each statement is converted in dictionary being used by pg_engine.
	"""
	def __init__(self):
		self.statements=[]
		self.tokens=[]

	def tokenise(self, sql_string):
		"""
			Splits the sql string in statements using the conventional end of statement marker ;
			A regular expression greps the words and parentesis and a split converts them in
			a list. Each list of words is then stored in the list token_list.

			:param sql_string: The sql string with the sql statements.
		"""
		token_list=[]

		self.statements=sql_string.split(';')
		for statement in self.statements:
			token_list=re.sub("[^\w][(][)]", " ",  statement).split()
			self.tokens.append(token_list)
		print self.tokens

<|endoftext|>


<commit_before><commit_msg>Trying to get the basedir set properly for heroku's static files serving (try 12).<commit_after><|endoftext|>