<commit_before>from django.http import HttpRequest
from django.http import HttpResponse
from django.utils.cache import cc_delim_re
from django.utils.http import http_date
import fudge
import hashlib
import random
import re
import urllib

from ._utils import TestCase
from ._utils import with_fake_request

from .. import middleware
from ..middleware import IncludeEsiMiddleware, StoreEsiStatusMiddleware

def full_process_response(request, response):
    response = StoreEsiStatusMiddleware().process_response(request, response)
    response = IncludeEsiMiddleware().process_response(request, response)
    return response

class TestMiddleware(TestCase):
    @with_fake_request
    def test_returns_unmodified_response_on_non_esi_response(self, request):
        original_content = str(random.randint(1000, 2000))
        response = HttpResponse(original_content)
        new_response = full_process_response(request, response)
        self.assertEqual(original_content, new_response.content)

    @with_fake_request
    def test_replaces_esi_tags_with_actual_response(self, request):
        rand = random.randint(100, 200)
        url = '/hello/%d/' % rand

        request.provides('get_full_path').returns('/')
        request.provides('build_absolute_uri').returns('http://example.com/')
        request.has_attr(_esi={'used': True})

        response = fudge.Fake(HttpResponse)
        esi_tag = '<esi:include src="%s" />' % url
        response.content = esi_tag
        fudge.clear_calls()

        result = full_process_response(request, response)
        self.assertFalse(re.search(esi_tag, result.content), msg='sanity check')
        self.assertEquals(result.content, str(rand))

    def get_cookie_test_objs(self, request):
        number = random.randint(100, 200)
        fragment_url = '/cookies/%d/' % number
        request_url = '/page-with-esi-tag/'
        response = HttpResponse('<esi:include src="%s" />' % fragment_url)
        request.has_attr(_esi={'used': True})
        request.provides('get_full_path').returns(request_url)
        request.provides('build_absolute_uri').returns(
            'http://example.com%s' % request_url)
        return request, response, number

    @with_fake_request
    def test_merges_cookies(self, request):
        request, response, number = self.get_cookie_test_objs(request)
        result = full_process_response(request, response)

        self.assertEqual(result.cookies['a'].value, 'apple')
        self.assertEqual(result.cookies['b'].value, 'banana')
        self.assertEqual(result.cookies['number'].value, str(number))
        self.assertEqual(result.cookies['b']['path'], '/cookies/')

    @with_fake_request
    def test_main_response_cookies_take_precedence(self, request):
        request, response, number = self.get_cookie_test_objs(request)
        response.set_cookie('a', 'alligator')
        result = full_process_response(request, response)

        self.assertEqual(result.cookies['a'].value, 'alligator')

    @with_fake_request
    def check_if_max_time_is_last_modified(self, request, main_response_time,
      fragment_time_1, fragment_time_2, max_time):
        fragment_urls = ['/last-modified/%s/' % num for num in
            (fragment_time_1, fragment_time_2)]
        html = '\n'.join('<esi:include src="%s" />' % url for url in
            fragment_urls)
        response = HttpResponse(html)
        response['Last-Modified'] = http_date(main_response_time)

        request_url = '/page-with-esi-tags/'
        request.has_attr(_esi={'used': True})
        request.provides('get_full_path').returns(request_url)
        request.provides('build_absolute_uri').returns(
            'http://example.com%s' % request_url)

        result = response = full_process_response(request, response)
        self.assertTrue(response['Last-Modified'], max_time)

    def test_merges_last_modified(self):
        time_a = random.randint(10000, 20000)
        time_b = random.randint(40000, 50000)
        time_c = random.randint(60000, 70000)

        max_time = http_date(time_c)
        self.check_if_max_time_is_last_modified(time_a, time_b, time_c, max_time)
        self.check_if_max_time_is_last_modified(time_b, time_c, time_a, max_time)
        self.check_if_max_time_is_last_modified(time_c, time_a, time_b, max_time)

    @with_fake_request
    def check_merged_vary_header(self, request, main_header, fragment_header_1,
      fragment_header_2):
        fragment_urls = ['/vary/?headers=%s' % urllib.quote_plus(vary) for vary in
            (fragment_header_1, fragment_header_2) if vary]
        html = '\n'.join('<esi:include src="%s" />' % url for url in
            fragment_urls)
        response = HttpResponse(html)
        if main_header:
            response['Vary'] = main_header

        request_url = '/page-with-esi-tags/'
        request.has_attr(_esi={'used': True})
        request.provides('get_full_path').returns(request_url)
        request.provides('build_absolute_uri').returns(
            'http://example.com%s' % request_url)

        result = full_process_response(request, response)
        vary_result = result.get('Vary', '')

        for header in (main_header, fragment_header_1, fragment_header_2):
            if not header:
                continue
            vary_fields = cc_delim_re.split(header)
            for field in vary_fields:
                self.assertEqual(len(re.findall(field, vary_result, re.I)), 1)

    def test_merges_vary(self):
        vary_sets = (
            (None, None, 'Cookie'),
            (None, 'cookie', 'Cookie'),
            (None, 'Accept-Encoding', 'Cookie'),
            (None, 'Accept-Encoding', 'Cookie, Accept-Language'),
            (None, 'Accept-Encoding, Accept-Language', 'Cookie'),
            ('Accept-Encoding', 'Cookie', 'Cookie'),
        )

        for headers in vary_sets:
            self.check_merged_vary_header(*headers)
<commit_msg>Verify that multiple ESI tags interspersed within content get replaced correctly<commit_after>from django.http import HttpRequest
from django.http import HttpResponse
from django.utils.cache import cc_delim_re
from django.utils.http import http_date
import fudge
import hashlib
import itertools
import math
import random
import re
import urllib

from ._utils import TestCase
from ._utils import with_fake_request

from .. import middleware
from ..middleware import IncludeEsiMiddleware, StoreEsiStatusMiddleware

def full_process_response(request, response):
    response = StoreEsiStatusMiddleware().process_response(request, response)
    response = IncludeEsiMiddleware().process_response(request, response)
    return response

class TestMiddleware(TestCase):
    @with_fake_request
    def test_returns_unmodified_response_on_non_esi_response(self, request):
        original_content = str(random.randint(1000, 2000))
        response = HttpResponse(original_content)
        new_response = full_process_response(request, response)
        self.assertEqual(original_content, new_response.content)

    @with_fake_request
    def test_replaces_esi_tags_with_actual_response(self, request):
        rand = random.randint(100, 200)
        url = '/hello/%d/' % rand

        request.provides('get_full_path').returns('/')
        request.provides('build_absolute_uri').returns('http://example.com/')
        request.has_attr(_esi={'used': True})

        response = fudge.Fake(HttpResponse)
        esi_tag = '<esi:include src="%s" />' % url
        response.content = esi_tag
        fudge.clear_calls()

        result = full_process_response(request, response)
        self.assertFalse(re.search(esi_tag, result.content), msg='sanity check')
        self.assertEquals(result.content, str(rand))

    @with_fake_request
    def check_content_permutation(self, request, permutation, chunk_results):
        request_url = '/page-with-esi-tags/'
        request.has_attr(_esi={'used': True})
        request.provides('get_full_path').returns(request_url)
        request.provides('build_absolute_uri').returns(
            'http://example.com%s' % request_url)

        response = HttpResponse(''.join(permutation))
        expected_result = ''.join(chunk_results[chunk] for chunk in permutation)
        result = full_process_response(request, response)
        self.assertEqual(result.content, expected_result)

    def test_replaces_multiple_esi_tags_correctly(self):
        static_text_options = (
            '',  # Allow ESI tags to end up next to each other
            'abcdefg',
            'abcdefghijklmnopqrstuvwxyz' * 10,
        )
        main_page_chunks = dict(zip(static_text_options, static_text_options))

        # Make sure some of the includes generate content longer than the ESI
        # tag itself and some shorter.
        number_lengths = (1, 10, 50, 100)
        for length in number_lengths:
            num = random.randint(math.pow(10, length-1), math.pow(10, length) - 1)
            content = '<esi:include src="/hello/%s/" />' % num
            main_page_chunks[content] = str(num)

        for permutation in itertools.permutations(main_page_chunks, 4):
            self.check_content_permutation(permutation, main_page_chunks)

    def get_cookie_test_objs(self, request):
        number = random.randint(100, 200)
        fragment_url = '/cookies/%d/' % number
        request_url = '/page-with-esi-tag/'
        response = HttpResponse('<esi:include src="%s" />' % fragment_url)
        request.has_attr(_esi={'used': True})
        request.provides('get_full_path').returns(request_url)
        request.provides('build_absolute_uri').returns(
            'http://example.com%s' % request_url)
        return request, response, number

    @with_fake_request
    def test_merges_cookies(self, request):
        request, response, number = self.get_cookie_test_objs(request)
        result = full_process_response(request, response)

        self.assertEqual(result.cookies['a'].value, 'apple')
        self.assertEqual(result.cookies['b'].value, 'banana')
        self.assertEqual(result.cookies['number'].value, str(number))
        self.assertEqual(result.cookies['b']['path'], '/cookies/')

    @with_fake_request
    def test_main_response_cookies_take_precedence(self, request):
        request, response, number = self.get_cookie_test_objs(request)
        response.set_cookie('a', 'alligator')
        result = full_process_response(request, response)

        self.assertEqual(result.cookies['a'].value, 'alligator')

    @with_fake_request
    def check_if_max_time_is_last_modified(self, request, main_response_time,
      fragment_time_1, fragment_time_2, max_time):
        fragment_urls = ['/last-modified/%s/' % num for num in
            (fragment_time_1, fragment_time_2)]
        html = '\n'.join('<esi:include src="%s" />' % url for url in
            fragment_urls)
        response = HttpResponse(html)
        response['Last-Modified'] = http_date(main_response_time)

        request_url = '/page-with-esi-tags/'
        request.has_attr(_esi={'used': True})
        request.provides('get_full_path').returns(request_url)
        request.provides('build_absolute_uri').returns(
            'http://example.com%s' % request_url)

        response = full_process_response(request, response)
        self.assertTrue(response['Last-Modified'], max_time)

    def test_merges_last_modified(self):
        time_a = random.randint(10000, 20000)
        time_b = random.randint(40000, 50000)
        time_c = random.randint(60000, 70000)

        max_time = http_date(time_c)
        self.check_if_max_time_is_last_modified(time_a, time_b, time_c, max_time)
        self.check_if_max_time_is_last_modified(time_b, time_c, time_a, max_time)
        self.check_if_max_time_is_last_modified(time_c, time_a, time_b, max_time)

    @with_fake_request
    def check_merged_vary_header(self, request, main_header, fragment_header_1,
      fragment_header_2):
        fragment_urls = ['/vary/?headers=%s' % urllib.quote_plus(vary) for vary in
            (fragment_header_1, fragment_header_2) if vary]
        html = '\n'.join('<esi:include src="%s" />' % url for url in
            fragment_urls)
        response = HttpResponse(html)
        if main_header:
            response['Vary'] = main_header

        request_url = '/page-with-esi-tags/'
        request.has_attr(_esi={'used': True})
        request.provides('get_full_path').returns(request_url)
        request.provides('build_absolute_uri').returns(
            'http://example.com%s' % request_url)

        result = full_process_response(request, response)
        vary_result = result.get('Vary', '')

        for header in (main_header, fragment_header_1, fragment_header_2):
            if not header:
                continue
            vary_fields = cc_delim_re.split(header)
            for field in vary_fields:
                self.assertEqual(len(re.findall(field, vary_result, re.I)), 1)

    def test_merges_vary(self):
        vary_sets = (
            (None, None, 'Cookie'),
            (None, 'cookie', 'Cookie'),
            (None, 'Accept-Encoding', 'Cookie'),
            (None, 'Accept-Encoding', 'Cookie, Accept-Language'),
            (None, 'Accept-Encoding, Accept-Language', 'Cookie'),
            ('Accept-Encoding', 'Cookie', 'Cookie'),
        )

        for headers in vary_sets:
            self.check_merged_vary_header(*headers)
<|endoftext|>


<commit_before>import time
from cumulusci.robotframework.pageobjects import ListingPage
from cumulusci.robotframework.pageobjects import pageobject
from BaseObjects import BaseNPSPPage
from NPSP import npsp_lex_locators
from logging import exception

@pageobject("Listing", "Batch_Gift_Entry")
class BatchGiftEntryPage(BaseNPSPPage, ListingPage):

    def _go_to_page(self, filter_name=None):
        """To go to NPSP Settings page"""
        url_template = "{root}/lightning/n/{object}"
        name = self._object_name
        object_name = "{}{}".format(self.cumulusci.get_namespace_prefix(), name)
        url = url_template.format(root=self.cumulusci.org.lightning_base_url, object=object_name)
        self.selenium.go_to(url)
        self.salesforce.wait_until_loading_is_complete()
<commit_msg>Update BatchGiftEntryPageObject.py<commit_after>import time
from cumulusci.robotframework.pageobjects import ListingPage
from cumulusci.robotframework.pageobjects import DetailPage
from cumulusci.robotframework.pageobjects import pageobject
from BaseObjects import BaseNPSPPage
from NPSP import npsp_lex_locators
from logging import exception

@pageobject("Listing", "Batch_Gift_Entry")
class BatchGiftEntryListPage(BaseNPSPPage, ListingPage):

    def _go_to_page(self, filter_name=None):
        """To go to Batch Gift Entry List View page"""
        url_template = "{root}/lightning/n/{object}"
        name = self._object_name
        object_name = "{}{}".format(self.cumulusci.get_namespace_prefix(), name)
        url = url_template.format(root=self.cumulusci.org.lightning_base_url, object=object_name)
        self.selenium.go_to(url)
        self.salesforce.wait_until_loading_is_complete()

@pageobject("Details", "DataImportBatch__c")
class BatchGiftEntryDetailPage(BaseNPSPPage, ListingPage):

    def _go_to_page(self, filter_name=None):
        """To go to gift batch record page"""
        url_template = "{root}/lightning/r/{object}"
        name = self._object_name
        object_name = "{}{}".format(self.cumulusci.get_namespace_prefix(), name)
        url = url_template.format(root=self.cumulusci.org.lightning_base_url, object=object_name)
        self.selenium.go_to(url)
        self.salesforce.wait_until_loading_is_complete()

    def _is_current_page(self):
        """ Verify we are on the BGE detail page
            by verifying that the url contains 'DataImportBatch__c'
        """
        self.selenium.wait_until_location_contains("DataImportBatch__c/",message="Current page is not a BGE batch record detail view")<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
from os import environ
from flask import Flask, request, render_template, redirect, url_for
from flask.ext.sqlalchemy import SQLAlchemy
import json
import itertools
import datetime
from dateutil.parser import parse as dparse

def bool_from_str(s):
    if isinstance(s, basestring):
        s = s.lower()
    if s in ['true', 't', '1', 'y']:
        return True
    if s in ['false', 'f', '0', 'n']:
        return False
    return bool(s)

AENEAS_DEBUG = bool_from_str(environ.get('AENEAS_DEBUG', False))

DEFAULT_AENEAS_PORT = 4935
AENEAS_PORT = environ.get('AENEAS_PORT', DEFAULT_AENEAS_PORT)
try:
    AENEAS_PORT = int(AENEAS_PORT)
except:
    AENEAS_PORT = DEFAULT_AENEAS_PORT

AENEAS_DB_URI = environ.get('AENEAS_DB_URI', 'sqlite://')

DEFAULT_AENEAS_MAX_CONTENT_LENGTH = 4000
AENEAS_MAX_CONTENT_LENGTH = environ.get('AENEAS_MAX_CONTENT_LENGTH',
                                        DEFAULT_AENEAS_MAX_CONTENT_LENGTH)
try:
    AENEAS_MAX_CONTENT_LENGTH = int(AENEAS_MAX_CONTENT_LENGTH)
except:
    AENEAS_MAX_CONTENT_LENGTH = DEFAULT_AENEAS_MAX_CONTENT_LENGTH

def generate_app(db_uri=AENEAS_DB_URI,
                 max_content_length=AENEAS_MAX_CONTENT_LENGTH):

    app = Flask(__name__)
    app.config['SQLALCHEMY_DATABASE_URI'] = db_uri
    app.config['MAX_CONTENT_LENGTH'] = max_content_length
    db = app.db = SQLAlchemy(app)

    class Report(db.Model):
        id = db.Column(db.Integer, primary_key=True)
        raw = db.Column(db.Text(128000), nullable=False)
        product = db.Column(db.String(100), nullable=False)
        version = db.Column(db.String(100), nullable=False)
        timestamp = db.Column(db.DateTime, nullable=False)

        def __init__(self, raw, product, version, timestamp=None):
            self.raw = raw
            self.product = product
            self.version = version
            if timestamp is None:
                timestamp = datetime.datetime.utcnow()
            if isinstance(timestamp, basestring):
                timestamp = dparse(timestamp)
            self.timestamp = timestamp

        def to_dict(self):
            return {'id': self.id,
                    'raw': self.raw,
                    'product': self.product,
                    'version': self.version,
                    'timestamp': self.timestamp}

    @app.route('/v1.0/reports', methods=['POST'])
    def submit_report():
        if request.content_type != 'application/json':
            return ('Content-Type was "{}", but only "application/json" is '
                    'supported.'.format(request.content_type), 415)
        if len(request.data) > app.config['MAX_CONTENT_LENGTH']:
            return '', 413

        report_json = request.json

        if 'product' not in report_json:
            return 'No product specified', 400
        product = report_json['product']
        if not isinstance(product, basestring):
            return 'Product is wrong type', 400

        if 'version' not in report_json:
            return 'No version specified', 400
        version = report_json['version']
        if not isinstance(version, basestring):
            return 'Version is wrong type', 400

        if 'X-Real-IP' in request.headers:
            report_json['server_remote_ip'] = request.headers['X-Real-IP']
        else:
            report_json['server_remote_ip'] = request.remote_addr

        raw = json.dumps(report_json)

        report = Report(raw, product, version)
        db.session.add(report)
        db.session.commit()

        clean_up_report(report)
        db.session.add(report)
        db.session.commit()

        return '', 201

    def get_accept_type():
        best = request.accept_mimetypes.best_match(['application/json',
                                                    'text/html'])
        if (best == 'text/html' and request.accept_mimetypes[best] >=
                request.accept_mimetypes['application/json']):
            return 'html'
        elif (best == 'application/json' and request.accept_mimetypes[best] >=
                request.accept_mimetypes['text/html']):
            return 'json'
        else:
            return None

    @app.route('/v1.0/reports', methods=['GET'])
    def list_reports():
        accept = get_accept_type()
        if accept is None:
            return '', 406

        reports = Report.query.all()

        if accept == 'html':
            return render_template('list_reports.html', reports=reports,
                                   cycle=itertools.cycle)
        else:
            jreports = [json.loads(r.raw) for r in reports]
            return json.dumps(jreports), 200

    def clean_up_report(report):
        jraw = json.loads(report.raw)
        jraw['id'] = report.id
        report.raw = json.dumps(jraw)
        return report

    @app.route('/v1.0/reports/clean-up-all', methods=['GET'])
    def clean_up_all_reports():
        reports = Report.query.all()
        for report in reports:
            clean_up_report(report)
            db.session.add(report)
        db.session.commit()
        return redirect(url_for('list_reports'))

    @app.route('/v1.0/reports/<int:id>', methods=['GET'])
    def show_report(id):
        accept = get_accept_type()
        if accept is None:
            return '', 406

        report = Report.query.get(id)
        if report is None:
            return '', 404
        if accept == 'html':
            raw = json.dumps(json.loads(report.raw), indent=2)
            return render_template('show_report.html', report=report, raw=raw)
        return report.raw, 200

    @app.route('/v1.0/reports/<int:id>/download', methods=['GET'])
    def download_report(id):
        report = Report.query.get(id)
        if report is None:
            return '', 404
        return report.raw, 200

    @app.route('/v1.0/diagnostics/show-full-request')
    def show_full_request():
        req = request
        # raise ZeroDivisionError
        return render_template('show_full_request.html', request=request,
                               keys=dir(request), getattr=getattr)

    return app

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('--debug',
                        help='Run Flask in debug mode, with auto-reload and '
                             'debugger page on errors. Default is {}. '
                             'Environment variable is AENEAS_DEBUG.'.format(
                                AENEAS_DEBUG),
                        action='store_true',
                        default=AENEAS_DEBUG)
    parser.add_argument('--port',
                        help='The port on which to accept incoming HTTP '
                             'requests. Default is {}. Environment variables '
                             'is AENEAS_PORT.'.format(AENEAS_PORT),
                        action='store', default=AENEAS_PORT, type=int)
    parser.add_argument('--db-uri',
                        help='The URI for the database, to be passed to '
                             'SQLAlchemy. Default is {}. Environment variable '
                             'is AENEAS_DB_URI.'.format(AENEAS_DB_URI),
                        action='store', default=AENEAS_DB_URI)
    parser.add_argument('--create-db', help='Initialize the database schema '
                                            'and then exit.',
                        action='store_true')
    parser.add_argument('--max-content-length',
                        help='Limit the maximum size of an incoming request '
                             'body that will be accepted. Default is {}. '
                             'Environment variable is '
                             'AENEAS_MAX_CONTENT_LENGTH'.format(
                                AENEAS_MAX_CONTENT_LENGTH),
                        action='store', default=AENEAS_MAX_CONTENT_LENGTH,
                        type=int)

    args = parser.parse_args()

    print('Debug: {}'.format(args.debug))
    print('Port: {}'.format(args.port))
    print('DB URI: {}'.format(args.db_uri))
    print('Max Content Length: {}'.format(args.max_content_length))

    app = generate_app(db_uri=args.db_uri,
                       max_content_length=args.max_content_length)

    if args.create_db:
        print('Setting up the database')
        app.db.create_all()
    else:
        app.run(debug=args.debug, port=args.port)
<commit_msg>Increase the column size. -- DB SCHEMA CHANGE<commit_after>#!/usr/bin/env python

import argparse
from os import environ
from flask import Flask, request, render_template, redirect, url_for
from flask.ext.sqlalchemy import SQLAlchemy
import json
import itertools
import datetime
from dateutil.parser import parse as dparse

def bool_from_str(s):
    if isinstance(s, basestring):
        s = s.lower()
    if s in ['true', 't', '1', 'y']:
        return True
    if s in ['false', 'f', '0', 'n']:
        return False
    return bool(s)

AENEAS_DEBUG = bool_from_str(environ.get('AENEAS_DEBUG', False))

DEFAULT_AENEAS_PORT = 4935
AENEAS_PORT = environ.get('AENEAS_PORT', DEFAULT_AENEAS_PORT)
try:
    AENEAS_PORT = int(AENEAS_PORT)
except:
    AENEAS_PORT = DEFAULT_AENEAS_PORT

AENEAS_DB_URI = environ.get('AENEAS_DB_URI', 'sqlite://')

DEFAULT_AENEAS_MAX_CONTENT_LENGTH = 4000
AENEAS_MAX_CONTENT_LENGTH = environ.get('AENEAS_MAX_CONTENT_LENGTH',
                                        DEFAULT_AENEAS_MAX_CONTENT_LENGTH)
try:
    AENEAS_MAX_CONTENT_LENGTH = int(AENEAS_MAX_CONTENT_LENGTH)
except:
    AENEAS_MAX_CONTENT_LENGTH = DEFAULT_AENEAS_MAX_CONTENT_LENGTH

def generate_app(db_uri=AENEAS_DB_URI,
                 max_content_length=AENEAS_MAX_CONTENT_LENGTH):

    app = Flask(__name__)
    app.config['SQLALCHEMY_DATABASE_URI'] = db_uri
    app.config['MAX_CONTENT_LENGTH'] = max_content_length
    db = app.db = SQLAlchemy(app)

    class Report(db.Model):
        id = db.Column(db.Integer, primary_key=True)
        raw = db.Column(db.Text(256000), nullable=False)
        product = db.Column(db.String(100), nullable=False)
        version = db.Column(db.String(100), nullable=False)
        timestamp = db.Column(db.DateTime, nullable=False)

        def __init__(self, raw, product, version, timestamp=None):
            self.raw = raw
            self.product = product
            self.version = version
            if timestamp is None:
                timestamp = datetime.datetime.utcnow()
            if isinstance(timestamp, basestring):
                timestamp = dparse(timestamp)
            self.timestamp = timestamp

        def to_dict(self):
            return {'id': self.id,
                    'raw': self.raw,
                    'product': self.product,
                    'version': self.version,
                    'timestamp': self.timestamp}

    @app.route('/v1.0/reports', methods=['POST'])
    def submit_report():
        if request.content_type != 'application/json':
            return ('Content-Type was "{}", but only "application/json" is '
                    'supported.'.format(request.content_type), 415)
        if len(request.data) > app.config['MAX_CONTENT_LENGTH']:
            return '', 413

        report_json = request.json

        if 'product' not in report_json:
            return 'No product specified', 400
        product = report_json['product']
        if not isinstance(product, basestring):
            return 'Product is wrong type', 400

        if 'version' not in report_json:
            return 'No version specified', 400
        version = report_json['version']
        if not isinstance(version, basestring):
            return 'Version is wrong type', 400

        if 'X-Real-IP' in request.headers:
            report_json['server_remote_ip'] = request.headers['X-Real-IP']
        else:
            report_json['server_remote_ip'] = request.remote_addr

        raw = json.dumps(report_json)

        report = Report(raw, product, version)
        db.session.add(report)
        db.session.commit()

        clean_up_report(report)
        db.session.add(report)
        db.session.commit()

        return '', 201

    def get_accept_type():
        best = request.accept_mimetypes.best_match(['application/json',
                                                    'text/html'])
        if (best == 'text/html' and request.accept_mimetypes[best] >=
                request.accept_mimetypes['application/json']):
            return 'html'
        elif (best == 'application/json' and request.accept_mimetypes[best] >=
                request.accept_mimetypes['text/html']):
            return 'json'
        else:
            return None

    @app.route('/v1.0/reports', methods=['GET'])
    def list_reports():
        accept = get_accept_type()
        if accept is None:
            return '', 406

        reports = Report.query.all()

        if accept == 'html':
            return render_template('list_reports.html', reports=reports,
                                   cycle=itertools.cycle)
        else:
            jreports = [json.loads(r.raw) for r in reports]
            return json.dumps(jreports), 200

    def clean_up_report(report):
        jraw = json.loads(report.raw)
        jraw['id'] = report.id
        report.raw = json.dumps(jraw)
        return report

    @app.route('/v1.0/reports/clean-up-all', methods=['GET'])
    def clean_up_all_reports():
        reports = Report.query.all()
        for report in reports:
            clean_up_report(report)
            db.session.add(report)
        db.session.commit()
        return redirect(url_for('list_reports'))

    @app.route('/v1.0/reports/<int:id>', methods=['GET'])
    def show_report(id):
        accept = get_accept_type()
        if accept is None:
            return '', 406

        report = Report.query.get(id)
        if report is None:
            return '', 404
        if accept == 'html':
            raw = json.dumps(json.loads(report.raw), indent=2)
            return render_template('show_report.html', report=report, raw=raw)
        return report.raw, 200

    @app.route('/v1.0/reports/<int:id>/download', methods=['GET'])
    def download_report(id):
        report = Report.query.get(id)
        if report is None:
            return '', 404
        return report.raw, 200

    @app.route('/v1.0/diagnostics/show-full-request')
    def show_full_request():
        req = request
        # raise ZeroDivisionError
        return render_template('show_full_request.html', request=request,
                               keys=dir(request), getattr=getattr)

    return app

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('--debug',
                        help='Run Flask in debug mode, with auto-reload and '
                             'debugger page on errors. Default is {}. '
                             'Environment variable is AENEAS_DEBUG.'.format(
                                AENEAS_DEBUG),
                        action='store_true',
                        default=AENEAS_DEBUG)
    parser.add_argument('--port',
                        help='The port on which to accept incoming HTTP '
                             'requests. Default is {}. Environment variables '
                             'is AENEAS_PORT.'.format(AENEAS_PORT),
                        action='store', default=AENEAS_PORT, type=int)
    parser.add_argument('--db-uri',
                        help='The URI for the database, to be passed to '
                             'SQLAlchemy. Default is {}. Environment variable '
                             'is AENEAS_DB_URI.'.format(AENEAS_DB_URI),
                        action='store', default=AENEAS_DB_URI)
    parser.add_argument('--create-db', help='Initialize the database schema '
                                            'and then exit.',
                        action='store_true')
    parser.add_argument('--max-content-length',
                        help='Limit the maximum size of an incoming request '
                             'body that will be accepted. Default is {}. '
                             'Environment variable is '
                             'AENEAS_MAX_CONTENT_LENGTH'.format(
                                AENEAS_MAX_CONTENT_LENGTH),
                        action='store', default=AENEAS_MAX_CONTENT_LENGTH,
                        type=int)

    args = parser.parse_args()

    print('Debug: {}'.format(args.debug))
    print('Port: {}'.format(args.port))
    print('DB URI: {}'.format(args.db_uri))
    print('Max Content Length: {}'.format(args.max_content_length))

    app = generate_app(db_uri=args.db_uri,
                       max_content_length=args.max_content_length)

    if args.create_db:
        print('Setting up the database')
        app.db.create_all()
    else:
        app.run(debug=args.debug, port=args.port)
<|endoftext|>


<commit_before>#! /usr/bin/env python
"""
Affine spatial transformation
"""
import numpy as np
from numpy.linalg import inv
import argparse, sys, csv, os, time

def getArgs():
	parser = argparse.ArgumentParser(
		description = """Affine spatial transformation"""
	)
	parser.add_argument(
		"-c",
		"--controlPoints",
		type = str,
		required = True,
		help = "Control points csv file. See README doc for formating instructions."
	)
	parser.add_argument(
		"-u",
		"--unprojectPoints",
		type = str,
		required = True,
		help = "Unprojected points on plot coordinate system"
	)
	parser.add_argument(
		"-x",
		"--xname",
		type = str,
		required = True,
		help = "field name for x-coordinate in unprojected points file"

	)
	parser.add_argument(
		"-y",
		"--yname",
		type = str,
		required = True,
		help = "field name for y-coordinate in unprojected points file"

	)
	parser.add_argument(
		"-i",
		"--uid",
		type = str,
		required = True,
		help = "field name for unique identifier of point in unprojected points file"

	)
	parser.add_argument(
		"-n",
		"--nsims",
		type = str,
		required = False,
		help = """Number of simulations"""
	)
	parser.add_argument(
		"-o",
		"--output",
		type = str,
		required = True,
		help = """Output file prefix"""
	)
	parser.add_argument(
		"-v",
		"--verbose",
		action = "store_true",
		help = "Print status updates while executing"
	)
	return parser.parse_args()

def getIndex(header, item):
	"""Get index of variable
	"""
	for i, elem in enumerate(header):
		if elem.lower() == item.lower():
			return i
	return None

def getData(path, fields):
	"""Read unprojected points file as numpy array"""
	data = []
	with open(path, 'rUb') as f:
		indata = csv.reader(f)
		var_indices = []
		for i, line in enumerate(indata):
			if i == 0:
				header = line
				indices = [getIndex(header, item) for item in fields]
				continue
			row = [line[index] for index in indices]
			data.append(row)
	data = np.array(data, dtype = np.float64)
	header = [header[index] for index in indices]
	return header, data

def getControl(path):
	"""Read control points file as numpy array
	"""
	data = []
	with open(path, "rU") as f:
		for i, line in enumerate(f):
			ls = line.strip().split(',')
			if i == 0:
				continue
			data.append(ls)
	data = np.array(data, dtype = np.float64)
	return data

def update(line):
	"""Convert uid or nsim column to integer
	"""
	line[0]=int(line[0])
	if len(line)==6:
		line[1]=int(line[1])
	return line

def writeOut(data, header, filename, verbose):
	"""Write data to a file.
	"""
	if verbose:
		print "\nInitializing {0} file output".format(filename)
	if header is not None:

		with open(filename, "w") as f:
			header_str = ",".join(header)+"\n"
			f.write(header_str)
			for line in data:
				line = update(line)
				row = ",".join([str(elem) for elem in line])+"\n"
				f.write(row)
	else:
		with open(filename, "a") as f:
			for line in data:
				line = update(line)
				row = ",".join([str(elem) for elem in line])+"\n"
				f.write(row)
	if verbose:
		print "Wrote {0} to disk\n".format(filename)

def fit(X, y):
	"""Fit using matrix algebra
	"""
	XTXinv = inv(np.dot(X.T, X))
	XTy = np.dot(X.T, y)
	B = np.dot(XTXinv, XTy)
	return B

def affine_parameterization(utm_e, utm_n, x, y):
	"""Parameterize affine function
	"""
	X = np.column_stack(
		(
			x,
			y,
			np.ones(len(x))
		)
	)
	affine_x = fit(X, utm_e)
	affine_y = fit(X, utm_n)
	return affine_x, affine_y

def affine_transformation(X_unprj, affine_x, affine_y, args, header):
	"""Predict with affine function
	"""
	x_pred = np.dot(X_unprj, affine_x)
	y_pred = np.dot(X_unprj, affine_y)
	return x_pred, y_pred

def main():
	t_i = time.time()
	args = getArgs()
	initDir = os.getcwd()

	fields = [
		args.uid,
		args.xname,
		args.yname
	]

	np.random.seed(10)

	header, data = getData(args.unprojectPoints, fields)
	cp = getControl(args.controlPoints)

	# Determine whether or not to run simulation
	if cp.shape[1] == 6 and args.nsims is not None:
		sim = True
		if args.verbose:
			print "Will simulate {} realizations of corners.".format(args.nsims)
	elif args.nsims is None:
		sim = False
	elif cp.shape[1] == 4:
		sim = False
		if args.nsims is not None:
			if args.verbose:
				print "Cannot simulate error because no utm_e and utm_n se's"
				print  "are included in control points file."
	else:
		print "Incorrect dimensions for control points file. Exiting."
		sys.exit(1)

	# Affine Spatial Transformation Parameterization
	# x' = Ax + By + C
	# y' = Dx + Ey + F
	utm_e, utm_n = cp[:,0].flatten(), cp[:,1].flatten()
	x, y = cp[:,2].flatten(), cp[:,3].flatten()
	affine_x, affine_y = affine_parameterization(utm_e, utm_n, x, y )

	uids = data[:, getIndex(header, args.uid)].flatten()
	gx = data[:, getIndex(header, args.xname)].flatten()
	gy = data[:, getIndex(header, args.yname)].flatten()
	X_unprj = np.column_stack(
		(
			gx,
			gy,
			np.ones(len(gx))
		)
	)
	x_pred, y_pred = affine_transformation(X_unprj, affine_x, affine_y, args, header)

	projected_data = np.column_stack(
		(
			uids,
			gx,
			gy,
			x_pred,
			y_pred
		)
	)

	projected_data_header = ["uid", "gx", "gy", "x_pred", "y_pred"]
	writeOut(
		projected_data,
		projected_data_header,
		args.output+".csv",
		args.verbose
	)

	# Simulated locations
	if sim:
		utm_se_e = cp[:,4]
		utm_se_n = cp[:,5]
		sim_data_header = ["iter", "uid", "gx", "gy", "x_pred", "y_pred"]
		for i in range(int(args.nsims)):
			if args.verbose:
				print "Simulation number {}".format(i+1)
			evec = np.zeros(cp.shape[0])
			nvec = np.zeros(cp.shape[0])
			for j, row in enumerate(cp):
				evec[j] = np.random.normal(row[0],row[4], 1)
				nvec[j] = np.random.normal(row[1],row[5], 1)
			affine_x, affine_y = affine_parameterization(evec, nvec, x, y)
			x_pred, y_pred = affine_transformation(
				X_unprj,
				affine_x,
				affine_y,
				args,
				header
			)
			iteration = i + np.zeros(x_pred.shape[0])
			sim_data = np.column_stack(
				(
				iteration,
				uids,
				gx,
				gy,
				x_pred,
				y_pred
				)
			)
			if i == 0:
				writeOut(
					sim_data,
					sim_data_header,
					args.output+"-sim.csv",
					args.verbose
				)
			else:
				writeOut(
					sim_data,
					None,
					args.output+"-sim.csv",
					args.verbose
				)
	t_f = time.time()
	if args.verbose:
		print "Spatially adjusted data in {} seconds".format(t_f - t_i)

if __name__ == "__main__":
	main()
<commit_msg>add header return to getControl<commit_after>#! /usr/bin/env python
"""
Affine spatial transformation
"""
import numpy as np
from numpy.linalg import inv
import argparse, sys, csv, os, time

def getArgs():
	parser = argparse.ArgumentParser(
		description = """Affine spatial transformation"""
	)
	parser.add_argument(
		"-c",
		"--controlPoints",
		type = str,
		required = True,
		help = "Control points csv file. See README doc for formating instructions."
	)
	parser.add_argument(
		"-u",
		"--unprojectPoints",
		type = str,
		required = True,
		help = "Unprojected points on plot coordinate system"
	)
	parser.add_argument(
		"-x",
		"--xname",
		type = str,
		required = True,
		help = "field name for x-coordinate in unprojected points file"

	)
	parser.add_argument(
		"-y",
		"--yname",
		type = str,
		required = True,
		help = "field name for y-coordinate in unprojected points file"

	)
	parser.add_argument(
		"-i",
		"--uid",
		type = str,
		required = True,
		help = "field name for unique identifier of point in unprojected points file"

	)
	parser.add_argument(
		"-n",
		"--nsims",
		type = str,
		required = False,
		help = """Number of simulations"""
	)
	parser.add_argument(
		"-o",
		"--output",
		type = str,
		required = True,
		help = """Output file prefix"""
	)
	parser.add_argument(
		"-v",
		"--verbose",
		action = "store_true",
		help = "Print status updates while executing"
	)
	return parser.parse_args()

def getIndex(header, item):
	"""Get index of variable
	"""
	for i, elem in enumerate(header):
		if elem.lower() == item.lower():
			return i
	return None

def getData(path, fields):
	"""Read unprojected points file as numpy array"""
	data = []
	with open(path, 'rUb') as f:
		indata = csv.reader(f)
		var_indices = []
		for i, line in enumerate(indata):
			if i == 0:
				header = line
				indices = [getIndex(header, item) for item in fields]
				continue
			row = [line[index] for index in indices]
			data.append(row)
	data = np.array(data, dtype = np.float64)
	header = [header[index] for index in indices]
	return header, data

def getControl(path):
	"""Read control points file as numpy array
	"""
	data = []
	with open(path, "rU") as f:
		for i, line in enumerate(f):
			ls = line.strip().split(',')
			if i == 0:
				header = ls
				continue
			data.append(ls)
	data = np.array(data, dtype = np.float64)
	return header, data

def update(line):
	"""Convert uid or nsim column to integer
	"""
	line[0]=int(line[0])
	if len(line)==6:
		line[1]=int(line[1])
	return line

def writeOut(data, header, filename, verbose):
	"""Write data to a file.
	"""
	if verbose:
		print "\nInitializing {0} file output".format(filename)
	if header is not None:

		with open(filename, "w") as f:
			header_str = ",".join(header)+"\n"
			f.write(header_str)
			for line in data:
				line = update(line)
				row = ",".join([str(elem) for elem in line])+"\n"
				f.write(row)
	else:
		with open(filename, "a") as f:
			for line in data:
				line = update(line)
				row = ",".join([str(elem) for elem in line])+"\n"
				f.write(row)
	if verbose:
		print "Wrote {0} to disk\n".format(filename)

def fit(X, y):
	"""Fit using matrix algebra
	"""
	XTXinv = inv(np.dot(X.T, X))
	XTy = np.dot(X.T, y)
	B = np.dot(XTXinv, XTy)
	return B

def affine_parameterization(utm_e, utm_n, x, y):
	"""Parameterize affine function
	"""
	X = np.column_stack(
		(
			x,
			y,
			np.ones(len(x))
		)
	)
	affine_x = fit(X, utm_e)
	affine_y = fit(X, utm_n)
	return affine_x, affine_y

def affine_transformation(X_unprj, affine_x, affine_y, args, header):
	"""Predict with affine function
	"""
	x_pred = np.dot(X_unprj, affine_x)
	y_pred = np.dot(X_unprj, affine_y)
	return x_pred, y_pred

def main():
	t_i = time.time()
	args = getArgs()
	initDir = os.getcwd()

	fields = [
		args.uid,
		args.xname,
		args.yname
	]

	np.random.seed(10)

	header, data = getData(args.unprojectPoints, fields)
	cp_header, cp = getControl(args.controlPoints)

	# Determine whether or not to run simulation
	if cp.shape[1] == 6 and args.nsims is not None:
		sim = True
		if args.verbose:
			print "Will simulate {} realizations of corners.".format(args.nsims)
	elif args.nsims is None:
		sim = False
	elif cp.shape[1] == 4:
		sim = False
		if args.nsims is not None:
			if args.verbose:
				print "Cannot simulate error because no utm_e and utm_n se's"
				print  "are included in control points file."
	else:
		print "Incorrect dimensions for control points file. Exiting."
		sys.exit(1)

	# Affine Spatial Transformation Parameterization
	# x' = Ax + By + C
	# y' = Dx + Ey + F
	utm_e, utm_n = cp[:,0].flatten(), cp[:,1].flatten()
	x, y = cp[:,2].flatten(), cp[:,3].flatten()
	affine_x, affine_y = affine_parameterization(utm_e, utm_n, x, y )

	uids = data[:, getIndex(header, args.uid)].flatten()
	gx = data[:, getIndex(header, args.xname)].flatten()
	gy = data[:, getIndex(header, args.yname)].flatten()
	X_unprj = np.column_stack(
		(
			gx,
			gy,
			np.ones(len(gx))
		)
	)
	x_pred, y_pred = affine_transformation(X_unprj, affine_x, affine_y, args, header)

	projected_data = np.column_stack(
		(
			uids,
			gx,
			gy,
			x_pred,
			y_pred
		)
	)

	projected_data_header = ["uid", "gx", "gy", cp_header[0], cp_header[1]]
	writeOut(
		projected_data,
		projected_data_header,
		args.output+".csv",
		args.verbose
	)

	# Simulated locations
	if sim:
		utm_se_e = cp[:,4]
		utm_se_n = cp[:,5]
		sim_data_header = ["iter", "uid", "gx", "gy", cp_header[0], cp_header[1]]
		for i in range(int(args.nsims)):
			if args.verbose:
				print "Simulation number {}".format(i+1)
			evec = np.zeros(cp.shape[0])
			nvec = np.zeros(cp.shape[0])
			for j, row in enumerate(cp):
				evec[j] = np.random.normal(row[0],row[4], 1)
				nvec[j] = np.random.normal(row[1],row[5], 1)
			affine_x, affine_y = affine_parameterization(evec, nvec, x, y)
			x_pred, y_pred = affine_transformation(
				X_unprj,
				affine_x,
				affine_y,
				args,
				header
			)
			iteration = i + np.zeros(x_pred.shape[0])
			sim_data = np.column_stack(
				(
				iteration,
				uids,
				gx,
				gy,
				x_pred,
				y_pred
				)
			)
			if i == 0:
				writeOut(
					sim_data,
					sim_data_header,
					args.output+"-sim.csv",
					args.verbose
				)
			else:
				writeOut(
					sim_data,
					None,
					args.output+"-sim.csv",
					args.verbose
				)
	t_f = time.time()
	if args.verbose:
		print "Spatially adjusted data in {} seconds".format(t_f - t_i)

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before>from trac.core import *
from trac.perm import IPermissionPolicy, IPermissionRequestor

revision = "$Rev$"
url = "$URL$"

class SecurityTicketsPolicy(Component):
    """Prevent public access to security sensitive tickets.

    Add the VULNERABILITY_VIEW permission as a pre-requisite for any
    other permission check done on tickets that have the words
    "security" or "vulnerability" in the summary or keywords fields.

    Once this plugin is enabled, you'll have to insert it at the appropriate
    place in your list of permission policies, e.g.
    {{{
    [trac]
    permission_policies = SecurityTicketsPolicy, AuthzPolicy,
                          DefaultPermissionPolicy, LegacyAttachmentPolicy
    }}}
    """

    implements(IPermissionPolicy, IPermissionRequestor)

    # IPermissionPolicy methods

    def check_permission(self, action, username, resource, perm):
        # We add the 'VULNERABILITY_VIEW' pre-requisite for any action
        # other than 'VULNERABILITY_VIEW' itself, as this would lead
        # to recursion.
        if action == 'VULNERABILITY_VIEW':
            return

        # Check whether we're dealing with a ticket resource
        while resource:
            if resource.realm == 'ticket':
                break
            resource = resource.parent

        if resource and resource.realm == 'ticket' and resource.id is not None:
            db = self.env.get_db_cnx()
            cursor = db.cursor()
            for keywords, summary in cursor.execute("SELECT keywords, summary "
                                                    " FROM ticket WHERE id=%s",
                                                    (resource.id,)):
                fields = ''.join([f for f in (keywords, summary) if f]).lower()
                if 'security' in fields or 'vulnerability' in fields:
                    if 'VULNERABILITY_VIEW' not in perm:
                        return False

    # IPermissionRequestor methods

    def get_permission_actions(self):
        yield 'VULNERABILITY_VIEW'
<commit_msg>Fix vulnerability_ticket sample plugin.<commit_after>from trac.core import *
from trac.perm import IPermissionPolicy, IPermissionRequestor

revision = "$Rev$"
url = "$URL$"

class SecurityTicketsPolicy(Component):
    """Prevent public access to security sensitive tickets.

    Add the VULNERABILITY_VIEW permission as a pre-requisite for any
    other permission check done on tickets that have the words
    "security" or "vulnerability" in the summary or keywords fields.

    Once this plugin is enabled, you'll have to insert it at the appropriate
    place in your list of permission policies, e.g.
    {{{
    [trac]
    permission_policies = SecurityTicketsPolicy, AuthzPolicy,
                          DefaultPermissionPolicy, LegacyAttachmentPolicy
    }}}
    """

    implements(IPermissionPolicy, IPermissionRequestor)

    # IPermissionPolicy methods

    def check_permission(self, action, username, resource, perm):
        # We add the 'VULNERABILITY_VIEW' pre-requisite for any action
        # other than 'VULNERABILITY_VIEW' itself, as this would lead
        # to recursion.
        if action == 'VULNERABILITY_VIEW':
            return

        # Check whether we're dealing with a ticket resource
        while resource:
            if resource.realm == 'ticket':
                break
            resource = resource.parent

        if resource and resource.realm == 'ticket' and resource.id is not None:
            db = self.env.get_db_cnx()
            cursor = db.cursor()
            cursor.execute("SELECT keywords, summary FROM ticket WHERE id=%s",
                           (resource.id,))
            for keywords, summary in cursor:
                fields = ''.join([f for f in (keywords, summary) if f]).lower()
                if 'security' in fields or 'vulnerability' in fields:
                    if 'VULNERABILITY_VIEW' not in perm:
                        return False

    # IPermissionRequestor methods

    def get_permission_actions(self):
        yield 'VULNERABILITY_VIEW'
<|endoftext|>


<commit_before>import datetime
import json

from django.conf import settings
from django.core.paginator import InvalidPage
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from django.http import Http404, JsonResponse
from django.shortcuts import render, redirect, get_object_or_404
from django.utils.translation import ugettext_lazy as _, activate
from django.views import static
from django.views.decorators.cache import cache_page

from django_hosts.resolvers import reverse

from elasticsearch_dsl import query

from .forms import DocSearchForm
from .models import DocumentRelease, Document
from .search import DocumentDocType, SearchPaginator
from .utils import get_doc_root_or_404, get_doc_path_or_404

CURRENT_LTS = '1.4'
UNSUPPORTED_THRESHOLD = '1.6'
SIMPLE_SEARCH_OPERATORS = ['+', '|', '-', '"', '*', '(', ')', '~']

def version_is_unsupported(version):
    # TODO: would be nice not to hardcode this.
    return version != CURRENT_LTS and version < UNSUPPORTED_THRESHOLD

def index(request):
    return redirect(DocumentRelease.objects.current())

def language(request, lang):
    return redirect(DocumentRelease.objects.current(lang))

def stable(request, lang, version, url):
    path = request.get_full_path()
    current_version = DocumentRelease.objects.current_version()
    return redirect(path.replace(version, current_version, 1))

def document(request, lang, version, url):
    # If either of these can't be encoded as ascii then later on down the line an
    # exception will be emitted by unipath, proactively check for bad data (mostly
    # from the Googlebot) so we can give a nice 404 error.
    try:
        version.encode("ascii")
        url.encode("ascii")
    except UnicodeEncodeError:
        raise Http404

    if lang != 'en':
        activate(lang)

    docroot = get_doc_root_or_404(lang, version)
    doc_path = get_doc_path_or_404(docroot, url)

    if version == 'dev':
        rtd_version = 'latest'
    elif version >= '1.5':
        rtd_version = version + '.x'
    else:
        rtd_version = version + '.X'

    template_names = [
        'docs/%s.html' % docroot.rel_path_to(doc_path).replace(doc_path.ext, ''),
        'docs/doc.html',
    ]
    context = {
        'doc': json.load(open(doc_path, 'rb')),
        'env': json.load(open(docroot.child('globalcontext.json'), 'rb')),
        'lang': lang,
        'version': version,
        'version_is_dev': version == 'dev',
        'version_is_unsupported': version_is_unsupported(version),
        'rtd_version': rtd_version,
        'docurl': url,
        'update_date': datetime.datetime.fromtimestamp(docroot.child('last_build').mtime()),
        'redirect_from': request.GET.get('from', None),
    }
    return render(request, template_names, context)

def pot_file(request, pot_name):
    version = DocumentRelease.objects.current().version
    doc_root = get_doc_root_or_404('en', version, subroot='gettext')
    return static.serve(request, document_root=doc_root, path=pot_name)

def sphinx_static(request, lang, version, path, subpath=None):
    """
    Serve Sphinx static assets from a subdir of the build location.
    """
    document_root = get_doc_root_or_404(lang, version).child(subpath)
    return static.serve(request, document_root=document_root, path=path)

def objects_inventory(request, lang, version):
    response = static.serve(request,
                            document_root=get_doc_root_or_404(lang, version),
                            path="objects.inv")
    response['Content-Type'] = "text/plain"
    return response

def redirect_index(request, *args, **kwargs):
    assert request.path.endswith('index/')
    return redirect(request.path[:-6])

def redirect_search(request):
    """
    Legacy search view to handle old queries correctly, e.g. in scraping
    sites, command line interface etc.
    """
    release = DocumentRelease.objects.current()
    kwargs = {
        'lang': release.lang,
        'version': release.version,
    }
    search_url = reverse('document-search', host='docs', kwargs=kwargs)
    q = request.GET.get('q') or None
    if q:
        search_url += '?q=%s' % q
    return redirect(search_url)

def search_results(request, lang, version, per_page=10, orphans=3):
    """
    Search view to handle language and version specific queries.
    The old search view is being redirected here.
    """
    release = get_object_or_404(DocumentRelease, version=version, lang=lang)
    form = DocSearchForm(request.GET or None, release=release)

    context = {
        'form': form,
        'lang': release.lang,
        'version': release.version,
        'release': release,
        'searchparams': request.GET.urlencode(),
        'version_is_dev': version == 'dev',
        'version_is_unsupported': version_is_unsupported(version),
    }

    if form.is_valid():
        q = form.cleaned_data.get('q')

        if q:
            # catch queries that are coming from browser search bars
            exact = Document.objects.filter(release=release, title=q).first()
            if exact is not None:
                return redirect(exact)

            should = []
            if any(operator in q for operator in SIMPLE_SEARCH_OPERATORS):
                should.append(query.SimpleQueryString(fields=['title',
                                                              'content^5'],
                                                      query=q,
                                                      analyzer='stop',
                                                      default_operator='and'))
            else:
                # let's just use simple queries since they allow some
                # neat syntaxes for exclusion etc. For more info see
                # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html
                should = [query.MultiMatch(fields=['title^10', 'content'],
                                           query=q,
                                           type='phrase_prefix'),
                          query.Match(query=q),
                          query.MultiMatch(fields=['title^5', 'content'],
                                           query=q,
                                           fuzziness=1)]

            # then apply the queries and filter out anything not matching
            # the wanted version and language, also highlight the content
            # and order the highlighted snippets by score so that the most
            # fitting result is used
            results = (DocumentDocType.search()
                                      .query(query.Bool(should=should))
                                      .filter('term', release__lang=release.lang)
                                      .filter('term', release__version=release.version)
                                      .highlight_options(order='score')
                                      .highlight('content'))

            page_number = request.GET.get('page') or 1
            paginator = SearchPaginator(results, per_page=per_page, orphans=orphans)

            try:
                page_number = int(page_number)
            except ValueError:
                if page_number == 'last':
                    page_number = paginator.num_pages
                else:
                    raise Http404(_("Page is not 'last', "
                                    "nor can it be converted to an int."))

            try:
                page = paginator.page(page_number)
            except InvalidPage as e:
                raise Http404(_('Invalid page (%(page_number)s): %(message)s') % {
                    'page_number': page_number,
                    'message': str(e)
                })

            context.update({
                'query': q,
                'page': page,
                'paginator': paginator,
            })

    if release.lang != 'en':
        activate(release.lang)

    return render(request, 'docs/search_results.html', context)

def search_suggestions(request, lang, version, per_page=20):
    """
    The endpoint for the OpenSearch browser integration.

    This will do a simple prefix match against the title to catch
    documents with a meaningful title.

    The link list contains redirect URLs so that IE will correctly
    redirect to those documents.
    """
    release = get_object_or_404(DocumentRelease, version=version, lang=lang)

    form = DocSearchForm(request.GET or None, release=release)
    suggestions = []

    if form.is_valid():
        q = form.cleaned_data.get('q')
        if q:
            search = DocumentDocType.search()
            search = (search.query(query.SimpleQueryString(fields=['title^10',
                                                                   'content'],
                                                           query=q,
                                                           analyzer='stop',
                                                           default_operator='and'))
                            .filter('term', release__lang=release.lang)
                            .filter('term', release__version=release.version)
                            .fields(['title', '_source']))

            suggestions.append(q)
            titles = []
            links = []
            content_type = ContentType.objects.get_for_model(Document)
            results = search[0:per_page].execute()
            for result in results:
                titles.append(result.title)
                kwargs = {
                    'content_type_id': content_type.pk,
                    'object_id': result.meta.id,
                }
                links.append(reverse('django.contrib.contenttypes.views.shortcut',
                                     kwargs=kwargs))
            suggestions.append(titles)
            suggestions.append([])
            suggestions.append(links)

    return JsonResponse(suggestions, safe=False)

if not settings.DEBUG:
    # 1 hour to handle the many requests
    search_suggestions = cache_page(60 * 60)(search_suggestions)

def search_description(request, lang, version):
    """
    Render an OpenSearch description.
    """
    release = get_object_or_404(DocumentRelease, version=version, lang=lang)
    context = {
        'site': Site.objects.get_current(),
        'release': release,
    }
    return render(request, 'docs/search_description.xml', context,
                  content_type='application/opensearchdescription+xml')

if not settings.DEBUG:
    # 1 week because there is no need to render it more often
    search_description = cache_page(60 * 60 * 24 * 7)(search_description)
<commit_msg>Use same queryset for doing the exact match to decide weather to redirect or not.<commit_after>import datetime
import json

from django.conf import settings
from django.core.paginator import InvalidPage
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from django.http import Http404, JsonResponse
from django.shortcuts import render, redirect, get_object_or_404
from django.utils.translation import ugettext_lazy as _, activate
from django.views import static
from django.views.decorators.cache import cache_page

from django_hosts.resolvers import reverse

from elasticsearch_dsl import query

from .forms import DocSearchForm
from .models import DocumentRelease, Document
from .search import DocumentDocType, SearchPaginator
from .utils import get_doc_root_or_404, get_doc_path_or_404

CURRENT_LTS = '1.4'
UNSUPPORTED_THRESHOLD = '1.6'
SIMPLE_SEARCH_OPERATORS = ['+', '|', '-', '"', '*', '(', ')', '~']

def version_is_unsupported(version):
    # TODO: would be nice not to hardcode this.
    return version != CURRENT_LTS and version < UNSUPPORTED_THRESHOLD

def index(request):
    return redirect(DocumentRelease.objects.current())

def language(request, lang):
    return redirect(DocumentRelease.objects.current(lang))

def stable(request, lang, version, url):
    path = request.get_full_path()
    current_version = DocumentRelease.objects.current_version()
    return redirect(path.replace(version, current_version, 1))

def document(request, lang, version, url):
    # If either of these can't be encoded as ascii then later on down the line an
    # exception will be emitted by unipath, proactively check for bad data (mostly
    # from the Googlebot) so we can give a nice 404 error.
    try:
        version.encode("ascii")
        url.encode("ascii")
    except UnicodeEncodeError:
        raise Http404

    if lang != 'en':
        activate(lang)

    docroot = get_doc_root_or_404(lang, version)
    doc_path = get_doc_path_or_404(docroot, url)

    if version == 'dev':
        rtd_version = 'latest'
    elif version >= '1.5':
        rtd_version = version + '.x'
    else:
        rtd_version = version + '.X'

    template_names = [
        'docs/%s.html' % docroot.rel_path_to(doc_path).replace(doc_path.ext, ''),
        'docs/doc.html',
    ]
    context = {
        'doc': json.load(open(doc_path, 'rb')),
        'env': json.load(open(docroot.child('globalcontext.json'), 'rb')),
        'lang': lang,
        'version': version,
        'version_is_dev': version == 'dev',
        'version_is_unsupported': version_is_unsupported(version),
        'rtd_version': rtd_version,
        'docurl': url,
        'update_date': datetime.datetime.fromtimestamp(docroot.child('last_build').mtime()),
        'redirect_from': request.GET.get('from', None),
    }
    return render(request, template_names, context)

def pot_file(request, pot_name):
    version = DocumentRelease.objects.current().version
    doc_root = get_doc_root_or_404('en', version, subroot='gettext')
    return static.serve(request, document_root=doc_root, path=pot_name)

def sphinx_static(request, lang, version, path, subpath=None):
    """
    Serve Sphinx static assets from a subdir of the build location.
    """
    document_root = get_doc_root_or_404(lang, version).child(subpath)
    return static.serve(request, document_root=document_root, path=path)

def objects_inventory(request, lang, version):
    response = static.serve(request,
                            document_root=get_doc_root_or_404(lang, version),
                            path="objects.inv")
    response['Content-Type'] = "text/plain"
    return response

def redirect_index(request, *args, **kwargs):
    assert request.path.endswith('index/')
    return redirect(request.path[:-6])

def redirect_search(request):
    """
    Legacy search view to handle old queries correctly, e.g. in scraping
    sites, command line interface etc.
    """
    release = DocumentRelease.objects.current()
    kwargs = {
        'lang': release.lang,
        'version': release.version,
    }
    search_url = reverse('document-search', host='docs', kwargs=kwargs)
    q = request.GET.get('q') or None
    if q:
        search_url += '?q=%s' % q
    return redirect(search_url)

def search_results(request, lang, version, per_page=10, orphans=3):
    """
    Search view to handle language and version specific queries.
    The old search view is being redirected here.
    """
    release = get_object_or_404(DocumentRelease, version=version, lang=lang)
    form = DocSearchForm(request.GET or None, release=release)

    context = {
        'form': form,
        'lang': release.lang,
        'version': release.version,
        'release': release,
        'searchparams': request.GET.urlencode(),
        'version_is_dev': version == 'dev',
        'version_is_unsupported': version_is_unsupported(version),
    }

    if form.is_valid():
        q = form.cleaned_data.get('q')

        if q:
            # catch queries that are coming from browser search bars
            exact = (DocumentDocType.index_queryset()
                                    .filter(release=release, title=q)
                                    .first())
            if exact is not None:
                return redirect(exact)

            should = []
            if any(operator in q for operator in SIMPLE_SEARCH_OPERATORS):
                should.append(query.SimpleQueryString(fields=['title',
                                                              'content^5'],
                                                      query=q,
                                                      analyzer='stop',
                                                      default_operator='and'))
            else:
                # let's just use simple queries since they allow some
                # neat syntaxes for exclusion etc. For more info see
                # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html
                should = [query.MultiMatch(fields=['title^10', 'content'],
                                           query=q,
                                           type='phrase_prefix'),
                          query.Match(query=q),
                          query.MultiMatch(fields=['title^5', 'content'],
                                           query=q,
                                           fuzziness=1)]

            # then apply the queries and filter out anything not matching
            # the wanted version and language, also highlight the content
            # and order the highlighted snippets by score so that the most
            # fitting result is used
            results = (DocumentDocType.search()
                                      .query(query.Bool(should=should))
                                      .filter('term', release__lang=release.lang)
                                      .filter('term', release__version=release.version)
                                      .highlight_options(order='score')
                                      .highlight('content'))

            page_number = request.GET.get('page') or 1
            paginator = SearchPaginator(results, per_page=per_page, orphans=orphans)

            try:
                page_number = int(page_number)
            except ValueError:
                if page_number == 'last':
                    page_number = paginator.num_pages
                else:
                    raise Http404(_("Page is not 'last', "
                                    "nor can it be converted to an int."))

            try:
                page = paginator.page(page_number)
            except InvalidPage as e:
                raise Http404(_('Invalid page (%(page_number)s): %(message)s') % {
                    'page_number': page_number,
                    'message': str(e)
                })

            context.update({
                'query': q,
                'page': page,
                'paginator': paginator,
            })

    if release.lang != 'en':
        activate(release.lang)

    return render(request, 'docs/search_results.html', context)

def search_suggestions(request, lang, version, per_page=20):
    """
    The endpoint for the OpenSearch browser integration.

    This will do a simple prefix match against the title to catch
    documents with a meaningful title.

    The link list contains redirect URLs so that IE will correctly
    redirect to those documents.
    """
    release = get_object_or_404(DocumentRelease, version=version, lang=lang)

    form = DocSearchForm(request.GET or None, release=release)
    suggestions = []

    if form.is_valid():
        q = form.cleaned_data.get('q')
        if q:
            search = DocumentDocType.search()
            search = (search.query(query.SimpleQueryString(fields=['title^10',
                                                                   'content'],
                                                           query=q,
                                                           analyzer='stop',
                                                           default_operator='and'))
                            .filter('term', release__lang=release.lang)
                            .filter('term', release__version=release.version)
                            .fields(['title', '_source']))

            suggestions.append(q)
            titles = []
            links = []
            content_type = ContentType.objects.get_for_model(Document)
            results = search[0:per_page].execute()
            for result in results:
                titles.append(result.title)
                kwargs = {
                    'content_type_id': content_type.pk,
                    'object_id': result.meta.id,
                }
                links.append(reverse('django.contrib.contenttypes.views.shortcut',
                                     kwargs=kwargs))
            suggestions.append(titles)
            suggestions.append([])
            suggestions.append(links)

    return JsonResponse(suggestions, safe=False)

if not settings.DEBUG:
    # 1 hour to handle the many requests
    search_suggestions = cache_page(60 * 60)(search_suggestions)

def search_description(request, lang, version):
    """
    Render an OpenSearch description.
    """
    release = get_object_or_404(DocumentRelease, version=version, lang=lang)
    context = {
        'site': Site.objects.get_current(),
        'release': release,
    }
    return render(request, 'docs/search_description.xml', context,
                  content_type='application/opensearchdescription+xml')

if not settings.DEBUG:
    # 1 week because there is no need to render it more often
    search_description = cache_page(60 * 60 * 24 * 7)(search_description)
<|endoftext|>


<commit_before>from scrapy import log
from scrapy.contrib.spiders.sitemap import SitemapSpider
from scrapy.http import Request, HtmlResponse
from scrapy.contrib.linkextractors.lxmlhtml import LxmlLinkExtractor

import re
import chardet
import mimetypes
import magic

from os2webscanner.models import Url

class ScannerSpider(SitemapSpider):
    name = 'scanner'
    magic = magic.Magic(mime=True)

    def __init__(self, scanner, *a, **kw):
        # Create scanner
        self.scanner = scanner

        self.allowed_domains = self.scanner.get_domains()
        self.sitemap_urls = self.scanner.get_sitemap_urls()

        # Follow alternate links in sitemaps (links to the same content in
        # different languages)
        self.sitemap_alternate_links = True

        SitemapSpider.__init__(self, *a, **kw)

        self.start_urls = []
        # TODO: Starting URLs and domains should be specified separately?
        for url in self.allowed_domains:
            if (not url.startswith('http://')
                and not url.startswith('https://')):
                url = 'http://%s/' % url
            self.start_urls.append(url)

        # TODO: Add more tags to extract links from?
        self.link_extractor = LxmlLinkExtractor(
            deny_extensions=(),
            tags=('a', 'area', 'frame', 'iframe'),
            attrs=('href', 'src')
        )

    def start_requests(self):
        """Start the spider with requests for all starting URLs
        AND sitemap URLs"""
        requests = [Request(url, callback=self.parse)
                    for url in self.start_urls]
        requests.extend(list(SitemapSpider.start_requests(self)))
        return requests

    def parse(self, response):
        """Process a response and follow all links"""
        r = []
        r.extend(self._extract_requests(response))
        self.scan(response)
        return r

    def _extract_requests(self, response):
        """Extract requests from the response"""
        r = []
        if isinstance(response, HtmlResponse):
            links = self.link_extractor.extract_links(response)
            log.msg("Extracted links: %s" % links, level=log.DEBUG)
            r.extend(Request(x.url, callback=self.parse) for x in links)
        return r

    def scan(self, response):
        """Scan a response, returning any matches."""
        content_type = response.headers.get('content-type')
        if content_type:
            mime_type = parse_content_type(content_type)
            log.msg("Content-Type: " + content_type, level=log.DEBUG)
        else:
            log.msg("Guessing mime-type based on file extension",
                    level=log.DEBUG)
            mime_type, encoding = mimetypes.guess_type(response.url)
            # Scrapy already guesses the encoding.. we don't need it

        if hasattr(response, "encoding"):
            try:
                data = response.body.decode(response.encoding)
            except UnicodeDecodeError, e:
                try:
                    # Encoding specified in Content-Type header was wrong, try
                    # to detect the encoding and decode again
                    encoding = chardet.detect(response.body).get('encoding')
                    if encoding is not None:
                        data = response.body.decode(encoding)
                        log.msg(("Error decoding response as %s. " +
                                 "Detected the encoding as %s.") % (
                                    response.encoding, encoding))
                    else:
                        mime_type = self.magic.from_buffer(response.body)
                        data = response.body
                        log.msg(("Error decoding response as %s. " +
                                 "Detected the mime " +
                                 "type as %s.") % (response.encoding,
                                                   mime_type))
                except UnicodeDecodeError, e:
                    # Could not decode with the detected encoding, so assume
                    # the file is binary and try to guess the mimetype from
                    # the file
                    mime_type = self.magic.from_buffer(response.body)
                    data = response.body
                    log.msg(
                        ("Error decoding response as %s. Detected the "
                         "mime type as %s.") % (response.encoding,
                                                mime_type)
                    )

        else:
            data = response.body

        # Save the URL item to the database
        url_object = Url(url=response.request.url, mime_type=mime_type,
                         scan=self.scanner.scan_object)
        url_object.save()
        result = self.scanner.scan(data, url_object)

def parse_content_type(content_type):
    """Parses and returns the mime-type from a Content-Type header value"""
    m = re.search('([^/]+/[^;\s]+)', content_type)
    return m.group(1)
<commit_msg>Added scanning of JavaScript.<commit_after>from scrapy import log
from scrapy.contrib.spiders.sitemap import SitemapSpider
from scrapy.http import Request, HtmlResponse
from scrapy.contrib.linkextractors.lxmlhtml import LxmlLinkExtractor

import re
import chardet
import mimetypes
import magic

from os2webscanner.models import Url

class ScannerSpider(SitemapSpider):
    name = 'scanner'
    magic = magic.Magic(mime=True)

    def __init__(self, scanner, *a, **kw):
        # Create scanner
        self.scanner = scanner

        self.allowed_domains = self.scanner.get_domains()
        self.sitemap_urls = self.scanner.get_sitemap_urls()

        # Follow alternate links in sitemaps (links to the same content in
        # different languages)
        self.sitemap_alternate_links = True

        SitemapSpider.__init__(self, *a, **kw)

        self.start_urls = []
        # TODO: Starting URLs and domains should be specified separately?
        for url in self.allowed_domains:
            if (not url.startswith('http://')
                and not url.startswith('https://')):
                url = 'http://%s/' % url
            self.start_urls.append(url)

        # TODO: Add more tags to extract links from?
        self.link_extractor = LxmlLinkExtractor(
            deny_extensions=(),
            tags=('a', 'area', 'frame', 'iframe', 'script'),
            attrs=('href', 'src')
        )

    def start_requests(self):
        """Start the spider with requests for all starting URLs
        AND sitemap URLs"""
        requests = [Request(url, callback=self.parse)
                    for url in self.start_urls]
        requests.extend(list(SitemapSpider.start_requests(self)))
        return requests

    def parse(self, response):
        """Process a response and follow all links"""
        r = []
        r.extend(self._extract_requests(response))
        self.scan(response)
        return r

    def _extract_requests(self, response):
        """Extract requests from the response"""
        r = []
        if isinstance(response, HtmlResponse):
            links = self.link_extractor.extract_links(response)
            log.msg("Extracted links: %s" % links, level=log.DEBUG)
            r.extend(Request(x.url, callback=self.parse) for x in links)
        return r

    def scan(self, response):
        """Scan a response, returning any matches."""
        content_type = response.headers.get('content-type')
        if content_type:
            mime_type = parse_content_type(content_type)
            log.msg("Content-Type: " + content_type, level=log.DEBUG)
        else:
            log.msg("Guessing mime-type based on file extension",
                    level=log.DEBUG)
            mime_type, encoding = mimetypes.guess_type(response.url)
            # Scrapy already guesses the encoding.. we don't need it

        if hasattr(response, "encoding"):
            try:
                data = response.body.decode(response.encoding)
            except UnicodeDecodeError, e:
                try:
                    # Encoding specified in Content-Type header was wrong, try
                    # to detect the encoding and decode again
                    encoding = chardet.detect(response.body).get('encoding')
                    if encoding is not None:
                        data = response.body.decode(encoding)
                        log.msg(("Error decoding response as %s. " +
                                 "Detected the encoding as %s.") % (
                                    response.encoding, encoding))
                    else:
                        mime_type = self.magic.from_buffer(response.body)
                        data = response.body
                        log.msg(("Error decoding response as %s. " +
                                 "Detected the mime " +
                                 "type as %s.") % (response.encoding,
                                                   mime_type))
                except UnicodeDecodeError, e:
                    # Could not decode with the detected encoding, so assume
                    # the file is binary and try to guess the mimetype from
                    # the file
                    mime_type = self.magic.from_buffer(response.body)
                    data = response.body
                    log.msg(
                        ("Error decoding response as %s. Detected the "
                         "mime type as %s.") % (response.encoding,
                                                mime_type)
                    )

        else:
            data = response.body

        # Save the URL item to the database
        url_object = Url(url=response.request.url, mime_type=mime_type,
                         scan=self.scanner.scan_object)
        url_object.save()
        result = self.scanner.scan(data, url_object)

def parse_content_type(content_type):
    """Parses and returns the mime-type from a Content-Type header value"""
    m = re.search('([^/]+/[^;\s]+)', content_type)
    return m.group(1)
<|endoftext|>


<commit_before>import datetime
import logging
import itertools
from logging import  handlers
from django.core.management.base import BaseCommand
import traceback
from rapidsms.models import Contact, Connection, Backend

from rapidsms_httprouter.models import Message

from django.db import transaction

from rapidsms.messages.outgoing import OutgoingMessage

from script.utils.outgoing import check_progress
from script.models import ScriptProgress, Email, Script
from optparse import OptionParser, make_option
import datetime
from django.core.mail import send_mail
from django.conf import settings
from django.template import Context, Template

# Make sure a NullHandler is available
# This was added in Python 2.7/3.2
try:
    from logging import NullHandler
except ImportError:
    class NullHandler(logging.Handler):
        def emit(self, record):
            pass
logger = logging.getLogger(__name__)
logging.basicConfig(filename="script.log", level=logging.INFO)
# Add the log message handler to the logger
handler = logging.handlers.RotatingFileHandler("script.log", maxBytes=5242880, backupCount=5)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)

class Command(BaseCommand):

    option_list = BaseCommand.option_list + (
        make_option("-e", "--early", dest="e"),
        make_option("-l", "--late", dest="l")
    )

    @transaction.commit_manually
    def handle(self, **options):

        current = datetime.datetime.now()
        recipients = getattr(settings, 'ADMINS', None)
        if recipients:
            recipients = [email for name, email in recipients]
        if current.hour in range(int(options['e']), int(options['l'])):
            for script in Script.objects.filter(enabled=True):
                try:
                    check_progress(script)
                    transaction.commit()
                except Exception, exc:
                    transaction.rollback()
                    print traceback.format_exc(exc)
                    logger.debug(str(exc))
                    if recipients:
                        send_mail('[Django] Error: check_script_progress cron', str(traceback.format_exc(exc)), 'root@uganda.rapidsms.org', recipients, fail_silently=True)<commit_msg>Take care of the transaction if there are no enabled scripts.<commit_after>import datetime
import logging
import itertools
from logging import  handlers
from django.core.management.base import BaseCommand
import traceback
from rapidsms.models import Contact, Connection, Backend

from rapidsms_httprouter.models import Message

from django.db import transaction

from rapidsms.messages.outgoing import OutgoingMessage

from script.utils.outgoing import check_progress
from script.models import ScriptProgress, Email, Script
from optparse import OptionParser, make_option
import datetime
from django.core.mail import send_mail
from django.conf import settings
from django.template import Context, Template

# Make sure a NullHandler is available
# This was added in Python 2.7/3.2
try:
    from logging import NullHandler
except ImportError:
    class NullHandler(logging.Handler):
        def emit(self, record):
            pass
logger = logging.getLogger(__name__)
logging.basicConfig(filename="script.log", level=logging.INFO)
# Add the log message handler to the logger
handler = logging.handlers.RotatingFileHandler("script.log", maxBytes=5242880, backupCount=5)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)

class Command(BaseCommand):

    option_list = BaseCommand.option_list + (
        make_option("-e", "--early", dest="e"),
        make_option("-l", "--late", dest="l")
    )

    @transaction.commit_manually
    def handle(self, **options):

        current = datetime.datetime.now()
        recipients = getattr(settings, 'ADMINS', None)
        if recipients:
            recipients = [email for name, email in recipients]
        if current.hour in range(int(options['e']), int(options['l'])):
            for script in Script.objects.filter(enabled=True):
                try:
                    check_progress(script)
                    transaction.commit()
                except Exception, exc:
                    transaction.rollback()
                    print traceback.format_exc(exc)
                    logger.debug(str(exc))
                    if recipients:
                        send_mail('[Django] Error: check_script_progress cron', str(traceback.format_exc(exc)), 'root@uganda.rapidsms.org', recipients, fail_silently=True)
            else:
                transaction.rollback()
<|endoftext|>