<commit_before>#!/usr/bin/env python3

from collections import Counter, defaultdict
import re

from xdfile.utils import progress, open_output, get_args, args_parser, COLUMN_SEPARATOR
from xdfile.utils import br_with_n
from xdfile import html, utils, catalog, pubyear
from xdfile import metadatabase as metadb
from xdfile.html import GridCalendar, mktag, year_widget
from xdfile.xdfile import num_cells
import xdfile

class PublicationStats:
    def __init__(self, pubid):
        self.pubid = pubid
        self.copyrights = Counter()  # [copyright_text] -> number of xd
        self.editors = Counter()  # [editor_name] -> number of xd
        self.formats = Counter()  # ["15x15 RS"] -> number of xd
        self.mindate = ""
        self.maxdate = ""
        self.num_xd = 0

        self.puzzles_meta = []

    def add(self, puzrow):
        self.copyrights[puzrow.Copyright.strip()] += 1
        self.editors[puzrow.Editor.strip()] += 1
        self.formats[puzrow.Size] += 1
        datestr = puzrow.Date
        if datestr:
            if not self.mindate:
                self.mindate = datestr
            else:
                self.mindate = min(self.mindate, datestr)
            if not self.maxdate:
                self.maxdate = datestr
            else:
                self.maxdate = max(self.maxdate, datestr)
        self.num_xd += 1

        self.puzzles_meta.append(puzrow)

    def meta(self):
        return 'pubid num dates formats copyrights editors'.split()

    def row(self):
        return [
                self.pubid,
                html.mkhref(str(self.num_xd), self.pubid),
                "%s — %s" % (self.mindate, self.maxdate),
                html_select_options(self.formats),
                html_select_options(self.copyrights),
                html_select_options(self.editors),
               ]

def tally_to_cell(d):
    freq_sorted = sorted([(v, k) for k, v in list(d.items())], reverse=True)

    if not freq_sorted:
        return ""
    elif len(freq_sorted) == 1:
        return "
".join("%s [x%s]" % (k, v) for v, k in freq_sorted)
    else:
        return "<select><option>" + "</option><option>".join("%s [x%s]" % (k, v) for v, k in freq_sorted) + "</select>"

def publication_header():
    return "PubId NumCollected DatesCollected Formats Copyrights Editors".split()

def main():
    parser = args_parser("generate publishers index html pages and index")

    args = get_args(parser=parser)

    outf = open_output()
    all_pubs = {}  # [(pubid,year)] -> PublicationStats
    pubyear_rows = {}
    similar = metadb.xd_similar()
    puzzles = metadb.xd_puzzles()
    outf.write_html('pub/index.html', pubyear.pubyear_html(), title='The xd crossword puzzle corpus')

    utils.log("collating puzzles")
    for puzrow in puzzles.values():
            pubid = utils.parse_pubid(puzrow.xdid)
            year = xdfile.year_from_date(puzrow.Date)
            k = (pubid, year or 9999)
            if k not in all_pubs:
                all_pubs[k] = PublicationStats(pubid)

            pubyear_rows[pubid] = []

            all_pubs[k].add(puzrow)

    pubyear_header = [ 'xdid', 'Date', 'Size', 'Title', 'Author', 'Editor', 'Copyright', 'Grid_1A_1D', 'ReusedCluePct', 'SimilarGrids' ]
    utils.log('generating index pages')

    # dict to generate pub page with calendars
    pub_grids = defaultdict(dict)

    for pair, pub in sorted(list(all_pubs.items())):
        c_grids = {}
        pubid, year = pair
        progress(pubid)

        reused_clues = 0
        reused_answers = 0
        total_clues = 0
        total_similar = []

        rows = []

        # Assign class based on xdid and similars
        def get_cell_classes(r):
            """ Return cell classes based on parameters """
            # TODO: Implement check that authors same
            classes = []
            rsim = similar.get(r.xdid)
            if rsim and float(rsim.similar_grid_pct) > 0:
                matches = [x.split('=') for x in rsim.matches.split()]
                # Get max for matches for class definition
                max_pct = int(max([ x[1] for x in matches ]))
                # < 40%: 'theme', 40-90%: 'partial', 90-99%: 'full', 100%: 'exact'
                if max_pct > 0 and max_pct < 40:
                    classes.append('pctfilled')
                if max_pct >= 40 and max_pct < 90:
                    classes.append('partial')
                if max_pct >= 90 and max_pct <= 99:
                    classes.append('full')
                if max_pct >= 100:
                    classes.append('exact')

                # Highlight only grids sized > 400 cells
                if num_cells(r.Size) >= 400:
                    classes.append('biggrid')

            return ' '.join(classes)

        for r in pub.puzzles_meta:
            similar_text = ""
            reused_clue_pct = "n/a"

            rsim = similar.get(r.xdid)
            if rsim:
                similar_pct = float(rsim.similar_grid_pct)
                if similar_pct > 0:
                    matches = [x.split('=') for x in rsim.matches.split()]
                    for xdid, pct in matches:
                        similar_text += '(%s%%) %s [%s]
' % (pct, puzzles[xdid].Author, xdid)
                    total_similar.append(similar_pct)
                else:
                    similar_text = "0"

                nclues = int(rsim.total_clues)

                reused_clues += int(rsim.reused_clues)
                reused_answers += int(rsim.reused_answers)
                total_clues += nclues

                if nclues:
                    reused_clue_pct = int(100*(float(rsim.reused_clues) / float(nclues)))
                else:
                    reused_clue_pct = ''

            row_dict = {} # Map row and style
            if similar_text and similar_text != "0":
                # http://stackoverflow.com/questions/1418838/html-making-a-link-lead-to-the-anchor-centered-in-the-middle-of-the-page
                pubidtext = '' % r.xdid
                pubidtext += ''
                pubidtext += html.mkhref(r.xdid, '/pub/' + r.xdid)
                c_grids[r.Date] = {
                        'link' : '/pub/%s%s/index.html#' % (pubid, year) + r.xdid,
                        'class': get_cell_classes(r),
                        #'class': 'pctfilled',
                        'title': br_with_n(similar_text),
                        }
                print(c_grids[r.Date])
                row_dict['tag_params'] = {
                    'onclick': 'location.href=\'/pub/%s\'' % r.xdid,
                    'class': 'puzzlehl'
                    }
            else:
                pubidtext = r.xdid
                row_dict['class'] = 'puzzle'

            row = [
                pubidtext,
                r.Date,
                r.Size,
                r.Title,
                r.Author,
                r.Editor,
                r.Copyright,
                r.A1_D1,
                reused_clue_pct,
                similar_text
              ]

            outf.write_row('pub/%s%s.tsv' % (pubid, year), " ".join(pubyear_header), row)
            row_dict['row'] = row
            rows.append(row_dict)

        pub_grids[pubid][year] = c_grids

        # Generate calendar
        onepubyear_html = GridCalendar(c_grids).formatyear(year, 6) + "
"

        # Generate html table sorted by 2nd element of row (date)
        onepubyear_html += html.html_table(sorted(rows , key=lambda x: x['row'][1]), pubyear_header, "puzzle", "puzzles")
        outf.write_html("pub/%s%s/index.html" % (pubid, year), onepubyear_html, title="%s %s" % (pubid, year))

        cluepct = ""
        wordpct = ""
        if total_clues:
            cluepct = "%d%%" % int(100.0*(total_clues-reused_clues)/total_clues)
            wordpct = "%.2f%%" % int(100.0*(total_clues-reused_answers)/total_clues)

        pubyear_rows[pubid].append([
            pubid,
            str(year),
            len(rows),
            "%.2f/%d" % (sum(total_similar)/100.0, len(total_similar)),
            wordpct,
            cluepct
            ])

    # Generate /pub/[publisher][year] page
    for pubid in pub_grids.keys():
        body = []
        for y in sorted(pub_grids[pubid], reverse=True):
            body.append(GridCalendar(pub_grids[pubid][y]).formatyear(y, 12, vertical=True) + mktag('br','break'))
        outf.write_html("pub/%s/index.html" % pubid, ''.join(body), title="%s" % pubid)

    progress()

if __name__ == "__main__":
    main()
<commit_msg>Stolen puzzles will be hightlighted<commit_after>#!/usr/bin/env python3

from collections import Counter, defaultdict
import re

from xdfile.utils import progress, open_output, get_args, args_parser, COLUMN_SEPARATOR
from xdfile.utils import br_with_n
from xdfile import html, utils, catalog, pubyear
from xdfile import metadatabase as metadb
from xdfile.html import GridCalendar, mktag, year_widget
from xdfile.xdfile import num_cells
import xdfile

class PublicationStats:
    def __init__(self, pubid):
        self.pubid = pubid
        self.copyrights = Counter()  # [copyright_text] -> number of xd
        self.editors = Counter()  # [editor_name] -> number of xd
        self.formats = Counter()  # ["15x15 RS"] -> number of xd
        self.mindate = ""
        self.maxdate = ""
        self.num_xd = 0

        self.puzzles_meta = []

    def add(self, puzrow):
        self.copyrights[puzrow.Copyright.strip()] += 1
        self.editors[puzrow.Editor.strip()] += 1
        self.formats[puzrow.Size] += 1
        datestr = puzrow.Date
        if datestr:
            if not self.mindate:
                self.mindate = datestr
            else:
                self.mindate = min(self.mindate, datestr)
            if not self.maxdate:
                self.maxdate = datestr
            else:
                self.maxdate = max(self.maxdate, datestr)
        self.num_xd += 1

        self.puzzles_meta.append(puzrow)

    def meta(self):
        return 'pubid num dates formats copyrights editors'.split()

    def row(self):
        return [
                self.pubid,
                html.mkhref(str(self.num_xd), self.pubid),
                "%s — %s" % (self.mindate, self.maxdate),
                html_select_options(self.formats),
                html_select_options(self.copyrights),
                html_select_options(self.editors),
               ]

def tally_to_cell(d):
    freq_sorted = sorted([(v, k) for k, v in list(d.items())], reverse=True)

    if not freq_sorted:
        return ""
    elif len(freq_sorted) == 1:
        return "
".join("%s [x%s]" % (k, v) for v, k in freq_sorted)
    else:
        return "<select><option>" + "</option><option>".join("%s [x%s]" % (k, v) for v, k in freq_sorted) + "</select>"

def publication_header():
    return "PubId NumCollected DatesCollected Formats Copyrights Editors".split()

def main():
    parser = args_parser("generate publishers index html pages and index")

    args = get_args(parser=parser)

    outf = open_output()
    all_pubs = {}  # [(pubid,year)] -> PublicationStats
    pubyear_rows = {}
    similar = metadb.xd_similar()
    puzzles = metadb.xd_puzzles()
    outf.write_html('pub/index.html', pubyear.pubyear_html(), title='The xd crossword puzzle corpus')

    utils.log("collating puzzles")
    for puzrow in puzzles.values():
            pubid = utils.parse_pubid(puzrow.xdid)
            year = xdfile.year_from_date(puzrow.Date)
            k = (pubid, year or 9999)
            if k not in all_pubs:
                all_pubs[k] = PublicationStats(pubid)

            pubyear_rows[pubid] = []

            all_pubs[k].add(puzrow)

    pubyear_header = [ 'xdid', 'Date', 'Size', 'Title', 'Author', 'Editor', 'Copyright', 'Grid_1A_1D', 'ReusedCluePct', 'SimilarGrids' ]
    utils.log('generating index pages')

    # dict to generate pub page with calendars
    pub_grids = defaultdict(dict)

    for pair, pub in sorted(list(all_pubs.items())):
        c_grids = {}
        pubid, year = pair
        progress(pubid)

        reused_clues = 0
        reused_answers = 0
        total_clues = 0
        total_similar = []

        rows = []

        # Assign class based on xdid and similars
        def get_cell_classes(r):
            """ Return cell classes based on parameters """
            # TODO: Implement check that authors same
            classes = []
            rsim = similar.get(r.xdid)
            if rsim and float(rsim.similar_grid_pct) > 0:
                matches = [x.split('=') for x in rsim.matches.split()]
                # Get max for matches for class definition
                max_pct = int(max([ x[1] for x in matches ]))
                # < 40%: 'theme', 40-90%: 'partial', 90-99%: 'full', 100%: 'exact'
                if max_pct > 0 and max_pct < 40:
                    classes.append('pctfilled')
                if max_pct >= 40 and max_pct < 90:
                    classes.append('partial')
                if max_pct >= 90 and max_pct <= 99:
                    classes.append('full')
                if max_pct >= 100:
                    classes.append('exact')
                # Highlight only grids sized > 400 cells
                if num_cells(r.Size) >= 400:
                    classes.append('biggrid')
                # Check for pub similarity
                pubid, y, m, d = utils.split_xdid(r.xdid)
                ymd = '%s%s%s' % (y, m, d)
                if pubid not in [ x[0] for x in matches ]:
                    for m in [ x[0] for x in matches ]:
                        p, y1, m1, d1 = utils.split_xdid(m)
                        ymd1 = '%s%s%s' % (y1, m1, d1)
                        if ymd1 < ymd and p != pubid:
                            classes.append('stolen')

            return ' '.join(classes)

        for r in pub.puzzles_meta:
            similar_text = ""
            reused_clue_pct = "n/a"

            rsim = similar.get(r.xdid)
            if rsim:
                similar_pct = float(rsim.similar_grid_pct)
                if similar_pct > 0:
                    matches = [x.split('=') for x in rsim.matches.split()]
                    for xdid, pct in matches:
                        similar_text += '(%s%%) %s [%s]
' % (pct, puzzles[xdid].Author, xdid)
                    total_similar.append(similar_pct)
                else:
                    similar_text = "0"

                nclues = int(rsim.total_clues)

                reused_clues += int(rsim.reused_clues)
                reused_answers += int(rsim.reused_answers)
                total_clues += nclues

                if nclues:
                    reused_clue_pct = int(100*(float(rsim.reused_clues) / float(nclues)))
                else:
                    reused_clue_pct = ''

            row_dict = {} # Map row and style
            if similar_text and similar_text != "0":
                # http://stackoverflow.com/questions/1418838/html-making-a-link-lead-to-the-anchor-centered-in-the-middle-of-the-page
                pubidtext = '' % r.xdid
                pubidtext += ''
                pubidtext += html.mkhref(r.xdid, '/pub/' + r.xdid)
                c_grids[r.Date] = {
                        'link' : '/pub/%s%s/index.html#' % (pubid, year) + r.xdid,
                        'class': get_cell_classes(r),
                        'title': br_with_n(similar_text),
                        }
                print(c_grids[r.Date])
                row_dict['tag_params'] = {
                    'onclick': 'location.href=\'/pub/%s\'' % r.xdid,
                    'class': 'puzzlehl'
                    }
            else:
                pubidtext = r.xdid
                row_dict['class'] = 'puzzle'

            row = [
                pubidtext,
                r.Date,
                r.Size,
                r.Title,
                r.Author,
                r.Editor,
                r.Copyright,
                r.A1_D1,
                reused_clue_pct,
                similar_text
              ]

            outf.write_row('pub/%s%s.tsv' % (pubid, year), " ".join(pubyear_header), row)
            row_dict['row'] = row
            rows.append(row_dict)

        pub_grids[pubid][year] = c_grids

        # Generate calendar
        onepubyear_html = GridCalendar(c_grids).formatyear(year, 6) + "
"

        # Generate html table sorted by 2nd element of row (date)
        onepubyear_html += html.html_table(sorted(rows , key=lambda x: x['row'][1]), pubyear_header, "puzzle", "puzzles")
        outf.write_html("pub/%s%s/index.html" % (pubid, year), onepubyear_html, title="%s %s" % (pubid, year))

        cluepct = ""
        wordpct = ""
        if total_clues:
            cluepct = "%d%%" % int(100.0*(total_clues-reused_clues)/total_clues)
            wordpct = "%.2f%%" % int(100.0*(total_clues-reused_answers)/total_clues)

        pubyear_rows[pubid].append([
            pubid,
            str(year),
            len(rows),
            "%.2f/%d" % (sum(total_similar)/100.0, len(total_similar)),
            wordpct,
            cluepct
            ])

    # Generate /pub/[publisher][year] page
    for pubid in pub_grids.keys():
        body = []
        for y in sorted(pub_grids[pubid], reverse=True):
            body.append(GridCalendar(pub_grids[pubid][y]).formatyear(y, 12, vertical=True) + mktag('br','break'))
        outf.write_html("pub/%s/index.html" % pubid, ''.join(body), title="%s" % pubid)

    progress()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from flask import Blueprint
from flask import flash, session, redirect, render_template, request, url_for
from flask import abort
from flask_babel import gettext as _
from flask_login import current_user
from fuzzywuzzy import fuzz

from app.decorators import require_role, require_membership
from app.forms import init_form
from app.forms.examination import EditForm
from app.models.examination import test_types
from app.roles import Roles
from app.service import role_service, examination_service, file_service
from app.enums import FileCategory

blueprint = Blueprint('examination', __name__, url_prefix='/examination')

@blueprint.route('/view/<int:exam_id>/<any(exam,answers):doc_type>/',
                 methods=['GET'])
@require_membership
def view(exam_id, doc_type):
    exam = examination_service.get_examination_by_id(exam_id)

    if doc_type == 'exam':
        _file = exam.examination_file
        fn = exam.examination_filename
    elif doc_type == 'answers' and exam.answers_file is not None:
        _file = exam.answers_file
        fn = exam.answers_filename
    else:
        return abort(404)

    content = file_service.get_file_content(_file)
    headers = file_service.get_file_content_headers(_file, display_name=fn)

    return content, headers

@blueprint.route('/add/', methods=['GET', 'POST'])
@require_membership
@require_role(Roles.EXAMINATION_WRITE)
def add():
    form = EditForm(request.form)

    courses = examination_service.find_all_courses()
    educations = examination_service.find_all_educations()

    form.course.choices = [(c.id, c.name) for c in courses]
    form.education.choices = [(e.id, e.name) for e in educations]
    form.test_type.choices = test_types.items()

    if form.validate_on_submit():
        exam_file_data = request.files.get('examination', None)
        answer_file_data = request.files.get('answers', None)

        # Exam file is required
        if exam_file_data is not None:
            exam_file = file_service.add_file(FileCategory.EXAMINATION,
                                              exam_file_data,
                                              exam_file_data.filename)
        else:
            flash(_('No examination uploaded.'), 'danger')
            return render_template('examination/edit.htm',
                                   courses=courses,
                                   educations=educations,
                                   form=form,
                                   test_types=test_types, new_exam=True)

        # Answer file is optional
        if answer_file_data is not None:
            answers_file = file_service.add_file(FileCategory.EXAMINATION,
                                                 answer_file_data,
                                                 answer_file_data.filename)
        else:
            answers_file = None
            flash(_('No answers uploaded.'), 'warning')

        examination_service.add_examination(
            exam_file, form.date.data,
            form.comment.data, form.course.data,
            form.education.data, form.test_type.data,
            answers_file)

        flash(_('Examination successfully uploaded.'), 'success')
        return redirect(url_for('examination.view_examination'))

    return render_template('examination/edit.htm',
                           courses=courses,
                           educations=educations,
                           form=form,
                           test_types=test_types, new_exam=True)

@blueprint.route('/edit/<int:exam_id>/', methods=['GET', 'POST'])
@require_role(Roles.EXAMINATION_WRITE)
def edit(exam_id):
    exam = examination_service.get_examination_by_id(exam_id)

    session['examination_edit_id'] = exam_id

    form = init_form(EditForm, obj=exam)

    courses = examination_service.find_all_courses()
    educations = examination_service.find_all_educations()

    form.course.choices = [(c.id, c.name) for c in courses]
    form.education.choices = [(e.id, e.name) for e in educations]
    form.test_type.choices = test_types.items()

    if form.validate_on_submit():
        exam_file_data = request.files.get('examination', None)
        answer_file_data = request.files.get('answers', None)

        old_examination_file = None
        old_answers_file = None

        if exam_file_data is not None:
            exam_file = file_service.add_file(FileCategory.EXAMINATION,
                                              exam_file_data,
                                              exam_file_data.filename)
            old_examination_file = exam.examination_file
        else:
            exam_file = exam.examination_file
            flash(_('No examination uploaded.'), 'warning')

        if answer_file_data is not None:
            answers_file = file_service.add_file(FileCategory.EXAMINATION,
                                                 answer_file_data,
                                                 answer_file_data.filename)
            old_answers_file = exam.answers_file
        else:
            answers_file = exam.answers_file
            flash(_('No answers uploaded.'), 'warning')

        date = form.date.data
        comment = form.comment.data
        course_id = form.course.data
        education_id = form.education.data
        test_type = form.test_type.data

        examination_service.update_examination(exam_id, exam_file, date,
                                               comment, course_id,
                                               education_id, test_type,
                                               answers_file)

        if old_examination_file is not None:
            file_service.delete_file(old_examination_file)

        if old_answers_file is not None:
            file_service.delete_file(old_answers_file)

        flash(_('Examination successfully changed.'), 'success')

        return redirect(url_for('examination.edit', exam_id=exam_id))
    else:
        # Load the existing field values
        form.course.data = exam.course_id
        form.education.data = exam.education_id
        form.test_type.data = exam.test_type
        form.comment.data = exam.comment

    return render_template('examination/edit.htm',
                           title=_('Examinations'),
                           form=form, examination=exam,
                           courses=courses, educations=educations,
                           new_exam=False)

@blueprint.route('/delete/<int:exam_id>/', methods=['POST'])
@require_role(Roles.EXAMINATION_WRITE)
def delete(exam_id):
    search = request.args.get('search', None)

    exam = examination_service.get_examination_by_id(exam_id)
    if exam is not None:
        examination_service.delete_examination(exam_id)
        file_service.delete_file(exam.examination_file)
        if exam.answers_file is not None:
            file_service.delete_file(exam.answers_file)

        flash(_('Examination successfully deleted.'), 'success')
    else:
        flash(_('Examination does not exist.'), 'danger')

    return redirect(url_for('.view_examination', search=search))

@blueprint.route('/', methods=['GET', 'POST'])
@blueprint.route('/<int:page_nr>/', methods=['GET', 'POST'])
def view_examination(page_nr=1):
    if request.args.get('search'):
        search = request.args.get('search')

        courses = examination_service.find_all_courses()
        course_scores = {}

        search_lower = search.lower().strip()

        # Search in all courses for matches in the course name
        for course in courses:
            course_score = fuzz. \
                partial_ratio(search_lower, course.name.lower())
            # If score is higher than a certain threshold, display in results
            if course_score > 75:
                course_scores[course.id] = course_score

        # Make a list of all course ids, sorted by their similarity score
        ranked_courses = sorted(course_scores,
                                key=course_scores.get, reverse=True)

        if len(ranked_courses) == 0:
            examinations = None
        else:

            # Query the exams. The filter part makes sure only the ranked
            # courses are returned. The order_by clause makes sure the exams
            # are first sorted by course according to the ranking and then
            # sorted by date of the exam
            examinations = examination_service \
                .search_examinations_by_courses(ranked_courses, page_nr, 15)
    else:
        search = ""
        # Query the exams. The order_by part makes sure the exams are sorted
        # by course and within a course are sorted by date
        examinations = examination_service\
            .find_all_examinations(page_nr, 15)

    can_write = role_service.user_has_role(current_user,
                                           Roles.EXAMINATION_WRITE)

    return render_template('examination/view.htm',
                           examinations=examinations, search=search,
                           title=_('Examinations'), test_types=test_types,
                           can_write=can_write)
<commit_msg>Correct presence check of examination files<commit_after>from flask import Blueprint
from flask import flash, session, redirect, render_template, request, url_for
from flask import abort
from flask_babel import gettext as _
from flask_login import current_user
from fuzzywuzzy import fuzz

from app.decorators import require_role, require_membership
from app.forms import init_form
from app.forms.examination import EditForm
from app.models.examination import test_types
from app.roles import Roles
from app.service import role_service, examination_service, file_service
from app.enums import FileCategory

blueprint = Blueprint('examination', __name__, url_prefix='/examination')

@blueprint.route('/view/<int:exam_id>/<any(exam,answers):doc_type>/',
                 methods=['GET'])
@require_membership
def view(exam_id, doc_type):
    exam = examination_service.get_examination_by_id(exam_id)

    if doc_type == 'exam':
        _file = exam.examination_file
        fn = exam.examination_filename
    elif doc_type == 'answers' and exam.answers_file is not None:
        _file = exam.answers_file
        fn = exam.answers_filename
    else:
        return abort(404)

    content = file_service.get_file_content(_file)
    headers = file_service.get_file_content_headers(_file, display_name=fn)

    return content, headers

@blueprint.route('/add/', methods=['GET', 'POST'])
@require_membership
@require_role(Roles.EXAMINATION_WRITE)
def add():
    form = EditForm(request.form)

    courses = examination_service.find_all_courses()
    educations = examination_service.find_all_educations()

    form.course.choices = [(c.id, c.name) for c in courses]
    form.education.choices = [(e.id, e.name) for e in educations]
    form.test_type.choices = test_types.items()

    if form.validate_on_submit():
        exam_file_data = request.files.get('examination', None)
        answer_file_data = request.files.get('answers', None)

        # Exam file is required
        if len(exam_file_data.name) > 0:
            exam_file = file_service.add_file(FileCategory.EXAMINATION,
                                              exam_file_data,
                                              exam_file_data.filename)
        else:
            flash(_('No examination uploaded.'), 'danger')
            return render_template('examination/edit.htm',
                                   courses=courses,
                                   educations=educations,
                                   form=form,
                                   test_types=test_types, new_exam=True)

        # Answer file is optional
        if len(answer_file_data.name) > 0:
            answers_file = file_service.add_file(FileCategory.EXAMINATION,
                                                 answer_file_data,
                                                 answer_file_data.filename)
        else:
            answers_file = None
            flash(_('No answers uploaded.'), 'warning')

        examination_service.add_examination(
            exam_file, form.date.data,
            form.comment.data, form.course.data,
            form.education.data, form.test_type.data,
            answers_file)

        flash(_('Examination successfully uploaded.'), 'success')
        return redirect(url_for('examination.view_examination'))

    return render_template('examination/edit.htm',
                           courses=courses,
                           educations=educations,
                           form=form,
                           test_types=test_types, new_exam=True)

@blueprint.route('/edit/<int:exam_id>/', methods=['GET', 'POST'])
@require_role(Roles.EXAMINATION_WRITE)
def edit(exam_id):
    exam = examination_service.get_examination_by_id(exam_id)

    session['examination_edit_id'] = exam_id

    form = init_form(EditForm, obj=exam)

    courses = examination_service.find_all_courses()
    educations = examination_service.find_all_educations()

    form.course.choices = [(c.id, c.name) for c in courses]
    form.education.choices = [(e.id, e.name) for e in educations]
    form.test_type.choices = test_types.items()

    if form.validate_on_submit():
        exam_file_data = request.files.get('examination', None)
        answer_file_data = request.files.get('answers', None)

        old_examination_file = None
        old_answers_file = None

        if exam_file_data is not None:
            exam_file = file_service.add_file(FileCategory.EXAMINATION,
                                              exam_file_data,
                                              exam_file_data.filename)
            old_examination_file = exam.examination_file
        else:
            exam_file = exam.examination_file
            flash(_('No examination uploaded.'), 'warning')

        if answer_file_data is not None:
            answers_file = file_service.add_file(FileCategory.EXAMINATION,
                                                 answer_file_data,
                                                 answer_file_data.filename)
            old_answers_file = exam.answers_file
        else:
            answers_file = exam.answers_file
            flash(_('No answers uploaded.'), 'warning')

        date = form.date.data
        comment = form.comment.data
        course_id = form.course.data
        education_id = form.education.data
        test_type = form.test_type.data

        examination_service.update_examination(exam_id, exam_file, date,
                                               comment, course_id,
                                               education_id, test_type,
                                               answers_file)

        if old_examination_file is not None:
            file_service.delete_file(old_examination_file)

        if old_answers_file is not None:
            file_service.delete_file(old_answers_file)

        flash(_('Examination successfully changed.'), 'success')

        return redirect(url_for('examination.edit', exam_id=exam_id))
    else:
        # Load the existing field values
        form.course.data = exam.course_id
        form.education.data = exam.education_id
        form.test_type.data = exam.test_type
        form.comment.data = exam.comment

    return render_template('examination/edit.htm',
                           title=_('Examinations'),
                           form=form, examination=exam,
                           courses=courses, educations=educations,
                           new_exam=False)

@blueprint.route('/delete/<int:exam_id>/', methods=['POST'])
@require_role(Roles.EXAMINATION_WRITE)
def delete(exam_id):
    search = request.args.get('search', None)

    exam = examination_service.get_examination_by_id(exam_id)
    if exam is not None:
        examination_service.delete_examination(exam_id)
        file_service.delete_file(exam.examination_file)
        if exam.answers_file is not None:
            file_service.delete_file(exam.answers_file)

        flash(_('Examination successfully deleted.'), 'success')
    else:
        flash(_('Examination does not exist.'), 'danger')

    return redirect(url_for('.view_examination', search=search))

@blueprint.route('/', methods=['GET', 'POST'])
@blueprint.route('/<int:page_nr>/', methods=['GET', 'POST'])
def view_examination(page_nr=1):
    if request.args.get('search'):
        search = request.args.get('search')

        courses = examination_service.find_all_courses()
        course_scores = {}

        search_lower = search.lower().strip()

        # Search in all courses for matches in the course name
        for course in courses:
            course_score = fuzz. \
                partial_ratio(search_lower, course.name.lower())
            # If score is higher than a certain threshold, display in results
            if course_score > 75:
                course_scores[course.id] = course_score

        # Make a list of all course ids, sorted by their similarity score
        ranked_courses = sorted(course_scores,
                                key=course_scores.get, reverse=True)

        if len(ranked_courses) == 0:
            examinations = None
        else:

            # Query the exams. The filter part makes sure only the ranked
            # courses are returned. The order_by clause makes sure the exams
            # are first sorted by course according to the ranking and then
            # sorted by date of the exam
            examinations = examination_service \
                .search_examinations_by_courses(ranked_courses, page_nr, 15)
    else:
        search = ""
        # Query the exams. The order_by part makes sure the exams are sorted
        # by course and within a course are sorted by date
        examinations = examination_service\
            .find_all_examinations(page_nr, 15)

    can_write = role_service.user_has_role(current_user,
                                           Roles.EXAMINATION_WRITE)

    return render_template('examination/view.htm',
                           examinations=examinations, search=search,
                           title=_('Examinations'), test_types=test_types,
                           can_write=can_write)
<|endoftext|>


<commit_before>#! /usr/bin/env python
import sys
import os
import time
import calendar

up_time_quanta = 500

f = open(sys.argv[1])

announce_histogram = {}

#TODO: make this histogram into a CDF

node_uptime_histogram = {}

counter = 0;

# maps search_id to a list of events. Each event is a dict containing:
#  t: timestamp
#  d: distance (from target)
#  o: outstanding searches
#  e: event (NEW, COMPLETED, ADD, INVOKE, TIMEOUT)
outstanding_searches = {}

# list of completed searches
searches = []

def convert_timestamp(t):
	parts = t.split('.')
	posix = time.strptime(parts[0], '%H:%M:%S')
	return (posix.tm_hour * 3600 + posix.tm_min * 60 + posix.tm_sec) * 1000 + int(parts[1])

for line in f:
	counter += 1
#	if counter % 1000 == 0:
#		print '\r%d' % counter,
	try:
		l = line.split(' ')
		if 'announce-distance:' in line:
			idx = l.index('announce-distance:')

			d = int(l[idx+1].strip())
			if not d in announce_histogram: announce_histogram[d] = 0
			announce_histogram[d] += 1
		if 'NODE FAILED' in line:
			idx = l.index('fails:')
			if int(l[idx+1].strip()) != 1: continue;
			idx = l.index('up-time:')
			d = int(l[idx+1].strip())
			# quantize
			d = d - (d % up_time_quanta)
			if not d in node_uptime_histogram: node_uptime_histogram[d] = 0
			node_uptime_histogram[d] += 1

		search_id = l[2]
		ts = l[0]
		event = l[3]

		if event == 'NEW':
			outstanding_searches[search_id] = [{ 't': ts, 'd': 160, 'o': 0, 'e': 'NEW'}]
		elif event == 'INVOKE' or event == 'ADD' or event == '1ST_TIMEOUT' or event == 'TIMEOUT' or event == 'PEERS':
			if not search_id in outstanding_searches:
				print 'orphaned event: %s' % line
			else:
				outstanding = int(l[l.index('invoke-count:')+1])
				distance = int(l[l.index('distance:')+1])
				outstanding_searches[search_id].append({ 't': ts, 'd': distance, 'o': outstanding + 1, 'e': event})
		elif event == 'COMPLETED':
				distance = int(l[l.index('distance:')+1])
				outstanding_searches[search_id].append({ 't': ts, 'd': distance, 'o': 0, 'e': event})

				s = outstanding_searches[search_id]

				try:
					start_time = convert_timestamp(s[0]['t'])
					for i in range(len(s)):
						s[i]['t'] = convert_timestamp(s[i]['t']) - start_time
				except:
					pass
				searches.append(s)
				del outstanding_searches[search_id]

	except Exception, e:
		print line.split(' ')

lookup_times_min = []
lookup_times_max = []

# these are the timestamps for lookups crossing distance
# to target boundaries
lookup_distance = []
for i in range(0, 15):
	lookup_distance.append([])

for s in searches:
	for i in s:
		if not 'last_dist' in i:
			i['last_dist'] = -1
		cur_dist = 160 - i['d']
		last_dist = i['last_dist']
		if cur_dist > last_dist:
			for j in range(last_dist + 1, cur_dist + 1):
				if j >= len(lookup_distance): break
				lookup_distance[j].append(i['t'])
			i['last_dist'] = cur_dist
		if i['e'] != 'PEERS': continue
		lookup_times_min.append(i['t'])
		break
	for i in reversed(s):
		if i['e'] != 'PEERS': continue
		lookup_times_max.append(i['t'])
		break

lookup_times_min.sort()
lookup_times_max.sort()
out = open('dht_lookup_times_cdf.txt', 'w+')
counter = 0
for i in range(len(lookup_times_min)):
	counter += 1
	print >>out, '%d\t%d\t%f' % (lookup_times_min[i], lookup_times_max[i], counter / float(len(lookup_times_min)))
out.close()

for i in lookup_distance:
	i.sort()

dist = 0
for i in lookup_distance:
	out = open('dht_lookup_distance_%d.txt' % dist, 'w+')
	dist += 1
	counter = 0
	for j in i:
		counter += 1
		print >>out, '%d\t%f' % (j, counter / float(len(i)))
	out.close()

out = open('dht_lookups.txt', 'w+')
for s in searches:
	for i in s:
		if i['e'] == 'INVOKE':
			print >>out, ' ->', i['t'], 160 - i['d']
		elif i['e'] == '1ST_TIMEOUT':
			print >>out, ' x ', i['t'], 160 - i['d']
		elif i['e'] == 'TIMEOUT':
			print >>out, ' X ', i['t'], 160 - i['d']
		elif i['e'] == 'PEERS':
			print >>out, ' <-', i['t'], 160 - i['d']
		elif i['e'] == 'COMPLETED':
			print >>out, '***', i['t'], 160 - i['d'], '\n'
			break
out.close()

out = open('dht_announce_distribution.dat', 'w+')
print 'announce distribution items: %d' % len(announce_histogram)
for k,v in announce_histogram.items():
	print >>out, '%d %d' % (k, v)
	print '%d %d' % (k, v)
out.close()

out = open('dht_node_uptime_cdf.txt', 'w+')
s = 0

total_uptime_nodes = 0
for k,v in node_uptime_histogram.items():
	total_uptime_nodes += v

for k,v in sorted(node_uptime_histogram.items()):
	s += v
	print >>out, '%f %f' % (k / float(60), s / float(total_uptime_nodes))
	print '%f %f' % (k / float(60), s / float(total_uptime_nodes))
out.close()

out = open('dht.gnuplot', 'w+')
out.write('''
set term png size 1200,700 small
set output "dht_lookup_times_cdf.png"
set title "portion of lookups that have received at least one data response"
set ylabel "portion of lookups"
set xlabel "time from start of lookup (ms)"
set grid
plot "dht_lookup_times_cdf.txt" using 1:3 with lines title "time to first result", \
	"dht_lookup_times_cdf.txt" using 2:3 with lines title "time to last result"

set terminal postscript
set output "dht_lookup_times_cdf.ps"
replot

set term png size 1200,700 small
set xtics 100
set xrange [0:2000]
set output "dht_min_lookup_times_cdf.png"
plot "dht_lookup_times_cdf.txt" using 1:3 with lines title "time to first result"

set terminal postscript
set output "dht_min_lookup_times_cdf.ps"
replot

set term png size 1200,700 small
set output "dht_node_uptime_cdf.png"
set xrange [0:*]
set title "node up time"
set ylabel "portion of nodes being offline"
set xlabel "time from first seeing the node (minutes)"
set xtics 10
unset grid
plot  "dht_node_uptime_cdf.txt" using 1:2 title "nodes" with lines

set term png size 1200,700 small
set output "dht_announce_distribution.png"
set xrange [0:30]
set xtics 1
set title "bucket # announces are made against relative to target node-id"
set ylabel "# of announces"
set boxwidth 1
set xlabel "bit prefix of nodes in announces"
set style fill solid border -1 pattern 2
plot  "dht_announce_distribution.dat" using 1:2 title "announces" with boxes

set terminal postscript
set output "dht_announce_distribution.ps"
replot

set term png size 1200,700 small
set output "dht_lookup_distance_cdf.png"
set title "portion of lookups that have reached a certain distance in their lookups"
set ylabel "portion of lookups"
set xlabel "time from start of lookup (ms)"
set xrange [0:2000]
set xtics 100
set grid
plot ''')

dist = 0
for i in lookup_distance:
	if dist > 0: out.write(', ')
	out.write('"dht_lookup_distance_%d.txt" using 1:2 title "%d" with lines' % (dist, dist))
	dist += 1

out.close()

os.system('gnuplot dht.gnuplot');

<commit_msg>count unique nodes by client in dht log parser<commit_after>#! /usr/bin/env python
import sys
import os
import time
import calendar
import pprint

pp = pprint.PrettyPrinter(indent=4)

up_time_quanta = 500

f = open(sys.argv[1])

announce_histogram = {}

#TODO: make this histogram into a CDF

node_uptime_histogram = {}

counter = 0;

# maps search_id to a list of events. Each event is a dict containing:
#  t: timestamp
#  d: distance (from target)
#  o: outstanding searches
#  e: event (NEW, COMPLETED, ADD, INVOKE, TIMEOUT)
outstanding_searches = {}

# list of completed searches
searches = []

def convert_timestamp(t):
	parts = t.split('.')
	posix = time.strptime(parts[0], '%H:%M:%S')
	return (posix.tm_hour * 3600 + posix.tm_min * 60 + posix.tm_sec) * 1000 + int(parts[1])

unique_ips = set()
client_version_histogram = {}
client_histogram = {}

for line in f:
	counter += 1
#	if counter % 1000 == 0:
#		print '\r%d' % counter,
	try:
		l = line.split(' ')
		try:
			if len(l) > 4 and l[2] == '<==' and l[1] == '[dht_tracker]':
				ip = l[3].split(':')[0]
				if ip not in unique_ips:
					unique_ips.add(ip)
					json_blob = line.split(l[3])[1]
					version = json_blob.split("'v': '")[1].split("'")[0]
					if len(version) == 4:
						v = '%s-%d' % (version[0:2], (ord(version[2]) << 8) + ord(version[3]))
					elif len(version) == 8:
						v = '%c%c-%d' % (chr(int(version[0:2], 16)), chr(int(version[2:4], 16)), int(version[4:8], 16))
					else:
						v = 'unknown'

					if not v in client_version_histogram:
						client_version_histogram[v] = 1
					else:
						client_version_histogram[v] += 1

					if not v[0:2] in client_histogram:
						client_histogram[v[0:2]] = 1
					else:
						client_histogram[v[0:2]] += 1
		except: pass

		if 'announce-distance:' in line:
			idx = l.index('announce-distance:')

			d = int(l[idx+1].strip())
			if not d in announce_histogram: announce_histogram[d] = 0
			announce_histogram[d] += 1
		if 'NODE FAILED' in line:
			idx = l.index('fails:')
			if int(l[idx+1].strip()) != 1: continue;
			idx = l.index('up-time:')
			d = int(l[idx+1].strip())
			# quantize
			d = d - (d % up_time_quanta)
			if not d in node_uptime_histogram: node_uptime_histogram[d] = 0
			node_uptime_histogram[d] += 1

		search_id = l[2]
		ts = l[0]
		event = l[3]

		if event == 'NEW':
			outstanding_searches[search_id] = [{ 't': ts, 'd': 160, 'o': 0, 'e': 'NEW'}]
		elif event == 'INVOKE' or event == 'ADD' or event == '1ST_TIMEOUT' or event == 'TIMEOUT' or event == 'PEERS':
			if not search_id in outstanding_searches:
				print 'orphaned event: %s' % line
			else:
				outstanding = int(l[l.index('invoke-count:')+1])
				distance = int(l[l.index('distance:')+1])
				outstanding_searches[search_id].append({ 't': ts, 'd': distance, 'o': outstanding + 1, 'e': event})
		elif event == 'COMPLETED':
				distance = int(l[l.index('distance:')+1])
				outstanding_searches[search_id].append({ 't': ts, 'd': distance, 'o': 0, 'e': event})

				s = outstanding_searches[search_id]

				try:
					start_time = convert_timestamp(s[0]['t'])
					for i in range(len(s)):
						s[i]['t'] = convert_timestamp(s[i]['t']) - start_time
				except:
					pass
				searches.append(s)
				del outstanding_searches[search_id]

	except Exception, e:
		print e
		print 'ERROR: ', line.split(' ')

lookup_times_min = []
lookup_times_max = []

# these are the timestamps for lookups crossing distance
# to target boundaries
lookup_distance = []
for i in range(0, 15):
	lookup_distance.append([])

for s in searches:
	for i in s:
		if not 'last_dist' in i:
			i['last_dist'] = -1
		cur_dist = 160 - i['d']
		last_dist = i['last_dist']
		if cur_dist > last_dist:
			for j in range(last_dist + 1, cur_dist + 1):
				if j >= len(lookup_distance): break
				lookup_distance[j].append(i['t'])
			i['last_dist'] = cur_dist
		if i['e'] != 'PEERS': continue
		lookup_times_min.append(i['t'])
		break
	for i in reversed(s):
		if i['e'] != 'PEERS': continue
		lookup_times_max.append(i['t'])
		break

lookup_times_min.sort()
lookup_times_max.sort()
out = open('dht_lookup_times_cdf.txt', 'w+')
counter = 0
for i in range(len(lookup_times_min)):
	counter += 1
	print >>out, '%d\t%d\t%f' % (lookup_times_min[i], lookup_times_max[i], counter / float(len(lookup_times_min)))
out.close()

for i in lookup_distance:
	i.sort()

dist = 0
for i in lookup_distance:
	out = open('dht_lookup_distance_%d.txt' % dist, 'w+')
	dist += 1
	counter = 0
	for j in i:
		counter += 1
		print >>out, '%d\t%f' % (j, counter / float(len(i)))
	out.close()

out = open('dht_lookups.txt', 'w+')
for s in searches:
	for i in s:
		if i['e'] == 'INVOKE':
			print >>out, ' ->', i['t'], 160 - i['d']
		elif i['e'] == '1ST_TIMEOUT':
			print >>out, ' x ', i['t'], 160 - i['d']
		elif i['e'] == 'TIMEOUT':
			print >>out, ' X ', i['t'], 160 - i['d']
		elif i['e'] == 'PEERS':
			print >>out, ' <-', i['t'], 160 - i['d']
		elif i['e'] == 'COMPLETED':
			print >>out, '***', i['t'], 160 - i['d'], '\n'
			break
out.close()

out = open('dht_announce_distribution.dat', 'w+')
print 'announce distribution items: %d' % len(announce_histogram)
for k,v in announce_histogram.items():
	print >>out, '%d %d' % (k, v)
	print '%d %d' % (k, v)
out.close()

out = open('dht_node_uptime_cdf.txt', 'w+')
s = 0

total_uptime_nodes = 0
for k,v in node_uptime_histogram.items():
	total_uptime_nodes += v

for k,v in sorted(node_uptime_histogram.items()):
	s += v
	print >>out, '%f %f' % (k / float(60), s / float(total_uptime_nodes))
	print '%f %f' % (k / float(60), s / float(total_uptime_nodes))
out.close()

print 'clients by version'
client_version_histogram = sorted(client_version_histogram.items(), key=lambda x: x[1], reverse=True)
pp.pprint(client_version_histogram)

print 'clients'
client_histogram = sorted(client_histogram.items(), key=lambda x: x[1], reverse=True)
pp.pprint(client_histogram)

out = open('dht.gnuplot', 'w+')
out.write('''
set term png size 1200,700 small
set output "dht_lookup_times_cdf.png"
set title "portion of lookups that have received at least one data response"
set ylabel "portion of lookups"
set xlabel "time from start of lookup (ms)"
set grid
plot "dht_lookup_times_cdf.txt" using 1:3 with lines title "time to first result", \
	"dht_lookup_times_cdf.txt" using 2:3 with lines title "time to last result"

set terminal postscript
set output "dht_lookup_times_cdf.ps"
replot

set term png size 1200,700 small
set xtics 100
set xrange [0:2000]
set output "dht_min_lookup_times_cdf.png"
plot "dht_lookup_times_cdf.txt" using 1:3 with lines title "time to first result"

set terminal postscript
set output "dht_min_lookup_times_cdf.ps"
replot

set term png size 1200,700 small
set output "dht_node_uptime_cdf.png"
set xrange [0:*]
set title "node up time"
set ylabel "portion of nodes being offline"
set xlabel "time from first seeing the node (minutes)"
set xtics 10
unset grid
plot  "dht_node_uptime_cdf.txt" using 1:2 title "nodes" with lines

set term png size 1200,700 small
set output "dht_announce_distribution.png"
set xrange [0:30]
set xtics 1
set title "bucket # announces are made against relative to target node-id"
set ylabel "# of announces"
set boxwidth 1
set xlabel "bit prefix of nodes in announces"
set style fill solid border -1 pattern 2
plot  "dht_announce_distribution.dat" using 1:2 title "announces" with boxes

set terminal postscript
set output "dht_announce_distribution.ps"
replot

set term png size 1200,700 small
set output "dht_lookup_distance_cdf.png"
set title "portion of lookups that have reached a certain distance in their lookups"
set ylabel "portion of lookups"
set xlabel "time from start of lookup (ms)"
set xrange [0:2000]
set xtics 100
set grid
plot ''')

dist = 0
for i in lookup_distance:
	if dist > 0: out.write(', ')
	out.write('"dht_lookup_distance_%d.txt" using 1:2 title "%d" with lines' % (dist, dist))
	dist += 1

out.close()

os.system('gnuplot dht.gnuplot');

<|endoftext|>


<commit_before>"""
spaceshooter.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:
Write and submit a program that implements the spacewar game:
https://github.com/HHS-IntroProgramming/Spacewar
"""
from ggame import App, RectangleAsset, ImageAsset, Sprite, LineStyle, Color, Frame

SCREEN_WIDTH = 640
SCREEN_HEIGHT = 480

class SpaceShip(Sprite):
    """
    Animated space ship
    """
    asset = ImageAsset("images/four_spaceship_by_albertov_with_thrust.png",
        Frame(227,0,292-227,125), 4, 'vertical')

    def __init__(self, position):
        super().__init__(SpaceShip.asset, position)
class SpaceGame(App):
    """
    Tutorial4 space game example.
    """
    def __init__(self, width, height):
        super().__init__(width, height)
        black = Color(0, 1)
        noline = LineStyle(0, black)
        bg_asset = RectangleAsset(width, height, noline, black)
        bg = Sprite(bg_asset, (0,0))
        SpaceShip((100,100))
        SpaceShip((150,150))
        SpaceShip((200,50))
myapp = SpaceGame(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()<commit_msg>Updated from Brython Server: 3/10/2016 11:19:35 AM<commit_after>"""
spaceshooter.py
Author: Payton
Credit: Morgan

Assignment:
Write and submit a program that implements the spacewar game:
https://github.com/HHS-IntroProgramming/Spacewar
"""
from ggame import App, RectangleAsset, ImageAsset, Sprite, LineStyle, Color, Frame

SCREEN_WIDTH = 640
SCREEN_HEIGHT = 480

class SpaceShip(Sprite):
    """
    Animated space ship
    """
    asset = ImageAsset("images/four_spaceship_by_albertov_with_thrust.png",
        Frame(227,0,292-227,125), 4, 'vertical')

    def __init__(self, position):
        super().__init__(SpaceShip.asset, position)
class SpaceGame(App):
    """
    Tutorial4 space game example.
    """
    def __init__(self, width, height):
        super().__init__(width, height)
        black = Color(0, 1)
        noline = LineStyle(0, black)
        bg_asset = RectangleAsset(width, height, noline, black)
        bg = Sprite(bg_asset, (0,0))
        SpaceShip((100,100))
        SpaceShip((150,150))
        SpaceShip((200,50))
myapp = SpaceGame(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()<|endoftext|>


<commit_before>from carrot.connection import DjangoAMQPConnection
from celery.messaging import TaskConsumer
from celery.conf import DAEMON_CONCURRENCY, DAEMON_LOG_FILE
from celery.conf import QUEUE_WAKEUP_AFTER, EMPTY_MSG_EMIT_EVERY
from celery.log import setup_logger
from celery.registry import tasks
from celery.process import ProcessQueue
from celery.models import PeriodicTaskMeta
import multiprocessing
import simplejson
import traceback
import logging
import time

class EmptyQueue(Exception):
    """The message queue is currently empty."""

class UnknownTask(Exception):
    """Got an unknown task in the queue. The message is requeued and
    ignored."""

class TaskWrapper(object):
    def __init__(self, task_name, task_id, task_func, args, kwargs):
        self.task_name = task_name
        self.task_id = task_id
        self.task_func = task_func
        self.args = args
        self.kwargs = kwargs

    @classmethod
    def from_message(cls, message):
        message_data = simplejson.loads(message.body)
        task_name = message_data.pop("task")
        task_id = message_data.pop("id")
        args = message_data.pop("args")
        kwargs = message_data.pop("kwargs")
        if task_name not in tasks:
            message.reject()
            raise UnknownTask(task_name)
        task_func = tasks[task_name]
        return cls(task_name, task_id, task_func, args, kwargs)

    def extend_kwargs_with_logging(self, loglevel, logfile):
        task_func_kwargs = {"logfile": logfile,
                            "loglevel": loglevel}
        task_func_kwargs.update(self.kwargs)
        return task_func_kwargs

    def execute(self, loglevel, logfile):
        task_func_kwargs = self.extend_kwargs_with_logging(logfile, loglevel)
        return self.task_func(*self.args, **task_func_kwargs)

    def execute_using_pool(self, pool, loglevel, logfile):
        task_func_kwargs = self.extend_kwargs_with_logging(logfile, loglevel)
        return pool.apply_async(self.task_func, self.args, task_func_kwargs)

class TaskDaemon(object):
    """Executes tasks waiting in the task queue.

    ``concurrency`` is the number of simultaneous processes.
    """
    loglevel = logging.ERROR
    concurrency = DAEMON_CONCURRENCY
    logfile = DAEMON_LOG_FILE
    queue_wakeup_after = QUEUE_WAKEUP_AFTER
    empty_msg_emit_every = EMPTY_MSG_EMIT_EVERY

    def __init__(self, concurrency=None, logfile=None, loglevel=None,
            queue_wakeup_after=None):
        self.loglevel = loglevel or self.loglevel
        self.concurrency = concurrency or self.concurrency
        self.logfile = logfile or self.logfile
        self.queue_wakeup_after = queue_wakeup_after or \
                                    self.queue_wakeup_after
        self.logger = setup_logger(loglevel, logfile)
        self.pool = multiprocessing.Pool(self.concurrency)
        self.task_consumer = TaskConsumer(connection=DjangoAMQPConnection)

    def fetch_next_task(self):
        message = self.task_consumer.fetch()
        if message is None: # No messages waiting.
            raise EmptyQueue()

        task = TaskWrapper.from_message(message)
        self.logger.info("Got task from broker: %s[%s]" % (
                            task.task_name, task.task_id))

        return task, message

    def execute_next_task(self):
        task, message = self.fetch_next_task()

        try:
            result = task.execute_using_pool(self.pool, self.loglevel,
                                             self.logfile)
        except Exception, error:
            self.logger.critical("Worker got exception %s: %s\n%s" % (
                error.__class__, error, traceback.format_exc()))
            return

        message.ack()
        return result, task.task_name, task.task_id

    def run_periodic_tasks(self):
        """Schedule all waiting periodic tasks for execution.

        Returns list of :class:`celery.models.PeriodicTaskMeta` objects.
        """
        waiting_tasks = PeriodicTaskMeta.objects.get_waiting_tasks()
        [waiting_task.delay()
                for waiting_task in waiting_tasks]
        return waiting_tasks

    def run(self):
        """Run the worker server."""
        results = ProcessQueue(self.concurrency, logger=self.logger,
                done_msg="Task %(name)s[%(id)s] processed: %(return_value)s")
        last_empty_emit = None

        while True:
            self.run_periodic_tasks()
            try:
                result, task_name, task_id = self.execute_next_task()
            except ValueError:
                # execute_next_task didn't return a r/name/id tuple,
                # probably because it got an exception.
                continue
            except EmptyQueue:
                emit_every = self.empty_msg_emit_every
                if emit_every:
                    if not last_empty_emit or \
                            time.time() > last_empty_emit + emit_every:
                        self.logger.info("Waiting for queue.")
                        last_empty_emit = time.time()
                time.sleep(self.queue_wakeup_after)
                continue
            except UnknownTask, e:
                self.logger.info("Unknown task ignored: %s" % (e))
                continue
            except Exception, e:
                self.logger.critical("Message queue raised %s: %s\n%s" % (
                             e.__class__, e, traceback.format_exc()))
                continue

            results.add(result, task_name, task_id)
<commit_msg>Mixed up loglevel/logfile arguments :(<commit_after>from carrot.connection import DjangoAMQPConnection
from celery.messaging import TaskConsumer
from celery.conf import DAEMON_CONCURRENCY, DAEMON_LOG_FILE
from celery.conf import QUEUE_WAKEUP_AFTER, EMPTY_MSG_EMIT_EVERY
from celery.log import setup_logger
from celery.registry import tasks
from celery.process import ProcessQueue
from celery.models import PeriodicTaskMeta
import multiprocessing
import simplejson
import traceback
import logging
import time

class EmptyQueue(Exception):
    """The message queue is currently empty."""

class UnknownTask(Exception):
    """Got an unknown task in the queue. The message is requeued and
    ignored."""

class TaskWrapper(object):
    def __init__(self, task_name, task_id, task_func, args, kwargs):
        self.task_name = task_name
        self.task_id = task_id
        self.task_func = task_func
        self.args = args
        self.kwargs = kwargs

    @classmethod
    def from_message(cls, message):
        message_data = simplejson.loads(message.body)
        task_name = message_data.pop("task")
        task_id = message_data.pop("id")
        args = message_data.pop("args")
        kwargs = message_data.pop("kwargs")
        if task_name not in tasks:
            message.reject()
            raise UnknownTask(task_name)
        task_func = tasks[task_name]
        return cls(task_name, task_id, task_func, args, kwargs)

    def extend_kwargs_with_logging(self, loglevel, logfile):
        task_func_kwargs = {"logfile": logfile,
                            "loglevel": loglevel}
        task_func_kwargs.update(self.kwargs)
        return task_func_kwargs

    def execute(self, loglevel, logfile):
        task_func_kwargs = self.extend_kwargs_with_logging(loglevel, logfile)
        return self.task_func(*self.args, **task_func_kwargs)

    def execute_using_pool(self, pool, loglevel, logfile):
        task_func_kwargs = self.extend_kwargs_with_logging(loglevel, logfile)
        return pool.apply_async(self.task_func, self.args, task_func_kwargs)

class TaskDaemon(object):
    """Executes tasks waiting in the task queue.

    ``concurrency`` is the number of simultaneous processes.
    """
    loglevel = logging.ERROR
    concurrency = DAEMON_CONCURRENCY
    logfile = DAEMON_LOG_FILE
    queue_wakeup_after = QUEUE_WAKEUP_AFTER
    empty_msg_emit_every = EMPTY_MSG_EMIT_EVERY

    def __init__(self, concurrency=None, logfile=None, loglevel=None,
            queue_wakeup_after=None):
        self.loglevel = loglevel or self.loglevel
        self.concurrency = concurrency or self.concurrency
        self.logfile = logfile or self.logfile
        self.queue_wakeup_after = queue_wakeup_after or \
                                    self.queue_wakeup_after
        self.logger = setup_logger(loglevel, logfile)
        self.pool = multiprocessing.Pool(self.concurrency)
        self.task_consumer = TaskConsumer(connection=DjangoAMQPConnection)

    def fetch_next_task(self):
        message = self.task_consumer.fetch()
        if message is None: # No messages waiting.
            raise EmptyQueue()

        task = TaskWrapper.from_message(message)
        self.logger.info("Got task from broker: %s[%s]" % (
                            task.task_name, task.task_id))

        return task, message

    def execute_next_task(self):
        task, message = self.fetch_next_task()

        try:
            result = task.execute_using_pool(self.pool, self.loglevel,
                                             self.logfile)
        except Exception, error:
            self.logger.critical("Worker got exception %s: %s\n%s" % (
                error.__class__, error, traceback.format_exc()))
            return

        message.ack()
        return result, task.task_name, task.task_id

    def run_periodic_tasks(self):
        """Schedule all waiting periodic tasks for execution.

        Returns list of :class:`celery.models.PeriodicTaskMeta` objects.
        """
        waiting_tasks = PeriodicTaskMeta.objects.get_waiting_tasks()
        [waiting_task.delay()
                for waiting_task in waiting_tasks]
        return waiting_tasks

    def run(self):
        """Run the worker server."""
        results = ProcessQueue(self.concurrency, logger=self.logger,
                done_msg="Task %(name)s[%(id)s] processed: %(return_value)s")
        last_empty_emit = None

        while True:
            self.run_periodic_tasks()
            try:
                result, task_name, task_id = self.execute_next_task()
            except ValueError:
                # execute_next_task didn't return a r/name/id tuple,
                # probably because it got an exception.
                continue
            except EmptyQueue:
                emit_every = self.empty_msg_emit_every
                if emit_every:
                    if not last_empty_emit or \
                            time.time() > last_empty_emit + emit_every:
                        self.logger.info("Waiting for queue.")
                        last_empty_emit = time.time()
                time.sleep(self.queue_wakeup_after)
                continue
            except UnknownTask, e:
                self.logger.info("Unknown task ignored: %s" % (e))
                continue
            except Exception, e:
                self.logger.critical("Message queue raised %s: %s\n%s" % (
                             e.__class__, e, traceback.format_exc()))
                continue

            results.add(result, task_name, task_id)
<|endoftext|>


<commit_before>'''
GotoNewest

A tool to quickly transfer to the latest directory
created in a base directory, provided the name of
the base directory is supplied as an argument
'''

import os

def transfer(basepath=None):
    ''' Transfer to the newest directory in the basepath
    directory
    '''
    if basepath is None:
        raise AttributeError
    subdirs = os.listdir(basepath)
    if len(subdirs) == 0:
        raise AttributeError
    if len(subdirs) == 1:
        return subdirs.pop()
    latest_subdir = get_latest_modified_subdir(basepath)
    return latest_subdir

def get_latest_modified_subdir(basepath):
    ''' Iterate through all the subdirectories in a given path
    and return the one with the latest modified time
    '''
    # TODO: There has to be a better way of doing this
    all_subdirs = os.listdir(basepath)
    latest = os.path.getmtime(basepath + '/' + all_subdirs[0])
    latest_subdir = all_subdirs[0]
    for subdir in all_subdirs:
        mtime = os.path.getmtime(basepath + '/' + subdir)
        if mtime > latest:
            latest = mtime
            latest_subdir = subdir

    return latest_subdir
<commit_msg>Add main method to test directory change<commit_after>'''
GotoNewest

A tool to quickly transfer to the latest directory
created in a base directory, provided the name of
the base directory is supplied as an argument
'''

import os
import sys
import gn_helper

def transfer(basepath=None):
    ''' Transfer to the newest directory in the basepath
    directory
    '''
    if basepath is None:
        raise AttributeError
    subdirs = os.listdir(basepath)
    if len(subdirs) == 0:
        raise AttributeError
    if len(subdirs) == 1:
        return subdirs.pop()
    latest_subdir = get_latest_modified_subdir(basepath)
    return latest_subdir

def get_latest_modified_subdir(basepath):
    ''' Iterate through all the subdirectories in a given path
    and return the one with the latest modified time
    '''
    # TODO: There has to be a better way of doing this
    all_subdirs = os.listdir(basepath)
    latest = os.path.getmtime(basepath + '/' + all_subdirs[0])
    latest_subdir = all_subdirs[0]
    for subdir in all_subdirs:
        mtime = os.path.getmtime(basepath + '/' + subdir)
        if mtime > latest:
            latest = mtime
            latest_subdir = subdir

    return latest_subdir

def main():
    ''' Get the latest directory created in the directory
    set as basepath in the config file, and then cd into it
    '''
    basepath = gn_helper.get_basepath()
    if basepath is not None:
        os.chdir(basepath)
        print 'Testing: in', os.getcwd()
    else:
        print 'Sorry, the basepath is not valid'

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/
# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#

import logging
import json
from requests.sessions import Session
from requests.utils import get_environ_proxies

from botocore.auth import AUTH_TYPE_MAPS, UnknownSignatureVersionError
import botocore.response
import botocore.exceptions
from botocore.awsrequest import AWSRequest

try:
    from urllib.parse import urljoin
except ImportError:
    from urlparse import urljoin

logger = logging.getLogger(__name__)

class Endpoint(object):
    """
    Represents an endpoint for a particular service in a specific
    region.  Only an endpoint can make requests.

    :ivar service: The Service object that describes this endpoints
        service.
    :ivar host: The fully qualified endpoint hostname.
    :ivar session: The session object.
    """

    def __init__(self, service, region_name, host, auth, proxies=None):
        self.service = service
        self.session = self.service.session
        self.region_name = region_name
        self.host = host
        self.verify = True
        self.auth = auth
        if proxies is None:
            proxies = {}
        self.proxies = proxies
        self.http_session = Session()

    def __repr__(self):
        return '%s(%s)' % (self.service.endpoint_prefix, self.host)

    def make_request(self, params, list_marker=None):
        raise NotImplementedError("make_request")

    def prepare_request(self, request):
        logger.debug('prepare_request')
        self.auth.add_auth(request=request)
        prepared_request = request.prepare()
        return prepared_request

    def _send_request(self, request, operation):
        return self.http_session.send(request, verify=self.verify,
                                      stream=operation.is_streaming(),
                                      proxies=self.proxies)

class QueryEndpoint(Endpoint):
    """
    This class handles only AWS/Query style services.
    """

    def make_request(self, operation, params):
        """
        Send a request to the endpoint and parse the response
        and return it and long with the HTTP response object
        from requests.
        """
        logger.debug(params)
        logger.debug('SSL Verify: %s' % self.verify)
        params['Action'] = operation.name
        params['Version'] = self.service.api_version
        user_agent = self.session.user_agent()
        request = AWSRequest(method='POST', url=self.host,
                             data=params, headers={'User-Agent': user_agent})
        prepared_request = self.prepare_request(request)
        http_response = self._send_request(prepared_request, operation)
        return botocore.response.get_response(operation, http_response)

class JSONEndpoint(Endpoint):
    """
    This class handles only AWS/JSON style services.
    """

    ResponseContentTypes = ['application/x-amz-json-1.1',
                            'application/json']

    def make_request(self, operation, params):
        """
        Send a request to the endpoint and parse the response
        and return it and long with the HTTP response object
        from requests.
        """
        logger.debug(params)
        logger.debug('SSL Verify: %s' % self.verify)
        user_agent = self.session.user_agent()
        target = '%s.%s' % (self.service.target_prefix, operation.name)
        json_version = '1.0'
        if hasattr(self.service, 'json_version'):
            json_version = str(self.service.json_version)
        content_type = 'application/x-amz-json-%s' % json_version
        content_encoding = 'amz-1.0'
        data = json.dumps(params)
        request = AWSRequest(method='POST', url=self.host,
                             data=data,
                             headers={'User-Agent': user_agent,
                                      'X-Amz-Target': target,
                                      'Content-Type': content_type,
                                      'Content-Encoding': content_encoding})
        prepared_request = self.prepare_request(request)
        http_response = self._send_request(prepared_request, operation)
        return botocore.response.get_response(operation, http_response)

class RestEndpoint(Endpoint):

    def build_uri(self, operation, params):
        uri = operation.http['uri']
        if '?' in uri:
            path, query_params = uri.split('?')
        else:
            path = uri
            query_params = ''
        logger.debug('path: %s' % path)
        logger.debug('query_params: %s' % query_params)
        path_components = []
        for pc in path.split('/'):
            if pc:
                pc = pc.format(**params['uri_params'])
            path_components.append(pc)
        path = '/'.join(path_components)
        query_param_components = []
        for qpc in query_params.split('&'):
            if qpc:
                if '=' in qpc:
                    key_name, value_name = qpc.split('=')
                else:
                    key_name = qpc
                    value_name = None
                if value_name:
                    value_name = value_name.strip('{}')
                    if value_name in params['uri_params']:
                        value = params['uri_params'][value_name]
                        query_param_components.append('%s=%s' % (key_name,
                                                                 value))
                else:
                    query_param_components.append(key_name)
        query_params = '&'.join(query_param_components)
        logger.debug('path: %s' % path)
        logger.debug('query_params: %s' % query_params)
        return path + '?' + query_params

    def make_request(self, operation, params):
        """
        Send a request to the endpoint and parse the response
        and return it and long with the HTTP response object
        from requests.
        """
        logger.debug(params)
        logger.debug('SSL Verify: %s' % self.verify)
        user_agent = self.session.user_agent()
        params['headers']['User-Agent'] = user_agent
        uri = self.build_uri(operation, params)
        uri = urljoin(self.host, uri)
        if params['payload'] is None:
            request = AWSRequest(method=operation.http['method'],
                                 url=uri, headers=params['headers'])
        else:
            request = AWSRequest(method=operation.http['method'],
                                 url=uri, headers=params['headers'],
                                 data=params['payload'])
        prepared_request = self.prepare_request(request)
        http_response = self._send_request(prepared_request, operation)
        return botocore.response.get_response(operation, http_response)

def _get_proxies(url):
    # We could also support getting proxies from a config file,
    # but for now proxy support is taken from the environment.
    return get_environ_proxies(url)

def get_endpoint(service, region_name, endpoint_url):
    service_to_endpoint = {
        'query': QueryEndpoint,
        'json': JSONEndpoint,
        'rest-xml': RestEndpoint,
        'rest-json': RestEndpoint,
    }
    service_type = service.type
    if service_type not in service_to_endpoint:
        raise botocore.exceptions.UnknownServiceStyle(
            service_style=service.type)
    cls = service_to_endpoint.get(service_type)
    if cls is None:
        raise NotImplementedError("%s service type is not yet implemented" %
                                  service_type)
    service_name = getattr(service, 'signing_name', service.endpoint_prefix)
    auth = _get_auth(service.signature_version,
                     credentials=service.session.get_credentials(),
                     service_name=service_name,
                     region_name=region_name)
    proxies = _get_proxies(endpoint_url)
    return cls(service, region_name, endpoint_url, auth=auth, proxies=proxies)

def _get_auth(signature_version, credentials, service_name, region_name):
    cls = AUTH_TYPE_MAPS.get(signature_version)
    if cls is None:
        raise UnknownSignatureVersionError(signature_version=signature_version)
    else:
        return cls(credentials=credentials,
                   service_name=service_name,
                   region_name=region_name)
<commit_msg>Move service type mapping to module level<commit_after># Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/
# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#

import logging
import json
from requests.sessions import Session
from requests.utils import get_environ_proxies

from botocore.auth import AUTH_TYPE_MAPS, UnknownSignatureVersionError
import botocore.response
import botocore.exceptions
from botocore.awsrequest import AWSRequest

try:
    from urllib.parse import urljoin
except ImportError:
    from urlparse import urljoin

logger = logging.getLogger(__name__)

class Endpoint(object):
    """
    Represents an endpoint for a particular service in a specific
    region.  Only an endpoint can make requests.

    :ivar service: The Service object that describes this endpoints
        service.
    :ivar host: The fully qualified endpoint hostname.
    :ivar session: The session object.
    """

    def __init__(self, service, region_name, host, auth, proxies=None):
        self.service = service
        self.session = self.service.session
        self.region_name = region_name
        self.host = host
        self.verify = True
        self.auth = auth
        if proxies is None:
            proxies = {}
        self.proxies = proxies
        self.http_session = Session()

    def __repr__(self):
        return '%s(%s)' % (self.service.endpoint_prefix, self.host)

    def make_request(self, params, list_marker=None):
        raise NotImplementedError("make_request")

    def prepare_request(self, request):
        logger.debug('prepare_request')
        self.auth.add_auth(request=request)
        prepared_request = request.prepare()
        return prepared_request

    def _send_request(self, request, operation):
        return self.http_session.send(request, verify=self.verify,
                                      stream=operation.is_streaming(),
                                      proxies=self.proxies)

class QueryEndpoint(Endpoint):
    """
    This class handles only AWS/Query style services.
    """

    def make_request(self, operation, params):
        """
        Send a request to the endpoint and parse the response
        and return it and long with the HTTP response object
        from requests.
        """
        logger.debug(params)
        logger.debug('SSL Verify: %s' % self.verify)
        params['Action'] = operation.name
        params['Version'] = self.service.api_version
        user_agent = self.session.user_agent()
        request = AWSRequest(method='POST', url=self.host,
                             data=params, headers={'User-Agent': user_agent})
        prepared_request = self.prepare_request(request)
        http_response = self._send_request(prepared_request, operation)
        return botocore.response.get_response(operation, http_response)

class JSONEndpoint(Endpoint):
    """
    This class handles only AWS/JSON style services.
    """

    ResponseContentTypes = ['application/x-amz-json-1.1',
                            'application/json']

    def make_request(self, operation, params):
        """
        Send a request to the endpoint and parse the response
        and return it and long with the HTTP response object
        from requests.
        """
        logger.debug(params)
        logger.debug('SSL Verify: %s' % self.verify)
        user_agent = self.session.user_agent()
        target = '%s.%s' % (self.service.target_prefix, operation.name)
        json_version = '1.0'
        if hasattr(self.service, 'json_version'):
            json_version = str(self.service.json_version)
        content_type = 'application/x-amz-json-%s' % json_version
        content_encoding = 'amz-1.0'
        data = json.dumps(params)
        request = AWSRequest(method='POST', url=self.host,
                             data=data,
                             headers={'User-Agent': user_agent,
                                      'X-Amz-Target': target,
                                      'Content-Type': content_type,
                                      'Content-Encoding': content_encoding})
        prepared_request = self.prepare_request(request)
        http_response = self._send_request(prepared_request, operation)
        return botocore.response.get_response(operation, http_response)

class RestEndpoint(Endpoint):

    def build_uri(self, operation, params):
        uri = operation.http['uri']
        if '?' in uri:
            path, query_params = uri.split('?')
        else:
            path = uri
            query_params = ''
        logger.debug('path: %s' % path)
        logger.debug('query_params: %s' % query_params)
        path_components = []
        for pc in path.split('/'):
            if pc:
                pc = pc.format(**params['uri_params'])
            path_components.append(pc)
        path = '/'.join(path_components)
        query_param_components = []
        for qpc in query_params.split('&'):
            if qpc:
                if '=' in qpc:
                    key_name, value_name = qpc.split('=')
                else:
                    key_name = qpc
                    value_name = None
                if value_name:
                    value_name = value_name.strip('{}')
                    if value_name in params['uri_params']:
                        value = params['uri_params'][value_name]
                        query_param_components.append('%s=%s' % (key_name,
                                                                 value))
                else:
                    query_param_components.append(key_name)
        query_params = '&'.join(query_param_components)
        logger.debug('path: %s' % path)
        logger.debug('query_params: %s' % query_params)
        return path + '?' + query_params

    def make_request(self, operation, params):
        """
        Send a request to the endpoint and parse the response
        and return it and long with the HTTP response object
        from requests.
        """
        logger.debug(params)
        logger.debug('SSL Verify: %s' % self.verify)
        user_agent = self.session.user_agent()
        params['headers']['User-Agent'] = user_agent
        uri = self.build_uri(operation, params)
        uri = urljoin(self.host, uri)
        if params['payload'] is None:
            request = AWSRequest(method=operation.http['method'],
                                 url=uri, headers=params['headers'])
        else:
            request = AWSRequest(method=operation.http['method'],
                                 url=uri, headers=params['headers'],
                                 data=params['payload'])
        prepared_request = self.prepare_request(request)
        http_response = self._send_request(prepared_request, operation)
        return botocore.response.get_response(operation, http_response)

def _get_proxies(url):
    # We could also support getting proxies from a config file,
    # but for now proxy support is taken from the environment.
    return get_environ_proxies(url)

def get_endpoint(service, region_name, endpoint_url):
    service_type = service.type
    cls = SERVICE_TO_ENDPOINT.get(service_type)
    if cls is None:
        raise botocore.exceptions.UnknownServiceStyle(
            service_style=service.type)
    service_name = getattr(service, 'signing_name', service.endpoint_prefix)
    auth = _get_auth(service.signature_version,
                     credentials=service.session.get_credentials(),
                     service_name=service_name,
                     region_name=region_name)
    proxies = _get_proxies(endpoint_url)
    return cls(service, region_name, endpoint_url, auth=auth, proxies=proxies)

def _get_auth(signature_version, credentials, service_name, region_name):
    cls = AUTH_TYPE_MAPS.get(signature_version)
    if cls is None:
        raise UnknownSignatureVersionError(signature_version=signature_version)
    else:
        return cls(credentials=credentials,
                   service_name=service_name,
                   region_name=region_name)

SERVICE_TO_ENDPOINT = {
    'query': QueryEndpoint,
    'json': JSONEndpoint,
    'rest-xml': RestEndpoint,
    'rest-json': RestEndpoint,
}
<|endoftext|>


<commit_before># Monkey-patch Quantity
import astropy.units as u

@classmethod
def _parse_quantity(cls, q):
    try:
        val,unit = q.split()
    except:
        val = q
        unit = u.dimensionless_unscaled

    return u.Quantity(float(val), unit)

u.Quantity.from_string = _parse_quantity

from . import coordinates
from . import dynamics
from . import integrate
from . import io
from . import potential

# Add a custom log level
from astropy import log as logger
import logging

def debug_factory(logger, debug_level):
    def custom_debug(msg, *args, **kwargs):
        if logger.level >= debug_level:
           return
        logger._log(debug_level, msg, args, kwargs)
    return custom_debug

logging.addLevelName(logging.INFO+5, 'IMPORTANT')
setattr(logger, 'important', debug_factory(logger, logging.INFO+5))
<commit_msg>default log level above info<commit_after># Monkey-patch Quantity
import astropy.units as u

@classmethod
def _parse_quantity(cls, q):
    try:
        val,unit = q.split()
    except:
        val = q
        unit = u.dimensionless_unscaled

    return u.Quantity(float(val), unit)

u.Quantity.from_string = _parse_quantity

from . import coordinates
from . import dynamics
from . import integrate
from . import io
from . import potential

# Add a custom log level
from astropy import log as logger
import logging

def debug_factory(logger, debug_level):
    def custom_debug(msg, *args, **kwargs):
        if logger.level >= debug_level:
           return
        logger._log(debug_level, msg, args, kwargs)
    return custom_debug

logging.addLevelName(logging.INFO+5, 'IMPORTANT')
setattr(logger, 'important', debug_factory(logger, logging.INFO+5))
logger.setLevel(logging.INFO+1)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from django.core.exceptions import PermissionDenied
from django.db.models import Count
from rest_framework import response, viewsets, permissions, status, decorators, mixins

from nodeconductor.core import serializers as core_serializers, filters as core_filters, permissions as core_permissions
from nodeconductor.core.views import BaseSummaryView
from nodeconductor.logging import elasticsearch_client, models, serializers, filters, log

class EventViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet):

    permission_classes = (permissions.IsAuthenticated, core_permissions.IsAdminOrReadOnly)
    filter_backends = (filters.EventFilterBackend,)
    serializer_class = serializers.EventSerializer

    def get_queryset(self):
        return elasticsearch_client.ElasticsearchResultList()

    def list(self, request, *args, **kwargs):
        self.queryset = self.filter_queryset(self.get_queryset())

        page = self.paginate_queryset(self.queryset)
        if page is not None:
            return self.get_paginated_response(page)
        return response.Response(self.queryset)

    def get(self, request, *args, **kwargs):
        return self.list(request, *args, **kwargs)

    def perform_create(self, serializer):
        scope = serializer.validated_data.get('scope')
        context = {'scope': scope} if scope is not None else {}

        log.event_logger.custom.process(
            level=serializer.validated_data.get('level'),
            message_template=serializer.validated_data.get('message'),
            event_type='custom_notification',
            event_context=context
        )

    @decorators.list_route()
    def count(self, request, *args, **kwargs):
        """ Returns a count of events.
            Response example: ``{"count": 12321}``
        """

        self.queryset = self.filter_queryset(self.get_queryset())
        return response.Response({'count': self.queryset.count()}, status=status.HTTP_200_OK)

    @decorators.list_route()
    def count_history(self, request, *args, **kwargs):
        queryset = self.filter_queryset(self.get_queryset())

        mapped = {
            'start': request.query_params.get('start'),
            'end': request.query_params.get('end'),
            'points_count': request.query_params.get('points_count'),
            'point_list': request.query_params.getlist('point'),
        }
        serializer = core_serializers.HistorySerializer(data={k: v for k, v in mapped.items() if v})
        serializer.is_valid(raise_exception=True)

        timestamp_ranges = [{'end': point_date} for point_date in serializer.get_filter_data()]
        aggregated_count = queryset.aggregated_count(timestamp_ranges)

        return response.Response(
            [{'point': int(ac['end']), 'object': {'count': ac['count']}} for ac in aggregated_count],
            status=status.HTTP_200_OK)

class AlertViewSet(mixins.CreateModelMixin,
                   viewsets.ReadOnlyModelViewSet):

    queryset = models.Alert.objects.all()
    serializer_class = serializers.AlertSerializer
    lookup_field = 'uuid'
    permission_classes = (permissions.IsAuthenticated,)
    filter_backends = (
        core_filters.DjangoMappingFilterBackend,
        filters.AdditionalAlertFilterBackend,
        filters.ExternalAlertFilterBackend,
        filters.AlertScopeFilterBackend,
    )
    filter_class = filters.AlertFilter

    def get_queryset(self):
        return models.Alert.objects.filtered_for_user(self.request.user).order_by('-created')

    @decorators.detail_route(methods=['post'])
    def close(self, request, *args, **kwargs):
        if not request.user.is_staff:
            raise PermissionDenied()
        alert = self.get_object()
        alert.close()

        return response.Response(status=status.HTTP_204_NO_CONTENT)

    @decorators.detail_route(methods=['post'])
    def acknowledge(self, request, *args, **kwargs):
        alert = self.get_object()
        if not alert.acknowledged:
            alert.acknowledge()
            return response.Response(status=status.HTTP_200_OK)
        else:
            return response.Response({'detail': 'Alert is already acknowledged'}, status=status.HTTP_409_CONFLICT)

        return response.Response(status=status.HTTP_200_OK)

    @decorators.detail_route(methods=['post'])
    def cancel_acknowledgment(self, request, *args, **kwargs):
        alert = self.get_object()
        if alert.acknowledged:
            alert.cancel_acknowledgment()
            return response.Response(status=status.HTTP_200_OK)
        else:
            return response.Response({'detail': 'Alert is not acknowledged'}, status=status.HTTP_409_CONFLICT)

    @decorators.list_route()
    def stats(self, request, *args, **kwargs):
        queryset = self.filter_queryset(self.get_queryset())
        alerts_severities_count = queryset.values('severity').annotate(count=Count('severity'))

        severity_names = dict(models.Alert.SeverityChoices.CHOICES)
        # For consistency with all other endpoint we need to return severity names in lower case.
        alerts_severities_count = {
            severity_names[asc['severity']].lower(): asc['count'] for asc in alerts_severities_count}
        for severity_name in severity_names.values():
            if severity_name.lower() not in alerts_severities_count:
                alerts_severities_count[severity_name.lower()] = 0

        return response.Response(alerts_severities_count, status=status.HTTP_200_OK)

class BaseHookViewSet(viewsets.ModelViewSet):
    permission_classes = (permissions.IsAuthenticated,)
    filter_backends = (core_filters.StaffOrUserFilter,)
    lookup_field = 'uuid'

class WebHookViewSet(BaseHookViewSet):
    queryset = models.WebHook.objects.all()
    serializer_class = serializers.WebHookSerializer

class EmailHookViewSet(BaseHookViewSet):
    queryset = models.EmailHook.objects.all()
    serializer_class = serializers.EmailHookSerializer

class HookSummary(BaseSummaryView):
    def get_urls(self, request):
        return ('webhook-list', 'emailhook-list')
<commit_msg>Fix alerts viewset permissions<commit_after>from __future__ import unicode_literals

from django.core.exceptions import PermissionDenied
from django.db.models import Count
from rest_framework import response, viewsets, permissions, status, decorators, mixins

from nodeconductor.core import serializers as core_serializers, filters as core_filters, permissions as core_permissions
from nodeconductor.core.views import BaseSummaryView
from nodeconductor.logging import elasticsearch_client, models, serializers, filters, log

class EventViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet):

    permission_classes = (permissions.IsAuthenticated, core_permissions.IsAdminOrReadOnly)
    filter_backends = (filters.EventFilterBackend,)
    serializer_class = serializers.EventSerializer

    def get_queryset(self):
        return elasticsearch_client.ElasticsearchResultList()

    def list(self, request, *args, **kwargs):
        self.queryset = self.filter_queryset(self.get_queryset())

        page = self.paginate_queryset(self.queryset)
        if page is not None:
            return self.get_paginated_response(page)
        return response.Response(self.queryset)

    def get(self, request, *args, **kwargs):
        return self.list(request, *args, **kwargs)

    def perform_create(self, serializer):
        scope = serializer.validated_data.get('scope')
        context = {'scope': scope} if scope is not None else {}

        log.event_logger.custom.process(
            level=serializer.validated_data.get('level'),
            message_template=serializer.validated_data.get('message'),
            event_type='custom_notification',
            event_context=context
        )

    @decorators.list_route()
    def count(self, request, *args, **kwargs):
        """ Returns a count of events.
            Response example: ``{"count": 12321}``
        """

        self.queryset = self.filter_queryset(self.get_queryset())
        return response.Response({'count': self.queryset.count()}, status=status.HTTP_200_OK)

    @decorators.list_route()
    def count_history(self, request, *args, **kwargs):
        queryset = self.filter_queryset(self.get_queryset())

        mapped = {
            'start': request.query_params.get('start'),
            'end': request.query_params.get('end'),
            'points_count': request.query_params.get('points_count'),
            'point_list': request.query_params.getlist('point'),
        }
        serializer = core_serializers.HistorySerializer(data={k: v for k, v in mapped.items() if v})
        serializer.is_valid(raise_exception=True)

        timestamp_ranges = [{'end': point_date} for point_date in serializer.get_filter_data()]
        aggregated_count = queryset.aggregated_count(timestamp_ranges)

        return response.Response(
            [{'point': int(ac['end']), 'object': {'count': ac['count']}} for ac in aggregated_count],
            status=status.HTTP_200_OK)

class AlertViewSet(mixins.CreateModelMixin,
                   viewsets.ReadOnlyModelViewSet):

    queryset = models.Alert.objects.all()
    serializer_class = serializers.AlertSerializer
    lookup_field = 'uuid'
    permission_classes = (permissions.IsAuthenticated, permissions.DjangoObjectPermissions)
    filter_backends = (
        core_filters.DjangoMappingFilterBackend,
        filters.AdditionalAlertFilterBackend,
        filters.ExternalAlertFilterBackend,
        filters.AlertScopeFilterBackend,
    )
    filter_class = filters.AlertFilter

    def get_queryset(self):
        return models.Alert.objects.filtered_for_user(self.request.user).order_by('-created')

    @decorators.detail_route(methods=['post'])
    def close(self, request, *args, **kwargs):
        if not request.user.is_staff:
            raise PermissionDenied()
        alert = self.get_object()
        alert.close()

        return response.Response(status=status.HTTP_204_NO_CONTENT)

    @decorators.detail_route(methods=['post'])
    def acknowledge(self, request, *args, **kwargs):
        alert = self.get_object()
        if not alert.acknowledged:
            alert.acknowledge()
            return response.Response(status=status.HTTP_200_OK)
        else:
            return response.Response({'detail': 'Alert is already acknowledged'}, status=status.HTTP_409_CONFLICT)

        return response.Response(status=status.HTTP_200_OK)

    @decorators.detail_route(methods=['post'])
    def cancel_acknowledgment(self, request, *args, **kwargs):
        alert = self.get_object()
        if alert.acknowledged:
            alert.cancel_acknowledgment()
            return response.Response(status=status.HTTP_200_OK)
        else:
            return response.Response({'detail': 'Alert is not acknowledged'}, status=status.HTTP_409_CONFLICT)

    @decorators.list_route()
    def stats(self, request, *args, **kwargs):
        queryset = self.filter_queryset(self.get_queryset())
        alerts_severities_count = queryset.values('severity').annotate(count=Count('severity'))

        severity_names = dict(models.Alert.SeverityChoices.CHOICES)
        # For consistency with all other endpoint we need to return severity names in lower case.
        alerts_severities_count = {
            severity_names[asc['severity']].lower(): asc['count'] for asc in alerts_severities_count}
        for severity_name in severity_names.values():
            if severity_name.lower() not in alerts_severities_count:
                alerts_severities_count[severity_name.lower()] = 0

        return response.Response(alerts_severities_count, status=status.HTTP_200_OK)

class BaseHookViewSet(viewsets.ModelViewSet):
    permission_classes = (permissions.IsAuthenticated,)
    filter_backends = (core_filters.StaffOrUserFilter,)
    lookup_field = 'uuid'

class WebHookViewSet(BaseHookViewSet):
    queryset = models.WebHook.objects.all()
    serializer_class = serializers.WebHookSerializer

class EmailHookViewSet(BaseHookViewSet):
    queryset = models.EmailHook.objects.all()
    serializer_class = serializers.EmailHookSerializer

class HookSummary(BaseSummaryView):
    def get_urls(self, request):
        return ('webhook-list', 'emailhook-list')
<|endoftext|>


<commit_before>
<commit_msg>Enable the faulthandler module, if available, when running tests<commit_after>
try:
    import faulthandler
except ImportError:
    pass
else:
    faulthandler.enable()
<|endoftext|>