<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import, unicode_literals

import operator
from itertools import chain

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.utils import six

from haystack.utils.geo import D, Point
from rest_framework.filters import BaseFilterBackend

class HaystackFilter(BaseFilterBackend):
    """
    A filter backend that compiles a haystack compatible
    filtering query.
    """

    @staticmethod
    def build_filter(view, filters=None):
        """
        Creates a single SQ filter from querystring parameters that
        correspond to the SearchIndex fields that have been "registered"
        in `view.fields`.

        Default behavior is to `OR` terms for the same parameters, and `AND`
        between parameters.

        Any querystring parameters that are not registered in
        `view.fields` will be ignored.
        """

        terms = []

        if filters is None:
            filters = {}  # pragma: no cover

        for param, value in filters.items():
            # Skip if the parameter is not listed in the serializer's `fields`
            # or if it's in the `exclude` list.
            if view.serializer_class:
                try:
                    if hasattr(view.serializer_class.Meta, "field_aliases"):
                        param = view.serializer_class.Meta.field_aliases.get(param, param)

                    fields = getattr(view.serializer_class.Meta, "fields", [])
                    exclude = getattr(view.serializer_class.Meta, "exclude", [])

                    if param not in fields or param in exclude or not value:
                        continue

                except AttributeError:
                    raise ImproperlyConfigured("%s must implement a Meta class." %
                                               view.serializer_class.__class__.__name__)

            tokens = value.split(view.lookup_sep)
            field_queries = []

            for token in tokens:
                if token:
                    field_queries.append(view.query_object((param, token)))

            terms.append(six.moves.reduce(operator.or_, filter(lambda x: x, field_queries)))

        return six.moves.reduce(operator.and_, filter(lambda x: x, terms)) if terms else []

    def filter_queryset(self, request, queryset, view):
        applicable_filters = self.build_filter(view, filters=request.GET.copy())
        if applicable_filters:
            queryset = queryset.filter(applicable_filters)
        return queryset

class HaystackAutocompleteFilter(HaystackFilter):
    """
    A filter backend to perform autocomplete search.

    Must be run against fields that are either `NgramField` or
    `EdgeNgramField`.
    """

    def filter_queryset(self, request, queryset, view):
        """
        Applying `applicable_filters` to the queryset by creating a
        single SQ filter using `AND`.
        """

        applicable_filters = self.build_filter(view, filters=request.GET.copy())

        if applicable_filters:
            query_bits = []
            for field_name, query in applicable_filters.children:
                for word in query.split(" "):
                    bit = queryset.query.clean(word.strip())
                    kwargs = {
                        field_name: bit
                    }
                    query_bits.append(view.query_object(**kwargs))
            queryset = queryset.filter(six.moves.reduce(operator.and_, filter(lambda x: x, query_bits)))

        return queryset

class HaystackGEOSpatialFilter(HaystackFilter):
    """
    A filter backend for doing geospatial filtering.
    If using this filter make sure your index has a `LocationField`
    named `coordinates`.

    We'll always do the somewhat slower but more accurate `dwithin`
    (radius) filter.
    """

    def unit_to_meters(self, distance_obj):
        """
        Emergency fix for https://github.com/toastdriven/django-haystack/issues/957
        According to Elasticsearch documentation, units are always measured in meters unless
        explicitly declared otherwise. It seems that the unit description is lost somewhere,
        so everything ends up in the query without any unit values, thus the value is calculated
        in meters.
        """
        return D(m=distance_obj.m * 1000)

    def geo_filter(self, queryset, filters=None):
        """
        Filter the queryset by looking up parameters from the query
        parameters.

        Expected query parameters are:
        - a `unit=value` parameter where the unit is a valid UNIT in the
          `django.contrib.gis.measure.Distance` class.
        - `from` which must be a comma separated longitude and latitude.

        Example query:
            /api/v1/search/?km=10&from=59.744076,10.152045

            Will perform a `dwithin` query within 10 km from the point
            with latitude 59.744076 and longitude 10.152045.
        """

        filters = dict((k, filters[k]) for k in chain(D.UNITS.keys(), ["from"]) if k in filters)
        distance = dict((k, v) for k, v in filters.items() if k in D.UNITS.keys())
        if "from" in filters and len(filters["from"].split(",")) == 2:
            try:
                latitude, longitude = map(float, filters["from"].split(","))
                point = Point(longitude, latitude, srid=getattr(settings, "GEO_SRID", 4326))
                if point and distance:
                    if queryset.query.backend.__class__.__name__ == "ElasticsearchSearchBackend":
                        # TODO: Make sure this is only applied if using a malfunction elasticsearch backend!
                        # NOTE: https://github.com/toastdriven/django-haystack/issues/957
                        # FIXME: Remove when upstream haystack bug is resolved
                        distance = self.unit_to_meters(D(**distance))
                    else:
                        distance = D(**distance)
                    queryset = queryset.dwithin("coordinates", point, distance).distance("coordinates", point)
            except ValueError:
                raise ValueError("Cannot convert `from=latitude,longitude` query parameter to "
                                 "float values. Make sure to provide numerical values only!")

        return queryset

    def filter_queryset(self, request, queryset, view):
        queryset = self.geo_filter(queryset, filters=request.GET.copy())
        return super(HaystackGEOSpatialFilter, self).filter_queryset(request, queryset, view)
<commit_msg>import D and Point in the GEO Spatial filter's __init__ in order to prevent drf-haystack to crash if not installed<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import, unicode_literals

import operator
import warnings
from itertools import chain

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.utils import six

from rest_framework.filters import BaseFilterBackend

class HaystackFilter(BaseFilterBackend):
    """
    A filter backend that compiles a haystack compatible
    filtering query.
    """

    @staticmethod
    def build_filter(view, filters=None):
        """
        Creates a single SQ filter from querystring parameters that
        correspond to the SearchIndex fields that have been "registered"
        in `view.fields`.

        Default behavior is to `OR` terms for the same parameters, and `AND`
        between parameters.

        Any querystring parameters that are not registered in
        `view.fields` will be ignored.
        """

        terms = []

        if filters is None:
            filters = {}  # pragma: no cover

        for param, value in filters.items():
            # Skip if the parameter is not listed in the serializer's `fields`
            # or if it's in the `exclude` list.
            if view.serializer_class:
                try:
                    if hasattr(view.serializer_class.Meta, "field_aliases"):
                        param = view.serializer_class.Meta.field_aliases.get(param, param)

                    fields = getattr(view.serializer_class.Meta, "fields", [])
                    exclude = getattr(view.serializer_class.Meta, "exclude", [])

                    if param not in fields or param in exclude or not value:
                        continue

                except AttributeError:
                    raise ImproperlyConfigured("%s must implement a Meta class." %
                                               view.serializer_class.__class__.__name__)

            tokens = value.split(view.lookup_sep)
            field_queries = []

            for token in tokens:
                if token:
                    field_queries.append(view.query_object((param, token)))

            terms.append(six.moves.reduce(operator.or_, filter(lambda x: x, field_queries)))

        return six.moves.reduce(operator.and_, filter(lambda x: x, terms)) if terms else []

    def filter_queryset(self, request, queryset, view):
        applicable_filters = self.build_filter(view, filters=request.GET.copy())
        if applicable_filters:
            queryset = queryset.filter(applicable_filters)
        return queryset

class HaystackAutocompleteFilter(HaystackFilter):
    """
    A filter backend to perform autocomplete search.

    Must be run against fields that are either `NgramField` or
    `EdgeNgramField`.
    """

    def filter_queryset(self, request, queryset, view):
        """
        Applying `applicable_filters` to the queryset by creating a
        single SQ filter using `AND`.
        """

        applicable_filters = self.build_filter(view, filters=request.GET.copy())

        if applicable_filters:
            query_bits = []
            for field_name, query in applicable_filters.children:
                for word in query.split(" "):
                    bit = queryset.query.clean(word.strip())
                    kwargs = {
                        field_name: bit
                    }
                    query_bits.append(view.query_object(**kwargs))
            queryset = queryset.filter(six.moves.reduce(operator.and_, filter(lambda x: x, query_bits)))

        return queryset

class HaystackGEOSpatialFilter(HaystackFilter):
    """
    A filter backend for doing geospatial filtering.
    If using this filter make sure your index has a `LocationField`
    named `coordinates`.

    We'll always do the somewhat slower but more accurate `dwithin`
    (radius) filter.
    """

    def __init__(self, *args, **kwargs):
        try:
            from haystack.utils.geo import D, Point
            self.D = D
            self.Point = Point
        except ImportError as e:
            warnings.warn("Make sure you've installed the `libgeos` library.\n "
                          "(`apt-get install libgeos` on linux, or `brew install geos` on OS X.)")
            raise e

    def unit_to_meters(self, distance_obj):
        """
        Emergency fix for https://github.com/toastdriven/django-haystack/issues/957
        According to Elasticsearch documentation, units are always measured in meters unless
        explicitly declared otherwise. It seems that the unit description is lost somewhere,
        so everything ends up in the query without any unit values, thus the value is calculated
        in meters.
        """
        return self.D(m=distance_obj.m * 1000)

    def geo_filter(self, queryset, filters=None):
        """
        Filter the queryset by looking up parameters from the query
        parameters.

        Expected query parameters are:
        - a `unit=value` parameter where the unit is a valid UNIT in the
          `django.contrib.gis.measure.Distance` class.
        - `from` which must be a comma separated longitude and latitude.

        Example query:
            /api/v1/search/?km=10&from=59.744076,10.152045

            Will perform a `dwithin` query within 10 km from the point
            with latitude 59.744076 and longitude 10.152045.
        """

        filters = dict((k, filters[k]) for k in chain(self.D.UNITS.keys(), ["from"]) if k in filters)
        distance = dict((k, v) for k, v in filters.items() if k in self.D.UNITS.keys())
        if "from" in filters and len(filters["from"].split(",")) == 2:
            try:
                latitude, longitude = map(float, filters["from"].split(","))
                point = self.Point(longitude, latitude, srid=getattr(settings, "GEO_SRID", 4326))
                if point and distance:
                    if queryset.query.backend.__class__.__name__ == "ElasticsearchSearchBackend":
                        # TODO: Make sure this is only applied if using a malfunction elasticsearch backend!
                        # NOTE: https://github.com/toastdriven/django-haystack/issues/957
                        # FIXME: Remove when upstream haystack bug is resolved
                        distance = self.unit_to_meters(self.D(**distance))
                    else:
                        distance = self.D(**distance)
                    queryset = queryset.dwithin("coordinates", point, distance).distance("coordinates", point)
            except ValueError:
                raise ValueError("Cannot convert `from=latitude,longitude` query parameter to "
                                 "float values. Make sure to provide numerical values only!")

        return queryset

    def filter_queryset(self, request, queryset, view):
        queryset = self.geo_filter(queryset, filters=request.GET.copy())
        return super(HaystackGEOSpatialFilter, self).filter_queryset(request, queryset, view)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Public section, including homepage and signup."""
from flask import (Blueprint, request, render_template, flash, url_for,
                    redirect, session, current_app, jsonify)
from flask_login import login_required, current_user

from dribdat.user.models import User, Event, Project, Resource
from dribdat.public.forms import (
    LoginForm, UserForm,
    ProjectNew, ProjectForm,
    ProjectPost, ResourceForm,
)
from dribdat.database import db
from dribdat.extensions import cache
from dribdat.aggregation import (
    SyncProjectData, GetProjectData,
    ProjectActivity, IsProjectStarred,
    GetEventUsers, SuggestionsByProgress,
)
from dribdat.user import projectProgressList, isUserActive

from datetime import datetime

blueprint = Blueprint('public', __name__, static_folder="../static")

def current_event(): return Event.query.filter_by(is_current=True).first()

# Renders a static dashboard
@blueprint.route("/dashboard/")
def dashboard():
    event = current_event()
    if not event: return 'No current event'
    wall = 'twitter.com' in event.community_url
    return render_template("public/dashboard.html", current_event=event, with_social_wall=wall)

# Outputs JSON-LD about the current event (see also api.py/info_event_hackathon_json)
@blueprint.route('/hackathon.json')
def info_current_hackathon_json():
    event = Event.query.filter_by(is_current=True).first() or Event.query.order_by(Event.id.desc()).first()
    return jsonify(event.get_schema(request.host_url))

# Renders a simple about page
@blueprint.route("/about/")
def about():
    return render_template("public/about.html", current_event=current_event())

# Favicon just points to a file
@blueprint.route("/favicon.ico")
def favicon():
    return redirect(url_for('static', filename='img/favicon.ico'))

# Home page
@blueprint.route("/")
def home():
    cur_event = current_event()
    if cur_event is not None:
        events = Event.query.filter(Event.id != cur_event.id)
    else:
        events = Event.query
    events = events.filter(Event.is_hidden.isnot(True))
    events = events.order_by(Event.starts_at.desc())
    today = datetime.utcnow()
    events_next = events.filter(Event.starts_at > today).all()
    events_past = events.filter(Event.ends_at < today).all()
    return render_template("public/home.html",
        events_next=events_next, events_past=events_past, current_event=cur_event)

@blueprint.route('/user/<username>', methods=['GET'])
def user(username):
    user = User.query.filter_by(username=username).first_or_404()
    if not isUserActive(user):
        # return "User deactivated. Please contact an event organizer."
        flash('This user account is under review. Please contact the organizing team if you have any questions.', 'warning')
    event = current_event()
    cert_path = user.get_cert_path(event)
    # projects = user.projects
    projects = user.joined_projects()
    posts = user.latest_posts()
    submissions = Resource.query.filter_by(user_id=user.id).order_by(Resource.id.desc()).all()
    return render_template("public/userprofile.html", active="profile",
        current_event=event, event=event, user=user, cert_path=cert_path,
        projects=projects, submissions=submissions, posts=posts)

@blueprint.route("/event/<int:event_id>")
def event(event_id):
    event = Event.query.filter_by(id=event_id).first_or_404()
    projects = Project.query.filter_by(event_id=event_id, is_hidden=False)
    if request.args.get('embed'):
        return render_template("public/embed.html", current_event=event, projects=projects)
    summaries = [ p.data for p in projects ]
    # Sort projects by reverse score, then name
    summaries.sort(key=lambda x: (
        -x['score'] if isinstance(x['score'], int) else 0,
        x['name'].lower()))
    project_count = projects.count()
    return render_template("public/event.html", current_event=event,
        projects=summaries, project_count=project_count, active="projects")

@blueprint.route("/event/<int:event_id>/participants")
def event_participants(event_id):
    event = Event.query.filter_by(id=event_id).first_or_404()
    users = GetEventUsers(event)
    usercount = len(users) if users else 0
    return render_template("public/eventusers.html",
        current_event=event, participants=users, usercount=usercount, active="participants")

@blueprint.route("/event/<int:event_id>/resources")
def event_resources(event_id):
    event = Event.query.filter_by(id=event_id).first_or_404()
    steps = []
    for ix, p in enumerate(projectProgressList(True, False)):
        steps.append({
            'index': ix + 1, 'name': p[1],
            'resources': SuggestionsByProgress(p[0])
        })
    steps.append({
        'name': '/etc', 'index': -1,
        'resources': SuggestionsByProgress(None)
    })
    return render_template("public/resources.html",
        current_event=event, steps=steps, active="resources")

@blueprint.route('/project/<int:project_id>')
def project(project_id):
    return project_action(project_id, None)

@blueprint.route('/project/<int:project_id>/edit', methods=['GET', 'POST'])
@login_required
def project_edit(project_id):
    project = Project.query.filter_by(id=project_id).first_or_404()
    event = project.event
    starred = IsProjectStarred(project, current_user)
    allow_edit = starred or (isUserActive(current_user) and current_user.is_admin)
    if not allow_edit:
        flash('You do not have access to edit this project.', 'warning')
        return project_action(project_id, None)
    form = ProjectForm(obj=project, next=request.args.get('next'))
    form.category_id.choices = [(c.id, c.name) for c in project.categories_all()]
    form.category_id.choices.insert(0, (-1, ''))
    if form.validate_on_submit():
        del form.id
        form.populate_obj(project)
        project.update()
        db.session.add(project)
        db.session.commit()
        cache.clear()
        flash('Project updated.', 'success')
        project_action(project_id, 'update', False)
        return redirect(url_for('public.project', project_id=project.id))
    return render_template('public/projectedit.html',
        current_event=event, project=project, form=form)

@blueprint.route('/project/<int:project_id>/post', methods=['GET', 'POST'])
@login_required
def project_post(project_id):
    project = Project.query.filter_by(id=project_id).first_or_404()
    event = project.event
    starred = IsProjectStarred(project, current_user)
    allow_edit = starred or (not current_user.is_anonymous and current_user.is_admin)
    if not allow_edit:
        flash('You do not have access to edit this project.', 'warning')
        return project_action(project_id, None)
    form = ProjectPost(obj=project, next=request.args.get('next'))
    # Populate progress dialog
    form.progress.choices = projectProgressList(event.has_started or event.has_finished)
    # Populate resource list
    resources = Resource.query.filter_by(is_visible=True).order_by(Resource.type_id).all()
    resource_list = [(0, '')]
    resource_list.extend([(r.id, r.of_type + ': ' + r.name) for r in resources])
    form.resource.choices = resource_list
    # Process form
    if form.validate_on_submit():
        del form.id
        if form.resource.data == 0:
            form.resource.data = None
        form.populate_obj(project)
        project.update()
        db.session.add(project)
        db.session.commit()
        cache.clear()
        flash('Thanks for your commit!', 'success')
        project_action(project_id, 'update', action='post', text=form.note.data, resource=form.resource.data)
        return redirect(url_for('public.project', project_id=project.id))
    return render_template('public/projectpost.html', current_event=event, project=project, form=form)

def project_action(project_id, of_type=None, as_view=True, then_redirect=False, action=None, text=None, resource=None):
    project = Project.query.filter_by(id=project_id).first_or_404()
    event = project.event
    if of_type is not None:
        ProjectActivity(project, of_type, current_user, action, text, resource)
    if not as_view:
        return True
    if then_redirect:
        return redirect(url_for('public.project', project_id=project.id))
    starred = IsProjectStarred(project, current_user)
    allow_edit = starred or (not current_user.is_anonymous and current_user.is_admin)
    allow_edit = allow_edit and not event.lock_editing
    project_team = project.team()
    latest_activity = project.latest_activity()
    project_signals = project.all_signals()
    suggestions = SuggestionsByProgress(project.progress)
    return render_template('public/project.html', current_event=event, project=project,
        project_starred=starred, project_team=project_team, project_signals=project_signals, suggestions=suggestions,
        allow_edit=allow_edit, latest_activity=latest_activity)

@blueprint.route('/project/<int:project_id>/star', methods=['GET', 'POST'])
@login_required
def project_star(project_id):
    if not isUserActive(current_user):
        return "User not allowed. Please contact event organizers."
    flash('Welcome to the team!', 'success')
    return project_action(project_id, 'star', then_redirect=True)

@blueprint.route('/project/<int:project_id>/unstar', methods=['GET', 'POST'])
@login_required
def project_unstar(project_id):
    flash('You have left the project', 'success')
    return project_action(project_id, 'unstar', then_redirect=True)

@blueprint.route('/event/<int:event_id>/project/new', methods=['GET', 'POST'])
def project_new(event_id):
    if not isUserActive(current_user):
        flash("User not activated. Please contact event organizers.", 'warning')
        return redirect(url_for('public.event', event_id=event_id))
    form = None
    event = Event.query.filter_by(id=event_id).first_or_404()
    if event.lock_starting:
        flash('Starting a new project is disabled for this event.', 'error')
        return redirect(url_for('public.event', event_id=event.id))
    if isUserActive(current_user):
        project = Project()
        project.user_id = current_user.id
        form = ProjectNew(obj=project, next=request.args.get('next'))
        form.category_id.choices = [(c.id, c.name) for c in project.categories_all(event)]
        form.category_id.choices.insert(0, (-1, ''))
        if form.validate_on_submit():
            del form.id
            form.populate_obj(project)
            project.event = event
            if event.has_started:
                project.progress = 0
            else:
                project.progress = -1 # Start as challenge
            project.update()
            db.session.add(project)
            db.session.commit()
            flash('New challenge added.', 'success')
            project_action(project.id, 'create', False)
            cache.clear()
            if event.has_started:
                project_action(project.id, 'star', False) # Join team
            if len(project.autotext_url)>1:
                return project_autoupdate(project.id)
            else:
                return redirect(url_for('public.project', project_id=project.id))
    return render_template('public/projectnew.html', active="projectnew", current_event=event, form=form)

@blueprint.route('/project/<int:project_id>/autoupdate')
@login_required
def project_autoupdate(project_id):
    project = Project.query.filter_by(id=project_id).first_or_404()
    starred = IsProjectStarred(project, current_user)
    allow_edit = starred or (not current_user.is_anonymous and current_user.is_admin)
    if not allow_edit or project.is_hidden or not project.is_autoupdate:
        flash('You may not sync this project.', 'warning')
        return project_action(project_id)
    data = GetProjectData(project.autotext_url)
    if not 'name' in data:
        flash("Could not sync: check that the Remote Link contains a README.", 'warning')
        return project_action(project_id)
    SyncProjectData(project, data)
    project_action(project.id, 'update', action='sync', text=str(len(project.autotext)) + ' bytes')
    flash("Project data synced from %s" % data['type'], 'success')
    return redirect(url_for('public.project', project_id=project.id))

@blueprint.route('/resource/<int:resource_id>', methods=['GET'])
def resource(resource_id):
    resource = Resource.query.filter_by(id=resource_id).first_or_404()
    projects = [ c.project for c in resource.get_comments() ]
    event = current_event()
    return render_template('public/resource.html',
        current_event=event, resource=resource, projects=projects)

@blueprint.route('/resource/post', methods=['GET', 'POST'])
@login_required
def resource_post():
    if not isUserActive(current_user):
        return "User not allowed. Please contact event organizers."
    resource = Resource()
    event = current_event()
    form = ResourceForm(obj=resource)
    if form.validate_on_submit():
        form.populate_obj(resource)
        resource.user_id = current_user.id
        resource.is_visible = not current_app.config['DRIBDAT_TOOL_APPROVE']
        db.session.add(resource)
        db.session.commit()
        flash('Thanks for the tip! Your suggestions are visible in your profile.', 'success')
        return redirect(url_for('public.resource', resource_id=resource.id))
    return render_template('public/resourcenew.html',
                current_event=event, form=form)

@blueprint.route('/resource/<int:resource_id>/edit', methods=['GET', 'POST'])
@login_required
def resource_edit(resource_id):
    resource = Resource.query.filter_by(id=resource_id).first_or_404()
    event = current_event()
    allow_edit = isUserActive(current_user) and current_user == resource.user or current_user.is_admin
    if not allow_edit:
        flash('You do not have access to edit this resource.', 'warning')
        return redirect(url_for('public.home'))
    form = ResourceForm(obj=resource)
    if form.validate_on_submit():
        form.populate_obj(resource)
        db.session.add(resource)
        db.session.commit()
        flash('Changes saved.', 'success')
        return redirect(url_for('public.resource', resource_id=resource.id))
    return render_template('public/resourceedit.html',
                current_event=event, resource=resource, form=form)
<commit_msg>Log in message new project<commit_after># -*- coding: utf-8 -*-
"""Public section, including homepage and signup."""
from flask import (Blueprint, request, render_template, flash, url_for,
                    redirect, session, current_app, jsonify)
from flask_login import login_required, current_user

from dribdat.user.models import User, Event, Project, Resource
from dribdat.public.forms import (
    LoginForm, UserForm,
    ProjectNew, ProjectForm,
    ProjectPost, ResourceForm,
)
from dribdat.database import db
from dribdat.extensions import cache
from dribdat.aggregation import (
    SyncProjectData, GetProjectData,
    ProjectActivity, IsProjectStarred,
    GetEventUsers, SuggestionsByProgress,
)
from dribdat.user import projectProgressList, isUserActive

from datetime import datetime

blueprint = Blueprint('public', __name__, static_folder="../static")

def current_event(): return Event.query.filter_by(is_current=True).first()

# Renders a static dashboard
@blueprint.route("/dashboard/")
def dashboard():
    event = current_event()
    if not event: return 'No current event'
    wall = 'twitter.com' in event.community_url
    return render_template("public/dashboard.html", current_event=event, with_social_wall=wall)

# Outputs JSON-LD about the current event (see also api.py/info_event_hackathon_json)
@blueprint.route('/hackathon.json')
def info_current_hackathon_json():
    event = Event.query.filter_by(is_current=True).first() or Event.query.order_by(Event.id.desc()).first()
    return jsonify(event.get_schema(request.host_url))

# Renders a simple about page
@blueprint.route("/about/")
def about():
    return render_template("public/about.html", current_event=current_event())

# Favicon just points to a file
@blueprint.route("/favicon.ico")
def favicon():
    return redirect(url_for('static', filename='img/favicon.ico'))

# Home page
@blueprint.route("/")
def home():
    cur_event = current_event()
    if cur_event is not None:
        events = Event.query.filter(Event.id != cur_event.id)
    else:
        events = Event.query
    events = events.filter(Event.is_hidden.isnot(True))
    events = events.order_by(Event.starts_at.desc())
    today = datetime.utcnow()
    events_next = events.filter(Event.starts_at > today).all()
    events_past = events.filter(Event.ends_at < today).all()
    return render_template("public/home.html",
        events_next=events_next, events_past=events_past, current_event=cur_event)

@blueprint.route('/user/<username>', methods=['GET'])
def user(username):
    user = User.query.filter_by(username=username).first_or_404()
    if not isUserActive(user):
        # return "User deactivated. Please contact an event organizer."
        flash('This user account is under review. Please contact the organizing team if you have any questions.', 'warning')
    event = current_event()
    cert_path = user.get_cert_path(event)
    # projects = user.projects
    projects = user.joined_projects()
    posts = user.latest_posts()
    submissions = Resource.query.filter_by(user_id=user.id).order_by(Resource.id.desc()).all()
    return render_template("public/userprofile.html", active="profile",
        current_event=event, event=event, user=user, cert_path=cert_path,
        projects=projects, submissions=submissions, posts=posts)

@blueprint.route("/event/<int:event_id>")
def event(event_id):
    event = Event.query.filter_by(id=event_id).first_or_404()
    projects = Project.query.filter_by(event_id=event_id, is_hidden=False)
    if request.args.get('embed'):
        return render_template("public/embed.html", current_event=event, projects=projects)
    summaries = [ p.data for p in projects ]
    # Sort projects by reverse score, then name
    summaries.sort(key=lambda x: (
        -x['score'] if isinstance(x['score'], int) else 0,
        x['name'].lower()))
    project_count = projects.count()
    return render_template("public/event.html", current_event=event,
        projects=summaries, project_count=project_count, active="projects")

@blueprint.route("/event/<int:event_id>/participants")
def event_participants(event_id):
    event = Event.query.filter_by(id=event_id).first_or_404()
    users = GetEventUsers(event)
    usercount = len(users) if users else 0
    return render_template("public/eventusers.html",
        current_event=event, participants=users, usercount=usercount, active="participants")

@blueprint.route("/event/<int:event_id>/resources")
def event_resources(event_id):
    event = Event.query.filter_by(id=event_id).first_or_404()
    steps = []
    for ix, p in enumerate(projectProgressList(True, False)):
        steps.append({
            'index': ix + 1, 'name': p[1],
            'resources': SuggestionsByProgress(p[0])
        })
    steps.append({
        'name': '/etc', 'index': -1,
        'resources': SuggestionsByProgress(None)
    })
    return render_template("public/resources.html",
        current_event=event, steps=steps, active="resources")

@blueprint.route('/project/<int:project_id>')
def project(project_id):
    return project_action(project_id, None)

@blueprint.route('/project/<int:project_id>/edit', methods=['GET', 'POST'])
@login_required
def project_edit(project_id):
    project = Project.query.filter_by(id=project_id).first_or_404()
    event = project.event
    starred = IsProjectStarred(project, current_user)
    allow_edit = starred or (isUserActive(current_user) and current_user.is_admin)
    if not allow_edit:
        flash('You do not have access to edit this project.', 'warning')
        return project_action(project_id, None)
    form = ProjectForm(obj=project, next=request.args.get('next'))
    form.category_id.choices = [(c.id, c.name) for c in project.categories_all()]
    form.category_id.choices.insert(0, (-1, ''))
    if form.validate_on_submit():
        del form.id
        form.populate_obj(project)
        project.update()
        db.session.add(project)
        db.session.commit()
        cache.clear()
        flash('Project updated.', 'success')
        project_action(project_id, 'update', False)
        return redirect(url_for('public.project', project_id=project.id))
    return render_template('public/projectedit.html',
        current_event=event, project=project, form=form)

@blueprint.route('/project/<int:project_id>/post', methods=['GET', 'POST'])
@login_required
def project_post(project_id):
    project = Project.query.filter_by(id=project_id).first_or_404()
    event = project.event
    starred = IsProjectStarred(project, current_user)
    allow_edit = starred or (not current_user.is_anonymous and current_user.is_admin)
    if not allow_edit:
        flash('You do not have access to edit this project.', 'warning')
        return project_action(project_id, None)
    form = ProjectPost(obj=project, next=request.args.get('next'))
    # Populate progress dialog
    form.progress.choices = projectProgressList(event.has_started or event.has_finished)
    # Populate resource list
    resources = Resource.query.filter_by(is_visible=True).order_by(Resource.type_id).all()
    resource_list = [(0, '')]
    resource_list.extend([(r.id, r.of_type + ': ' + r.name) for r in resources])
    form.resource.choices = resource_list
    # Process form
    if form.validate_on_submit():
        del form.id
        if form.resource.data == 0:
            form.resource.data = None
        form.populate_obj(project)
        project.update()
        db.session.add(project)
        db.session.commit()
        cache.clear()
        flash('Thanks for your commit!', 'success')
        project_action(project_id, 'update', action='post', text=form.note.data, resource=form.resource.data)
        return redirect(url_for('public.project', project_id=project.id))
    return render_template('public/projectpost.html', current_event=event, project=project, form=form)

def project_action(project_id, of_type=None, as_view=True, then_redirect=False, action=None, text=None, resource=None):
    project = Project.query.filter_by(id=project_id).first_or_404()
    event = project.event
    if of_type is not None:
        ProjectActivity(project, of_type, current_user, action, text, resource)
    if not as_view:
        return True
    if then_redirect:
        return redirect(url_for('public.project', project_id=project.id))
    starred = IsProjectStarred(project, current_user)
    allow_edit = starred or (not current_user.is_anonymous and current_user.is_admin)
    allow_edit = allow_edit and not event.lock_editing
    project_team = project.team()
    latest_activity = project.latest_activity()
    project_signals = project.all_signals()
    suggestions = SuggestionsByProgress(project.progress)
    return render_template('public/project.html', current_event=event, project=project,
        project_starred=starred, project_team=project_team, project_signals=project_signals, suggestions=suggestions,
        allow_edit=allow_edit, latest_activity=latest_activity)

@blueprint.route('/project/<int:project_id>/star', methods=['GET', 'POST'])
@login_required
def project_star(project_id):
    if not isUserActive(current_user):
        return "User not allowed. Please contact event organizers."
    flash('Welcome to the team!', 'success')
    return project_action(project_id, 'star', then_redirect=True)

@blueprint.route('/project/<int:project_id>/unstar', methods=['GET', 'POST'])
@login_required
def project_unstar(project_id):
    flash('You have left the project', 'success')
    return project_action(project_id, 'unstar', then_redirect=True)

@blueprint.route('/event/<int:event_id>/project/new', methods=['GET', 'POST'])
@login_required
def project_new(event_id):
    if not isUserActive(current_user):
        flash("Your account needs to be activated: please contact an organizer.", 'warning')
        return redirect(url_for('public.event', event_id=event_id))
    form = None
    event = Event.query.filter_by(id=event_id).first_or_404()
    if event.lock_starting:
        flash('Starting a new project is disabled for this event.', 'error')
        return redirect(url_for('public.event', event_id=event.id))
    if isUserActive(current_user):
        project = Project()
        project.user_id = current_user.id
        form = ProjectNew(obj=project, next=request.args.get('next'))
        form.category_id.choices = [(c.id, c.name) for c in project.categories_all(event)]
        form.category_id.choices.insert(0, (-1, ''))
        if form.validate_on_submit():
            del form.id
            form.populate_obj(project)
            project.event = event
            if event.has_started:
                project.progress = 0
            else:
                project.progress = -1 # Start as challenge
            project.update()
            db.session.add(project)
            db.session.commit()
            flash('New challenge added.', 'success')
            project_action(project.id, 'create', False)
            cache.clear()
            if event.has_started:
                project_action(project.id, 'star', False) # Join team
            if len(project.autotext_url)>1:
                return project_autoupdate(project.id)
            else:
                return redirect(url_for('public.project', project_id=project.id))
    return render_template('public/projectnew.html', active="projectnew", current_event=event, form=form)

@blueprint.route('/project/<int:project_id>/autoupdate')
@login_required
def project_autoupdate(project_id):
    project = Project.query.filter_by(id=project_id).first_or_404()
    starred = IsProjectStarred(project, current_user)
    allow_edit = starred or (not current_user.is_anonymous and current_user.is_admin)
    if not allow_edit or project.is_hidden or not project.is_autoupdate:
        flash('You may not sync this project.', 'warning')
        return project_action(project_id)
    data = GetProjectData(project.autotext_url)
    if not 'name' in data:
        flash("Could not sync: check that the Remote Link contains a README.", 'warning')
        return project_action(project_id)
    SyncProjectData(project, data)
    project_action(project.id, 'update', action='sync', text=str(len(project.autotext)) + ' bytes')
    flash("Project data synced from %s" % data['type'], 'success')
    return redirect(url_for('public.project', project_id=project.id))

@blueprint.route('/resource/<int:resource_id>', methods=['GET'])
def resource(resource_id):
    resource = Resource.query.filter_by(id=resource_id).first_or_404()
    projects = [ c.project for c in resource.get_comments() ]
    event = current_event()
    return render_template('public/resource.html',
        current_event=event, resource=resource, projects=projects)

@blueprint.route('/resource/post', methods=['GET', 'POST'])
@login_required
def resource_post():
    if not isUserActive(current_user):
        return "User not allowed. Please contact event organizers."
    resource = Resource()
    event = current_event()
    form = ResourceForm(obj=resource)
    if form.validate_on_submit():
        form.populate_obj(resource)
        resource.user_id = current_user.id
        resource.is_visible = not current_app.config['DRIBDAT_TOOL_APPROVE']
        db.session.add(resource)
        db.session.commit()
        flash('Thanks for the tip! Your suggestions are visible in your profile.', 'success')
        return redirect(url_for('public.resource', resource_id=resource.id))
    return render_template('public/resourcenew.html',
                current_event=event, form=form)

@blueprint.route('/resource/<int:resource_id>/edit', methods=['GET', 'POST'])
@login_required
def resource_edit(resource_id):
    resource = Resource.query.filter_by(id=resource_id).first_or_404()
    event = current_event()
    allow_edit = isUserActive(current_user) and current_user == resource.user or current_user.is_admin
    if not allow_edit:
        flash('You do not have access to edit this resource.', 'warning')
        return redirect(url_for('public.home'))
    form = ResourceForm(obj=resource)
    if form.validate_on_submit():
        form.populate_obj(resource)
        db.session.add(resource)
        db.session.commit()
        flash('Changes saved.', 'success')
        return redirect(url_for('public.resource', resource_id=resource.id))
    return render_template('public/resourceedit.html',
                current_event=event, resource=resource, form=form)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4

# pylint: disable=no-member
# pylint: disable=import-error

"""
driloader.downloader
--------------------

Module which performs the driver download.
"""

import os
import platform
import zipfile

import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning

from driloader.commands import Commands

from .browsers import Browser

class Downloader:

    def __init__(self, driver):
        self.os_name = platform.system()
        self.drivers_path = self._create_driver_folder()
        self.browser = Browser(driver, self.os_name)

    def _create_driver_folder(self):
        drivers_path = os.path.expanduser('~{0}Driloader{0}Drivers{0}'.format(os.sep))
        if self.os_name == "Windows":
            if not os.path.exists(drivers_path):
                os.makedirs(drivers_path)
            import ctypes
            # This hides the folder in Windows.
            ctypes.windll.kernel32.SetFileAttributesW(drivers_path, 2)
            return drivers_path
        else:
            # This hides the folder in Linux
            hidden_name = drivers_path.replace("Drivers", ".Drivers")
            if not os.path.exists(hidden_name):
                os.makedirs(hidden_name)
            return hidden_name

    @staticmethod
    def download_file(url, path_to_download, proxy):
        if not os.path.exists(path_to_download):
            requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
            response = requests.get(url, verify=False, proxy=proxy)
            path = path_to_download.rpartition("/")[0]
            if not os.path.exists(path):
                os.makedirs(path)
            with open(path_to_download, "wb") as file:
                file.write(response.content)

    @staticmethod
    def unzip(zip_file, path_to_extract, delete_after_extract=False):
        """
        Extract a 'zip' or 'gz' file content to the same path as file is in.
        :param zip_file: file to be extracted.
        :param delete_after_extract: deletes original zipped file after it's extracted.
        :param path_to_extract: the path to extract the file.
        """
        if zip_file.endswith("zip"):
            zfile = zipfile.ZipFile(zip_file, "r")
            zfile.extractall(path_to_extract)
            zfile.close()
        if zip_file.endswith("gz"):
            Commands.run("tar -zxvf {} -C {}".format(zip_file, zip_file.rpartition("/")[0]))
        if delete_after_extract:
            os.remove(zip_file)

    @staticmethod
    def check_driver_exists(path_to_file):
        return os.path.isfile(path_to_file)

    def get_default_path(self):
        """
        Get the full unzipped file's path.
        :return: unzipped file's path.
        """
        return self.drivers_path
<commit_msg>Fixing parameter name from 'proxy' to 'proxies'<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4

# pylint: disable=no-member
# pylint: disable=import-error

"""
driloader.downloader
--------------------

Module which performs the driver download.
"""

import os
import platform
import zipfile

import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning

from driloader.commands import Commands

from .browsers import Browser

class Downloader:

    def __init__(self, driver):
        self.os_name = platform.system()
        self.drivers_path = self._create_driver_folder()
        self.browser = Browser(driver, self.os_name)

    def _create_driver_folder(self):
        drivers_path = os.path.expanduser('~{0}Driloader{0}Drivers{0}'.format(os.sep))
        if self.os_name == "Windows":
            if not os.path.exists(drivers_path):
                os.makedirs(drivers_path)
            import ctypes
            # This hides the folder in Windows.
            ctypes.windll.kernel32.SetFileAttributesW(drivers_path, 2)
            return drivers_path
        else:
            # This hides the folder in Linux
            hidden_name = drivers_path.replace("Drivers", ".Drivers")
            if not os.path.exists(hidden_name):
                os.makedirs(hidden_name)
            return hidden_name

    @staticmethod
    def download_file(url, path_to_download, proxy):
        if not os.path.exists(path_to_download):
            requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
            response = requests.get(url, verify=False, proxies=proxy)
            path = path_to_download.rpartition("/")[0]
            if not os.path.exists(path):
                os.makedirs(path)
            with open(path_to_download, "wb") as file:
                file.write(response.content)

    @staticmethod
    def unzip(zip_file, path_to_extract, delete_after_extract=False):
        """
        Extract a 'zip' or 'gz' file content to the same path as file is in.
        :param zip_file: file to be extracted.
        :param delete_after_extract: deletes original zipped file after it's extracted.
        :param path_to_extract: the path to extract the file.
        """
        if zip_file.endswith("zip"):
            zfile = zipfile.ZipFile(zip_file, "r")
            zfile.extractall(path_to_extract)
            zfile.close()
        if zip_file.endswith("gz"):
            Commands.run("tar -zxvf {} -C {}".format(zip_file, zip_file.rpartition("/")[0]))
        if delete_after_extract:
            os.remove(zip_file)

    @staticmethod
    def check_driver_exists(path_to_file):
        return os.path.isfile(path_to_file)

    def get_default_path(self):
        """
        Get the full unzipped file's path.
        :return: unzipped file's path.
        """
        return self.drivers_path
<|endoftext|>


<commit_before># ! /usr/bin/python
# Copyright 2014 Jeremy Carroll
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collectd
import json
import urllib2
import socket
import collections
from distutils.version import StrictVersion

PREFIX = "elasticsearch"
ES_CLUSTER = "elasticsearch"
ES_HOST = "localhost"
ES_PORT = 9200
VERBOSE_LOGGING = False

Stat = collections.namedtuple('Stat', ('type', 'path'))

STATS_CUR = {}

# DICT: ElasticSearch 1.0.0
STATS_ES1 = {
    ## STORE
    'indices.store.throttle-time': Stat("counter", "nodes.%s.indices.store.throttle_time_in_millis"),

    ##SEARCH
    'indices.search.open-contexts': Stat("gauge", "nodes.%s.indices.search.open_contexts"),

    ##CACHE
    'indices.cache.field.eviction': Stat("counter", "nodes.%s.indices.fielddata.evictions"),
    'indices.cache.field.size': Stat("bytes", "nodes.%s.indices.fielddata.memory_size_in_bytes"),
    'indices.cache.filter.evictions': Stat("counter", "nodes.%s.indices.filter_cache.evictions"),
    'indices.cache.filter.size': Stat("bytes", "nodes.%s.indices.filter_cache.memory_size_in_bytes"),

    ## FLUSH
    'indices.flush.total': Stat("counter", "nodes.%s.indices.flush.total"),
    'indices.flush.time': Stat("counter", "nodes.%s.indices.flush.total_time_in_millis"),

    ## MERGES
    'indices.merges.current': Stat("gauge", "nodes.%s.indices.merges.current"),
    'indices.merges.current-docs': Stat("gauge", "nodes.%s.indices.merges.current_docs"),
    'indices.merges.current-size': Stat("bytes", "nodes.%s.indices.merges.current_size_in_bytes"),
    'indices.merges.total': Stat("counter", "nodes.%s.indices.merges.total"),
    'indices.merges.total-docs': Stat("gauge", "nodes.%s.indices.merges.total_docs"),
    'indices.merges.total-size': Stat("bytes", "nodes.%s.indices.merges.total_size_in_bytes"),
    'indices.merges.time': Stat("counter", "nodes.%s.indices.merges.total_time_in_millis"),

    ## REFRESH
    'indices.refresh.total': Stat("counter", "nodes.%s.indices.refresh.total"),
    'indices.refresh.time': Stat("counter", "nodes.%s.indices.refresh.total_time_in_millis"),
}

# DICT: ElasticSearch 0.90.x
STATS_ES09 = {

    ##CPU
    'process.cpu.percent': Stat("gauge", "nodes.%s.process.cpu.percent"),
}

# DICT: Common stuff
STATS = {

    ## DOCS
    'indices.docs.count': Stat("gauge", "nodes.%s.indices.docs.count"),
    'indices.docs.deleted': Stat("counter", "nodes.%s.indices.docs.deleted"),

    ## STORE
    'indices.store.size': Stat("bytes", "nodes.%s.indices.store.size_in_bytes"),

    ## INDEXING
    'indices.indexing.index-total': Stat("counter", "nodes.%s.indices.indexing.index_total"),
    'indices.indexing.index-time': Stat("counter", "nodes.%s.indices.indexing.index_time_in_millis"),
    'indices.indexing.delete-total': Stat("counter", "nodes.%s.indices.indexing.delete_total"),
    'indices.indexing.delete-time': Stat("counter", "nodes.%s.indices.indexing.delete_time_in_millis"),
    'indices.indexing.index-current': Stat("gauge", "nodes.%s.indices.indexing.index_current"),
    'indices.indexing.delete-current': Stat("gauge", "nodes.%s.indices.indexing.delete_current"),

    # SEGMENTS
    'indices.segments.count': Stat('counter', 'nodes.%s.indices.segments.count'),
    'indices.segments.memory_in_bytes': Stat('bytes', 'nodes.%s.indices.segments.memory_in_bytes'),
    'indices.segments.fixed_bit_set_memory_in_bytes': Stat('bytes',
                                                           'nodes.%s.indices.segments.fixed_bit_set_memory_in_bytes'),

    ## GET
    'indices.get.total': Stat("counter", "nodes.%s.indices.get.total"),
    'indices.get.time': Stat("counter", "nodes.%s.indices.get.time_in_millis"),
    'indices.get.exists-total': Stat("counter", "nodes.%s.indices.get.exists_total"),
    'indices.get.exists-time': Stat("counter", "nodes.%s.indices.get.exists_time_in_millis"),
    'indices.get.missing-total': Stat("counter", "nodes.%s.indices.get.missing_total"),
    'indices.get.missing-time': Stat("counter", "nodes.%s.indices.get.missing_time_in_millis"),
    'indices.get.current': Stat("gauge", "nodes.%s.indices.get.current"),

    ## SEARCH
    'indices.search.query-current': Stat("gauge", "nodes.%s.indices.search.query_current"),
    'indices.search.query-total': Stat("counter", "nodes.%s.indices.search.query_total"),
    'indices.search.query-time': Stat("counter", "nodes.%s.indices.search.query_time_in_millis"),
    'indices.search.fetch-current': Stat("gauge", "nodes.%s.indices.search.fetch_current"),
    'indices.search.fetch-total': Stat("counter", "nodes.%s.indices.search.fetch_total"),
    'indices.search.fetch-time': Stat("counter", "nodes.%s.indices.search.fetch_time_in_millis"),

    # JVM METRICS #
    ##GC
    'jvm.gc.time': Stat("counter", "nodes.%s.jvm.gc.collectors.young.collection_time_in_millis"),
    'jvm.gc.count': Stat("counter", "nodes.%s.jvm.gc.collectors.young.collection_count"),
    'jvm.gc.old-time': Stat("counter", "nodes.%s.jvm.gc.collectors.old.collection_time_in_millis"),
    'jvm.gc.old-count': Stat("counter", "nodes.%s.jvm.gc.collectors.old.collection_count"),

    ## MEM
    'jvm.mem.heap-committed': Stat("bytes", "nodes.%s.jvm.mem.heap_committed_in_bytes"),
    'jvm.mem.heap-used': Stat("bytes", "nodes.%s.jvm.mem.heap_used_in_bytes"),
    'jvm.mem.heap-used-percent': Stat("percent", "nodes.%s.jvm.mem.heap_used_percent"),
    'jvm.mem.non-heap-committed': Stat("bytes", "nodes.%s.jvm.mem.non_heap_committed_in_bytes"),
    'jvm.mem.non-heap-used': Stat("bytes", "nodes.%s.jvm.mem.non_heap_used_in_bytes"),

    ## THREADS
    'jvm.threads.count': Stat("gauge", "nodes.%s.jvm.threads.count"),
    'jvm.threads.peak': Stat("gauge", "nodes.%s.jvm.threads.peak_count"),

    # TRANSPORT METRICS #
    'transport.server_open': Stat("gauge", "nodes.%s.transport.server_open"),
    'transport.rx.count': Stat("counter", "nodes.%s.transport.rx_count"),
    'transport.rx.size': Stat("bytes", "nodes.%s.transport.rx_size_in_bytes"),
    'transport.tx.count': Stat("counter", "nodes.%s.transport.tx_count"),
    'transport.tx.size': Stat("bytes", "nodes.%s.transport.tx_size_in_bytes"),

    # HTTP METRICS #
    'http.current_open': Stat("gauge", "nodes.%s.http.current_open"),
    'http.total_open': Stat("counter", "nodes.%s.http.total_opened"),

    # PROCESS METRICS #
    'process.open_file_descriptors': Stat("gauge", "nodes.%s.process.open_file_descriptors"),
}

# FUNCTION: Collect stats from JSON result
def lookup_stat(stat, json):
    node_names = json['nodes'].keys()
    results = []
    for node in node_names:
        val = dig_it_up(json, STATS_CUR[stat].path % node)
        # Check to make sure we have a valid result
        # dig_it_up returns False if no match found
        if not isinstance(val, bool):
            results.append(int(val))
        else:
            results.append(None)
    return results

def configure_callback(conf):
    """Received configuration information"""
    global ES_HOST, ES_PORT, VERBOSE_LOGGING
    for node in conf.children:
        if node.key == 'Host':
            ES_HOST = node.values[0]
        elif node.key == 'Port':
            ES_PORT = int(node.values[0])
        elif node.key == 'Verbose':
            VERBOSE_LOGGING = bool(node.values[0])
        elif node.key == 'Cluster':
            ES_CLUSTER = node.values[0]
        else:
            collectd.warning('elasticsearch plugin: Unknown config key: %s.'
                             % node.key)

    log_verbose('Configured with host=%s, port=%s' % (ES_HOST, ES_PORT))

def fetch_url(url):
    try:
        result = json.load(urllib2.urlopen(url, timeout=10))
    except urllib2.URLError, e:
        collectd.error('elasticsearch plugin: Error connecting to %s - %r' % (url, e))
        return None
    except socket.timeout as e:
        collectd.error('elasticsearch plugin: Timeout connecting to %s - %r' % (url, e))
        return None
    except socket.error as e:  # for example : ECONNRESET
        collectd.error('elasticsearch plugin: Connection error to %s - %r' % (url, e))
        return None
    return result

def fetch_stats():
    global ES_CLUSTER, ES_HOST, ES_PORT, STATS_CUR

    base_url = 'http://' + ES_HOST + ':' + str(ES_PORT) + '/'
    server_info = fetch_url(base_url)
    if server_info is None:
        return
    version = server_info['version']['number']

    if StrictVersion(version) >= StrictVersion('1.0.0'):
        ES_URL = base_url + '_nodes/stats/transport,http,process,jvm,indices'
        STATS_CUR = dict(STATS.items() + STATS_ES1.items())
    else:
        ES_URL = base_url + '_cluster/nodes/_local/stats?http=true&process=true&jvm=true&transport=true'
        STATS_CUR = dict(STATS.items() + STATS_ES09.items())

    # add info on thread pools
    for pool in ['generic', 'index', 'get', 'snapshot', 'merge', 'optimize', 'bulk', 'warmer', 'flush', 'search', 'refresh']:
      for attr in ['threads', 'queue', 'active', 'largest']:
        path = 'thread_pool.{0}.{1}'.format(pool, attr)
        STATS_CUR[path] = Stat("gauge", 'nodes.%s.{0}'.format(path))
      for attr in ['completed', 'rejected']:
        path = 'thread_pool.{0}.{1}'.format(pool, attr)
        STATS_CUR[path] = Stat("counter", 'nodes.%s.{0}'.format(path))

    result = fetch_url(ES_URL)
    if result is None:
        return

    ES_CLUSTER = result['cluster_name']
    return parse_stats(result)

def parse_stats(json):
    """Parse stats response from ElasticSearch"""
    for name, key in STATS_CUR.iteritems():
        result = lookup_stat(name, json)
        count = len(result)
        if count == 1:
            dispatch_stat(result[0], name, key, "")
        else:
            total = 0
            for index, value in enumerate(result):
                dispatch_stat(value, name, key, (json['nodes'].values()[index]['host']) or ("node%d" % index))
                total += value
            if key.type != "bytes":
                total = total / count
            dispatch_stat(total, name, key, "total")

def dispatch_stat(result, name, key, node_index):
    """Read a key from info response data and dispatch a value"""
    if result is None:
        collectd.warning('elasticsearch plugin: Value not found for %s' % name)
        return
    estype = key.type
    value = int(result)
    log_verbose('Sending value[%s]: %s=%s' % (estype, name, value))

    val = collectd.Values(plugin='elasticsearch')
    val.plugin_instance = ES_CLUSTER + "_" + node_index
    if len(node_index) == 0:
        val.plugin_instance = ES_CLUSTER
    val.type = estype
    val.type_instance = name
    val.values = [value]
    val.dispatch()

def read_callback():
    log_verbose('Read callback called')
    stats = fetch_stats()

def dig_it_up(obj, path):
    try:
        if type(path) in (str, unicode):
            path = path.split('.')
        return reduce(lambda x, y: x[y], path, obj)
    except:
        return False

def log_verbose(msg):
    if not VERBOSE_LOGGING:
        return
    collectd.info('elasticsearch plugin [verbose]: %s' % msg)

collectd.register_config(configure_callback)
collectd.register_read(read_callback)
<commit_msg>ensure value is not null<commit_after># ! /usr/bin/python
# Copyright 2014 Jeremy Carroll
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collectd
import json
import urllib2
import socket
import collections
from distutils.version import StrictVersion

PREFIX = "elasticsearch"
ES_CLUSTER = "elasticsearch"
ES_HOST = "localhost"
ES_PORT = 9200
VERBOSE_LOGGING = False

Stat = collections.namedtuple('Stat', ('type', 'path'))

STATS_CUR = {}

# DICT: ElasticSearch 1.0.0
STATS_ES1 = {
    ## STORE
    'indices.store.throttle-time': Stat("counter", "nodes.%s.indices.store.throttle_time_in_millis"),

    ##SEARCH
    'indices.search.open-contexts': Stat("gauge", "nodes.%s.indices.search.open_contexts"),

    ##CACHE
    'indices.cache.field.eviction': Stat("counter", "nodes.%s.indices.fielddata.evictions"),
    'indices.cache.field.size': Stat("bytes", "nodes.%s.indices.fielddata.memory_size_in_bytes"),
    'indices.cache.filter.evictions': Stat("counter", "nodes.%s.indices.filter_cache.evictions"),
    'indices.cache.filter.size': Stat("bytes", "nodes.%s.indices.filter_cache.memory_size_in_bytes"),

    ## FLUSH
    'indices.flush.total': Stat("counter", "nodes.%s.indices.flush.total"),
    'indices.flush.time': Stat("counter", "nodes.%s.indices.flush.total_time_in_millis"),

    ## MERGES
    'indices.merges.current': Stat("gauge", "nodes.%s.indices.merges.current"),
    'indices.merges.current-docs': Stat("gauge", "nodes.%s.indices.merges.current_docs"),
    'indices.merges.current-size': Stat("bytes", "nodes.%s.indices.merges.current_size_in_bytes"),
    'indices.merges.total': Stat("counter", "nodes.%s.indices.merges.total"),
    'indices.merges.total-docs': Stat("gauge", "nodes.%s.indices.merges.total_docs"),
    'indices.merges.total-size': Stat("bytes", "nodes.%s.indices.merges.total_size_in_bytes"),
    'indices.merges.time': Stat("counter", "nodes.%s.indices.merges.total_time_in_millis"),

    ## REFRESH
    'indices.refresh.total': Stat("counter", "nodes.%s.indices.refresh.total"),
    'indices.refresh.time': Stat("counter", "nodes.%s.indices.refresh.total_time_in_millis"),
}

# DICT: ElasticSearch 0.90.x
STATS_ES09 = {

    ##CPU
    'process.cpu.percent': Stat("gauge", "nodes.%s.process.cpu.percent"),
}

# DICT: Common stuff
STATS = {

    ## DOCS
    'indices.docs.count': Stat("gauge", "nodes.%s.indices.docs.count"),
    'indices.docs.deleted': Stat("counter", "nodes.%s.indices.docs.deleted"),

    ## STORE
    'indices.store.size': Stat("bytes", "nodes.%s.indices.store.size_in_bytes"),

    ## INDEXING
    'indices.indexing.index-total': Stat("counter", "nodes.%s.indices.indexing.index_total"),
    'indices.indexing.index-time': Stat("counter", "nodes.%s.indices.indexing.index_time_in_millis"),
    'indices.indexing.delete-total': Stat("counter", "nodes.%s.indices.indexing.delete_total"),
    'indices.indexing.delete-time': Stat("counter", "nodes.%s.indices.indexing.delete_time_in_millis"),
    'indices.indexing.index-current': Stat("gauge", "nodes.%s.indices.indexing.index_current"),
    'indices.indexing.delete-current': Stat("gauge", "nodes.%s.indices.indexing.delete_current"),

    # SEGMENTS
    'indices.segments.count': Stat('counter', 'nodes.%s.indices.segments.count'),
    'indices.segments.memory_in_bytes': Stat('bytes', 'nodes.%s.indices.segments.memory_in_bytes'),
    'indices.segments.fixed_bit_set_memory_in_bytes': Stat('bytes',
                                                           'nodes.%s.indices.segments.fixed_bit_set_memory_in_bytes'),

    ## GET
    'indices.get.total': Stat("counter", "nodes.%s.indices.get.total"),
    'indices.get.time': Stat("counter", "nodes.%s.indices.get.time_in_millis"),
    'indices.get.exists-total': Stat("counter", "nodes.%s.indices.get.exists_total"),
    'indices.get.exists-time': Stat("counter", "nodes.%s.indices.get.exists_time_in_millis"),
    'indices.get.missing-total': Stat("counter", "nodes.%s.indices.get.missing_total"),
    'indices.get.missing-time': Stat("counter", "nodes.%s.indices.get.missing_time_in_millis"),
    'indices.get.current': Stat("gauge", "nodes.%s.indices.get.current"),

    ## SEARCH
    'indices.search.query-current': Stat("gauge", "nodes.%s.indices.search.query_current"),
    'indices.search.query-total': Stat("counter", "nodes.%s.indices.search.query_total"),
    'indices.search.query-time': Stat("counter", "nodes.%s.indices.search.query_time_in_millis"),
    'indices.search.fetch-current': Stat("gauge", "nodes.%s.indices.search.fetch_current"),
    'indices.search.fetch-total': Stat("counter", "nodes.%s.indices.search.fetch_total"),
    'indices.search.fetch-time': Stat("counter", "nodes.%s.indices.search.fetch_time_in_millis"),

    # JVM METRICS #
    ##GC
    'jvm.gc.time': Stat("counter", "nodes.%s.jvm.gc.collectors.young.collection_time_in_millis"),
    'jvm.gc.count': Stat("counter", "nodes.%s.jvm.gc.collectors.young.collection_count"),
    'jvm.gc.old-time': Stat("counter", "nodes.%s.jvm.gc.collectors.old.collection_time_in_millis"),
    'jvm.gc.old-count': Stat("counter", "nodes.%s.jvm.gc.collectors.old.collection_count"),

    ## MEM
    'jvm.mem.heap-committed': Stat("bytes", "nodes.%s.jvm.mem.heap_committed_in_bytes"),
    'jvm.mem.heap-used': Stat("bytes", "nodes.%s.jvm.mem.heap_used_in_bytes"),
    'jvm.mem.heap-used-percent': Stat("percent", "nodes.%s.jvm.mem.heap_used_percent"),
    'jvm.mem.non-heap-committed': Stat("bytes", "nodes.%s.jvm.mem.non_heap_committed_in_bytes"),
    'jvm.mem.non-heap-used': Stat("bytes", "nodes.%s.jvm.mem.non_heap_used_in_bytes"),

    ## THREADS
    'jvm.threads.count': Stat("gauge", "nodes.%s.jvm.threads.count"),
    'jvm.threads.peak': Stat("gauge", "nodes.%s.jvm.threads.peak_count"),

    # TRANSPORT METRICS #
    'transport.server_open': Stat("gauge", "nodes.%s.transport.server_open"),
    'transport.rx.count': Stat("counter", "nodes.%s.transport.rx_count"),
    'transport.rx.size': Stat("bytes", "nodes.%s.transport.rx_size_in_bytes"),
    'transport.tx.count': Stat("counter", "nodes.%s.transport.tx_count"),
    'transport.tx.size': Stat("bytes", "nodes.%s.transport.tx_size_in_bytes"),

    # HTTP METRICS #
    'http.current_open': Stat("gauge", "nodes.%s.http.current_open"),
    'http.total_open': Stat("counter", "nodes.%s.http.total_opened"),

    # PROCESS METRICS #
    'process.open_file_descriptors': Stat("gauge", "nodes.%s.process.open_file_descriptors"),
}

# FUNCTION: Collect stats from JSON result
def lookup_stat(stat, json):
    node_names = json['nodes'].keys()
    results = []
    for node in node_names:
        val = dig_it_up(json, STATS_CUR[stat].path % node)
        # Check to make sure we have a valid result
        # dig_it_up returns False if no match found
        if not isinstance(val, bool):
            results.append(int(val))
        else:
            results.append(None)
    return results

def configure_callback(conf):
    """Received configuration information"""
    global ES_HOST, ES_PORT, VERBOSE_LOGGING
    for node in conf.children:
        if node.key == 'Host':
            ES_HOST = node.values[0]
        elif node.key == 'Port':
            ES_PORT = int(node.values[0])
        elif node.key == 'Verbose':
            VERBOSE_LOGGING = bool(node.values[0])
        elif node.key == 'Cluster':
            ES_CLUSTER = node.values[0]
        else:
            collectd.warning('elasticsearch plugin: Unknown config key: %s.'
                             % node.key)

    log_verbose('Configured with host=%s, port=%s' % (ES_HOST, ES_PORT))

def fetch_url(url):
    try:
        result = json.load(urllib2.urlopen(url, timeout=10))
    except urllib2.URLError, e:
        collectd.error('elasticsearch plugin: Error connecting to %s - %r' % (url, e))
        return None
    except socket.timeout as e:
        collectd.error('elasticsearch plugin: Timeout connecting to %s - %r' % (url, e))
        return None
    except socket.error as e:  # for example : ECONNRESET
        collectd.error('elasticsearch plugin: Connection error to %s - %r' % (url, e))
        return None
    return result

def fetch_stats():
    global ES_CLUSTER, ES_HOST, ES_PORT, STATS_CUR

    base_url = 'http://' + ES_HOST + ':' + str(ES_PORT) + '/'
    server_info = fetch_url(base_url)
    if server_info is None:
        return
    version = server_info['version']['number']

    if StrictVersion(version) >= StrictVersion('1.0.0'):
        ES_URL = base_url + '_nodes/stats/transport,http,process,jvm,indices'
        STATS_CUR = dict(STATS.items() + STATS_ES1.items())
    else:
        ES_URL = base_url + '_cluster/nodes/_local/stats?http=true&process=true&jvm=true&transport=true'
        STATS_CUR = dict(STATS.items() + STATS_ES09.items())

    # add info on thread pools
    for pool in ['generic', 'index', 'get', 'snapshot', 'merge', 'optimize', 'bulk', 'warmer', 'flush', 'search', 'refresh']:
      for attr in ['threads', 'queue', 'active', 'largest']:
        path = 'thread_pool.{0}.{1}'.format(pool, attr)
        STATS_CUR[path] = Stat("gauge", 'nodes.%s.{0}'.format(path))
      for attr in ['completed', 'rejected']:
        path = 'thread_pool.{0}.{1}'.format(pool, attr)
        STATS_CUR[path] = Stat("counter", 'nodes.%s.{0}'.format(path))

    result = fetch_url(ES_URL)
    if result is None:
        return

    ES_CLUSTER = result['cluster_name']
    return parse_stats(result)

def parse_stats(json):
    """Parse stats response from ElasticSearch"""
    for name, key in STATS_CUR.iteritems():
        result = lookup_stat(name, json)
        count = len(result)
        if count == 1:
            dispatch_stat(result[0], name, key, "")
        else:
            total = 0
            for index, value in enumerate(result):
                dispatch_stat(value, name, key, (json['nodes'].values()[index]['host']) or ("node%d" % index))
                if value is not None:
                    total += value
            if key.type != "bytes":
                total = total / count
            dispatch_stat(total, name, key, "total")

def dispatch_stat(result, name, key, node_index):
    """Read a key from info response data and dispatch a value"""
    if result is None:
        collectd.warning('elasticsearch plugin: Value not found for %s' % name)
        return
    estype = key.type
    value = int(result)
    log_verbose('Sending value[%s]: %s=%s' % (estype, name, value))

    val = collectd.Values(plugin='elasticsearch')
    val.plugin_instance = ES_CLUSTER + "_" + node_index
    if len(node_index) == 0:
        val.plugin_instance = ES_CLUSTER
    val.type = estype
    val.type_instance = name
    val.values = [value]
    val.dispatch()

def read_callback():
    log_verbose('Read callback called')
    stats = fetch_stats()

def dig_it_up(obj, path):
    try:
        if type(path) in (str, unicode):
            path = path.split('.')
        return reduce(lambda x, y: x[y], path, obj)
    except:
        return False

def log_verbose(msg):
    if not VERBOSE_LOGGING:
        return
    collectd.info('elasticsearch plugin [verbose]: %s' % msg)

collectd.register_config(configure_callback)
collectd.register_read(read_callback)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (c) 2016, Neil Booth
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.

'''Script to send RPC commands to a running ElectrumX server.'''

import argparse
import asyncio
import json
from functools import partial
from os import environ

from lib.jsonrpc import JSONSession, JSONRPCv2
from server.controller import Controller

class RPCClient(JSONSession):

    def __init__(self):
        super().__init__(version=JSONRPCv2)
        self.max_send = 0
        self.max_buffer_size = 5*10**6

    async def wait_for_response(self):
        await self.items_event.wait()
        await self.process_pending_items()

    def send_rpc_request(self, method, params):
        handler = partial(self.handle_response, method)
        self.send_request(handler, method, params)

    def handle_response(self, method, result, error):
        if method in ('groups', 'peers', 'sessions') and not error:
            lines_func = getattr(Controller, '{}_text_lines'.format(method))
            for line in lines_func(result):
                print(line)
        elif error:
            print('error: {} (code {:d})'
                  .format(error['message'], error['code']))
        else:
            print(json.dumps(result, indent=4, sort_keys=True))

def rpc_send_and_wait(port, method, params, timeout=15):
    loop = asyncio.get_event_loop()
    coro = loop.create_connection(RPCClient, 'localhost', port)
    try:
        transport, rpc_client = loop.run_until_complete(coro)
        rpc_client.send_rpc_request(method, params)
        try:
            coro = rpc_client.wait_for_response()
            loop.run_until_complete(asyncio.wait_for(coro, timeout))
        except asyncio.TimeoutError:
            print('request timed out after {}s'.format(timeout))
    except OSError:
        print('cannot connect - is ElectrumX catching up, not running, or '
              'is {:d} the wrong RPC port?'.format(port))
    finally:
        loop.close()

def main():
    '''Send the RPC command to the server and print the result.'''
    parser = argparse.ArgumentParser('Send electrumx an RPC command')
    parser.add_argument('-p', '--port', metavar='port_num', type=int,
                        help='RPC port number')
    parser.add_argument('command', nargs=1, default=[],
                        help='command to send')
    parser.add_argument('param', nargs='*', default=[],
                        help='params to send')
    args = parser.parse_args()

    port = args.port
    if port is None:
        port = int(environ.get('RPC_PORT', 8000))

    # Get the RPC request.
    method = args.command[0]
    params = args.param
    if method in ('log', 'disconnect'):
        params = [params]

    rpc_send_and_wait(port, method, params)

if __name__ == '__main__':
    main()
<commit_msg>Use aiorpcx in electrumx_rpc script<commit_after>#!/usr/bin/env python3
#
# Copyright (c) 2016-2018, Neil Booth
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.

'''Script to send RPC commands to a running ElectrumX server.'''

import argparse
import asyncio
import json
from os import environ

from aiorpcx import ClientSession

from server.controller import Controller

def main():
    '''Send the RPC command to the server and print the result.'''
    parser = argparse.ArgumentParser('Send electrumx an RPC command')
    parser.add_argument('-p', '--port', metavar='port_num', type=int,
                        help='RPC port number')
    parser.add_argument('command', nargs=1, default=[],
                        help='command to send')
    parser.add_argument('param', nargs='*', default=[],
                        help='params to send')
    args = parser.parse_args()

    port = args.port
    if port is None:
        port = int(environ.get('RPC_PORT', 8000))

    # Get the RPC request.
    method = args.command[0]
    params = args.param
    if method in ('log', 'disconnect'):
        params = [params]

    async def send_request():
        # aiorpcX makes this so easy...
        async with ClientSession('localhost', port) as session:
            result = await session.send_request(method, params, timeout=15)
            if method in ('groups', 'peers', 'sessions'):
                lines_func = getattr(Controller, f'{method}_text_lines')
                for line in lines_func(result):
                    print(line)
            else:
                print(json.dumps(result, indent=4, sort_keys=True))

    loop = asyncio.get_event_loop()
    try:
        loop.run_until_complete(send_request())
    except OSError:
        print('cannot connect - is ElectrumX catching up, not running, or '
              f'is {port} the wrong RPC port?')
    except Exception as e:
        print(f'error making request: {e}')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import random
from MoMMI.commands import command

@command("pick", r"(?:pick|choose)\s*\((.*?)\)")
async def pick_command(channel, match, message):
    choices = [x.strip() for x in match.group(1).split(",")]
    if len(choices) < 2:
        await channel.send("You need to provide at least 2 options.")
        return

    choice = random.choice(choices)
    await channel.send(f"**{choice}**")
<commit_msg>adds 1d6 (#16)<commit_after>import random
from MoMMI.commands import command

@command("pick", r"(?:pick|choose)\s*\((.*?)\)")
async def pick_command(channel, match, message):
    choices = [x.strip() for x in match.group(1).split(",")]
    if len(choices) < 2:
        await channel.send("You need to provide at least 2 options.")
        return

    choice = random.choice(choices)
    await channel.send(f"**{choice}**")

@command("roll", r"(\d+)d(\d+)")
async def roll_command(channel, match, message):
    result = "Results: "
    for i in range(0, int(match.group(1))):
        if i > 0:
            result += ", "
        result += str(random.randint(1, int(match.group(2))))
    await channel.send(result)
<|endoftext|>