<commit_before>"""All views for the extension."""

# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.core.urlresolvers import reverse
from django.views.generic import CreateView, FormView, TemplateView
from django.shortcuts import redirect
from django.utils.safestring import mark_safe
from django.contrib import messages

from rest_framework import status
from rest_framework.views import APIView
from rest_framework.response import Response

from braces.views import LoginRequiredMixin

from geokey.core.decorators import handle_exceptions_for_ajax
from geokey.projects.models import Project
from geokey.projects.views import ProjectContext

from .helpers.context_helpers import does_not_exist_msg
from .base import STATUS, FORMAT
from .models import WebResource
from .forms import WebResourceForm
from .serializers import WebResourceSerializer

# ###########################
# ADMIN PAGES
# ###########################

class IndexPage(LoginRequiredMixin, TemplateView):
    """Main index page."""

    template_name = 'wr_index.html'

    def get_context_data(self, *args, **kwargs):
        """
        GET method for the template.

        Return the context to render the view. Overwrite the method by adding
        all projects (where user is an administrator) to the context.

        Returns
        -------
        dict
            Context.
        """
        projects = Project.objects.filter(admins=self.request.user)

        return super(IndexPage, self).get_context_data(
            projects=projects,
            *args,
            **kwargs
        )

class AllWebResourcesPage(LoginRequiredMixin, ProjectContext, TemplateView):
    """All web resources page."""

    template_name = 'wr_all_webresources.html'

class AddWebResourcePage(LoginRequiredMixin, ProjectContext, CreateView):
    """Add new web resource page."""

    template_name = 'wr_add_webresource.html'
    form_class = WebResourceForm

    def get_context_data(self, *args, **kwargs):
        """
        GET method for the template.

        Return the context to render the view. Overwrite the method by adding
        project ID and available data formats to the context.

        Returns
        -------
        dict
            Context.
        """
        project_id = self.kwargs['project_id']

        return super(AddWebResourcePage, self).get_context_data(
            project_id,
            data_formats=FORMAT,
            *args,
            **kwargs
        )

    def form_valid(self, form):
        """
        Add web resource when form data is valid.

        Parameters
        ----------
        form : geokey_webresource.forms.WebResourceForm
            Represents the user input.

        Returns
        -------
        django.http.HttpResponse
            Rendered template.
        """
        context = self.get_context_data(form=form)
        project = context.get('project')

        if project:
            if project.islocked:
                messages.error(
                    self.request,
                    'The project is locked. New web resources cannot be added.'
                )
            else:
                form.instance.project = project
                form.instance.creator = self.request.user

                add_another_url = reverse(
                    'geokey_webresources:webresource_add',
                    kwargs={
                        'project_id': project.id
                    }
                )
                messages.success(
                    self.request,
                    mark_safe(
                        'The web resource has been added. Add '
                        'another web resource.' % add_another_url
                    )
                )
                return super(AddWebResourcePage, self).form_valid(form)

        return self.render_to_response(context)

    def form_invalid(self, form):
        """
        Display an error message when form data is invalid.

        Parameters
        ----------
        form : geokey_webresource.forms.WebResourceForm
            Represents the user input.

        Returns
        -------
        dict
            Context.
        """
        messages.error(self.request, 'An error occurred.')
        return self.render_to_response(self.get_context_data(form=form))

    def get_success_url(self):
        """
        Set URL redirection when web resource created successfully.

        Returns
        -------
        str
            URL for redirection.
        """
        return reverse(
            'geokey_webresources:all_webresources',
            kwargs={
                'project_id': self.kwargs['project_id']
            }
        )

class WebResourceContext(LoginRequiredMixin, ProjectContext):
    """Get web resource mixin."""

    def get_context_data(self, project_id, webresource_id, *args, **kwargs):
        """
        GET method for the template.

        Return the context to render the view. Overwrite the method by adding
        a web resource and available status types & data formats to the
        context.

        Parameters
        ----------
        project_id : int
            Identifies the project in the database.
        webresource_id : int
            Identifies the web resource in the database.

        Returns
        -------
        dict
            Context.
        """
        context = super(WebResourceContext, self).get_context_data(
            project_id,
            *args,
            **kwargs
        )

        context['status_types'] = STATUS
        context['data_formats'] = FORMAT

        try:
            context['webresource'] = WebResource.objects.get(
                pk=webresource_id,
                project=context.get('project')
            )

            return context
        except WebResource.DoesNotExist:
            return {
                'error': 'Not found.',
                'error_description': does_not_exist_msg('Web resource')
            }

class SingleWebResourcePage(WebResourceContext, FormView):
    """Single web resource page."""

    template_name = 'wr_single_webresource.html'
    form_class = WebResourceForm

    def get_object(self):
        """
        Get and return web resource object.

        Returns
        -------
        geokey_webresource.models.WebResource
            Web resource object.
        """
        try:
            return WebResource.objects.get(
                pk=self.kwargs['webresource_id']
            )
        except WebResource.DoesNotExist:
            return None

    def get_context_data(self, *args, **kwargs):
        """
        GET method for the template.

        Return the context to render the view. Overwrite the method by adding
        project ID and web resource ID to the context.

        Returns
        -------
        dict
            Context.
        """
        project_id = self.kwargs['project_id']
        webresource_id = self.kwargs['webresource_id']

        return super(SingleWebResourcePage, self).get_context_data(
            project_id,
            webresource_id,
            *args,
            **kwargs
        )

    def get_form(self, form_class):
        """Attach instance object to form data."""
        return form_class(instance=self.get_object(), **self.get_form_kwargs())

    def form_valid(self, form):
        """
        Update web resource when form data is valid.

        Parameters
        ----------
        form : geokey_webresource.forms.WebResourceForm
            Represents the user input.

        Returns
        -------
        django.http.HttpResponse
            Rendered template.
        """
        context = self.get_context_data(form=form)
        project = context.get('project')

        if project:
            if project.islocked:
                messages.error(
                    self.request,
                    'The project is locked. Web resources cannot be updated.'
                )
            else:
                if self.request.POST.get('symbol_clear') == 'true':
                    form.instance.symbol = None
                form.save()

                messages.success(
                    self.request,
                    mark_safe('The web resource has been updated.')
                )
                return super(SingleWebResourcePage, self).form_valid(form)

        return self.render_to_response(context)

    def form_invalid(self, form):
        """
        Display an error message when form data is invalid.

        Parameters
        ----------
        form : geokey_webresource.forms.WebResourceForm
            Represents the user input.

        Returns
        -------
        dict
            Context.
        """
        messages.error(self.request, 'An error occurred.')
        return self.render_to_response(self.get_context_data(form=form))

    def get_success_url(self):
        """
        Set URL redirection when web resource updated successfully.

        Returns
        -------
        str
            URL for redirection.
        """
        return reverse(
            'geokey_webresources:all_webresources',
            kwargs={
                'project_id': self.kwargs['project_id']
            }
        )

class RemoveWebResourcePage(WebResourceContext, TemplateView):
    """Remove web resource page."""

    template_name = 'base.html'

    def get(self, request, project_id, webresource_id):
        """
        GET method for removing web resource.

        Parameters
        ----------
        request : django.http.HttpRequest
            Object representing the request.
        project_id : int
            Identifies the project in the database.
        webresource_id : int
            Identifies the web resource in the database.

        Returns
        -------
        django.http.HttpResponseRedirect
            Redirects to all web resources if web resource is removed, single
            web resource page if project is locked.
        django.http.HttpResponse
            Rendered template if project or web resource does not exist.
        """
        context = self.get_context_data(project_id, webresource_id)
        webresource = context.get('webresource')

        if webresource:
            if webresource.project.islocked:
                messages.error(
                    request,
                    'The project is locked. Web resource cannot be removed.'
                )
                return redirect(
                    'geokey_webresources:single_webresource',
                    project_id=project_id,
                    webresource_id=webresource_id
                )
            else:
                webresource.delete()
                messages.success(
                    request,
                    'The web resource has been removed.'
                )
                return redirect(
                    'geokey_webresources:all_webresources',
                    project_id=project_id
                )

        return self.render_to_response(context)

# ###########################
# ADMIN AJAX
# ###########################

class ReorderWebResourcesAjax(APIView):
    """Reorder web resources via Ajax."""

    @handle_exceptions_for_ajax
    def post(self, request, project_id):
        """
        POST method for reordering web resources.

        Parameters
        ----------
        request : rest_framework.request.Request
            Object representing the request.
        project_id : int
            Identifies the project in the database.

        Returns
        -------
        rest_framework.response.Response
            Response to the request.
        """
        project = Project.objects.as_admin(request.user, project_id)

        if project.islocked:
            return Response(
                {'error': 'Project is locked.'},
                status=status.HTTP_403_FORBIDDEN
            )
        elif not project.webresources.exists():
            return Response(
                {'error': 'Project has no web resources.'},
                status=status.HTTP_404_NOT_FOUND
            )

        try:
            webresources = []

            for order, webresource_id in enumerate(request.data.get('order')):
                webresource = project.webresources.get(pk=webresource_id)
                webresource.order = order
                webresources.append(webresource)

            for webresource in webresources:
                webresource.save()

            serializer = WebResourceSerializer(
                project.webresources,
                many=True
            )
            return Response(serializer.data)
        except WebResource.DoesNotExist:
            return Response(
                {'error': 'One or more web resources were not found.'},
                status=status.HTTP_400_BAD_REQUEST
            )

class UpdateWebResourceAjax(APIView):
    """Update web resource via Ajax."""

    @handle_exceptions_for_ajax
    def put(self, request, project_id, webresource_id):
        """
        PUT method for updating web resource.

        Parameters
        ----------
        request : rest_framework.request.Request
            Object representing the request.
        project_id : int
            Identifies the project in the database.
        webresource_id : int
            Identifies the web resource in the database.

        Returns
        -------
        rest_framework.response.Response
            Response to the request.
        """
        project = Project.objects.as_admin(request.user, project_id)

        if project.islocked:
            return Response(
                {'error': 'Project is locked.'},
                status=status.HTTP_403_FORBIDDEN
            )

        try:
            webresource = project.webresources.get(pk=webresource_id)
            serializer = WebResourceSerializer(
                webresource,
                data=request.data,
                partial=True,
                fields=('id', 'status')
            )

            if serializer.is_valid():
                serializer.save()
                return Response(serializer.data)
            else:
                return Response(
                    serializer.errors,
                    status=status.HTTP_400_BAD_REQUEST
                )
        except WebResource.DoesNotExist, error:
            return Response(
                {'error': str(error)},
                status=status.HTTP_404_NOT_FOUND
            )

# ###########################
# PUBLIC API
# ###########################

class AllWebResourcesAPI(APIView):
    """All web resources via API."""

    @handle_exceptions_for_ajax
    def get(self, request, project_id):
        """
        GET method for all web resources of a project.

        Parameters
        ----------
        request : rest_framework.request.Request
            Object representing the request.
        project_id : int
            Identifies the project in the database.

        Returns
        -------
        rest_framework.response.Response
            Response to the request.
        """
        project = Project.objects.get_single(request.user, project_id)
        serializer = WebResourceSerializer(
            project.webresources.filter(status=STATUS.active),
            many=True
        )
        return Response(serializer.data)

class SingleWebResourceAPI(APIView):
    """Single web resource via API."""

    @handle_exceptions_for_ajax
    def get(self, request, project_id, webresource_id):
        """
        GET method for a single web resource of a project.

        Only active web resources are returned to anyone who has access to the
        project.

        Parameters
        ----------
        request : rest_framework.request.Request
            Object representing the request.
        project_id : int
            Identifies the project in the database.
        webresource_id : int
            Identifies the web resource in the database.

        Returns
        -------
        rest_framework.response.Response
            Response to the request.
        """
        project = Project.objects.get_single(request.user, project_id)

        try:
            webresource = project.webresources.get(
                pk=webresource_id,
                status=STATUS.active
            )
            serializer = WebResourceSerializer(webresource)
            return Response(serializer.data)
        except WebResource.DoesNotExist:
            return Response(
                {'error': 'Web resource not found.'},
                status=status.HTTP_404_NOT_FOUND
            )
<commit_msg>get_form() provides a default form class<commit_after>"""All views for the extension."""

# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.core.urlresolvers import reverse
from django.views.generic import CreateView, FormView, TemplateView
from django.shortcuts import redirect
from django.utils.safestring import mark_safe
from django.contrib import messages

from rest_framework import status
from rest_framework.views import APIView
from rest_framework.response import Response

from braces.views import LoginRequiredMixin

from geokey.core.decorators import handle_exceptions_for_ajax
from geokey.projects.models import Project
from geokey.projects.views import ProjectContext

from .helpers.context_helpers import does_not_exist_msg
from .base import STATUS, FORMAT
from .models import WebResource
from .forms import WebResourceForm
from .serializers import WebResourceSerializer

# ###########################
# ADMIN PAGES
# ###########################

class IndexPage(LoginRequiredMixin, TemplateView):
    """Main index page."""

    template_name = 'wr_index.html'

    def get_context_data(self, *args, **kwargs):
        """
        GET method for the template.

        Return the context to render the view. Overwrite the method by adding
        all projects (where user is an administrator) to the context.

        Returns
        -------
        dict
            Context.
        """
        projects = Project.objects.filter(admins=self.request.user)

        return super(IndexPage, self).get_context_data(
            projects=projects,
            *args,
            **kwargs
        )

class AllWebResourcesPage(LoginRequiredMixin, ProjectContext, TemplateView):
    """All web resources page."""

    template_name = 'wr_all_webresources.html'

class AddWebResourcePage(LoginRequiredMixin, ProjectContext, CreateView):
    """Add new web resource page."""

    template_name = 'wr_add_webresource.html'
    form_class = WebResourceForm

    def get_context_data(self, *args, **kwargs):
        """
        GET method for the template.

        Return the context to render the view. Overwrite the method by adding
        project ID and available data formats to the context.

        Returns
        -------
        dict
            Context.
        """
        project_id = self.kwargs['project_id']

        return super(AddWebResourcePage, self).get_context_data(
            project_id,
            data_formats=FORMAT,
            *args,
            **kwargs
        )

    def form_valid(self, form):
        """
        Add web resource when form data is valid.

        Parameters
        ----------
        form : geokey_webresource.forms.WebResourceForm
            Represents the user input.

        Returns
        -------
        django.http.HttpResponse
            Rendered template.
        """
        context = self.get_context_data(form=form)
        project = context.get('project')

        if project:
            if project.islocked:
                messages.error(
                    self.request,
                    'The project is locked. New web resources cannot be added.'
                )
            else:
                form.instance.project = project
                form.instance.creator = self.request.user

                add_another_url = reverse(
                    'geokey_webresources:webresource_add',
                    kwargs={
                        'project_id': project.id
                    }
                )
                messages.success(
                    self.request,
                    mark_safe(
                        'The web resource has been added. Add '
                        'another web resource.' % add_another_url
                    )
                )
                return super(AddWebResourcePage, self).form_valid(form)

        return self.render_to_response(context)

    def form_invalid(self, form):
        """
        Display an error message when form data is invalid.

        Parameters
        ----------
        form : geokey_webresource.forms.WebResourceForm
            Represents the user input.

        Returns
        -------
        dict
            Context.
        """
        messages.error(self.request, 'An error occurred.')
        return self.render_to_response(self.get_context_data(form=form))

    def get_success_url(self):
        """
        Set URL redirection when web resource created successfully.

        Returns
        -------
        str
            URL for redirection.
        """
        return reverse(
            'geokey_webresources:all_webresources',
            kwargs={
                'project_id': self.kwargs['project_id']
            }
        )

class WebResourceContext(LoginRequiredMixin, ProjectContext):
    """Get web resource mixin."""

    def get_context_data(self, project_id, webresource_id, *args, **kwargs):
        """
        GET method for the template.

        Return the context to render the view. Overwrite the method by adding
        a web resource and available status types & data formats to the
        context.

        Parameters
        ----------
        project_id : int
            Identifies the project in the database.
        webresource_id : int
            Identifies the web resource in the database.

        Returns
        -------
        dict
            Context.
        """
        context = super(WebResourceContext, self).get_context_data(
            project_id,
            *args,
            **kwargs
        )

        context['status_types'] = STATUS
        context['data_formats'] = FORMAT

        try:
            context['webresource'] = WebResource.objects.get(
                pk=webresource_id,
                project=context.get('project')
            )

            return context
        except WebResource.DoesNotExist:
            return {
                'error': 'Not found.',
                'error_description': does_not_exist_msg('Web resource')
            }

class SingleWebResourcePage(WebResourceContext, FormView):
    """Single web resource page."""

    template_name = 'wr_single_webresource.html'

    def get_object(self):
        """
        Get and return web resource object.

        Returns
        -------
        geokey_webresource.models.WebResource
            Web resource object.
        """
        try:
            return WebResource.objects.get(
                pk=self.kwargs['webresource_id']
            )
        except WebResource.DoesNotExist:
            return None

    def get_context_data(self, *args, **kwargs):
        """
        GET method for the template.

        Return the context to render the view. Overwrite the method by adding
        project ID and web resource ID to the context.

        Returns
        -------
        dict
            Context.
        """
        project_id = self.kwargs['project_id']
        webresource_id = self.kwargs['webresource_id']

        return super(SingleWebResourcePage, self).get_context_data(
            project_id,
            webresource_id,
            *args,
            **kwargs
        )

    def get_form(self, form_class=WebResourceForm):
        """Attach instance object to form data."""
        return form_class(instance=self.get_object(), **self.get_form_kwargs())

    def form_valid(self, form):
        """
        Update web resource when form data is valid.

        Parameters
        ----------
        form : geokey_webresource.forms.WebResourceForm
            Represents the user input.

        Returns
        -------
        django.http.HttpResponse
            Rendered template.
        """
        context = self.get_context_data(form=form)
        project = context.get('project')

        if project:
            if project.islocked:
                messages.error(
                    self.request,
                    'The project is locked. Web resources cannot be updated.'
                )
            else:
                if self.request.POST.get('symbol_clear') == 'true':
                    form.instance.symbol = None
                form.save()

                messages.success(
                    self.request,
                    mark_safe('The web resource has been updated.')
                )
                return super(SingleWebResourcePage, self).form_valid(form)

        return self.render_to_response(context)

    def form_invalid(self, form):
        """
        Display an error message when form data is invalid.

        Parameters
        ----------
        form : geokey_webresource.forms.WebResourceForm
            Represents the user input.

        Returns
        -------
        dict
            Context.
        """
        messages.error(self.request, 'An error occurred.')
        return self.render_to_response(self.get_context_data(form=form))

    def get_success_url(self):
        """
        Set URL redirection when web resource updated successfully.

        Returns
        -------
        str
            URL for redirection.
        """
        return reverse(
            'geokey_webresources:all_webresources',
            kwargs={
                'project_id': self.kwargs['project_id']
            }
        )

class RemoveWebResourcePage(WebResourceContext, TemplateView):
    """Remove web resource page."""

    template_name = 'base.html'

    def get(self, request, project_id, webresource_id):
        """
        GET method for removing web resource.

        Parameters
        ----------
        request : django.http.HttpRequest
            Object representing the request.
        project_id : int
            Identifies the project in the database.
        webresource_id : int
            Identifies the web resource in the database.

        Returns
        -------
        django.http.HttpResponseRedirect
            Redirects to all web resources if web resource is removed, single
            web resource page if project is locked.
        django.http.HttpResponse
            Rendered template if project or web resource does not exist.
        """
        context = self.get_context_data(project_id, webresource_id)
        webresource = context.get('webresource')

        if webresource:
            if webresource.project.islocked:
                messages.error(
                    request,
                    'The project is locked. Web resource cannot be removed.'
                )
                return redirect(
                    'geokey_webresources:single_webresource',
                    project_id=project_id,
                    webresource_id=webresource_id
                )
            else:
                webresource.delete()
                messages.success(
                    request,
                    'The web resource has been removed.'
                )
                return redirect(
                    'geokey_webresources:all_webresources',
                    project_id=project_id
                )

        return self.render_to_response(context)

# ###########################
# ADMIN AJAX
# ###########################

class ReorderWebResourcesAjax(APIView):
    """Reorder web resources via Ajax."""

    @handle_exceptions_for_ajax
    def post(self, request, project_id):
        """
        POST method for reordering web resources.

        Parameters
        ----------
        request : rest_framework.request.Request
            Object representing the request.
        project_id : int
            Identifies the project in the database.

        Returns
        -------
        rest_framework.response.Response
            Response to the request.
        """
        project = Project.objects.as_admin(request.user, project_id)

        if project.islocked:
            return Response(
                {'error': 'Project is locked.'},
                status=status.HTTP_403_FORBIDDEN
            )
        elif not project.webresources.exists():
            return Response(
                {'error': 'Project has no web resources.'},
                status=status.HTTP_404_NOT_FOUND
            )

        try:
            webresources = []

            for order, webresource_id in enumerate(request.data.get('order')):
                webresource = project.webresources.get(pk=webresource_id)
                webresource.order = order
                webresources.append(webresource)

            for webresource in webresources:
                webresource.save()

            serializer = WebResourceSerializer(
                project.webresources,
                many=True
            )
            return Response(serializer.data)
        except WebResource.DoesNotExist:
            return Response(
                {'error': 'One or more web resources were not found.'},
                status=status.HTTP_400_BAD_REQUEST
            )

class UpdateWebResourceAjax(APIView):
    """Update web resource via Ajax."""

    @handle_exceptions_for_ajax
    def put(self, request, project_id, webresource_id):
        """
        PUT method for updating web resource.

        Parameters
        ----------
        request : rest_framework.request.Request
            Object representing the request.
        project_id : int
            Identifies the project in the database.
        webresource_id : int
            Identifies the web resource in the database.

        Returns
        -------
        rest_framework.response.Response
            Response to the request.
        """
        project = Project.objects.as_admin(request.user, project_id)

        if project.islocked:
            return Response(
                {'error': 'Project is locked.'},
                status=status.HTTP_403_FORBIDDEN
            )

        try:
            webresource = project.webresources.get(pk=webresource_id)
            serializer = WebResourceSerializer(
                webresource,
                data=request.data,
                partial=True,
                fields=('id', 'status')
            )

            if serializer.is_valid():
                serializer.save()
                return Response(serializer.data)
            else:
                return Response(
                    serializer.errors,
                    status=status.HTTP_400_BAD_REQUEST
                )
        except WebResource.DoesNotExist, error:
            return Response(
                {'error': str(error)},
                status=status.HTTP_404_NOT_FOUND
            )

# ###########################
# PUBLIC API
# ###########################

class AllWebResourcesAPI(APIView):
    """All web resources via API."""

    @handle_exceptions_for_ajax
    def get(self, request, project_id):
        """
        GET method for all web resources of a project.

        Parameters
        ----------
        request : rest_framework.request.Request
            Object representing the request.
        project_id : int
            Identifies the project in the database.

        Returns
        -------
        rest_framework.response.Response
            Response to the request.
        """
        project = Project.objects.get_single(request.user, project_id)
        serializer = WebResourceSerializer(
            project.webresources.filter(status=STATUS.active),
            many=True
        )
        return Response(serializer.data)

class SingleWebResourceAPI(APIView):
    """Single web resource via API."""

    @handle_exceptions_for_ajax
    def get(self, request, project_id, webresource_id):
        """
        GET method for a single web resource of a project.

        Only active web resources are returned to anyone who has access to the
        project.

        Parameters
        ----------
        request : rest_framework.request.Request
            Object representing the request.
        project_id : int
            Identifies the project in the database.
        webresource_id : int
            Identifies the web resource in the database.

        Returns
        -------
        rest_framework.response.Response
            Response to the request.
        """
        project = Project.objects.get_single(request.user, project_id)

        try:
            webresource = project.webresources.get(
                pk=webresource_id,
                status=STATUS.active
            )
            serializer = WebResourceSerializer(webresource)
            return Response(serializer.data)
        except WebResource.DoesNotExist:
            return Response(
                {'error': 'Web resource not found.'},
                status=status.HTTP_404_NOT_FOUND
            )
<|endoftext|>


<commit_before>"""
Base classes for Custom Authenticator to use OAuth with JupyterHub

Most of the code c/o Kyle Kelley (@rgbkrk)
"""

import base64
import json
import os
import uuid

from tornado import gen, web
from tornado.log import app_log

from jupyterhub.handlers import BaseHandler
from jupyterhub.auth import Authenticator
from jupyterhub.utils import url_path_join

from traitlets import Unicode, Bool, List

def guess_callback_uri(protocol, host, hub_server_url):
    return '{proto}://{host}{path}'.format(
        proto=protocol,
        host=host,
        path=url_path_join(
            hub_server_url,
            'oauth_callback'
        )
    )

STATE_COOKIE_NAME = 'oauthenticator-state'

def _serialize_state(state):
    """Serialize OAuth state to a base64 string after passing through JSON"""
    json_state = json.dumps(state)
    return base64.urlsafe_b64encode(
        json_state.encode('utf8')
    ).decode('ascii')

def _deserialize_state(b64_state):
    """Deserialize OAuth state as serialized in _serialize_state"""
    if isinstance(b64_state, str):
        b64_state = b64_state.encode('ascii')
    try:
        json_state = base64.urlsafe_b64decode(b64_state).decode('utf8')
    except ValueError:
        app_log.error("Failed to b64-decode state: %r", b64_state)
        return {}
    try:
        return json.loads(json_state)
    except ValueError:
        app_log.error("Failed to json-decode state: %r", json_state)
        return {}

class OAuthLoginHandler(BaseHandler):
    """Base class for OAuth login handler

    Typically subclasses will need
    """

    def set_state_cookie(self, state):
        self.set_secure_cookie(STATE_COOKIE_NAME,
            state, expires_days=1, httponly=True,
        )

    _state = None
    def get_state(self):
        next_url = self.get_argument('next', None)
        if self._state is None:
            self._state = _serialize_state({
                'state_id': uuid.uuid4().hex,
                'next_url': next_url,
            })
        return self._state

    def get(self):
        redirect_uri = self.authenticator.get_callback_url(self)
        self.log.info('OAuth redirect: %r', redirect_uri)
        state = self.get_state()
        self.set_state_cookie(state)
        self.authorize_redirect(
            redirect_uri=redirect_uri,
            client_id=self.authenticator.client_id,
            scope=self.authenticator.scope,
            extra_params={'state': state},
            response_type='code')

class OAuthCallbackHandler(BaseHandler):
    """Basic handler for OAuth callback. Calls authenticator to verify username."""

    _state_cookie = None

    def get_state_cookie(self):
        """Get OAuth state from cookies

        To be compared with the value in redirect URL
        """
        if self._state_cookie is None:
            self._state_cookie = (self.get_secure_cookie(STATE_COOKIE_NAME) or b'').decode('utf8', 'replace')
            self.clear_cookie(STATE_COOKIE_NAME)
        return self._state_cookie

    def get_state_url(self):
        """Get OAuth state from URL parameters

        to be compared with the value in cookies
        """
        return self.get_argument("state")

    def check_state(self):
        """Verify OAuth state

        compare value in cookie with redirect url param
        """
        cookie_state = self.get_state_cookie()
        url_state = self.get_state_url()
        if not cookie_state:
            raise web.HTTPError(400, "OAuth state missing from cookies")
        if not url_state:
            raise web.HTTPError(400, "OAuth state missing from URL")
        if cookie_state != url_state:
            self.log.warning("OAuth state mismatch: %s != %s", cookie_state, url_state)
            raise web.HTTPError(400, "OAuth state mismatch")

    def check_code(self):
        """Check the OAuth code"""
        if not self.get_argument("code", False):
            raise web.HTTPError(400, "OAuth callback made without a code")

    def check_arguments(self):
        """Validate the arguments of the redirect

        Default:

        - check that there's a code
        - check that state matches
        """
        self.check_code()
        self.check_state()

    def get_next_url(self, user=None):
        """Get the redirect target from the state field"""
        state = self.get_state_url()
        if state:
            next_url = _deserialize_state(state).get('next_url')
            if next_url:
                return next_url
        # JupyterHub 0.8 adds default .get_next_url for a fallback
        if hasattr(BaseHandler, 'get_next_url'):
            return super().get_next_url(user)
        return url_path_join(self.hub.server.base_url, 'home')

    @gen.coroutine
    def _login_user_pre_08(self):
        """login_user simplifies the login+cookie+auth_state process in JupyterHub 0.8

        _login_user_07 is for backward-compatibility with JupyterHub 0.7
        """
        user_info = yield self.authenticator.get_authenticated_user(self, None)
        if user_info is None:
            return
        if isinstance(user_info, dict):
            username = user_info['name']
        else:
            username = user_info
        user = self.user_from_username(username)
        self.set_login_cookie(user)
        return user

    if not hasattr(BaseHandler, 'login_user'):
        # JupyterHub 0.7 doesn't have .login_user
        login_user = _login_user_pre_08

    @gen.coroutine
    def get(self):
        self.check_arguments()
        user = yield self.login_user()
        if user is None:
            # todo: custom error page?
            raise web.HTTPError(403)
        self.redirect(self.get_next_url(user))

class OAuthenticator(Authenticator):
    """Base class for OAuthenticators

    Subclasses must override:

    login_service (string identifying the service provider)
    login_handler (likely a subclass of OAuthLoginHandler)
    authenticate (method takes one arg - the request handler handling the oauth callback)
    """

    scope = List(Unicode(), config=True,
        help="""The OAuth scopes to request.
        See the OAuth documentation of your OAuth provider for options.
        For GitHub in particular, you can see github_scopes.md in this repo.
        """
    )

    login_service = 'override in subclass'
    oauth_callback_url = Unicode(
        os.getenv('OAUTH_CALLBACK_URL', ''),
        config=True,
        help="""Callback URL to use.
        Typically `https://{host}/hub/oauth_callback`"""
    )

    client_id_env = ''
    client_id = Unicode(config=True)
    def _client_id_default(self):
        if self.client_id_env:
            client_id = os.getenv(self.client_id_env, '')
            if client_id:
                return client_id
        return os.getenv('OAUTH_CLIENT_ID', '')

    client_secret_env = ''
    client_secret = Unicode(config=True)
    def _client_secret_default(self):
        if self.client_secret_env:
            client_secret = os.getenv(self.client_secret_env, '')
            if client_secret:
                return client_secret
        return os.getenv('OAUTH_CLIENT_SECRET', '')

    validate_server_cert_env = 'OAUTH_TLS_VERIFY'
    validate_server_cert = Bool(config=True)
    def _validate_server_cert_default(self):
        env_value = os.getenv(self.validate_server_cert_env, '')
        if env_value == '0':
            return False
        else:
            return True

    def login_url(self, base_url):
        return url_path_join(base_url, 'oauth_login')

    login_handler = "Specify login handler class in subclass"
    callback_handler = OAuthCallbackHandler

    def get_callback_url(self, handler=None):
        """Get my OAuth redirect URL

        Either from config or guess based on the current request.
        """
        if self.oauth_callback_url:
            return self.oauth_callback_url
        elif handler:
            return guess_callback_uri(
                handler.request.protocol,
                handler.request.host,
                handler.hub.server.base_url
            )
        else:
            raise ValueError("Specify callback oauth_callback_url or give me a handler to guess with")

    def get_handlers(self, app):
        return [
            (r'/oauth_login', self.login_handler),
            (r'/oauth_callback', self.callback_handler),
        ]

    @gen.coroutine
    def authenticate(self, handler, data=None):
        raise NotImplementedError()
<commit_msg>handle common error, error_description params<commit_after>"""
Base classes for Custom Authenticator to use OAuth with JupyterHub

Most of the code c/o Kyle Kelley (@rgbkrk)
"""

import base64
import json
import os
import uuid

from tornado import gen, web
from tornado.log import app_log

from jupyterhub.handlers import BaseHandler
from jupyterhub.auth import Authenticator
from jupyterhub.utils import url_path_join

from traitlets import Unicode, Bool, List

def guess_callback_uri(protocol, host, hub_server_url):
    return '{proto}://{host}{path}'.format(
        proto=protocol,
        host=host,
        path=url_path_join(
            hub_server_url,
            'oauth_callback'
        )
    )

STATE_COOKIE_NAME = 'oauthenticator-state'

def _serialize_state(state):
    """Serialize OAuth state to a base64 string after passing through JSON"""
    json_state = json.dumps(state)
    return base64.urlsafe_b64encode(
        json_state.encode('utf8')
    ).decode('ascii')

def _deserialize_state(b64_state):
    """Deserialize OAuth state as serialized in _serialize_state"""
    if isinstance(b64_state, str):
        b64_state = b64_state.encode('ascii')
    try:
        json_state = base64.urlsafe_b64decode(b64_state).decode('utf8')
    except ValueError:
        app_log.error("Failed to b64-decode state: %r", b64_state)
        return {}
    try:
        return json.loads(json_state)
    except ValueError:
        app_log.error("Failed to json-decode state: %r", json_state)
        return {}

class OAuthLoginHandler(BaseHandler):
    """Base class for OAuth login handler

    Typically subclasses will need
    """

    def set_state_cookie(self, state):
        self.set_secure_cookie(STATE_COOKIE_NAME,
            state, expires_days=1, httponly=True,
        )

    _state = None
    def get_state(self):
        next_url = self.get_argument('next', None)
        if self._state is None:
            self._state = _serialize_state({
                'state_id': uuid.uuid4().hex,
                'next_url': next_url,
            })
        return self._state

    def get(self):
        redirect_uri = self.authenticator.get_callback_url(self)
        self.log.info('OAuth redirect: %r', redirect_uri)
        state = self.get_state()
        self.set_state_cookie(state)
        self.authorize_redirect(
            redirect_uri=redirect_uri,
            client_id=self.authenticator.client_id,
            scope=self.authenticator.scope,
            extra_params={'state': state},
            response_type='code')

class OAuthCallbackHandler(BaseHandler):
    """Basic handler for OAuth callback. Calls authenticator to verify username."""

    _state_cookie = None

    def get_state_cookie(self):
        """Get OAuth state from cookies

        To be compared with the value in redirect URL
        """
        if self._state_cookie is None:
            self._state_cookie = (self.get_secure_cookie(STATE_COOKIE_NAME) or b'').decode('utf8', 'replace')
            self.clear_cookie(STATE_COOKIE_NAME)
        return self._state_cookie

    def get_state_url(self):
        """Get OAuth state from URL parameters

        to be compared with the value in cookies
        """
        return self.get_argument("state")

    def check_state(self):
        """Verify OAuth state

        compare value in cookie with redirect url param
        """
        cookie_state = self.get_state_cookie()
        url_state = self.get_state_url()
        if not cookie_state:
            raise web.HTTPError(400, "OAuth state missing from cookies")
        if not url_state:
            raise web.HTTPError(400, "OAuth state missing from URL")
        if cookie_state != url_state:
            self.log.warning("OAuth state mismatch: %s != %s", cookie_state, url_state)
            raise web.HTTPError(400, "OAuth state mismatch")

    def check_error(self):
        """Check the OAuth code"""
        error = self.get_argument("error", False)
        if error:
            message = self.get_argument("error_description", error)
            raise web.HTTPError(400, "OAuth error: %s" % message)

    def check_code(self):
        """Check the OAuth code"""
        if not self.get_argument("code", False):
            raise web.HTTPError(400, "OAuth callback made without a code")

    def check_arguments(self):
        """Validate the arguments of the redirect

        Default:

        - check for oauth-standard error, error_description arguments
        - check that there's a code
        - check that state matches
        """
        self.check_error()
        self.check_code()
        self.check_state()

    def get_next_url(self, user=None):
        """Get the redirect target from the state field"""
        state = self.get_state_url()
        if state:
            next_url = _deserialize_state(state).get('next_url')
            if next_url:
                return next_url
        # JupyterHub 0.8 adds default .get_next_url for a fallback
        if hasattr(BaseHandler, 'get_next_url'):
            return super().get_next_url(user)
        return url_path_join(self.hub.server.base_url, 'home')

    @gen.coroutine
    def _login_user_pre_08(self):
        """login_user simplifies the login+cookie+auth_state process in JupyterHub 0.8

        _login_user_07 is for backward-compatibility with JupyterHub 0.7
        """
        user_info = yield self.authenticator.get_authenticated_user(self, None)
        if user_info is None:
            return
        if isinstance(user_info, dict):
            username = user_info['name']
        else:
            username = user_info
        user = self.user_from_username(username)
        self.set_login_cookie(user)
        return user

    if not hasattr(BaseHandler, 'login_user'):
        # JupyterHub 0.7 doesn't have .login_user
        login_user = _login_user_pre_08

    @gen.coroutine
    def get(self):
        self.check_arguments()
        user = yield self.login_user()
        if user is None:
            # todo: custom error page?
            raise web.HTTPError(403)
        self.redirect(self.get_next_url(user))

class OAuthenticator(Authenticator):
    """Base class for OAuthenticators

    Subclasses must override:

    login_service (string identifying the service provider)
    login_handler (likely a subclass of OAuthLoginHandler)
    authenticate (method takes one arg - the request handler handling the oauth callback)
    """

    scope = List(Unicode(), config=True,
        help="""The OAuth scopes to request.
        See the OAuth documentation of your OAuth provider for options.
        For GitHub in particular, you can see github_scopes.md in this repo.
        """
    )

    login_service = 'override in subclass'
    oauth_callback_url = Unicode(
        os.getenv('OAUTH_CALLBACK_URL', ''),
        config=True,
        help="""Callback URL to use.
        Typically `https://{host}/hub/oauth_callback`"""
    )

    client_id_env = ''
    client_id = Unicode(config=True)
    def _client_id_default(self):
        if self.client_id_env:
            client_id = os.getenv(self.client_id_env, '')
            if client_id:
                return client_id
        return os.getenv('OAUTH_CLIENT_ID', '')

    client_secret_env = ''
    client_secret = Unicode(config=True)
    def _client_secret_default(self):
        if self.client_secret_env:
            client_secret = os.getenv(self.client_secret_env, '')
            if client_secret:
                return client_secret
        return os.getenv('OAUTH_CLIENT_SECRET', '')

    validate_server_cert_env = 'OAUTH_TLS_VERIFY'
    validate_server_cert = Bool(config=True)
    def _validate_server_cert_default(self):
        env_value = os.getenv(self.validate_server_cert_env, '')
        if env_value == '0':
            return False
        else:
            return True

    def login_url(self, base_url):
        return url_path_join(base_url, 'oauth_login')

    login_handler = "Specify login handler class in subclass"
    callback_handler = OAuthCallbackHandler

    def get_callback_url(self, handler=None):
        """Get my OAuth redirect URL

        Either from config or guess based on the current request.
        """
        if self.oauth_callback_url:
            return self.oauth_callback_url
        elif handler:
            return guess_callback_uri(
                handler.request.protocol,
                handler.request.host,
                handler.hub.server.base_url
            )
        else:
            raise ValueError("Specify callback oauth_callback_url or give me a handler to guess with")

    def get_handlers(self, app):
        return [
            (r'/oauth_login', self.login_handler),
            (r'/oauth_callback', self.callback_handler),
        ]

    @gen.coroutine
    def authenticate(self, handler, data=None):
        raise NotImplementedError()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# title: Parse clusters and find the corresponding spcies
# in the cluster
# author: Peter Thorpe, Leighton Pritchard September 2016.
# The James Hutton Insitute, Dundee, UK.
# imports
import os
import argparse
from math import log
from Bio import SeqIO
import matplotlib
# this code added to prevent this error:
# self.tk = _tkinter.create(screenName, baseName,
# className, interactive, wantobjects, useTk, sync, use)
# _tkinter.TclError: no display name and
# no $DISPLAY environment variable
# Force matplotlib to not use any Xwindows backend.
matplotlib.use('Agg')

import matplotlib.pyplot as plt
import pylab
import numpy as np

# Turn off warning messages
import warnings
warnings.filterwarnings('ignore')

##########################################################
# drawing a histogram
# n, bins, patches = hist(data)

def get_names_from_Seq_db(seq_db):
    """function to get a list of name in the seq db"""
    names = []
    names_abudance_removed = []
    db = open(seq_db, "r")
    for seq_record in SeqIO.parse(db, "fasta"):
        if seq_record.id.endswith("_1"):
            names.append(seq_record.id)
            names_abudance_removed.append(("_").join(
                seq_record.id.split("_")[:-1]))
        else:
            names_abudance_removed.append(seq_record.id)
            names.append(seq_record.id + "_1")
    db.close()
    return names, names_abudance_removed

def parse_line(line):
    """function to parse a given line and return
    tab separated elements"""
    if not line.strip():
        return False  # if the last line is blank
    if line.startswith("#"):
        return False
    cluster_line_split = line.rstrip("\n").split()
    return cluster_line_split

def count_element_in_cluster(cluster_line_split,
                             names,
                             names_abudance_removed):
    """func to count to total memebers in each cluster.
    Returns a dic with """
    species_set = set([])
    # counters to keep track
    members_count = 0
    species_count = 0
    for member in cluster_line_split:
        members_count = members_count + 1
        member = member.rstrip()
        # check if it was a db entry
        if member in names or member in names_abudance_removed:
            if member not in species_set:
                species_count = species_count+1
                species_set.add(member)
        abun_remvd = member.split("_abundance=1")[0]
        if abun_remvd in names or abun_remvd in names_abudance_removed:
            if member not in species_set:
                species_count = species_count + 1
                species_set.add(abun_remvd)
    return members_count, species_count

try:
    # New in Python 3.4
    from statistics import mean
except ImportError:
    def mean(list_of_values):
        """Calculate the mean average of a list of numbers."""
        # Explicit float(...) to allow for Python 2 division.
        return sum(list_of_values) / float(len(list_of_values))

assert mean([1, 2, 3, 4, 5]) == 3

def covert_dict_to_list_of_value(in_dict):
    """function to convert a given dict, to convert it to a
    list of value. In dict should be something like this:
    count_dict= {1: 4, 2: 1}  """
    # need this value to plot the number of histogram bars
    number_of_keys = len(in_dict.keys())
    output_list = []
    key_list = []
    max_val = 0
    vals_for_bar_chart = []
    for key, val in in_dict.items():
        key_list.append(key)
        for i in range(0, val):
            output_list.append(key)
            # get the maximum val for graph
            if val > max_val:
                max_val = val
    for i in range(1, max(key_list)+1):
        try:
            val = in_dict[i]
            vals_for_bar_chart.append(val)
        except KeyError:
            vals_for_bar_chart.append(0)

    return sorted(output_list), number_of_keys, max_val, vals_for_bar_chart

def plot_hitstogram_graph(data_values, title,
                          number_of_keys,
                          max_val,
                          file_in):
    """function to draw a histogram of a given list of values.
    http://matplotlib.org/1.3.0/examples/pylab_examples/
    histogram_demo_extended.html
    https://github.com/widdowquinn/Teaching-Data-Visualisation
    /blob/master/exercises/one_variable_continuous/
    one_variable_continuous.ipynb
    """

    # bins = max(data_values)
    # pylab.hist(data_values, facecolor='blue')
    pylab.hist(data_values, facecolor='green', alpha=0.6)
    pylab.grid(True)
    pylab.title(title + "_histogram")
    pylab.xlabel('number in cluster')
    pylab.ylabel('Count')
    pylab.savefig(file_in + "_" + title + '_histogram.png')
    plt.close()
    pylab.close()
    os.chdir('..')

def plot_individual_bar_chart_graph(data_values, title,
                                    number_of_keys,
                                    max_val,
                                    vals_for_bar_chart,
                                    file_in):
    """function to draw a bar of a given list of values.
    FOR these data this IS the correct type of graph.
    http://matplotlib.org/examples/api/barchart_demo.html
    https://github.com/widdowquinn/
    Teaching-Data-Visualisation/blob/master/
    exercises/one_variable_continuous/
    one_variable_continuous.ipynb
    bar(left, height, width=0.8, bottom=None,
    hold=None, **kwargs)
    """

    n_groups = len(vals_for_bar_chart)
    fig, ax = plt.subplots()
    index = np.arange(n_groups)
    bar_width = 0.9
    opacity = 0.4
    # print vals_for_bar_chart
    rects1 = plt.bar(index,
                     vals_for_bar_chart,
                     bar_width,
                     alpha=opacity,
                     color='b')  # label='whatever'
    plt.xlabel('number in cluster')
    plt.ylabel('Count')
    plt.title(title+"_barchart")
    plt.legend()
    pylab.grid(True)
    ax.set_yscale('symlog')
    ax.set_xscale('symlog')
    plt.tight_layout()
    plt.show()
    pylab.savefig(file_in + "_" + title + '_barchart.png')
    plt.close()
    pylab.close()

def plot_multi_bar_chart_graph(title1, vals_for_bar_chart1,
                               title2, vals_for_bar_chart2,
                               title3, vals_for_bar_chart3,
                               file_in):
    """function to draw a bar of a given list of values.
    FOR these data this IS the correct type of graph.
    http://matplotlib.org/examples/api/barchart_demo.html
    https://github.com/widdowquinn/
    Teaching-Data-Visualisation/blob/master/
    exercises/one_variable_continuous/
    one_variable_continuous.ipynb
    bar(left, height, width=0.8, bottom=None,
    hold=None, **kwargs)
    """
    n1_groups = len(vals_for_bar_chart1)
    n2_groups = len(vals_for_bar_chart2)
    n3_groups = len(vals_for_bar_chart3)
    fig = plt.figure(figsize=(10, 8), dpi=1200)
    #    # Create subplot axes
    ax1 = fig.add_subplot(1, 3, 1)  # 1x3 grid, position 1
    ax2 = fig.add_subplot(1, 3, 2)  # 1x3 grid, position 2
    ax3 = fig.add_subplot(1, 3, 3)  # 1x3 grid, position 3

    index1 = np.arange(n1_groups)
    index2 = np.arange(n2_groups)
    index3 = np.arange(n3_groups)
    # print (index)
    bar_width = 0.9
    opacity = 0.6

    # graph1
    rects1 = ax1.bar(index1, vals_for_bar_chart1, bar_width,
                     alpha=opacity, color='green')  # label='whatever'
    ax1.set_xlabel('log number in cluster')
    ax1.set_ylabel('log Count')
    ax1.set_yscale('symlog')
    ax1.set_xscale('symlog')
    ax1.grid(True)
    ax1.set_title(title1)

    # graph 2
    rects2 = ax2.bar(index2, vals_for_bar_chart2, bar_width,
                     alpha=opacity, color='blue')  # label='whatever'
    ax2.set_xlabel('log number in cluster')
    ax2.set_ylabel('log Count')
    ax2.set_yscale('symlog')
    ax2.set_xscale('symlog')
    ax2.grid(True)
    ax2.set_title(title2)

    # graph 3
    rects3 = ax3.bar(index3, vals_for_bar_chart3, bar_width,
                     alpha=opacity, color='pink')  # label='whatever'
    ax3.set_xlabel('cluster number')
    ax3.set_ylabel('Number of sequences in cluster')
    pylab.grid(True)
    ax3.set_title(title3 + "_barchart")
    fig.tight_layout()
    fig
    pylab.savefig(file_in + '_barchart.png')
    pylab.close()

def parse_tab_file_get_clusters(filename1,
                                seq_db):
    """#script to open up a tab separeted clustering
    output and identify the
    species in the clustering """
    cluster_file = open(filename1, "r")
    # dictionaries for keeping the counts
    member_in_cluster_to_count_dict = dict()
    species_in_cluster_count_dict = dict()
    names, names_abudance_removed = get_names_from_Seq_db(seq_db)

    # a way of keeping track of the iteration
    interation_count = int(0)
    # iterate through the file
    for line in cluster_file:
        interation_count += 1
        # call the func to split up the line
        cluster_line_split = parse_line(line.rstrip())
        if not cluster_line_split:
            # this could be a blank line or starts with #
            continue
        # call the function to get the number of
        # elements and species.
        members_count, \
            species_count = count_element_in_cluster(cluster_line_split,
                                                     names,
                                                     names_abudance_removed)
        try:
            # if we have seen this count before,
            # then just add one to it.
            member_in_cluster_to_count_dict[members_count] += 1
        except KeyError:
            # not seen this before, set up a new dic element
            # and make the equal 1
            member_in_cluster_to_count_dict[members_count] = 1
        try:
            # if we have seen this count of species before,
            # then just add one to it.
            species_in_cluster_count_dict[species_count] += 1
        except KeyError:
            species_in_cluster_count_dict[species_count] = 1
    species_in_cluster_list, species_number_of_keys, species_max_val, \
        species_vals_for_bar_chart = covert_dict_to_list_of_value(
            species_in_cluster_count_dict)

    # print ("member_in_cluster_to_count_dict: ",
    #        member_in_cluster_to_count_dict)
    member_in_cluster_list, member_number_of_keys, member_max_val, \
        member_vals_for_bar_chart = covert_dict_to_list_of_value(
            member_in_cluster_to_count_dict)
    # plot_multi_bar_chart_graph
    plot_multi_bar_chart_graph("Barchart: database species clusters",
                               species_vals_for_bar_chart,
                               "Barchart: total members in all cluster",
                               member_vals_for_bar_chart,
                               "Barchart: cluster size",
                               member_vals_for_bar_chart,
                               filename1)

#############################################################################

# to run the script

usage = """usage :

script to graphically represent clustering output.

python draws_bar...py -i clustering_file -o summarise_clusters.out

requires:
use pip install ...

 matplotlib
 numpy
 Biopython
"""

def get_args():
    parser = argparse.ArgumentParser(description="draw bar charts of " +
                                     " clusters :  %s " % usage,
                                     add_help=False)
    optional = parser.add_argument_group('optional arguments')
    optional.add_argument("-i", "--in", dest="in_file",
                          default=None,
                          help="clustering out file")

    optional.add_argument("--db", dest="seq_db",
                          default=None,
                          help="db used for clustering")
    optional.add_argument("--heatmap", dest="heatmap",
                          default=False,
                          help="draw a heatmap of the species clustering " +
                          " currently not implemented")
    args = parser.parse_args()
    return args

args = get_args()
in_file = args.in_file
heatmap = args.heatmap
seq_db = args.seq_db

# run the program
parse_tab_file_get_clusters(in_file, seq_db)
<commit_msg>removed unused imports<commit_after>#!/usr/bin/env python
# title: Parse clusters and find the corresponding spcies
# in the cluster
# author: Peter Thorpe, Leighton Pritchard September 2016.
# The James Hutton Insitute, Dundee, UK.
# imports
import os
import argparse
from Bio import SeqIO
import matplotlib
# this code added to prevent this error:
# self.tk = _tkinter.create(screenName, baseName,
# className, interactive, wantobjects, useTk, sync, use)
# _tkinter.TclError: no display name and
# no $DISPLAY environment variable
# Force matplotlib to not use any Xwindows backend.
matplotlib.use('Agg')

import matplotlib.pyplot as plt
import pylab
import numpy as np

# Turn off warning messages
import warnings
warnings.filterwarnings('ignore')

##########################################################
# drawing a histogram
# n, bins, patches = hist(data)

def get_names_from_Seq_db(seq_db):
    """function to get a list of name in the seq db"""
    names = []
    names_abudance_removed = []
    db = open(seq_db, "r")
    for seq_record in SeqIO.parse(db, "fasta"):
        if seq_record.id.endswith("_1"):
            names.append(seq_record.id)
            names_abudance_removed.append(("_").join(
                seq_record.id.split("_")[:-1]))
        else:
            names_abudance_removed.append(seq_record.id)
            names.append(seq_record.id + "_1")
    db.close()
    return names, names_abudance_removed

def parse_line(line):
    """function to parse a given line and return
    tab separated elements"""
    if not line.strip():
        return False  # if the last line is blank
    if line.startswith("#"):
        return False
    cluster_line_split = line.rstrip("\n").split()
    return cluster_line_split

def count_element_in_cluster(cluster_line_split,
                             names,
                             names_abudance_removed):
    """func to count to total memebers in each cluster.
    Returns a dic with """
    species_set = set([])
    # counters to keep track
    members_count = 0
    species_count = 0
    for member in cluster_line_split:
        members_count = members_count + 1
        member = member.rstrip()
        # check if it was a db entry
        if member in names or member in names_abudance_removed:
            if member not in species_set:
                species_count = species_count+1
                species_set.add(member)
        abun_remvd = member.split("_abundance=1")[0]
        if abun_remvd in names or abun_remvd in names_abudance_removed:
            if member not in species_set:
                species_count = species_count + 1
                species_set.add(abun_remvd)
    return members_count, species_count

try:
    # New in Python 3.4
    from statistics import mean
except ImportError:
    def mean(list_of_values):
        """Calculate the mean average of a list of numbers."""
        # Explicit float(...) to allow for Python 2 division.
        return sum(list_of_values) / float(len(list_of_values))

assert mean([1, 2, 3, 4, 5]) == 3

def covert_dict_to_list_of_value(in_dict):
    """function to convert a given dict, to convert it to a
    list of value. In dict should be something like this:
    count_dict= {1: 4, 2: 1}  """
    # need this value to plot the number of histogram bars
    number_of_keys = len(in_dict.keys())
    output_list = []
    key_list = []
    max_val = 0
    vals_for_bar_chart = []
    for key, val in in_dict.items():
        key_list.append(key)
        for i in range(0, val):
            output_list.append(key)
            # get the maximum val for graph
            if val > max_val:
                max_val = val
    for i in range(1, max(key_list)+1):
        try:
            val = in_dict[i]
            vals_for_bar_chart.append(val)
        except KeyError:
            vals_for_bar_chart.append(0)

    return sorted(output_list), number_of_keys, max_val, vals_for_bar_chart

def plot_hitstogram_graph(data_values, title,
                          number_of_keys,
                          max_val,
                          file_in):
    """function to draw a histogram of a given list of values.
    http://matplotlib.org/1.3.0/examples/pylab_examples/
    histogram_demo_extended.html
    https://github.com/widdowquinn/Teaching-Data-Visualisation
    /blob/master/exercises/one_variable_continuous/
    one_variable_continuous.ipynb
    """

    # bins = max(data_values)
    # pylab.hist(data_values, facecolor='blue')
    pylab.hist(data_values, facecolor='green', alpha=0.6)
    pylab.grid(True)
    pylab.title(title + "_histogram")
    pylab.xlabel('number in cluster')
    pylab.ylabel('Count')
    pylab.savefig(file_in + "_" + title + '_histogram.png')
    plt.close()
    pylab.close()
    os.chdir('..')

def plot_individual_bar_chart_graph(data_values, title,
                                    number_of_keys,
                                    max_val,
                                    vals_for_bar_chart,
                                    file_in):
    """function to draw a bar of a given list of values.
    FOR these data this IS the correct type of graph.
    http://matplotlib.org/examples/api/barchart_demo.html
    https://github.com/widdowquinn/
    Teaching-Data-Visualisation/blob/master/
    exercises/one_variable_continuous/
    one_variable_continuous.ipynb
    bar(left, height, width=0.8, bottom=None,
    hold=None, **kwargs)
    """

    n_groups = len(vals_for_bar_chart)
    fig, ax = plt.subplots()
    index = np.arange(n_groups)
    bar_width = 0.9
    opacity = 0.4
    # print vals_for_bar_chart
    rects1 = plt.bar(index,
                     vals_for_bar_chart,
                     bar_width,
                     alpha=opacity,
                     color='b')  # label='whatever'
    plt.xlabel('number in cluster')
    plt.ylabel('Count')
    plt.title(title+"_barchart")
    plt.legend()
    pylab.grid(True)
    ax.set_yscale('symlog')
    ax.set_xscale('symlog')
    plt.tight_layout()
    plt.show()
    pylab.savefig(file_in + "_" + title + '_barchart.png')
    plt.close()
    pylab.close()

def plot_multi_bar_chart_graph(title1, vals_for_bar_chart1,
                               title2, vals_for_bar_chart2,
                               title3, vals_for_bar_chart3,
                               file_in):
    """function to draw a bar of a given list of values.
    FOR these data this IS the correct type of graph.
    http://matplotlib.org/examples/api/barchart_demo.html
    https://github.com/widdowquinn/
    Teaching-Data-Visualisation/blob/master/
    exercises/one_variable_continuous/
    one_variable_continuous.ipynb
    bar(left, height, width=0.8, bottom=None,
    hold=None, **kwargs)
    """
    n1_groups = len(vals_for_bar_chart1)
    n2_groups = len(vals_for_bar_chart2)
    n3_groups = len(vals_for_bar_chart3)
    fig = plt.figure(figsize=(10, 8), dpi=1200)
    #    # Create subplot axes
    ax1 = fig.add_subplot(1, 3, 1)  # 1x3 grid, position 1
    ax2 = fig.add_subplot(1, 3, 2)  # 1x3 grid, position 2
    ax3 = fig.add_subplot(1, 3, 3)  # 1x3 grid, position 3

    index1 = np.arange(n1_groups)
    index2 = np.arange(n2_groups)
    index3 = np.arange(n3_groups)
    # print (index)
    bar_width = 0.9
    opacity = 0.6

    # graph1
    rects1 = ax1.bar(index1, vals_for_bar_chart1, bar_width,
                     alpha=opacity, color='green')  # label='whatever'
    ax1.set_xlabel('log number in cluster')
    ax1.set_ylabel('log Count')
    ax1.set_yscale('symlog')
    ax1.set_xscale('symlog')
    ax1.grid(True)
    ax1.set_title(title1)

    # graph 2
    rects2 = ax2.bar(index2, vals_for_bar_chart2, bar_width,
                     alpha=opacity, color='blue')  # label='whatever'
    ax2.set_xlabel('log number in cluster')
    ax2.set_ylabel('log Count')
    ax2.set_yscale('symlog')
    ax2.set_xscale('symlog')
    ax2.grid(True)
    ax2.set_title(title2)

    # graph 3
    rects3 = ax3.bar(index3, vals_for_bar_chart3, bar_width,
                     alpha=opacity, color='pink')  # label='whatever'
    ax3.set_xlabel('cluster number')
    ax3.set_ylabel('Number of sequences in cluster')
    pylab.grid(True)
    ax3.set_title(title3 + "_barchart")
    fig.tight_layout()
    fig
    pylab.savefig(file_in + '_barchart.png')
    pylab.close()

def parse_tab_file_get_clusters(filename1,
                                seq_db):
    """#script to open up a tab separeted clustering
    output and identify the
    species in the clustering """
    cluster_file = open(filename1, "r")
    # dictionaries for keeping the counts
    member_in_cluster_to_count_dict = dict()
    species_in_cluster_count_dict = dict()
    names, names_abudance_removed = get_names_from_Seq_db(seq_db)

    # a way of keeping track of the iteration
    interation_count = int(0)
    # iterate through the file
    for line in cluster_file:
        interation_count += 1
        # call the func to split up the line
        cluster_line_split = parse_line(line.rstrip())
        if not cluster_line_split:
            # this could be a blank line or starts with #
            continue
        # call the function to get the number of
        # elements and species.
        members_count, \
            species_count = count_element_in_cluster(cluster_line_split,
                                                     names,
                                                     names_abudance_removed)
        try:
            # if we have seen this count before,
            # then just add one to it.
            member_in_cluster_to_count_dict[members_count] += 1
        except KeyError:
            # not seen this before, set up a new dic element
            # and make the equal 1
            member_in_cluster_to_count_dict[members_count] = 1
        try:
            # if we have seen this count of species before,
            # then just add one to it.
            species_in_cluster_count_dict[species_count] += 1
        except KeyError:
            species_in_cluster_count_dict[species_count] = 1
    species_in_cluster_list, species_number_of_keys, species_max_val, \
        species_vals_for_bar_chart = covert_dict_to_list_of_value(
            species_in_cluster_count_dict)

    # print ("member_in_cluster_to_count_dict: ",
    #        member_in_cluster_to_count_dict)
    member_in_cluster_list, member_number_of_keys, member_max_val, \
        member_vals_for_bar_chart = covert_dict_to_list_of_value(
            member_in_cluster_to_count_dict)
    # plot_multi_bar_chart_graph
    plot_multi_bar_chart_graph("Barchart: database species clusters",
                               species_vals_for_bar_chart,
                               "Barchart: total members in all cluster",
                               member_vals_for_bar_chart,
                               "Barchart: cluster size",
                               member_vals_for_bar_chart,
                               filename1)

#############################################################################

# to run the script

usage = """usage :

script to graphically represent clustering output.

python draws_bar...py -i clustering_file -o summarise_clusters.out

requires:
use pip install ...

 matplotlib
 numpy
 Biopython
"""

def get_args():
    parser = argparse.ArgumentParser(description="draw bar charts of " +
                                     " clusters :  %s " % usage,
                                     add_help=False)
    optional = parser.add_argument_group('optional arguments')
    optional.add_argument("-i", "--in", dest="in_file",
                          default=None,
                          help="clustering out file")

    optional.add_argument("--db", dest="seq_db",
                          default=None,
                          help="db used for clustering")
    optional.add_argument("--heatmap", dest="heatmap",
                          default=False,
                          help="draw a heatmap of the species clustering " +
                          " currently not implemented")
    args = parser.parse_args()
    return args

args = get_args()
in_file = args.in_file
heatmap = args.heatmap
seq_db = args.seq_db

# run the program
parse_tab_file_get_clusters(in_file, seq_db)
<|endoftext|>


<commit_before>import string
from org.geotools.feature.simple import SimpleFeatureBuilder
from org.opengis.feature import Feature as _Feature
from geoscript import core, geom

class Feature(object):
  """
  An object composed of a set of named attributes with values.

  A feature is constructed from a ``dict`` of name value pairs and an optional identifier.

  >>> f = Feature({ 'name': 'anvil', 'price': 100.0 }, 'widgets.1')
  >>> str(f.get('name'))
  'anvil'

  A feature can also be constructed optionally with a :class:`Schema`.

  >>> from schema import Schema
  >>> s = Schema('widgets', [('name', str), ('price', float)])
  >>> f = Feature({'name': 'anvil'}, '1', s)
  >>> f
  widgets.1 {name: anvil, price: None}

  When *schema* is specified feature values can be passed a ``list``.

  >>> s = Schema('widgets', [('name', str), ('price', float)])
  >>> f = Feature(['anvil', 100.0], '1', s)
  >>> f
  widgets.1 {name: anvil, price: 100.0}
  """
  def __init__(self, atts=None, id=None, schema=None, f=None):

    from schema import Schema
    if atts:
      # attributes specified directly

      # if list specified assume values in order of schema
      if isinstance(atts, list):
        if not schema:
          raise Exception('Values may be specified as list only when schema is supplied')

        natts = {}
        for i in range(len(atts)):
          natts[schema.fields[i].name] = atts[i]
        atts = natts

      # generate feature type if necessary
      self.schema = schema
      if not self.schema:
        self.schema = Schema('feature', [(att, type(val)) for att,val in atts.iteritems()])

      # generate feature
      b = SimpleFeatureBuilder(self.schema._type)
      for att, val in atts.iteritems():
        b.set(att, val)

      self._feature = b.buildFeature(str(id) if id else None)

    elif f:
      # feature specififed directly
      self._feature = f
      self.schema = schema if schema else Schema(ft=f.type)

    else:
      raise Exception('No attributes specified for feature')

  def getid(self):
    return self._feature.identifier.toString()

  id = property(getid, None)
  """
  Identifier of the feature as a ``str``

  >>> f = Feature({'name': 'anvil'}, 'widgets.1')
  >>> f.id
  'widgets.1'
  """

  def getgeom(self):
    return core.map(self._feature.defaultGeometry)

  def setgeom(self, g):
    self._feature.defaultGeometry = g

  geom = property(getgeom, setgeom)
  """
  The geometry of the feature.

  >>> import geom
  >>> f = Feature({'geom': geom.Point(1,1)})
  >>> f.geom
  POINT (1 1)
  >>> f.geom = geom.Point(2,2)
  >>> f.geom
  POINT (2 2)
  """

  def getbounds(self):
    bounds = geom.Bounds()
    for val in self._feature.getAttributes():
      if val is not None and isinstance(val, geom.Geometry):
        bounds.expand(val.envelopeInternal)

    if not bounds.isNull():
      return bounds

  bounds = property(getbounds)
  """
  The :class:`Bounds <geoscript.geom.bounds.Bounds>` of the feature geometry.
  Will return ``None`` if the feature does not contain any geometric attributes.
  """

  def get(self, name):
    """
    Returns a feature attribute value by name. ``KeyError`` is thrown if the attribute does not exist.

    *name* is the name of the attribute whose value to return.

    >>> f = Feature({'name': 'anvil', 'price': 100.0})
    >>> str(f.get('name'))
    'anvil'
    """
    self.schema.get(name)
    return self._feature.getAttribute(name)

  def set(self, name, value):
    """
    Sets a feature attribute value by name. ``KeyError`` is thrown is the attribute does not exist.

    *name* is the name of the attribute whose value to set.

    *value* is the new attribute value.

    >>> f = Feature({'name': 'anvil', 'price': 100.0})
    >>> str(f.get('name'))
    'anvil'
    >>> f.set('name', 'mallet')
    >>> str(f.get('name'))
    'mallet'
    """

    self.schema.get(name)
    self._feature.setAttribute(name, value)

  def getattributes(self):
    atts = {}
    for fld in self.schema.fields:
      atts[fld.name] = core.map(self._feature.getAttribute(fld.name))

    return atts

  attributes = property(getattributes, None)
  """
  A ``dict`` of name, value for the attributes of the feature.

  >>> f = Feature({'name': 'anvil', 'price': 100.0})
  >>> atts = f.attributes
  >>> str(atts['name'])
  'anvil'
  >>> atts['price']
  100.0
  """

  def __getitem__(self, key):
    return self.get(key)

  def __setitem__(self, key, value):
    self.set(key, value)

  def __iter__(self):
    return self.schema.__iter__()

  def iterkeys(self):
    return self.__iter__()

  def iteritems(self):
    return self.attributes.iteritems()

  def keys(self):
    return [f.name for f in self.schema.fields]

  def values(self):
    return [core.map(val) for val in self._feature.getAttributes()]

  def __repr__(self):
    atts = ['%s: %s' % (fld.name, self.get(fld.name)) for fld in self.schema.fields]

    id = self.id if self.id.startswith(self.schema.name) else '%s.%s' % (self.schema.name, self.id)
    return '%s {%s}' % (id, string.join(atts,', '))

  def __eq__(self, other):
    return other and self._feature == other._feature

core.registerTypeMapping(_Feature, Feature, lambda x: Feature(f=x))
core.registerTypeUnmapping(Feature, _Feature, lambda x: x._feature)
<commit_msg>Updated Feature.bounds to use underlying feature bounds.<commit_after>import string
from org.geotools.feature.simple import SimpleFeatureBuilder
from org.opengis.feature import Feature as _Feature
from geoscript import core, geom

class Feature(object):
  """
  An object composed of a set of named attributes with values.

  A feature is constructed from a ``dict`` of name value pairs and an optional identifier.

  >>> f = Feature({ 'name': 'anvil', 'price': 100.0 }, 'widgets.1')
  >>> str(f.get('name'))
  'anvil'

  A feature can also be constructed optionally with a :class:`Schema`.

  >>> from schema import Schema
  >>> s = Schema('widgets', [('name', str), ('price', float)])
  >>> f = Feature({'name': 'anvil'}, '1', s)
  >>> f
  widgets.1 {name: anvil, price: None}

  When *schema* is specified feature values can be passed a ``list``.

  >>> s = Schema('widgets', [('name', str), ('price', float)])
  >>> f = Feature(['anvil', 100.0], '1', s)
  >>> f
  widgets.1 {name: anvil, price: 100.0}
  """
  def __init__(self, atts=None, id=None, schema=None, f=None):

    from schema import Schema
    if atts:
      # attributes specified directly

      # if list specified assume values in order of schema
      if isinstance(atts, list):
        if not schema:
          raise Exception('Values may be specified as list only when schema is supplied')

        natts = {}
        for i in range(len(atts)):
          natts[schema.fields[i].name] = atts[i]
        atts = natts

      # generate feature type if necessary
      self.schema = schema
      if not self.schema:
        self.schema = Schema('feature', [(att, type(val)) for att,val in atts.iteritems()])

      # generate feature
      b = SimpleFeatureBuilder(self.schema._type)
      for att, val in atts.iteritems():
        b.set(att, val)

      self._feature = b.buildFeature(str(id) if id else None)

    elif f:
      # feature specififed directly
      self._feature = f
      self.schema = schema if schema else Schema(ft=f.type)

    else:
      raise Exception('No attributes specified for feature')

  def getid(self):
    return self._feature.identifier.toString()

  id = property(getid, None)
  """
  Identifier of the feature as a ``str``

  >>> f = Feature({'name': 'anvil'}, 'widgets.1')
  >>> f.id
  'widgets.1'
  """

  def getgeom(self):
    return core.map(self._feature.defaultGeometry)

  def setgeom(self, g):
    self._feature.defaultGeometry = g

  geom = property(getgeom, setgeom)
  """
  The geometry of the feature.

  >>> import geom
  >>> f = Feature({'geom': geom.Point(1,1)})
  >>> f.geom
  POINT (1 1)
  >>> f.geom = geom.Point(2,2)
  >>> f.geom
  POINT (2 2)
  """

  def getbounds(self):
    env = self._feature.getBounds()
    if env is not None and not env.isNull():
      return geom.Bounds(env=env)

  bounds = property(getbounds)
  """
  The :class:`Bounds <geoscript.geom.bounds.Bounds>` of the feature geometry.
  Will return ``None`` if the feature does not contain any geometric attributes.
  """

  def get(self, name):
    """
    Returns a feature attribute value by name. ``KeyError`` is thrown if the attribute does not exist.

    *name* is the name of the attribute whose value to return.

    >>> f = Feature({'name': 'anvil', 'price': 100.0})
    >>> str(f.get('name'))
    'anvil'
    """
    self.schema.get(name)
    return self._feature.getAttribute(name)

  def set(self, name, value):
    """
    Sets a feature attribute value by name. ``KeyError`` is thrown is the attribute does not exist.

    *name* is the name of the attribute whose value to set.

    *value* is the new attribute value.

    >>> f = Feature({'name': 'anvil', 'price': 100.0})
    >>> str(f.get('name'))
    'anvil'
    >>> f.set('name', 'mallet')
    >>> str(f.get('name'))
    'mallet'
    """

    self.schema.get(name)
    self._feature.setAttribute(name, value)

  def getattributes(self):
    atts = {}
    for fld in self.schema.fields:
      atts[fld.name] = core.map(self._feature.getAttribute(fld.name))

    return atts

  attributes = property(getattributes, None)
  """
  A ``dict`` of name, value for the attributes of the feature.

  >>> f = Feature({'name': 'anvil', 'price': 100.0})
  >>> atts = f.attributes
  >>> str(atts['name'])
  'anvil'
  >>> atts['price']
  100.0
  """

  def __getitem__(self, key):
    return self.get(key)

  def __setitem__(self, key, value):
    self.set(key, value)

  def __iter__(self):
    return self.schema.__iter__()

  def iterkeys(self):
    return self.__iter__()

  def iteritems(self):
    return self.attributes.iteritems()

  def keys(self):
    return [f.name for f in self.schema.fields]

  def values(self):
    return [core.map(val) for val in self._feature.getAttributes()]

  def __repr__(self):
    atts = ['%s: %s' % (fld.name, self.get(fld.name)) for fld in self.schema.fields]

    id = self.id if self.id.startswith(self.schema.name) else '%s.%s' % (self.schema.name, self.id)
    return '%s {%s}' % (id, string.join(atts,', '))

  def __eq__(self, other):
    return other and self._feature == other._feature

core.registerTypeMapping(_Feature, Feature, lambda x: Feature(f=x))
core.registerTypeUnmapping(Feature, _Feature, lambda x: x._feature)
<|endoftext|>