<commit_before># -*- coding: utf-8 -*-
"""Custom meta-class and MethodView for Swagger"""

import os
import logging
import yaml
from flask import current_app
from typing import Pattern
from importlib import import_module
from flask.views import MethodViewType
from flasgger.marshmallow_apispec import SwaggerView as OriginalSwaggerView
from flasgger.marshmallow_apispec import schema2jsonschema
from marshmallow_mongoengine import ModelSchema
from flask_mongorest.views import ResourceView
from mongoengine.queryset.visitor import Q
from werkzeug.exceptions import Unauthorized
from mpcontribs.api.config import SWAGGER

logger = logging.getLogger("app")

def get_specs(klass, method, collection):
    method_name = (
        method.__name__ if getattr(method, "__name__", None) is not None else method
    )
    default_response = {
        "description": "Error",
        "schema": {"type": "object", "properties": {"error": {"type": "string"}}},
    }
    fields_param = None
    if klass.resource.fields is not None:
        fields_avail = (
            klass.resource.fields + klass.resource.get_optional_fields() + ["_all"]
        )
        description = f"List of fields to include in response ({fields_avail})."
        description += " Use dot-notation for nested subfields."
        fields_param = {
            "name": "_fields",
            "in": "query",
            "default": klass.resource.fields,
            "type": "array",
            "items": {"type": "string"},
            "description": description,
        }

    field_pagination_params = []
    for field, limits in klass.resource.fields_to_paginate.items():
        field_pagination_params.append(
            {
                "name": f"{field}_page",
                "in": "query",
                "default": 1,
                "type": "integer",
                "description": f"page to retrieve for {field} field",
            }
        )
        field_pagination_params.append(
            {
                "name": f"{field}_per_page",
                "in": "query",
                "default": limits[0],
                "maximum": limits[1],
                "type": "integer",
                "description": f"number of items to retrieve per page for {field} field",
            }
        )

    limit_params = [
        {
            "name": "_skip",
            "in": "query",
            "type": "integer",
            "description": "number of items to skip",
        },
        {
            "name": "_limit",
            "in": "query",
            "type": "integer",
            "description": "maximum number of items to return",
        },
        {
            "name": "page",
            "in": "query",
            "type": "integer",
            "description": "page number to return (in batches of `per_page/_limit`; alternative to `_skip`)",
        },
        {
            "name": "per_page",
            "in": "query",
            "type": "integer",
            "description": "maximum number of items to return per page (same as `_limit`)",
        },
    ]

    filter_params = []
    if getattr(klass.resource, "filters", None) is not None:
        for k, v in klass.resource.filters.items():
            label = k.pattern if isinstance(k, Pattern) else k
            for op in v:
                filter_params.append(
                    {
                        "name": label if op.op == "exact" else f"{label}__{op.op}",
                        "in": "query",
                        "type": op.typ,
                        "description": f"filter {label}"
                        if op.op == "exact"
                        else f"filter {label} via ${op.op}",
                    }
                )
                if op.typ == "array":
                    filter_params[-1]["items"] = {"type": "string"}

    order_params = []
    if klass.resource.allowed_ordering:
        allowed_ordering = [
            o.pattern if isinstance(o, Pattern) else o
            for o in klass.resource.allowed_ordering
        ]
        order_params = [
            {
                "name": "_order_by",
                "in": "query",
                "type": "string",
                "description": f"order {collection} via {allowed_ordering}",
            },
            {
                "name": "order",
                "in": "query",
                "type": "string",
                "description": f"order {collection} *asc* or *desc*",
            },
        ]

    spec = None
    if method_name == "Fetch":
        params = [
            {
                "name": "pk",
                "in": "path",
                "type": "string",
                "required": True,
                "description": f"{collection[:-1]} (primary key)",
            }
        ]
        if fields_param is not None:
            params.append(fields_param)
        params += field_pagination_params
        spec = {
            "summary": f"Retrieve a {collection[:-1]}.",
            "operationId": "get_entry",
            "parameters": params,
            "responses": {
                200: {
                    "description": f"single {collection} entry",
                    "schema": {"$ref": f"#/definitions/{klass.schema_name}"},
                },
                "default": default_response,
            },
        }

    elif method_name == "BulkFetch":
        params = [fields_param] if fields_param is not None else []
        params += field_pagination_params
        params += order_params
        params += filter_params
        schema_props = {
            "data": {
                "type": "array",
                "items": {"$ref": f"#/definitions/{klass.schema_name}"},
            }
        }
        if klass.resource.paginate:
            schema_props["has_more"] = {"type": "boolean"}
            schema_props["total_count"] = {"type": "integer"}
            schema_props["total_pages"] = {"type": "integer"}
            params += limit_params
        spec = {
            "summary": f"Filter and retrieve {collection}.",
            "operationId": "get_entries",
            "parameters": params,
            "responses": {
                200: {
                    "description": f"list of {collection}",
                    "schema": {"type": "object", "properties": schema_props},
                },
                "default": default_response,
            },
        }

    elif method_name == "Download":
        params = [
            {
                "name": "short_mime",
                "in": "path",
                "type": "string",
                "required": True,
                "description": f"MIME Download Type: gz",
                "default": "gz",
            },
            {
                "name": "format",
                "in": "query",
                "type": "string",
                "required": True,
                "description": f"download {collection} in different formats: {klass.resource.download_formats}",
            },
        ]
        params += [fields_param] if fields_param is not None else []
        params += order_params
        params += filter_params
        spec = {
            "summary": f"Filter and download {collection}.",
            "operationId": "download_entries",
            "parameters": params,
            "produces": ["application/gzip"],
            "responses": {
                200: {
                    "description": f"{collection} download",
                    "schema": {"type": "file"},
                },
                "default": default_response,
            },
        }

    elif method_name == "Create":
        spec = {
            "summary": f"Create a new {collection[:-1]}.",
            "operationId": "create_entry",
            "parameters": [
                {
                    "name": f"{collection[:-1]}",
                    "in": "body",
                    "description": f"The object to use for {collection[:-1]} creation",
                    "schema": {"$ref": f"#/definitions/{klass.schema_name}"},
                }
            ],
            "responses": {
                200: {
                    "description": f"{collection[:-1]} created",
                    "schema": {"$ref": f"#/definitions/{klass.schema_name}"},
                },
                "default": default_response,
            },
        }

    elif method_name == "BulkCreate":
        spec = {
            "summary": f"Create new {collection[:-1]}(s).",
            "operationId": "create_entries",
            "parameters": [
                {
                    "name": f"{collection}",
                    "in": "body",
                    "description": f"The objects to use for {collection[:-1]} creation",
                    "schema": {
                        "type": "array",
                        "items": {"$ref": f"#/definitions/{klass.schema_name}"},
                    },
                }
            ],
            "responses": {
                200: {
                    "description": f"{collection} created",
                    "schema": {
                        "type": "object",
                        "properties": {
                            "count": {"type": "integer"},
                            "data": {
                                "type": "array",
                                "items": {"$ref": f"#/definitions/{klass.schema_name}"},
                            },
                        },
                    },
                },
                "default": default_response,
            },
        }

    elif method_name == "Update":
        spec = {
            "summary": f"Update a {collection[:-1]}.",
            "operationId": "update_entry",
            "parameters": [
                {
                    "name": "pk",
                    "in": "path",
                    "type": "string",
                    "required": True,
                    "description": f"The {collection[:-1]} (primary key) to update",
                },
                {
                    "name": f"{collection[:-1]}",
                    "in": "body",
                    "description": f"The object to use for {collection[:-1]} update",
                    "schema": {"type": "object"},
                },
            ],
            "responses": {
                200: {
                    "description": f"{collection[:-1]} updated",
                    "schema": {"$ref": f"#/definitions/{klass.schema_name}"},
                },
                "default": default_response,
            },
        }
    elif method_name == "BulkUpdate":
        params = filter_params
        params.append(
            {
                "name": f"{collection}",
                "in": "body",
                "description": f"The object to use for {collection} bulk update",
                "schema": {"type": "object"},
            }
        )
        schema_props = {"count": {"type": "integer"}}
        if klass.resource.paginate:
            schema_props["has_more"] = {"type": "boolean"}
            schema_props["total_count"] = {"type": "integer"}
            schema_props["total_pages"] = {"type": "integer"}
            params += limit_params
        spec = {
            "summary": f"Filter and update {collection}.",
            "operationId": "update_entries",
            "parameters": params,
            "responses": {
                200: {
                    "description": f"Number of {collection} updated",
                    "schema": {"type": "object", "properties": schema_props},
                },
                "default": default_response,
            },
        }

    elif method_name == "BulkDelete":
        params = filter_params
        schema_props = {"count": {"type": "integer"}}
        if klass.resource.paginate:
            schema_props["has_more"] = {"type": "boolean"}
            schema_props["total_count"] = {"type": "integer"}
            schema_props["total_pages"] = {"type": "integer"}
            params += limit_params
        spec = {
            "summary": f"Filter and delete {collection}.",
            "operationId": "delete_entries",
            "parameters": params,
            "responses": {
                200: {
                    "description": f"Number of {collection} deleted",
                    "schema": {"type": "object", "properties": schema_props},
                },
                "default": default_response,
            },
        }

    elif method_name == "Delete":
        spec = {
            "summary": f"Delete a {collection[:-1]}.",
            "operationId": "delete_entry",
            "parameters": [
                {
                    "name": "pk",
                    "in": "path",
                    "type": "string",
                    "required": True,
                    "description": f"The {collection[:-1]} (primary key) to delete",
                }
            ],
            "responses": {
                200: {"description": f"{collection[:-1]} deleted"},
                "default": default_response,
            },
        }

    return spec

# https://github.com/pallets/flask/blob/master/flask/views.py
# TODO FYI https://github.com/flasgger/flasgger/pull/396
class SwaggerViewType(MethodViewType):
    """Metaclass for `SwaggerView` defining custom attributes"""

    def __init__(cls, name, bases, d):
        """initialize Schema, decorators, definitions, and tags"""
        super(SwaggerViewType, cls).__init__(name, bases, d)

        if not __name__ == cls.__module__:
            # e.g.: cls.__module__ = mpcontribs.api.projects.views
            views_path = cls.__module__.split(".")
            doc_path = ".".join(views_path[:-1] + ["document"])
            cls.tags = [views_path[-2]]
            doc_filepath = doc_path.replace(".", os.sep) + ".py"
            if os.path.exists(doc_filepath):
                cls.doc_name = cls.tags[0].capitalize()
                Model = getattr(import_module(doc_path), cls.doc_name)
                cls.schema_name = cls.doc_name + "Schema"
                cls.Schema = type(
                    cls.schema_name,
                    (ModelSchema, object),
                    {
                        "Meta": type(
                            "Meta",
                            (object,),
                            dict(model=Model, ordered=True, model_build_obj=False),
                        )
                    },
                )
                cls.definitions = {cls.schema_name: schema2jsonschema(cls.Schema)}
                cls.resource.schema = cls.Schema

                # write flask-mongorest swagger specs
                for method in cls.methods:
                    spec = get_specs(cls, method, cls.tags[0])
                    if spec:
                        dir_path = os.path.join(SWAGGER["doc_dir"], cls.tags[0])
                        file_path = os.path.join(dir_path, method.__name__ + ".yml")
                        if not os.path.exists(file_path):
                            os.makedirs(dir_path, exist_ok=True)
                            with open(file_path, "w") as f:
                                yaml.dump(spec, f)
                                logger.warning(
                                    f"{cls.tags[0]}.{method.__name__} written to {file_path}"
                                )

def accessible_projects(username, groups):
    # only allow to read public approved projects, or
    # approved projects the user owns or is group member for
    qfilter = Q(is_public=True)
    if username:
        qfilter |= Q(owner=username)
    if groups:
        qfilter |= Q(name__in=list(groups))

    return Q(is_approved=True) & qfilter

class SwaggerView(OriginalSwaggerView, ResourceView, metaclass=SwaggerViewType):
    """A class-based view defining additional methods"""

    def get_groups(self, request):
        groups = request.headers.get("X-Authenticated-Groups", "").split(",")
        groups += request.headers.get("X-Consumer-Groups", "").split(",")
        return set(groups)

    def is_admin(self, groups):
        cname = current_app.config["PORTAL_CNAME"]
        return "admin" in groups or f"admin_{cname}" in groups

    def is_admin_or_project_user(self, request, obj):
        groups = self.get_groups(request)
        if self.is_admin(groups):
            return True

        if hasattr(obj, "is_approved"):
            is_approved = obj.is_approved
            owner = obj.owner
            project = obj.name
        elif hasattr(obj, "project"):
            is_approved = obj.project.is_approved
            owner = obj.project.owner
            project = obj.project.name
        else:
            raise Unauthorized(f"Unable to authorize {obj}")

        username = request.headers.get("X-Consumer-Username")
        return (project in groups or owner == username) and is_approved

    def has_read_permission(self, request, qs):
        groups = self.get_groups(request)
        if self.is_admin(groups):
            return qs  # admins can read all entries

        username = request.headers.get("X-Consumer-Username")
        qfilter = accessible_projects(username, groups)

        if request.path.startswith("/projects/"):
            return qs.filter(qfilter)
        elif request.path.startswith("/contributions/"):
            # project is LazyReferenceFields (multiple queries)
            module = import_module("mpcontribs.api.projects.document")
            Projects = getattr(module, "Projects")
            projects = Projects.objects.only("name").filter(qfilter)
            # now filter contributions
            q = qs._query
            if "project" in q and all(p.name != q["project"] for p in projects):
                qfilter = Q(is_public=True) | Q(project__in=projects)
                return qs.filter(qfilter)

        return qs

    def has_add_permission(self, request, obj):
        if not self.is_admin_or_project_user(request, obj):
            return False

        if hasattr(obj, "identifier") and obj.project.unique_identifiers:
            if self.resource.document.objects(
                project=obj.project.id, identifier=obj.identifier
            ).count():
                raise Unauthorized(
                    f"{obj.identifier} already added for {obj.project.id}"
                )

        return True

    def has_change_permission(self, request, obj):
        return self.is_admin_or_project_user(request, obj)

    def has_delete_permission(self, request, obj):
        return self.is_admin_or_project_user(request, obj)
<commit_msg>api: catch empty strings in groups<commit_after># -*- coding: utf-8 -*-
"""Custom meta-class and MethodView for Swagger"""

import os
import logging
import yaml
from flask import current_app
from typing import Pattern
from importlib import import_module
from flask.views import MethodViewType
from flasgger.marshmallow_apispec import SwaggerView as OriginalSwaggerView
from flasgger.marshmallow_apispec import schema2jsonschema
from marshmallow_mongoengine import ModelSchema
from flask_mongorest.views import ResourceView
from mongoengine.queryset.visitor import Q
from werkzeug.exceptions import Unauthorized
from mpcontribs.api.config import SWAGGER

logger = logging.getLogger("app")

def get_specs(klass, method, collection):
    method_name = (
        method.__name__ if getattr(method, "__name__", None) is not None else method
    )
    default_response = {
        "description": "Error",
        "schema": {"type": "object", "properties": {"error": {"type": "string"}}},
    }
    fields_param = None
    if klass.resource.fields is not None:
        fields_avail = (
            klass.resource.fields + klass.resource.get_optional_fields() + ["_all"]
        )
        description = f"List of fields to include in response ({fields_avail})."
        description += " Use dot-notation for nested subfields."
        fields_param = {
            "name": "_fields",
            "in": "query",
            "default": klass.resource.fields,
            "type": "array",
            "items": {"type": "string"},
            "description": description,
        }

    field_pagination_params = []
    for field, limits in klass.resource.fields_to_paginate.items():
        field_pagination_params.append(
            {
                "name": f"{field}_page",
                "in": "query",
                "default": 1,
                "type": "integer",
                "description": f"page to retrieve for {field} field",
            }
        )
        field_pagination_params.append(
            {
                "name": f"{field}_per_page",
                "in": "query",
                "default": limits[0],
                "maximum": limits[1],
                "type": "integer",
                "description": f"number of items to retrieve per page for {field} field",
            }
        )

    limit_params = [
        {
            "name": "_skip",
            "in": "query",
            "type": "integer",
            "description": "number of items to skip",
        },
        {
            "name": "_limit",
            "in": "query",
            "type": "integer",
            "description": "maximum number of items to return",
        },
        {
            "name": "page",
            "in": "query",
            "type": "integer",
            "description": "page number to return (in batches of `per_page/_limit`; alternative to `_skip`)",
        },
        {
            "name": "per_page",
            "in": "query",
            "type": "integer",
            "description": "maximum number of items to return per page (same as `_limit`)",
        },
    ]

    filter_params = []
    if getattr(klass.resource, "filters", None) is not None:
        for k, v in klass.resource.filters.items():
            label = k.pattern if isinstance(k, Pattern) else k
            for op in v:
                filter_params.append(
                    {
                        "name": label if op.op == "exact" else f"{label}__{op.op}",
                        "in": "query",
                        "type": op.typ,
                        "description": f"filter {label}"
                        if op.op == "exact"
                        else f"filter {label} via ${op.op}",
                    }
                )
                if op.typ == "array":
                    filter_params[-1]["items"] = {"type": "string"}

    order_params = []
    if klass.resource.allowed_ordering:
        allowed_ordering = [
            o.pattern if isinstance(o, Pattern) else o
            for o in klass.resource.allowed_ordering
        ]
        order_params = [
            {
                "name": "_order_by",
                "in": "query",
                "type": "string",
                "description": f"order {collection} via {allowed_ordering}",
            },
            {
                "name": "order",
                "in": "query",
                "type": "string",
                "description": f"order {collection} *asc* or *desc*",
            },
        ]

    spec = None
    if method_name == "Fetch":
        params = [
            {
                "name": "pk",
                "in": "path",
                "type": "string",
                "required": True,
                "description": f"{collection[:-1]} (primary key)",
            }
        ]
        if fields_param is not None:
            params.append(fields_param)
        params += field_pagination_params
        spec = {
            "summary": f"Retrieve a {collection[:-1]}.",
            "operationId": "get_entry",
            "parameters": params,
            "responses": {
                200: {
                    "description": f"single {collection} entry",
                    "schema": {"$ref": f"#/definitions/{klass.schema_name}"},
                },
                "default": default_response,
            },
        }

    elif method_name == "BulkFetch":
        params = [fields_param] if fields_param is not None else []
        params += field_pagination_params
        params += order_params
        params += filter_params
        schema_props = {
            "data": {
                "type": "array",
                "items": {"$ref": f"#/definitions/{klass.schema_name}"},
            }
        }
        if klass.resource.paginate:
            schema_props["has_more"] = {"type": "boolean"}
            schema_props["total_count"] = {"type": "integer"}
            schema_props["total_pages"] = {"type": "integer"}
            params += limit_params
        spec = {
            "summary": f"Filter and retrieve {collection}.",
            "operationId": "get_entries",
            "parameters": params,
            "responses": {
                200: {
                    "description": f"list of {collection}",
                    "schema": {"type": "object", "properties": schema_props},
                },
                "default": default_response,
            },
        }

    elif method_name == "Download":
        params = [
            {
                "name": "short_mime",
                "in": "path",
                "type": "string",
                "required": True,
                "description": f"MIME Download Type: gz",
                "default": "gz",
            },
            {
                "name": "format",
                "in": "query",
                "type": "string",
                "required": True,
                "description": f"download {collection} in different formats: {klass.resource.download_formats}",
            },
        ]
        params += [fields_param] if fields_param is not None else []
        params += order_params
        params += filter_params
        spec = {
            "summary": f"Filter and download {collection}.",
            "operationId": "download_entries",
            "parameters": params,
            "produces": ["application/gzip"],
            "responses": {
                200: {
                    "description": f"{collection} download",
                    "schema": {"type": "file"},
                },
                "default": default_response,
            },
        }

    elif method_name == "Create":
        spec = {
            "summary": f"Create a new {collection[:-1]}.",
            "operationId": "create_entry",
            "parameters": [
                {
                    "name": f"{collection[:-1]}",
                    "in": "body",
                    "description": f"The object to use for {collection[:-1]} creation",
                    "schema": {"$ref": f"#/definitions/{klass.schema_name}"},
                }
            ],
            "responses": {
                200: {
                    "description": f"{collection[:-1]} created",
                    "schema": {"$ref": f"#/definitions/{klass.schema_name}"},
                },
                "default": default_response,
            },
        }

    elif method_name == "BulkCreate":
        spec = {
            "summary": f"Create new {collection[:-1]}(s).",
            "operationId": "create_entries",
            "parameters": [
                {
                    "name": f"{collection}",
                    "in": "body",
                    "description": f"The objects to use for {collection[:-1]} creation",
                    "schema": {
                        "type": "array",
                        "items": {"$ref": f"#/definitions/{klass.schema_name}"},
                    },
                }
            ],
            "responses": {
                200: {
                    "description": f"{collection} created",
                    "schema": {
                        "type": "object",
                        "properties": {
                            "count": {"type": "integer"},
                            "data": {
                                "type": "array",
                                "items": {"$ref": f"#/definitions/{klass.schema_name}"},
                            },
                        },
                    },
                },
                "default": default_response,
            },
        }

    elif method_name == "Update":
        spec = {
            "summary": f"Update a {collection[:-1]}.",
            "operationId": "update_entry",
            "parameters": [
                {
                    "name": "pk",
                    "in": "path",
                    "type": "string",
                    "required": True,
                    "description": f"The {collection[:-1]} (primary key) to update",
                },
                {
                    "name": f"{collection[:-1]}",
                    "in": "body",
                    "description": f"The object to use for {collection[:-1]} update",
                    "schema": {"type": "object"},
                },
            ],
            "responses": {
                200: {
                    "description": f"{collection[:-1]} updated",
                    "schema": {"$ref": f"#/definitions/{klass.schema_name}"},
                },
                "default": default_response,
            },
        }
    elif method_name == "BulkUpdate":
        params = filter_params
        params.append(
            {
                "name": f"{collection}",
                "in": "body",
                "description": f"The object to use for {collection} bulk update",
                "schema": {"type": "object"},
            }
        )
        schema_props = {"count": {"type": "integer"}}
        if klass.resource.paginate:
            schema_props["has_more"] = {"type": "boolean"}
            schema_props["total_count"] = {"type": "integer"}
            schema_props["total_pages"] = {"type": "integer"}
            params += limit_params
        spec = {
            "summary": f"Filter and update {collection}.",
            "operationId": "update_entries",
            "parameters": params,
            "responses": {
                200: {
                    "description": f"Number of {collection} updated",
                    "schema": {"type": "object", "properties": schema_props},
                },
                "default": default_response,
            },
        }

    elif method_name == "BulkDelete":
        params = filter_params
        schema_props = {"count": {"type": "integer"}}
        if klass.resource.paginate:
            schema_props["has_more"] = {"type": "boolean"}
            schema_props["total_count"] = {"type": "integer"}
            schema_props["total_pages"] = {"type": "integer"}
            params += limit_params
        spec = {
            "summary": f"Filter and delete {collection}.",
            "operationId": "delete_entries",
            "parameters": params,
            "responses": {
                200: {
                    "description": f"Number of {collection} deleted",
                    "schema": {"type": "object", "properties": schema_props},
                },
                "default": default_response,
            },
        }

    elif method_name == "Delete":
        spec = {
            "summary": f"Delete a {collection[:-1]}.",
            "operationId": "delete_entry",
            "parameters": [
                {
                    "name": "pk",
                    "in": "path",
                    "type": "string",
                    "required": True,
                    "description": f"The {collection[:-1]} (primary key) to delete",
                }
            ],
            "responses": {
                200: {"description": f"{collection[:-1]} deleted"},
                "default": default_response,
            },
        }

    return spec

# https://github.com/pallets/flask/blob/master/flask/views.py
# TODO FYI https://github.com/flasgger/flasgger/pull/396
class SwaggerViewType(MethodViewType):
    """Metaclass for `SwaggerView` defining custom attributes"""

    def __init__(cls, name, bases, d):
        """initialize Schema, decorators, definitions, and tags"""
        super(SwaggerViewType, cls).__init__(name, bases, d)

        if not __name__ == cls.__module__:
            # e.g.: cls.__module__ = mpcontribs.api.projects.views
            views_path = cls.__module__.split(".")
            doc_path = ".".join(views_path[:-1] + ["document"])
            cls.tags = [views_path[-2]]
            doc_filepath = doc_path.replace(".", os.sep) + ".py"
            if os.path.exists(doc_filepath):
                cls.doc_name = cls.tags[0].capitalize()
                Model = getattr(import_module(doc_path), cls.doc_name)
                cls.schema_name = cls.doc_name + "Schema"
                cls.Schema = type(
                    cls.schema_name,
                    (ModelSchema, object),
                    {
                        "Meta": type(
                            "Meta",
                            (object,),
                            dict(model=Model, ordered=True, model_build_obj=False),
                        )
                    },
                )
                cls.definitions = {cls.schema_name: schema2jsonschema(cls.Schema)}
                cls.resource.schema = cls.Schema

                # write flask-mongorest swagger specs
                for method in cls.methods:
                    spec = get_specs(cls, method, cls.tags[0])
                    if spec:
                        dir_path = os.path.join(SWAGGER["doc_dir"], cls.tags[0])
                        file_path = os.path.join(dir_path, method.__name__ + ".yml")
                        if not os.path.exists(file_path):
                            os.makedirs(dir_path, exist_ok=True)
                            with open(file_path, "w") as f:
                                yaml.dump(spec, f)
                                logger.warning(
                                    f"{cls.tags[0]}.{method.__name__} written to {file_path}"
                                )

def accessible_projects(username, groups):
    # only allow to read public approved projects, or
    # approved projects the user owns or is group member for
    qfilter = Q(is_public=True)
    if username:
        qfilter |= Q(owner=username)
    if groups:
        qfilter |= Q(name__in=list(groups))

    return Q(is_approved=True) & qfilter

class SwaggerView(OriginalSwaggerView, ResourceView, metaclass=SwaggerViewType):
    """A class-based view defining additional methods"""

    def get_groups(self, request):
        groups = request.headers.get("X-Authenticated-Groups", "").split(",")
        groups += request.headers.get("X-Consumer-Groups", "").split(",")
        return set(g for g in groups if g)

    def is_admin(self, groups):
        cname = current_app.config["PORTAL_CNAME"]
        return "admin" in groups or f"admin_{cname}" in groups

    def is_admin_or_project_user(self, request, obj):
        groups = self.get_groups(request)
        if self.is_admin(groups):
            return True

        if hasattr(obj, "is_approved"):
            is_approved = obj.is_approved
            owner = obj.owner
            project = obj.name
        elif hasattr(obj, "project"):
            is_approved = obj.project.is_approved
            owner = obj.project.owner
            project = obj.project.name
        else:
            raise Unauthorized(f"Unable to authorize {obj}")

        username = request.headers.get("X-Consumer-Username")
        return (project in groups or owner == username) and is_approved

    def has_read_permission(self, request, qs):
        groups = self.get_groups(request)
        if self.is_admin(groups):
            return qs  # admins can read all entries

        username = request.headers.get("X-Consumer-Username")
        qfilter = accessible_projects(username, groups)

        if request.path.startswith("/projects/"):
            return qs.filter(qfilter)
        elif request.path.startswith("/contributions/"):
            # project is LazyReferenceFields (multiple queries)
            module = import_module("mpcontribs.api.projects.document")
            Projects = getattr(module, "Projects")
            projects = Projects.objects.only("name").filter(qfilter)
            # now filter contributions
            q = qs._query
            if "project" in q and all(p.name != q["project"] for p in projects):
                qfilter = Q(is_public=True) | Q(project__in=projects)
                return qs.filter(qfilter)

        return qs

    def has_add_permission(self, request, obj):
        if not self.is_admin_or_project_user(request, obj):
            return False

        if hasattr(obj, "identifier") and obj.project.unique_identifiers:
            if self.resource.document.objects(
                project=obj.project.id, identifier=obj.identifier
            ).count():
                raise Unauthorized(
                    f"{obj.identifier} already added for {obj.project.id}"
                )

        return True

    def has_change_permission(self, request, obj):
        return self.is_admin_or_project_user(request, obj)

    def has_delete_permission(self, request, obj):
        return self.is_admin_or_project_user(request, obj)
<|endoftext|>


<commit_before>import json
import arrow
from django.utils import timezone
from django.db import models
from .utils import datetime_to_iso
from moneywagon import get_single_transaction, get_block
from pricetick.models import PriceTick

class CachedTransaction(models.Model):
    txid = models.CharField(max_length=72, primary_key=True)
    content = models.TextField()
    crypto = models.CharField(max_length=8, default='btc')

    def __unicode__(self):
        txid = "?"
        if self.content:
            txid = json.loads(self.content)['txid']
        return "%s:%s" % (self.crypto.upper(), txid)

    @classmethod
    def fetch_full_tx(cls, crypto, txid, existing_tx_data=None, fiat=None):
        try:
            tx_obj = cls.objects.get(txid=txid)
            if tx_obj.content == "Pending":
                return None
            tx = json.loads(tx_obj.content)

            if existing_tx_data and existing_tx_data.get('confirmations', None):
                # update cached entry with updated confirmations if its available
                tx['confirmations'] = existing_tx_data['confirmations']
                tx_obj.content = json.dumps(tx)
                tx_obj.save()

        except cls.DoesNotExist:
            # not cached, fetch from API service.
            tx_obj, c = cls.objects.get_or_create(txid=txid, content="Pending")
            tx = get_single_transaction(crypto, txid, random=True)

            if existing_tx_data and existing_tx_data.get('counterparty', False):
                tx['inputs'] = []
                tx['outputs'] = []

                if existing_tx_data['amount'] > 0:
                    # send, add amount to outputs
                    which = "outputs"
                else:
                    # receive, add amount to inputs
                    which = "inputs"

                tx[which] = [{}]
                tx[which][0]['amount'] = existing_tx_data['amount'] / 1e8
                tx[which][0]['address'] = existing_tx_data['address']

            if tx['confirmations'] > 0:
                tx_obj.content = json.dumps(tx, default=datetime_to_iso)
                tx_obj.crypto = crypto
                tx_obj.save()

        if fiat:
            time = arrow.get(tx['time']).datetime
            tx['historical_price'] = cls.get_historical_fiat(crypto, fiat, time)

        tx['memos'] = Memo.get(txid=txid, crypto=crypto)
        return tx

    @classmethod
    def get_historical_fiat(cls, crypto, fiat, time):
        try:
            return PriceTick.nearest(crypto, fiat, time)
        except PriceTick.DoesNotExist:
            return None

class Memo(models.Model):
    crypto = models.CharField(max_length=8, default='btc')
    encrypted_text = models.TextField(blank=False)
    txid = models.CharField(max_length=72, db_index=True)
    pubkey = models.TextField()
    signature = models.TextField(blank=True)
    created = models.DateTimeField(default=timezone.now)

    @classmethod
    def get(cls, crypto, txid):
        memos = cls.objects.filter(txid=txid, crypto=crypto).exclude(encrypted_text="Please Delete")
        return [x.encrypted_text for x in memos]

    def as_dict(self):
        return {
            'encrypted_text': self.encrypted_text,
            'txid': self.txid,
            'signature': self.signature,
            'pubkey': self.pubkey,
            'currency': self.crypto
        }

class PullHistory(models.Model):
    """
    Records when the last time a memo server has been pulled
    """
    last_pulled = models.DateTimeField(default=timezone.now)
    pull_url = models.TextField()

    def __unicode__(self):
        ago = (timezone.now() - self.last_pulled)
        return "%s (%s Ago)" % (self.pull_url, ago)

class PushHistory(models.Model):
    last_pushed = models.DateTimeField(default=timezone.now)
<commit_msg>fixed error<commit_after>import json
import arrow
from django.utils import timezone
from django.db import models
from .utils import datetime_to_iso
from moneywagon import get_single_transaction, get_block
from pricetick.models import PriceTick

class CachedTransaction(models.Model):
    txid = models.CharField(max_length=72, primary_key=True)
    content = models.TextField()
    crypto = models.CharField(max_length=8, default='btc')

    def __unicode__(self):
        txid = "?"
        if self.content == "Pending":
            return "Pending"

        txid = json.loads(self.content)['txid']
        return "%s:%s" % (self.crypto.upper(), txid)

    @classmethod
    def fetch_full_tx(cls, crypto, txid, existing_tx_data=None, fiat=None):
        try:
            tx_obj = cls.objects.get(txid=txid)
            if tx_obj.content == "Pending":
                return None
            tx = json.loads(tx_obj.content)

            if existing_tx_data and existing_tx_data.get('confirmations', None):
                # update cached entry with updated confirmations if its available
                tx['confirmations'] = existing_tx_data['confirmations']
                tx_obj.content = json.dumps(tx)
                tx_obj.save()

        except cls.DoesNotExist:
            # not cached, fetch from API service.
            tx_obj, c = cls.objects.get_or_create(txid=txid, content="Pending")
            tx = get_single_transaction(crypto, txid, random=True)

            if existing_tx_data and existing_tx_data.get('counterparty', False):
                tx['inputs'] = []
                tx['outputs'] = []

                if existing_tx_data['amount'] > 0:
                    # send, add amount to outputs
                    which = "outputs"
                else:
                    # receive, add amount to inputs
                    which = "inputs"

                tx[which] = [{}]
                tx[which][0]['amount'] = existing_tx_data['amount'] / 1e8
                tx[which][0]['address'] = existing_tx_data['address']

            if tx['confirmations'] > 0:
                tx_obj.content = json.dumps(tx, default=datetime_to_iso)
                tx_obj.crypto = crypto
                tx_obj.save()

        if fiat:
            time = arrow.get(tx['time']).datetime
            tx['historical_price'] = cls.get_historical_fiat(crypto, fiat, time)

        tx['memos'] = Memo.get(txid=txid, crypto=crypto)
        return tx

    @classmethod
    def get_historical_fiat(cls, crypto, fiat, time):
        try:
            return PriceTick.nearest(crypto, fiat, time)
        except PriceTick.DoesNotExist:
            return None

class Memo(models.Model):
    crypto = models.CharField(max_length=8, default='btc')
    encrypted_text = models.TextField(blank=False)
    txid = models.CharField(max_length=72, db_index=True)
    pubkey = models.TextField()
    signature = models.TextField(blank=True)
    created = models.DateTimeField(default=timezone.now)

    @classmethod
    def get(cls, crypto, txid):
        memos = cls.objects.filter(txid=txid, crypto=crypto).exclude(encrypted_text="Please Delete")
        return [x.encrypted_text for x in memos]

    def as_dict(self):
        return {
            'encrypted_text': self.encrypted_text,
            'txid': self.txid,
            'signature': self.signature,
            'pubkey': self.pubkey,
            'currency': self.crypto
        }

class PullHistory(models.Model):
    """
    Records when the last time a memo server has been pulled
    """
    last_pulled = models.DateTimeField(default=timezone.now)
    pull_url = models.TextField()

    def __unicode__(self):
        ago = (timezone.now() - self.last_pulled)
        return "%s (%s Ago)" % (self.pull_url, ago)

class PushHistory(models.Model):
    last_pushed = models.DateTimeField(default=timezone.now)
<|endoftext|>


<commit_before># This file is part of OpenHatch.
# Copyright (C) 2010 Jack Grigg
# Copyright (C) 2010 OpenHatch, Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import urllib2
import re
import lxml.html # scraper library
import itertools
import csv
import datetime
import logging
import urlparse

import mysite.search.models
import mysite.customs.mechanize_helpers

def csv_url2bugs(csv_url):
    csv_fd = mysite.customs.mechanize_helpers.mechanize_get(
        csv_url).response()
    dict_reader = csv.DictReader(csv_fd)
    for thing in dict_reader:
        yield int(thing['id'])

class RoundupTracker(object):
    def __init__(self, root_url, tracker_name):
        assert root_url[-1] == '/'
        assert root_url[-2] != '/'
        self.root_url = unicode(root_url)
        # For now, just make the project name equal to the tracker name.
        bug_project_name = tracker_name
        self.project, _ = mysite.search.models.Project.objects.get_or_create(name=bug_project_name)

    @staticmethod
    def roundup_tree2metadata_dict(tree):
        '''
        Input: tree is a parsed HTML document that lxml.html can understand.

        Output: For each key	value	 in the tree,
        append {'key': 'value'} to a dictionary.
        Return the dictionary when done.'''

        ret = {}
        for th in tree.cssselect('th'):
            # Get next sibling
            key_with_colon = th.text_content().strip()
            key = key_with_colon.rsplit(':', 1)[0]
            try:
                td = th.itersiblings().next()
            except StopIteration:
                # If there isn't an adjacent TD, don't use this TH.
                continue
            value = td.text_content().strip()
            ret[key] = value

        return ret

    def str2datetime_obj(self, date_string, possibility_index=0):
        # FIXME: I make guesses as to the timezone.
        possible_date_strings = [
                "%Y-%m-%d %H:%M",
                "%Y-%m-%d %H:%M:%S",
                "%Y-%m-%d.%H:%M",
                "%Y-%m-%d.%H:%M:%S"]
        try:
            return datetime.datetime.strptime(date_string, possible_date_strings[possibility_index])
        except ValueError:
            # A keyerror raised here means we ran out of a possibilities.
            return self.str2datetime_obj(date_string, possibility_index=possibility_index+1)

    def get_remote_bug_ids_already_stored(self):
        bugs = mysite.search.models.Bug.all_bugs.filter(
                canonical_bug_link__contains=self.root_url)
        bug_ids = [self.get_remote_bug_id(bug) for bug in bugs]
        return bug_ids

    def get_all_submitter_realname_pairs(self, tree):
        '''Input: the tree
        Output: A dictionary mapping username=>realname'''

        ret = {}
        for th in tree.cssselect('th'):
            match = re.match("Author: ([^(]*) \(([^)]*)", th.text_content().strip())
            if match:
                realname, username = match.groups()
                ret[username] = realname
        return ret

    def get_submitter_realname(self, tree, submitter_username):
        try:
            return self.get_all_submitter_realname_pairs(tree)[submitter_username]
        except KeyError:
            return None

    def remote_bug_id2url(self, remote_bug_id):
        return urlparse.urljoin(self.root_url, "issue%d" % remote_bug_id)

    def get_remote_bug_id(self, bug):
        plus_slash_issue = urlparse.urljoin(self.root_url, 'issue')
        return int(bug.canonical_bug_link.split(plus_slash_issue)[1])

    def create_bug_object_for_remote_bug_id_if_necessary(self, remote_bug_id):
        """See if we have either no bug or only a stale one. If so,
        refresh."""
        logging.info(
            "Was asked to look at bug %d in %s" % (remote_bug_id, self.project.name))
        try:
            bug = mysite.search.models.Bug.all_bugs.get(
                canonical_bug_link=self.remote_bug_id2url(remote_bug_id))
            if bug.data_is_more_fresh_than_one_day():
                return False
        except mysite.search.models.Bug.DoesNotExist:
            bug = mysite.search.models.Bug()
            bug.canonical_bug_link = self.remote_bug_id2url(remote_bug_id)

        # otherwise, fill the bug object with fresh data
        bug = self._update_bug_object_for_remote_bug_id(
            bug_object=bug,
            remote_bug_id=remote_bug_id)
        bug.save()
        logging.info(
            "Actually loaded %d in from %s" % (
                remote_bug_id, self.project.name))
        return True

    def _update_bug_object_for_remote_bug_id(self, bug_object, remote_bug_id):
        """Create but don't save a bug."""
        remote_bug_url = self.remote_bug_id2url(remote_bug_id)
        tree = lxml.html.document_fromstring(urllib2.urlopen(remote_bug_url).read())

        metadata_dict = RoundupTracker.roundup_tree2metadata_dict(tree)

        date_reported, bug_object.submitter_username, last_touched, last_toucher = [
                x.text_content() for x in tree.cssselect(
                    'form[name=itemSynopsis] + p > b, form[name=itemSynopsis] + hr + p > b')]
        bug_object.submitter_realname = self.get_submitter_realname(tree, bug_object.submitter_username)
        bug_object.date_reported = self.str2datetime_obj(date_reported)
        bug_object.last_touched = self.str2datetime_obj(last_touched)
        bug_object.canonical_bug_link = remote_bug_url

        bug_object.status = metadata_dict['Status']
        bug_object.looks_closed = (metadata_dict['Status'] == 'closed')
        bug_object.title = metadata_dict['Title']
        bug_object.importance = metadata_dict['Priority']

        # For description, just grab the first "message"
        try:
            bug_object.description = tree.cssselect('table.messages td.content')[0].text_content().strip()
        except IndexError:
            # This Roundup issue has no messages.
            bug_object.description = ""

        bug_object.project = self.project

        # How many people participated?
        bug_object.people_involved = len(self.get_all_submitter_realname_pairs(tree))

        bug_object.last_polled = datetime.datetime.utcnow()

        self.extract_bug_tracker_specific_data(metadata_dict=metadata_dict,
                                               bug_object=bug_object)

        return bug_object

    def generate_list_of_bug_ids_to_look_at(self):
        # Override this in a project-tracker-specific subclass.
        raise NotImplementedError

    def extract_bug_tracker_specific_data(self, metadata_dict, bug_object):
        # Override this in a project-tracker-specific subclass.
        raise NotImplementedError

    def grab(self):
        """Loops over the Python bug tracker's easy bugs and stores/updates them in our DB.
        For now, just grab the easy bugs to be kind to their servers."""

        bug_ids = itertools.chain(self.get_remote_bug_ids_to_read(),
                                  self.get_remote_bug_ids_already_stored())

        for bug_id in bug_ids:
            print bug_id
            bug = self.create_bug_object_for_remote_bug_id(bug_id)

            # If there is already a bug with this canonical_bug_link in the DB, just delete it.
            bugs_this_one_replaces = mysite.search.models.Bug.all_bugs.filter(canonical_bug_link=
                                                        bug.canonical_bug_link)
            for delete_me in bugs_this_one_replaces:
                delete_me.delete()

            print bug
            # With the coast clear, we save the bug we just extracted from the Miro tracker.
            bug.save()

    def update(self):
        '''Call this nightly.'''
        logging.info("Learning about new bugs in %s" % self.project.name)

        # First, find an examine any new bugs.
        for bug_id in self.generate_list_of_bug_ids_to_look_at():
            self.create_bug_object_for_remote_bug_id_if_necessary(bug_id)
        # Second, make sure old bugs in the database aren't too stale.
        logging.info("Starting refreshing all bugs from %s." % self.project.name)
        count = 0
        for bug_id in self.get_remote_bug_ids_already_stored():
            self.create_bug_object_for_remote_bug_id_if_necessary(remote_bug_id=bug_id)
            count += 1
        logging.info("Okay, looked at %d bugs from %s." % (
                count, self.project.name))

    def __unicode__(self):
        return "<Roundup bug tracker for %s>" % self.root_url

class PythonTracker(RoundupTracker):
    enabled = True

    def __init__(self):
        RoundupTracker.__init__(self,
                                root_url='http://bugs.python.org/',
                                tracker_name='Python')

    def extract_bug_tracker_specific_data(self, metadata_dict, bug_object):
        bug_object.good_for_newcomers = (
            'easy' in metadata_dict['Keywords'])
        bug_object.concerns_just_documentation = (
            'Documentation' in metadata_dict['Components'])
        bug_object.status = metadata_dict['Status']
        bug_object.looks_closed = (
            metadata_dict['Status'] == 'closed' or
            'patch' in metadata_dict['Keywords'])
        bug_object.importance = metadata_dict['Priority']

    def generate_list_of_bug_ids_to_look_at(self):
        ### bug queries to look at
        queries = {
            'Documentation bugs':
                'http://bugs.python.org/issue?status=1%2C3&%40sort=activity&%40columns=id&%40startwith=0&%40group=priority&%40filter=status%2Ccomponents&components=4&%40action=export_csv',
            'Easy bugs':
                'http://bugs.python.org/issue?status=1%2C3&%40sort=activity&%40columns=id&%40startwith=0&%40group=priority&%40filter=status%2Ckeywords&keywords=6&%40action=export_csv',
            }

        for query_name in queries:
            query_url = queries[query_name]
            for bug_id in csv_url2bugs(query_url):
                yield bug_id

<commit_msg>customs/bugtrackers/roundup.py: Remove unnecessary line-ending whitespace<commit_after># This file is part of OpenHatch.
# Copyright (C) 2010 Jack Grigg
# Copyright (C) 2010 OpenHatch, Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import urllib2
import re
import lxml.html # scraper library
import itertools
import csv
import datetime
import logging
import urlparse

import mysite.search.models
import mysite.customs.mechanize_helpers

def csv_url2bugs(csv_url):
    csv_fd = mysite.customs.mechanize_helpers.mechanize_get(
        csv_url).response()
    dict_reader = csv.DictReader(csv_fd)
    for thing in dict_reader:
        yield int(thing['id'])

class RoundupTracker(object):
    def __init__(self, root_url, tracker_name):
        assert root_url[-1] == '/'
        assert root_url[-2] != '/'
        self.root_url = unicode(root_url)
        # For now, just make the project name equal to the tracker name.
        bug_project_name = tracker_name
        self.project, _ = mysite.search.models.Project.objects.get_or_create(name=bug_project_name)

    @staticmethod
    def roundup_tree2metadata_dict(tree):
        '''
        Input: tree is a parsed HTML document that lxml.html can understand.

        Output: For each key	value	 in the tree,
        append {'key': 'value'} to a dictionary.
        Return the dictionary when done.'''

        ret = {}
        for th in tree.cssselect('th'):
            # Get next sibling
            key_with_colon = th.text_content().strip()
            key = key_with_colon.rsplit(':', 1)[0]
            try:
                td = th.itersiblings().next()
            except StopIteration:
                # If there isn't an adjacent TD, don't use this TH.
                continue
            value = td.text_content().strip()
            ret[key] = value

        return ret

    def str2datetime_obj(self, date_string, possibility_index=0):
        # FIXME: I make guesses as to the timezone.
        possible_date_strings = [
                "%Y-%m-%d %H:%M",
                "%Y-%m-%d %H:%M:%S",
                "%Y-%m-%d.%H:%M",
                "%Y-%m-%d.%H:%M:%S"]
        try:
            return datetime.datetime.strptime(date_string, possible_date_strings[possibility_index])
        except ValueError:
            # A keyerror raised here means we ran out of a possibilities.
            return self.str2datetime_obj(date_string, possibility_index=possibility_index+1)

    def get_remote_bug_ids_already_stored(self):
        bugs = mysite.search.models.Bug.all_bugs.filter(
                canonical_bug_link__contains=self.root_url)
        bug_ids = [self.get_remote_bug_id(bug) for bug in bugs]
        return bug_ids

    def get_all_submitter_realname_pairs(self, tree):
        '''Input: the tree
        Output: A dictionary mapping username=>realname'''

        ret = {}
        for th in tree.cssselect('th'):
            match = re.match("Author: ([^(]*) \(([^)]*)", th.text_content().strip())
            if match:
                realname, username = match.groups()
                ret[username] = realname
        return ret

    def get_submitter_realname(self, tree, submitter_username):
        try:
            return self.get_all_submitter_realname_pairs(tree)[submitter_username]
        except KeyError:
            return None

    def remote_bug_id2url(self, remote_bug_id):
        return urlparse.urljoin(self.root_url, "issue%d" % remote_bug_id)

    def get_remote_bug_id(self, bug):
        plus_slash_issue = urlparse.urljoin(self.root_url, 'issue')
        return int(bug.canonical_bug_link.split(plus_slash_issue)[1])

    def create_bug_object_for_remote_bug_id_if_necessary(self, remote_bug_id):
        """See if we have either no bug or only a stale one. If so,
        refresh."""
        logging.info(
            "Was asked to look at bug %d in %s" % (remote_bug_id, self.project.name))
        try:
            bug = mysite.search.models.Bug.all_bugs.get(
                canonical_bug_link=self.remote_bug_id2url(remote_bug_id))
            if bug.data_is_more_fresh_than_one_day():
                return False
        except mysite.search.models.Bug.DoesNotExist:
            bug = mysite.search.models.Bug()
            bug.canonical_bug_link = self.remote_bug_id2url(remote_bug_id)

        # otherwise, fill the bug object with fresh data
        bug = self._update_bug_object_for_remote_bug_id(
            bug_object=bug,
            remote_bug_id=remote_bug_id)
        bug.save()
        logging.info(
            "Actually loaded %d in from %s" % (
                remote_bug_id, self.project.name))
        return True

    def _update_bug_object_for_remote_bug_id(self, bug_object, remote_bug_id):
        """Create but don't save a bug."""
        remote_bug_url = self.remote_bug_id2url(remote_bug_id)
        tree = lxml.html.document_fromstring(urllib2.urlopen(remote_bug_url).read())

        metadata_dict = RoundupTracker.roundup_tree2metadata_dict(tree)

        date_reported, bug_object.submitter_username, last_touched, last_toucher = [
                x.text_content() for x in tree.cssselect(
                    'form[name=itemSynopsis] + p > b, form[name=itemSynopsis] + hr + p > b')]
        bug_object.submitter_realname = self.get_submitter_realname(tree, bug_object.submitter_username)
        bug_object.date_reported = self.str2datetime_obj(date_reported)
        bug_object.last_touched = self.str2datetime_obj(last_touched)
        bug_object.canonical_bug_link = remote_bug_url

        bug_object.status = metadata_dict['Status']
        bug_object.looks_closed = (metadata_dict['Status'] == 'closed')
        bug_object.title = metadata_dict['Title']
        bug_object.importance = metadata_dict['Priority']

        # For description, just grab the first "message"
        try:
            bug_object.description = tree.cssselect('table.messages td.content')[0].text_content().strip()
        except IndexError:
            # This Roundup issue has no messages.
            bug_object.description = ""

        bug_object.project = self.project

        # How many people participated?
        bug_object.people_involved = len(self.get_all_submitter_realname_pairs(tree))

        bug_object.last_polled = datetime.datetime.utcnow()

        self.extract_bug_tracker_specific_data(metadata_dict=metadata_dict,
                                               bug_object=bug_object)

        return bug_object

    def generate_list_of_bug_ids_to_look_at(self):
        # Override this in a project-tracker-specific subclass.
        raise NotImplementedError

    def extract_bug_tracker_specific_data(self, metadata_dict, bug_object):
        # Override this in a project-tracker-specific subclass.
        raise NotImplementedError

    def grab(self):
        """Loops over the Python bug tracker's easy bugs and stores/updates them in our DB.
        For now, just grab the easy bugs to be kind to their servers."""

        bug_ids = itertools.chain(self.get_remote_bug_ids_to_read(),
                                  self.get_remote_bug_ids_already_stored())

        for bug_id in bug_ids:
            print bug_id
            bug = self.create_bug_object_for_remote_bug_id(bug_id)

            # If there is already a bug with this canonical_bug_link in the DB, just delete it.
            bugs_this_one_replaces = mysite.search.models.Bug.all_bugs.filter(canonical_bug_link=
                                                        bug.canonical_bug_link)
            for delete_me in bugs_this_one_replaces:
                delete_me.delete()

            print bug
            # With the coast clear, we save the bug we just extracted from the Miro tracker.
            bug.save()

    def update(self):
        '''Call this nightly.'''
        logging.info("Learning about new bugs in %s" % self.project.name)

        # First, find an examine any new bugs.
        for bug_id in self.generate_list_of_bug_ids_to_look_at():
            self.create_bug_object_for_remote_bug_id_if_necessary(bug_id)
        # Second, make sure old bugs in the database aren't too stale.
        logging.info("Starting refreshing all bugs from %s." % self.project.name)
        count = 0
        for bug_id in self.get_remote_bug_ids_already_stored():
            self.create_bug_object_for_remote_bug_id_if_necessary(remote_bug_id=bug_id)
            count += 1
        logging.info("Okay, looked at %d bugs from %s." % (
                count, self.project.name))

    def __unicode__(self):
        return "<Roundup bug tracker for %s>" % self.root_url

class PythonTracker(RoundupTracker):
    enabled = True

    def __init__(self):
        RoundupTracker.__init__(self,
                                root_url='http://bugs.python.org/',
                                tracker_name='Python')

    def extract_bug_tracker_specific_data(self, metadata_dict, bug_object):
        bug_object.good_for_newcomers = (
            'easy' in metadata_dict['Keywords'])
        bug_object.concerns_just_documentation = (
            'Documentation' in metadata_dict['Components'])
        bug_object.status = metadata_dict['Status']
        bug_object.looks_closed = (
            metadata_dict['Status'] == 'closed' or
            'patch' in metadata_dict['Keywords'])
        bug_object.importance = metadata_dict['Priority']

    def generate_list_of_bug_ids_to_look_at(self):
        ### bug queries to look at
        queries = {
            'Documentation bugs':
                'http://bugs.python.org/issue?status=1%2C3&%40sort=activity&%40columns=id&%40startwith=0&%40group=priority&%40filter=status%2Ccomponents&components=4&%40action=export_csv',
            'Easy bugs':
                'http://bugs.python.org/issue?status=1%2C3&%40sort=activity&%40columns=id&%40startwith=0&%40group=priority&%40filter=status%2Ckeywords&keywords=6&%40action=export_csv',
            }

        for query_name in queries:
            query_url = queries[query_name]
            for bug_id in csv_url2bugs(query_url):
                yield bug_id

<|endoftext|>


<commit_before>from concurrent import futures
import os
import logging
import time
import sys

from ghost import Ghost
from ghost.bindings import (
    QPainter,
    QPrinter,
    QtWebKit,
    QtCore,
)

import tornado.web
from tornado.httpserver import HTTPServer

from tornado.ioloop import IOLoop
from tornado.concurrent import run_on_executor

import nbformat

from PyPDF2 import (
    PdfFileReader,
    PdfFileWriter,
    PdfFileMerger,
)

from .base import DEFAULT_STATIC_FILES_PATH

class CaptureServer(HTTPServer):
    executor = futures.ThreadPoolExecutor(max_workers=1)

    def __init__(self, *args, **kwargs):
        super(CaptureServer, self).__init__(*args, **kwargs)

    @run_on_executor
    def capture(self):
        # DO SOME MAGIC
        ghost = Ghost(
            log_level=logging.DEBUG
        )
        session = ghost.start(
            # display=True,
            viewport_size=(1920, 1080),
        )
        merger = PdfFileMerger()
        join = lambda *bits: os.path.join(self.static_path, *bits)

        session.open("http://localhost:9999/index.html")

        try:
            session.wait_for_selector("#nbpresent-css")
            time.sleep(1)
        except Exception as err:
            print(err)

        for i, slide in enumerate(self.notebook.metadata.nbpresent.slides):
            print("\n\n\nprinting slide", i, slide)
            filename = join("notebook-{0:04d}.pdf".format(i))
            session.show()
            screenshot(self.notebook, session, filename)
            merger.append(PdfFileReader(filename, "rb"))
            result, resources = session.evaluate(
                """
                console.log(window.nbpresent);
                console.log(window.nbpresent.mode.presenter.speaker.advance());
                """)
            time.sleep(1)

        merger.write(join("notebook-unmeta.pdf"))

        unmeta = PdfFileReader(join("notebook-unmeta.pdf"), "rb")

        meta = PdfFileWriter()
        meta.appendPagesFromReader(unmeta)

        ipynb = "notebook.ipynb"

        with open(join(ipynb), "rb") as fp:
            meta.addAttachment(ipynb, fp.read())

        with open(join("notebook.pdf"), "wb") as fp:
            meta.write(fp)

        raise KeyboardInterrupt()

def screenshot(nb, session, dest, as_print=False):
    """
    big thanks to https://gist.github.com/jmaupetit/4217925
    """

    printer = QPrinter(mode=QPrinter.ScreenResolution)
    printer.setOutputFormat(QPrinter.PdfFormat)
    printer.setPaperSize(QtCore.QSizeF(1080, 1920), QPrinter.DevicePixel)
    printer.setOrientation(QPrinter.Landscape)
    printer.setOutputFileName(dest)
    printer.setPageMargins(0, 0, 0, 0, QPrinter.DevicePixel)

    if as_print:
        webview = QtWebKit.QWebView()
        webview.setPage(session.page)
        webview.print_(printer)
    else:
        painter = QPainter(printer)
        painter.scale(1.45, 1.45)
        session.main_frame.render(painter)
        painter.end()

def pdf_capture(static_path):
    settings = {
        "static_path": static_path
    }

    app = tornado.web.Application([
        (r"/components/(.*)", tornado.web.StaticFileHandler, {
            "path": os.path.join(DEFAULT_STATIC_FILES_PATH, "components")
        }),
        (r"/(.*)", tornado.web.StaticFileHandler, {
            "path": settings['static_path']
        }),
    ], **settings)

    server = CaptureServer(app)
    server.static_path = static_path

    with open(os.path.join(static_path, "notebook.ipynb")) as fp:
        server.notebook = nbformat.read(fp, 4)

    ioloop = IOLoop()
    ioloop.add_callback(server.capture)
    server.listen(9999)

    try:
        ioloop.start()
    except KeyboardInterrupt:
        print("stopped")

if __name__ == "__main__":
    pdf_capture(sys.argv[1])
<commit_msg>printing onto one giant page<commit_after>from concurrent import futures
import os
import logging
import time
import sys

from ghost import Ghost
from ghost.bindings import (
    QPainter,
    QPrinter,
    QtCore,
    QtWebKit,
)

import tornado.web
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
from tornado.concurrent import run_on_executor

from PyPDF2 import (
    PdfFileReader,
    PdfFileWriter,
)

import nbformat
from notebook import DEFAULT_STATIC_FILES_PATH

# the port on which to serve the fake server
PORT = 9999
VIEWPORT = (1200, 900)

class CaptureServer(HTTPServer):
    executor = futures.ThreadPoolExecutor(max_workers=1)
    pdf_name = "notebook.pdf"
    ipynb_name = "notebook.ipynb"
    embed_ipynb = True

    def __init__(self, *args, **kwargs):
        super(CaptureServer, self).__init__(*args, **kwargs)

    def init_ghost(self):
        return Ghost(
            log_level=logging.DEBUG
        )

    def init_session(self):
        return self.ghost.start(
            # display=True,
            # TODO: read this off config
            viewport_size=VIEWPORT,
            show_scrollbars=True,
        )

    def page_ready(self):
        self.session.wait_for_page_loaded()
        time.sleep(1)

    def post_process(self):
        if self.embed_ipynb:
            join = lambda *bits: os.path.join(self.static_path, *bits)

            unmeta = PdfFileReader(join(self.pdf_name), "rb")

            meta = PdfFileWriter()
            meta.appendPagesFromReader(unmeta)

            with open(join(self.ipynb_name), "rb") as fp:
                meta.addAttachment(self.ipynb_name, fp.read())

            with open(join("notebook.pdf"), "wb") as fp:
                meta.write(fp)

    @run_on_executor
    def capture(self):
        self.ghost = self.init_ghost()
        self.session = self.init_session()

        self.session.open("http://localhost:9999/index.html")

        try:
            self.page_ready()
        except Exception as err:
            print(err)

        self.print_to_pdf(self.pdf_name)

        self.post_process()

        raise KeyboardInterrupt()

    def print_to_pdf(self, path):
        """Saves page as a pdf file.
        See qt4 QPrinter documentation for more detailed explanations
        of options.
        :param path: The destination path.
        :param paper_size: A 2-tuple indicating size of page to print to.
        :param paper_margins: A 4-tuple indicating size of each margin.
        :param paper_units: Units for pager_size, pager_margins.
        :param zoom_factor: Scale the output content.
        """

        # TODO: read these from notebook metadata? args?
        paper_size = (8.5, 11.0)
        paper_margins = (0, 0, 0, 0)
        paper_units = QPrinter.Inch
        resolution = 1200

        printer = QPrinter(QPrinter.HighResolution)
        printer.setColorMode(QPrinter.Color)
        printer.setOutputFormat(QPrinter.PdfFormat)
        printer.setPageMargins(*(paper_margins + (paper_units,)))
        printer.setPaperSize(QtCore.QSizeF(*paper_size), paper_units)
        printer.setResolution(resolution)
        printer.setFullPage(True)

        printer.setOutputFileName(path)

        self.session.page.setViewportSize(QtCore.QSize(*VIEWPORT))
        content_size = self.session.main_frame.contentsSize()
        print("CONTENT SIZE", content_size)

        sizes, resources = self.session.evaluate(
            """(function(){
                var nb = $("#notebook")[0],
                    body = $("body")[0];
                return {
                    notebook: [nb.clientWidth, nb.clientHeight],
                    body: [body.clientWidth, body.clientHeight]
                };
            })();""")
        print("SIZES", sizes)

        self.session.page.setViewportSize(
            QtCore.QSize(VIEWPORT[0], sizes["notebook"][1] + 40)
        )

        ratio = paper_size[0] / sizes["body"][0]

        printer.setPaperSize(
            QtCore.QSizeF(paper_size[0], sizes["notebook"][1] * ratio),
            paper_units)

        painter = QPainter(printer)
        painter.scale(8, 8)
        self.session.main_frame.render(painter)
        painter.end()

def pdf_capture(static_path):
    settings = {
        "static_path": static_path
    }

    app = tornado.web.Application([
        (r"/components/(.*)", tornado.web.StaticFileHandler, {
            "path": os.path.join(DEFAULT_STATIC_FILES_PATH, "components")
        }),
        (r"/(.*)", tornado.web.StaticFileHandler, {
            "path": settings['static_path']
        }),
    ], **settings)

    server = CaptureServer(app)
    server.static_path = static_path

    with open(os.path.join(static_path, "notebook.ipynb")) as fp:
        server.notebook = nbformat.read(fp, 4)

    ioloop = IOLoop()
    ioloop.add_callback(server.capture)
    server.listen(9999)

    try:
        ioloop.start()
    except KeyboardInterrupt:
        print("stopped")

if __name__ == "__main__":
    pdf_capture(sys.argv[1])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Tests of neo.io.asciisignalio
"""

# needed for python 3 compatibility
from __future__ import absolute_import, division

import unittest

from neo.io import AsciiSignalIO
from neo.test.iotest.common_io_test import BaseTestIO

class TestAsciiSignalIO(BaseTestIO, unittest.TestCase, ):
    ioclass = AsciiSignalIO
    files_to_download = ['File_asciisignal_1.asc',
                         'File_asciisignal_2.txt',
                         'File_asciisignal_3.txt',
                         ]
    files_to_test = files_to_download

if __name__ == "__main__":
    unittest.main()
<commit_msg>Remove problematic file from testing<commit_after># -*- coding: utf-8 -*-
"""
Tests of neo.io.asciisignalio
"""

# needed for python 3 compatibility
from __future__ import absolute_import, division

import unittest

from neo.io import AsciiSignalIO
from neo.test.iotest.common_io_test import BaseTestIO

class TestAsciiSignalIO(BaseTestIO, unittest.TestCase, ):
    ioclass = AsciiSignalIO
    files_to_download = [#'File_asciisignal_1.asc',
                         'File_asciisignal_2.txt',
                         'File_asciisignal_3.txt',
                         ]
    files_to_test = files_to_download

if __name__ == "__main__":
    unittest.main()
<|endoftext|>