<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
"""a collection of model-related helper classes and functions"""
import json
import logging
import re
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional, Set, Union

# isort and pylint disagree, isort should win
# pylint: disable=ungrouped-imports
import humanize
import pandas as pd
import pytz
import sqlalchemy as sa
import yaml
from flask import escape, g, Markup
from flask_appbuilder.models.decorators import renders
from flask_appbuilder.models.mixins import AuditMixin
from flask_appbuilder.security.sqla.models import User
from sqlalchemy import and_, or_, UniqueConstraint
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.orm import Mapper, Session
from sqlalchemy.orm.exc import MultipleResultsFound

from superset.utils.core import QueryStatus

logger = logging.getLogger(__name__)

def json_to_dict(json_str: str) -> Dict[Any, Any]:
    if json_str:
        val = re.sub(",[ \t\r\n]+}", "}", json_str)
        val = re.sub(
            ",[ \t\r\n]+\]", "]", val  # pylint: disable=anomalous-backslash-in-string
        )
        return json.loads(val)

    return {}

class ImportMixin:
    export_parent: Optional[str] = None
    # The name of the attribute
    # with the SQL Alchemy back reference

    export_children: List[str] = []
    # List of (str) names of attributes
    # with the SQL Alchemy forward references

    export_fields: List[str] = []
    # The names of the attributes
    # that are available for import and export

    __mapper__: Mapper

    @classmethod
    def _parent_foreign_key_mappings(cls) -> Dict[str, str]:
        """Get a mapping of foreign name to the local name of foreign keys"""
        parent_rel = cls.__mapper__.relationships.get(cls.export_parent)
        if parent_rel:
            return {l.name: r.name for (l, r) in parent_rel.local_remote_pairs}
        return {}

    @classmethod
    def _unique_constrains(cls) -> List[Set[str]]:
        """Get all (single column and multi column) unique constraints"""
        unique = [
            {c.name for c in u.columns}
            for u in cls.__table_args__  # type: ignore
            if isinstance(u, UniqueConstraint)
        ]
        unique.extend(
            {c.name} for c in cls.__table__.columns if c.unique  # type: ignore
        )
        return unique

    @classmethod
    def export_schema(
        cls, recursive: bool = True, include_parent_ref: bool = False
    ) -> Dict[str, Any]:
        """Export schema as a dictionary"""
        parent_excludes = set()
        if not include_parent_ref:
            parent_ref = cls.__mapper__.relationships.get(cls.export_parent)
            if parent_ref:
                parent_excludes = {column.name for column in parent_ref.local_columns}

        def formatter(column: sa.Column) -> str:
            return (
                "{0} Default ({1})".format(str(column.type), column.default.arg)
                if column.default
                else str(column.type)
            )

        schema: Dict[str, Any] = {
            column.name: formatter(column)
            for column in cls.__table__.columns  # type: ignore
            if (column.name in cls.export_fields and column.name not in parent_excludes)
        }
        if recursive:
            for column in cls.export_children:
                child_class = cls.__mapper__.relationships[column].argument.class_
                schema[column] = [
                    child_class.export_schema(
                        recursive=recursive, include_parent_ref=include_parent_ref
                    )
                ]
        return schema

    @classmethod
    def import_from_dict(
        # pylint: disable=too-many-arguments,too-many-branches,too-many-locals
        cls,
        session: Session,
        dict_rep: Dict[Any, Any],
        parent: Optional[Any] = None,
        recursive: bool = True,
        sync: Optional[List[str]] = None,
    ) -> Any:  # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
        """Import obj from a dictionary"""
        if sync is None:
            sync = []
        parent_refs = cls._parent_foreign_key_mappings()
        export_fields = set(cls.export_fields) | set(parent_refs.keys())
        new_children = {c: dict_rep[c] for c in cls.export_children if c in dict_rep}
        unique_constrains = cls._unique_constrains()

        filters = []  # Using these filters to check if obj already exists

        # Remove fields that should not get imported
        for k in list(dict_rep):
            if k not in export_fields:
                del dict_rep[k]

        if not parent:
            if cls.export_parent:
                for prnt in parent_refs.keys():
                    if prnt not in dict_rep:
                        raise RuntimeError(
                            "{0}: Missing field {1}".format(cls.__name__, prnt)
                        )
        else:
            # Set foreign keys to parent obj
            for k, v in parent_refs.items():
                dict_rep[k] = getattr(parent, v)

        # Add filter for parent obj
        filters.extend([getattr(cls, k) == dict_rep.get(k) for k in parent_refs.keys()])

        # Add filter for unique constraints
        ucs = [
            and_(
                *[
                    getattr(cls, k) == dict_rep.get(k)
                    for k in cs
                    if dict_rep.get(k) is not None
                ]
            )
            for cs in unique_constrains
        ]
        filters.append(or_(*ucs))

        # Check if object already exists in DB, break if more than one is found
        try:
            obj_query = session.query(cls).filter(and_(*filters))
            obj = obj_query.one_or_none()
        except MultipleResultsFound as ex:
            logger.error(
                "Error importing %s \n %s \n %s",
                cls.__name__,
                str(obj_query),
                yaml.safe_dump(dict_rep),
            )
            raise ex

        if not obj:
            is_new_obj = True
            # Create new DB object
            obj = cls(**dict_rep)  # type: ignore
            logger.info("Importing new %s %s", obj.__tablename__, str(obj))
            if cls.export_parent and parent:
                setattr(obj, cls.export_parent, parent)
            session.add(obj)
        else:
            is_new_obj = False
            logger.info("Updating %s %s", obj.__tablename__, str(obj))
            # Update columns
            for k, v in dict_rep.items():
                setattr(obj, k, v)

        # Recursively create children
        if recursive:
            for child in cls.export_children:
                child_class = cls.__mapper__.relationships[child].argument.class_
                added = []
                for c_obj in new_children.get(child, []):
                    added.append(
                        child_class.import_from_dict(
                            session=session, dict_rep=c_obj, parent=obj, sync=sync
                        )
                    )
                # If children should get synced, delete the ones that did not
                # get updated.
                if child in sync and not is_new_obj:
                    back_refs = (
                        child_class._parent_foreign_key_mappings()  # pylint: disable=protected-access
                    )
                    delete_filters = [
                        getattr(child_class, k) == getattr(obj, back_refs.get(k))
                        for k in back_refs.keys()
                    ]
                    to_delete = set(
                        session.query(child_class).filter(and_(*delete_filters))
                    ).difference(set(added))
                    for o in to_delete:
                        logger.info("Deleting %s %s", child, str(obj))
                        session.delete(o)

        return obj

    def export_to_dict(
        self,
        recursive: bool = True,
        include_parent_ref: bool = False,
        include_defaults: bool = False,
    ) -> Dict[Any, Any]:
        """Export obj to dictionary"""
        cls = self.__class__
        parent_excludes = set()
        if recursive and not include_parent_ref:
            parent_ref = cls.__mapper__.relationships.get(cls.export_parent)
            if parent_ref:
                parent_excludes = {c.name for c in parent_ref.local_columns}
        dict_rep = {
            c.name: getattr(self, c.name)
            for c in cls.__table__.columns  # type: ignore
            if (
                c.name in self.export_fields
                and c.name not in parent_excludes
                and (
                    include_defaults
                    or (
                        getattr(self, c.name) is not None
                        and (not c.default or getattr(self, c.name) != c.default.arg)
                    )
                )
            )
        }
        if recursive:
            for cld in self.export_children:
                # sorting to make lists of children stable
                dict_rep[cld] = sorted(
                    [
                        child.export_to_dict(
                            recursive=recursive,
                            include_parent_ref=include_parent_ref,
                            include_defaults=include_defaults,
                        )
                        for child in getattr(self, cld)
                    ],
                    key=lambda k: sorted(str(k.items())),
                )

        return dict_rep

    def override(self, obj: Any) -> None:
        """Overrides the plain fields of the dashboard."""
        for field in obj.__class__.export_fields:
            setattr(self, field, getattr(obj, field))

    def copy(self) -> Any:
        """Creates a copy of the dashboard without relationships."""
        new_obj = self.__class__()
        new_obj.override(self)
        return new_obj

    def alter_params(self, **kwargs: Any) -> None:
        params = self.params_dict
        params.update(kwargs)
        self.params = json.dumps(params)

    def remove_params(self, param_to_remove: str) -> None:
        params = self.params_dict
        params.pop(param_to_remove, None)
        self.params = json.dumps(params)

    def reset_ownership(self) -> None:
        """ object will belong to the user the current user """
        # make sure the object doesn't have relations to a user
        # it will be filled by appbuilder on save
        self.created_by = None
        self.changed_by = None
        # flask global context might not exist (in cli or tests for example)
        try:
            if g.user:
                self.owners = [g.user]
        except Exception:  # pylint: disable=broad-except
            self.owners = []

    @property
    def params_dict(self) -> Dict[Any, Any]:
        return json_to_dict(self.params)

    @property
    def template_params_dict(self) -> Dict[Any, Any]:
        return json_to_dict(self.template_params)  # type: ignore

def _user_link(user: User) -> Union[Markup, str]:  # pylint: disable=no-self-use
    if not user:
        return ""
    url = "/superset/profile/{}/".format(user.username)
    return Markup('{}'.format(url, escape(user) or ""))

class AuditMixinNullable(AuditMixin):
    """Altering the AuditMixin to use nullable fields

    Allows creating objects programmatically outside of CRUD
    """

    created_on = sa.Column(sa.DateTime, default=datetime.now, nullable=True)
    changed_on = sa.Column(
        sa.DateTime, default=datetime.now, onupdate=datetime.now, nullable=True
    )

    @declared_attr
    def created_by_fk(self) -> sa.Column:
        return sa.Column(
            sa.Integer,
            sa.ForeignKey("ab_user.id"),
            default=self.get_user_id,
            nullable=True,
        )

    @declared_attr
    def changed_by_fk(self) -> sa.Column:
        return sa.Column(
            sa.Integer,
            sa.ForeignKey("ab_user.id"),
            default=self.get_user_id,
            onupdate=self.get_user_id,
            nullable=True,
        )

    @property
    def changed_by_name(self) -> str:
        if self.changed_by:
            return escape("{}".format(self.changed_by))
        return ""

    @renders("created_by")
    def creator(self) -> Union[Markup, str]:
        return _user_link(self.created_by)

    @property
    def changed_by_(self) -> Union[Markup, str]:
        return _user_link(self.changed_by)

    @renders("changed_on")
    def changed_on_(self) -> Markup:
        return Markup(f'{self.changed_on}')

    @renders("changed_on")
    def changed_on_delta_humanized(self) -> str:
        return self.changed_on_humanized

    @renders("changed_on")
    def changed_on_utc(self) -> str:
        # Convert naive datetime to UTC
        return self.changed_on.astimezone(pytz.utc).strftime("%Y-%m-%dT%H:%M:%S.%f%z")

    @property
    def changed_on_humanized(self) -> str:
        return humanize.naturaltime(datetime.now() - self.changed_on)

    @renders("changed_on")
    def modified(self) -> Markup:
        return Markup(f'{self.changed_on_humanized}')

class QueryResult:  # pylint: disable=too-few-public-methods

    """Object returned by the query interface"""

    def __init__(  # pylint: disable=too-many-arguments
        self,
        df: pd.DataFrame,
        query: str,
        duration: timedelta,
        status: str = QueryStatus.SUCCESS,
        error_message: Optional[str] = None,
        errors: Optional[List[Dict[str, Any]]] = None,
    ) -> None:
        self.df = df
        self.query = query
        self.duration = duration
        self.status = status
        self.error_message = error_message
        self.errors = errors or []

class ExtraJSONMixin:
    """Mixin to add an `extra` column (JSON) and utility methods"""

    extra_json = sa.Column(sa.Text, default="{}")

    @property
    def extra(self) -> Dict[str, Any]:
        try:
            return json.loads(self.extra_json)
        except Exception:  # pylint: disable=broad-except
            return {}

    def set_extra_json(self, extras: Dict[str, Any]) -> None:
        self.extra_json = json.dumps(extras)

    def set_extra_json_key(self, key: str, value: Any) -> None:
        extra = self.extra
        extra[key] = value
        self.extra_json = json.dumps(extra)
<commit_msg>fix: several disabled pylint rules in models/helpers.py (#10909)<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
"""a collection of model-related helper classes and functions"""
import json
import logging
import re
from datetime import datetime, timedelta
from json.decoder import JSONDecodeError
from typing import Any, Dict, List, Optional, Set, Union

import humanize
import pandas as pd
import pytz
import sqlalchemy as sa
import yaml
from flask import escape, g, Markup
from flask_appbuilder.models.decorators import renders
from flask_appbuilder.models.mixins import AuditMixin
from flask_appbuilder.security.sqla.models import User
from sqlalchemy import and_, or_, UniqueConstraint
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.orm import Mapper, Session
from sqlalchemy.orm.exc import MultipleResultsFound

from superset.utils.core import QueryStatus

logger = logging.getLogger(__name__)

def json_to_dict(json_str: str) -> Dict[Any, Any]:
    if json_str:
        val = re.sub(",[ \t\r\n]+}", "}", json_str)
        val = re.sub(",[ \t\r\n]+\\]", "]", val)
        return json.loads(val)

    return {}

class ImportMixin:
    export_parent: Optional[str] = None
    # The name of the attribute
    # with the SQL Alchemy back reference

    export_children: List[str] = []
    # List of (str) names of attributes
    # with the SQL Alchemy forward references

    export_fields: List[str] = []
    # The names of the attributes
    # that are available for import and export

    __mapper__: Mapper

    @classmethod
    def _parent_foreign_key_mappings(cls) -> Dict[str, str]:
        """Get a mapping of foreign name to the local name of foreign keys"""
        parent_rel = cls.__mapper__.relationships.get(cls.export_parent)
        if parent_rel:
            return {l.name: r.name for (l, r) in parent_rel.local_remote_pairs}
        return {}

    @classmethod
    def _unique_constrains(cls) -> List[Set[str]]:
        """Get all (single column and multi column) unique constraints"""
        unique = [
            {c.name for c in u.columns}
            for u in cls.__table_args__  # type: ignore
            if isinstance(u, UniqueConstraint)
        ]
        unique.extend(
            {c.name} for c in cls.__table__.columns if c.unique  # type: ignore
        )
        return unique

    @classmethod
    def parent_foreign_key_mappings(cls) -> Dict[str, str]:
        """Get a mapping of foreign name to the local name of foreign keys"""
        parent_rel = cls.__mapper__.relationships.get(cls.export_parent)
        if parent_rel:
            return {l.name: r.name for (l, r) in parent_rel.local_remote_pairs}
        return {}

    @classmethod
    def export_schema(
        cls, recursive: bool = True, include_parent_ref: bool = False
    ) -> Dict[str, Any]:
        """Export schema as a dictionary"""
        parent_excludes = set()
        if not include_parent_ref:
            parent_ref = cls.__mapper__.relationships.get(cls.export_parent)
            if parent_ref:
                parent_excludes = {column.name for column in parent_ref.local_columns}

        def formatter(column: sa.Column) -> str:
            return (
                "{0} Default ({1})".format(str(column.type), column.default.arg)
                if column.default
                else str(column.type)
            )

        schema: Dict[str, Any] = {
            column.name: formatter(column)
            for column in cls.__table__.columns  # type: ignore
            if (column.name in cls.export_fields and column.name not in parent_excludes)
        }
        if recursive:
            for column in cls.export_children:
                child_class = cls.__mapper__.relationships[column].argument.class_
                schema[column] = [
                    child_class.export_schema(
                        recursive=recursive, include_parent_ref=include_parent_ref
                    )
                ]
        return schema

    @classmethod
    def import_from_dict(
        # pylint: disable=too-many-arguments,too-many-branches,too-many-locals
        cls,
        session: Session,
        dict_rep: Dict[Any, Any],
        parent: Optional[Any] = None,
        recursive: bool = True,
        sync: Optional[List[str]] = None,
    ) -> Any:  # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
        """Import obj from a dictionary"""
        if sync is None:
            sync = []
        parent_refs = cls.parent_foreign_key_mappings()
        export_fields = set(cls.export_fields) | set(parent_refs.keys())
        new_children = {c: dict_rep[c] for c in cls.export_children if c in dict_rep}
        unique_constrains = cls._unique_constrains()

        filters = []  # Using these filters to check if obj already exists

        # Remove fields that should not get imported
        for k in list(dict_rep):
            if k not in export_fields:
                del dict_rep[k]

        if not parent:
            if cls.export_parent:
                for prnt in parent_refs.keys():
                    if prnt not in dict_rep:
                        raise RuntimeError(
                            "{0}: Missing field {1}".format(cls.__name__, prnt)
                        )
        else:
            # Set foreign keys to parent obj
            for k, v in parent_refs.items():
                dict_rep[k] = getattr(parent, v)

        # Add filter for parent obj
        filters.extend([getattr(cls, k) == dict_rep.get(k) for k in parent_refs.keys()])

        # Add filter for unique constraints
        ucs = [
            and_(
                *[
                    getattr(cls, k) == dict_rep.get(k)
                    for k in cs
                    if dict_rep.get(k) is not None
                ]
            )
            for cs in unique_constrains
        ]
        filters.append(or_(*ucs))

        # Check if object already exists in DB, break if more than one is found
        try:
            obj_query = session.query(cls).filter(and_(*filters))
            obj = obj_query.one_or_none()
        except MultipleResultsFound as ex:
            logger.error(
                "Error importing %s \n %s \n %s",
                cls.__name__,
                str(obj_query),
                yaml.safe_dump(dict_rep),
            )
            raise ex

        if not obj:
            is_new_obj = True
            # Create new DB object
            obj = cls(**dict_rep)  # type: ignore
            logger.info("Importing new %s %s", obj.__tablename__, str(obj))
            if cls.export_parent and parent:
                setattr(obj, cls.export_parent, parent)
            session.add(obj)
        else:
            is_new_obj = False
            logger.info("Updating %s %s", obj.__tablename__, str(obj))
            # Update columns
            for k, v in dict_rep.items():
                setattr(obj, k, v)

        # Recursively create children
        if recursive:
            for child in cls.export_children:
                child_class = cls.__mapper__.relationships[child].argument.class_
                added = []
                for c_obj in new_children.get(child, []):
                    added.append(
                        child_class.import_from_dict(
                            session=session, dict_rep=c_obj, parent=obj, sync=sync
                        )
                    )
                # If children should get synced, delete the ones that did not
                # get updated.
                if child in sync and not is_new_obj:
                    back_refs = child_class.parent_foreign_key_mappings()
                    delete_filters = [
                        getattr(child_class, k) == getattr(obj, back_refs.get(k))
                        for k in back_refs.keys()
                    ]
                    to_delete = set(
                        session.query(child_class).filter(and_(*delete_filters))
                    ).difference(set(added))
                    for o in to_delete:
                        logger.info("Deleting %s %s", child, str(obj))
                        session.delete(o)

        return obj

    def export_to_dict(
        self,
        recursive: bool = True,
        include_parent_ref: bool = False,
        include_defaults: bool = False,
    ) -> Dict[Any, Any]:
        """Export obj to dictionary"""
        cls = self.__class__
        parent_excludes = set()
        if recursive and not include_parent_ref:
            parent_ref = cls.__mapper__.relationships.get(cls.export_parent)
            if parent_ref:
                parent_excludes = {c.name for c in parent_ref.local_columns}
        dict_rep = {
            c.name: getattr(self, c.name)
            for c in cls.__table__.columns  # type: ignore
            if (
                c.name in self.export_fields
                and c.name not in parent_excludes
                and (
                    include_defaults
                    or (
                        getattr(self, c.name) is not None
                        and (not c.default or getattr(self, c.name) != c.default.arg)
                    )
                )
            )
        }
        if recursive:
            for cld in self.export_children:
                # sorting to make lists of children stable
                dict_rep[cld] = sorted(
                    [
                        child.export_to_dict(
                            recursive=recursive,
                            include_parent_ref=include_parent_ref,
                            include_defaults=include_defaults,
                        )
                        for child in getattr(self, cld)
                    ],
                    key=lambda k: sorted(str(k.items())),
                )

        return dict_rep

    def override(self, obj: Any) -> None:
        """Overrides the plain fields of the dashboard."""
        for field in obj.__class__.export_fields:
            setattr(self, field, getattr(obj, field))

    def copy(self) -> Any:
        """Creates a copy of the dashboard without relationships."""
        new_obj = self.__class__()
        new_obj.override(self)
        return new_obj

    def alter_params(self, **kwargs: Any) -> None:
        params = self.params_dict
        params.update(kwargs)
        self.params = json.dumps(params)

    def remove_params(self, param_to_remove: str) -> None:
        params = self.params_dict
        params.pop(param_to_remove, None)
        self.params = json.dumps(params)

    def reset_ownership(self) -> None:
        """ object will belong to the user the current user """
        # make sure the object doesn't have relations to a user
        # it will be filled by appbuilder on save
        self.created_by = None
        self.changed_by = None
        # flask global context might not exist (in cli or tests for example)
        self.owners = []
        if g and hasattr(g, "user"):
            self.owners = [g.user]

    @property
    def params_dict(self) -> Dict[Any, Any]:
        return json_to_dict(self.params)

    @property
    def template_params_dict(self) -> Dict[Any, Any]:
        return json_to_dict(self.template_params)  # type: ignore

def _user_link(user: User) -> Union[Markup, str]:
    if not user:
        return ""
    url = "/superset/profile/{}/".format(user.username)
    return Markup('{}'.format(url, escape(user) or ""))

class AuditMixinNullable(AuditMixin):
    """Altering the AuditMixin to use nullable fields

    Allows creating objects programmatically outside of CRUD
    """

    created_on = sa.Column(sa.DateTime, default=datetime.now, nullable=True)
    changed_on = sa.Column(
        sa.DateTime, default=datetime.now, onupdate=datetime.now, nullable=True
    )

    @declared_attr
    def created_by_fk(self) -> sa.Column:
        return sa.Column(
            sa.Integer,
            sa.ForeignKey("ab_user.id"),
            default=self.get_user_id,
            nullable=True,
        )

    @declared_attr
    def changed_by_fk(self) -> sa.Column:
        return sa.Column(
            sa.Integer,
            sa.ForeignKey("ab_user.id"),
            default=self.get_user_id,
            onupdate=self.get_user_id,
            nullable=True,
        )

    @property
    def changed_by_name(self) -> str:
        if self.changed_by:
            return escape("{}".format(self.changed_by))
        return ""

    @renders("created_by")
    def creator(self) -> Union[Markup, str]:
        return _user_link(self.created_by)

    @property
    def changed_by_(self) -> Union[Markup, str]:
        return _user_link(self.changed_by)

    @renders("changed_on")
    def changed_on_(self) -> Markup:
        return Markup(f'{self.changed_on}')

    @renders("changed_on")
    def changed_on_delta_humanized(self) -> str:
        return self.changed_on_humanized

    @renders("changed_on")
    def changed_on_utc(self) -> str:
        # Convert naive datetime to UTC
        return self.changed_on.astimezone(pytz.utc).strftime("%Y-%m-%dT%H:%M:%S.%f%z")

    @property
    def changed_on_humanized(self) -> str:
        return humanize.naturaltime(datetime.now() - self.changed_on)

    @renders("changed_on")
    def modified(self) -> Markup:
        return Markup(f'{self.changed_on_humanized}')

class QueryResult:  # pylint: disable=too-few-public-methods

    """Object returned by the query interface"""

    def __init__(  # pylint: disable=too-many-arguments
        self,
        df: pd.DataFrame,
        query: str,
        duration: timedelta,
        status: str = QueryStatus.SUCCESS,
        error_message: Optional[str] = None,
        errors: Optional[List[Dict[str, Any]]] = None,
    ) -> None:
        self.df = df
        self.query = query
        self.duration = duration
        self.status = status
        self.error_message = error_message
        self.errors = errors or []

class ExtraJSONMixin:
    """Mixin to add an `extra` column (JSON) and utility methods"""

    extra_json = sa.Column(sa.Text, default="{}")

    @property
    def extra(self) -> Dict[str, Any]:
        try:
            return json.loads(self.extra_json)
        except (TypeError, JSONDecodeError) as exc:
            logger.error(
                "Unable to load an extra json: %r. Leaving empty.", exc, exc_info=True
            )
            return {}

    def set_extra_json(self, extras: Dict[str, Any]) -> None:
        self.extra_json = json.dumps(extras)

    def set_extra_json_key(self, key: str, value: Any) -> None:
        extra = self.extra
        extra[key] = value
        self.extra_json = json.dumps(extra)
<|endoftext|>


<commit_before>"""
Processes for Species distribution
Author: Nils Hempelmann (nils.hempelmann@lsce.ipsl.fr)
"""

from flyingpigeon.sdm import _SDMINDICES_

from pywps import Process
from pywps import LiteralInput
from pywps import ComplexInput, ComplexOutput
from pywps import Format, FORMATS
from pywps.app.Common import Metadata
from flyingpigeon.log import init_process_logger

import logging
LOGGER = logging.getLogger("PYWPS")

class SDMgetindicesProcess(Process):
    def __init__(self):
        inputs = [
            ComplexInput('dataset', 'Dataset',
                         abstract="Enter either URL pointing to a NetCDF File"
                                  " or an archive (tar/zip) containing NetCDF files.",
                         min_occurs=1,
                         max_occurs=1000,
                         supported_formats=[
                             Format('application/x-netcdf'),
                             Format('application/x-tar'),
                             Format('application/zip'),
                         ]),

            # ComplexInput('resource', 'Resource',
            #              abstract='NetCDF Files or archive (tar/zip) containing netCDF files.',
            #              metadata=[Metadata('Info')],
            #              min_occurs=1,
            #              max_occurs=1000,
            #              supported_formats=[
            #                  Format('application/x-netcdf'),
            #                  Format('application/x-tar'),
            #                  Format('application/zip'),
            #              ]),

            LiteralInput("input_indices", "Indices",
                         abstract="Climate indices related to growth conditions \
                                    of tree species",
                         default=['TG_JJA', 'TNn_Jan'],
                         data_type='string',
                         min_occurs=1,
                         max_occurs=10,
                         allowed_values=_SDMINDICES_
                         ),

            LiteralInput("archive_format", "Archive format",
                         abstract="Result files will be compressed into archives. \
                                   Choose an appropriate format",
                         default="tar",
                         data_type='string',
                         min_occurs=1,
                         max_occurs=1,
                         allowed_values=['zip', 'tar']
                         )
        ]

        outputs = [
            ComplexOutput("output_indices", "Climate indices for growth conditions over all timesteps",
                          abstract="Archive (tar/zip) containing calculated climate indices",
                          supported_formats=[Format('application/x-tar'),
                                             Format('application/zip')
                                             ],
                          as_reference=True,
                          ),

            ComplexOutput('output_log', 'Logging information',
                          abstract="Collected logs during process run.",
                          as_reference=True,
                          supported_formats=[Format('text/plain')]
                          )
        ]

        super(SDMgetindicesProcess, self).__init__(
            self._handler,
            identifier="sdm_getindices",
            title="Species distribution Model (only indices calculation )",
            version="0.10",
            metadata=[
                Metadata("LWF", "http://www.lwf.bayern.de/"),
                Metadata(
                    "Doc",
                    "http://flyingpigeon.readthedocs.io/en/latest/descriptions/index.html#species-distribution-model"),
                Metadata("paper",
                         "http://www.hindawi.com/journals/jcli/2013/787250/"),
                Metadata("Tutorial",
                         "http://flyingpigeon.readthedocs.io/en/latest/tutorials/sdm.html"),
            ],
            abstract="Indices preparation for SDM process",
            inputs=inputs,
            outputs=outputs,
            status_supported=True,
            store_supported=True,
        )

    def _handler(self, request, response):
        from os.path import basename
        from flyingpigeon import sdm
        from flyingpigeon.utils import archive

        init_process_logger('log.txt')
        response.outputs['output_log'].file = 'log.txt'

        response.update_status('Start process', 0)

        try:
            LOGGER.info('reading the arguments')
            resources = archiveextract(
                resource=rename_complexinputs(request.inputs['dataset']))
            indices = request.inputs['input_indices']
            LOGGER.debug("indices = %s", indices)
            archive_format = request.inputs['archive_format']
        except:
            LOGGER.exception('failed to read in the arguments')
        LOGGER.info('indices %s ' % indices)

        #################################
        # calculate the climate indices
        #################################

        # indices calculation
        ncs_indices = None
        try:
            response.update_status('start calculation of climate indices for %s'
                                   % indices, 30)
            ncs_indices = sdm.get_indices(resources=resources, indices=indices)
            LOGGER.info('indice calculation done')
        except:
            msg = 'failed to calculate indices'
            LOGGER.exception(msg)
            raise Exception(msg)

        # archive multiple output files to one archive file
        try:
            archive_indices = archive(ncs_indices, format=archive_format)
            LOGGER.info('indices 3D added to tarfile')
        except:
            msg = 'failed adding indices to tar'
            LOGGER.exception(msg)
            raise Exception(msg)

        response.outputs['output_indices'].file = archive_indices
        response.update_status('done', 100)
        return response
<commit_msg>change default sdm indices simple<commit_after>"""
Processes for Species distribution
Author: Nils Hempelmann (nils.hempelmann@lsce.ipsl.fr)
"""

from flyingpigeon.sdm import _SDMINDICES_

from pywps import Process
from pywps import LiteralInput
from pywps import ComplexInput, ComplexOutput
from pywps import Format, FORMATS
from pywps.app.Common import Metadata
from flyingpigeon.log import init_process_logger

import logging
LOGGER = logging.getLogger("PYWPS")

class SDMgetindicesProcess(Process):
    def __init__(self):
        inputs = [
            ComplexInput('dataset', 'Dataset',
                         abstract="Enter either URL pointing to a NetCDF File"
                                  " or an archive (tar/zip) containing NetCDF files.",
                         min_occurs=1,
                         max_occurs=1000,
                         supported_formats=[
                             Format('application/x-netcdf'),
                             Format('application/x-tar'),
                             Format('application/zip'),
                         ]),

            # ComplexInput('resource', 'Resource',
            #              abstract='NetCDF Files or archive (tar/zip) containing netCDF files.',
            #              metadata=[Metadata('Info')],
            #              min_occurs=1,
            #              max_occurs=1000,
            #              supported_formats=[
            #                  Format('application/x-netcdf'),
            #                  Format('application/x-tar'),
            #                  Format('application/zip'),
            #              ]),

            LiteralInput("input_indices", "Indices",
                         abstract="Climate indices related to growth conditions \
                                    of tree species",
                         default='TG_JJA',
                         data_type='string',
                         min_occurs=1,
                         max_occurs=10,
                         allowed_values=_SDMINDICES_
                         ),

            LiteralInput("archive_format", "Archive format",
                         abstract="Result files will be compressed into archives. \
                                   Choose an appropriate format",
                         default="tar",
                         data_type='string',
                         min_occurs=1,
                         max_occurs=1,
                         allowed_values=['zip', 'tar']
                         )
        ]

        outputs = [
            ComplexOutput("output_indices", "Climate indices for growth conditions over all timesteps",
                          abstract="Archive (tar/zip) containing calculated climate indices",
                          supported_formats=[Format('application/x-tar'),
                                             Format('application/zip')
                                             ],
                          as_reference=True,
                          ),

            ComplexOutput('output_log', 'Logging information',
                          abstract="Collected logs during process run.",
                          as_reference=True,
                          supported_formats=[Format('text/plain')]
                          )
        ]

        super(SDMgetindicesProcess, self).__init__(
            self._handler,
            identifier="sdm_getindices",
            title="Species distribution Model (only indices calculation )",
            version="0.10",
            metadata=[
                Metadata("LWF", "http://www.lwf.bayern.de/"),
                Metadata(
                    "Doc",
                    "http://flyingpigeon.readthedocs.io/en/latest/descriptions/index.html#species-distribution-model"),
                Metadata("paper",
                         "http://www.hindawi.com/journals/jcli/2013/787250/"),
                Metadata("Tutorial",
                         "http://flyingpigeon.readthedocs.io/en/latest/tutorials/sdm.html"),
            ],
            abstract="Indices preparation for SDM process",
            inputs=inputs,
            outputs=outputs,
            status_supported=True,
            store_supported=True,
        )

    def _handler(self, request, response):
        from os.path import basename
        from flyingpigeon import sdm
        from flyingpigeon.utils import archive

        init_process_logger('log.txt')
        response.outputs['output_log'].file = 'log.txt'

        response.update_status('Start process', 0)

        try:
            LOGGER.info('reading the arguments')
            resources = archiveextract(
                resource=rename_complexinputs(request.inputs['dataset']))
            indices = request.inputs['input_indices']
            LOGGER.debug("indices = %s", indices)
            archive_format = request.inputs['archive_format']
        except:
            LOGGER.exception('failed to read in the arguments')
        LOGGER.info('indices %s ' % indices)

        #################################
        # calculate the climate indices
        #################################

        # indices calculation
        ncs_indices = None
        try:
            response.update_status('start calculation of climate indices for %s'
                                   % indices, 30)
            ncs_indices = sdm.get_indices(resources=resources, indices=indices)
            LOGGER.info('indice calculation done')
        except:
            msg = 'failed to calculate indices'
            LOGGER.exception(msg)
            raise Exception(msg)

        # archive multiple output files to one archive file
        try:
            archive_indices = archive(ncs_indices, format=archive_format)
            LOGGER.info('indices 3D added to tarfile')
        except:
            msg = 'failed adding indices to tar'
            LOGGER.exception(msg)
            raise Exception(msg)

        response.outputs['output_indices'].file = archive_indices
        response.update_status('done', 100)
        return response
<|endoftext|>


<commit_before>"""
Run all test cases.
"""

import sys
import os
import unittest

# Recommended minimum versions for optional packages
minimum_hdf5_version = "1.6.5"
minimum_numpy_version = "1.0.4"
minimum_numeric_version = "24.2"
minimum_numarray_version = "1.5.2"

import numpy

import tables
from tables.tests import common

def suite():
    test_modules = [
        'tables.tests.test_attributes',
        'tables.tests.test_basics',
        'tables.tests.test_create',
        'tables.tests.test_backcompat',
        'tables.tests.test_types',
        'tables.tests.test_lists',
        'tables.tests.test_tables',
        'tables.tests.test_tablesMD',
        'tables.tests.test_array',
        'tables.tests.test_earray',
        'tables.tests.test_carray',
        'tables.tests.test_vlarray',
        'tables.tests.test_tree',
        'tables.tests.test_timetype',
        'tables.tests.test_do_undo',
        'tables.tests.test_enum',
        'tables.tests.test_nestedtypes',
        'tables.tests.test_hdf5compat',
        'tables.tests.test_numpy',
        'tables.tests.test_queries',
        # Sub-packages
        'tables.nodes.tests.test_filenode',
        'tables.netcdf3.tests.test_netcdf3',
        'tables.numexpr.tests.test_numexpr',
        ]

    # Run indexing tests only under Pro
    if tables.is_pro:
        test_modules.append('tables.tests.test_indexes')
        test_modules.append('tables.tests.test_indexvalues')
        test_modules.append('tables.tests.test_index_backcompat')

    # Add test_Numeric only if Numeric is installed
    if common.numeric_imported:
        import Numeric
        print "Numeric (version %s) is present. Adding the Numeric test suite." % \
              (Numeric.__version__)
        if Numeric.__version__ < minimum_numeric_version:
            print "*Warning*: Numeric version is lower than recommended: %s < %s" % \
                  (Numeric.__version__, minimum_numeric_version)
        test_modules.append("tables.tests.test_Numeric")
    else:
        print "Skipping Numeric test suite."

    # Add test_numarray only if numarray is installed
    if common.numarray_imported:
        import numarray
        print \
"""numarray (version %s) is present. Adding the numarray test suite.""" % \
              (numarray.__version__)
        if numarray.__version__ < minimum_numarray_version:
            print \
"*Warning*: Numarray version is lower than recommended: %s < %s" % \
                  (numarray.__version__, minimum_numarray_version)
        test_modules.append("tables.tests.test_numarray")
        test_modules.append("tables.nra.tests.test_nestedrecords")
        test_modules.append("tables.nra.tests.test_nriterators")
    else:
        print "Skipping numarray test suite."
    print '-=' * 38

    # The test for garbage must be run *in the last place*.
    # Else, it is not as useful.
    test_modules.append('tables.tests.test_garbage')

    alltests = unittest.TestSuite()
    if common.show_memory:
        # Add a memory report at the beginning
        alltests.addTest(unittest.makeSuite(common.ShowMemTime))
    for name in test_modules:
        exec('from %s import suite as test_suite' % name)
        alltests.addTest(test_suite())
        if common.show_memory:
            # Add a memory report after each test module
            alltests.addTest(unittest.makeSuite(common.ShowMemTime))
    return alltests

def print_versions():
    """Print all the versions of software that PyTables relies on."""
    print '-=' * 38
    print "PyTables version:  %s" % tables.__version__
    print "HDF5 version:      %s" % tables.whichLibVersion("hdf5")[1]
    #print "HDF5 version:      %s" % tables.hdf5Version
    print "NumPy version:     %s" % numpy.__version__
    #print "Zlib version:      %s" % tables.whichLibVersion("zlib")[1]
    tinfo = tables.whichLibVersion("zlib")
    if tinfo is not None:
        print "Zlib version:      %s" % (tinfo[1])
    tinfo = tables.whichLibVersion("lzo")
    if tinfo is not None:
        print "LZO version:       %s (%s)" % (tinfo[1], tinfo[2])
    tinfo = tables.whichLibVersion("bzip2")
    if tinfo is not None:
        print "BZIP2 version:     %s (%s)" % (tinfo[1], tinfo[2])
    print 'Python version:    %s' % sys.version
    if os.name == 'posix':
        (sysname, nodename, release, version, machine) = os.uname()
        print 'Platform:          %s-%s' % (sys.platform, machine)
    print 'Byte-ordering:     %s' % sys.byteorder
    print '-=' * 38

def print_heavy(heavy):
    if heavy:
        print """\
Performing the complete test suite!"""
    else:
        print """\
Performing only a light (yet comprehensive) subset of the test suite.
If you want a more complete test, try passing the --heavy flag to this script
(or set the 'heavy' parameter in case you are using tables.test() call).
The whole suite will take more than 150 minutes to complete on a relatively
modern CPU and around 220 MB of main memory."""
    print '-=' * 38

def test(verbose=False, heavy=False):
    """
    Run all the tests in the test suite.

    If `verbose` is set, the test suite will emit messages with full
    verbosity (not recommended unless you are looking into a certain
    problem).

    If `heavy` is set, the test suite will be run in *heavy* mode (you
    should be careful with this because it can take a lot of time and
    resources from your computer).
    """
    print_versions()
    print_heavy(heavy)

    # What a context this is!
    oldverbose, common.verbose = common.verbose, verbose
    oldheavy, common.heavy = common.heavy, heavy
    try:
        unittest.TextTestRunner().run(suite())
    finally:
        common.verbose = oldverbose
        common.heavy = oldheavy  # there are pretty young heavies, too ;)

if __name__ == '__main__':

    hdf5_version = tables.whichLibVersion("hdf5")[1]
    if hdf5_version < minimum_hdf5_version:
        print "*Warning*: HDF5 version is lower than recommended: %s < %s" % \
              (hdf5_version, minimum_hdf5_version)

    if numpy.__version__ < minimum_numpy_version:
        print "*Warning*: NumPy version is lower than recommended: %s < %s" % \
              (numpy.__version__, minimum_numpy_version)

    # Handle some global flags (i.e. only useful for test_all.py)
    only_versions = 0
    args = sys.argv[:]
    for arg in args:
        if arg == '--show-versions':
            only_versions = True
            sys.argv.remove(arg)
        elif arg == '--show-memory':
            common.show_memory = True
            sys.argv.remove(arg)

    print_versions()
    if not only_versions:
        print_heavy(common.heavy)
        unittest.main(defaultTest='tables.tests.suite')
<commit_msg>Bettered estimation of memory taken by tests.<commit_after>"""
Run all test cases.
"""

import sys
import os
import unittest

# Recommended minimum versions for optional packages
minimum_hdf5_version = "1.6.5"
minimum_numpy_version = "1.0.4"
minimum_numeric_version = "24.2"
minimum_numarray_version = "1.5.2"

import numpy

import tables
from tables.tests import common

def suite():
    test_modules = [
        'tables.tests.test_attributes',
        'tables.tests.test_basics',
        'tables.tests.test_create',
        'tables.tests.test_backcompat',
        'tables.tests.test_types',
        'tables.tests.test_lists',
        'tables.tests.test_tables',
        'tables.tests.test_tablesMD',
        'tables.tests.test_array',
        'tables.tests.test_earray',
        'tables.tests.test_carray',
        'tables.tests.test_vlarray',
        'tables.tests.test_tree',
        'tables.tests.test_timetype',
        'tables.tests.test_do_undo',
        'tables.tests.test_enum',
        'tables.tests.test_nestedtypes',
        'tables.tests.test_hdf5compat',
        'tables.tests.test_numpy',
        'tables.tests.test_queries',
        # Sub-packages
        'tables.nodes.tests.test_filenode',
        'tables.netcdf3.tests.test_netcdf3',
        'tables.numexpr.tests.test_numexpr',
        ]

    # Run indexing tests only under Pro
    if tables.is_pro:
        test_modules.append('tables.tests.test_indexes')
        test_modules.append('tables.tests.test_indexvalues')
        test_modules.append('tables.tests.test_index_backcompat')

    # Add test_Numeric only if Numeric is installed
    if common.numeric_imported:
        import Numeric
        print "Numeric (version %s) is present. Adding the Numeric test suite." % \
              (Numeric.__version__)
        if Numeric.__version__ < minimum_numeric_version:
            print "*Warning*: Numeric version is lower than recommended: %s < %s" % \
                  (Numeric.__version__, minimum_numeric_version)
        test_modules.append("tables.tests.test_Numeric")
    else:
        print "Skipping Numeric test suite."

    # Add test_numarray only if numarray is installed
    if common.numarray_imported:
        import numarray
        print \
"""numarray (version %s) is present. Adding the numarray test suite.""" % \
              (numarray.__version__)
        if numarray.__version__ < minimum_numarray_version:
            print \
"*Warning*: Numarray version is lower than recommended: %s < %s" % \
                  (numarray.__version__, minimum_numarray_version)
        test_modules.append("tables.tests.test_numarray")
        test_modules.append("tables.nra.tests.test_nestedrecords")
        test_modules.append("tables.nra.tests.test_nriterators")
    else:
        print "Skipping numarray test suite."
    print '-=' * 38

    # The test for garbage must be run *in the last place*.
    # Else, it is not as useful.
    test_modules.append('tables.tests.test_garbage')

    alltests = unittest.TestSuite()
    if common.show_memory:
        # Add a memory report at the beginning
        alltests.addTest(unittest.makeSuite(common.ShowMemTime))
    for name in test_modules:
        exec('from %s import suite as test_suite' % name)
        alltests.addTest(test_suite())
        if common.show_memory:
            # Add a memory report after each test module
            alltests.addTest(unittest.makeSuite(common.ShowMemTime))
    return alltests

def print_versions():
    """Print all the versions of software that PyTables relies on."""
    print '-=' * 38
    print "PyTables version:  %s" % tables.__version__
    print "HDF5 version:      %s" % tables.whichLibVersion("hdf5")[1]
    #print "HDF5 version:      %s" % tables.hdf5Version
    print "NumPy version:     %s" % numpy.__version__
    #print "Zlib version:      %s" % tables.whichLibVersion("zlib")[1]
    tinfo = tables.whichLibVersion("zlib")
    if tinfo is not None:
        print "Zlib version:      %s" % (tinfo[1])
    tinfo = tables.whichLibVersion("lzo")
    if tinfo is not None:
        print "LZO version:       %s (%s)" % (tinfo[1], tinfo[2])
    tinfo = tables.whichLibVersion("bzip2")
    if tinfo is not None:
        print "BZIP2 version:     %s (%s)" % (tinfo[1], tinfo[2])
    print 'Python version:    %s' % sys.version
    if os.name == 'posix':
        (sysname, nodename, release, version, machine) = os.uname()
        print 'Platform:          %s-%s' % (sys.platform, machine)
    print 'Byte-ordering:     %s' % sys.byteorder
    print '-=' * 38

def print_heavy(heavy):
    if heavy:
        print """\
Performing the complete test suite!"""
    else:
        print """\
Performing only a light (yet comprehensive) subset of the test suite.
If you want a more complete test, try passing the --heavy flag to this script
(or set the 'heavy' parameter in case you are using tables.test() call).
The whole suite will take more than 150 minutes to complete on a relatively
modern CPU and around 180 MB of main memory."""
    print '-=' * 38

def test(verbose=False, heavy=False):
    """
    Run all the tests in the test suite.

    If `verbose` is set, the test suite will emit messages with full
    verbosity (not recommended unless you are looking into a certain
    problem).

    If `heavy` is set, the test suite will be run in *heavy* mode (you
    should be careful with this because it can take a lot of time and
    resources from your computer).
    """
    print_versions()
    print_heavy(heavy)

    # What a context this is!
    oldverbose, common.verbose = common.verbose, verbose
    oldheavy, common.heavy = common.heavy, heavy
    try:
        unittest.TextTestRunner().run(suite())
    finally:
        common.verbose = oldverbose
        common.heavy = oldheavy  # there are pretty young heavies, too ;)

if __name__ == '__main__':

    hdf5_version = tables.whichLibVersion("hdf5")[1]
    if hdf5_version < minimum_hdf5_version:
        print "*Warning*: HDF5 version is lower than recommended: %s < %s" % \
              (hdf5_version, minimum_hdf5_version)

    if numpy.__version__ < minimum_numpy_version:
        print "*Warning*: NumPy version is lower than recommended: %s < %s" % \
              (numpy.__version__, minimum_numpy_version)

    # Handle some global flags (i.e. only useful for test_all.py)
    only_versions = 0
    args = sys.argv[:]
    for arg in args:
        if arg == '--show-versions':
            only_versions = True
            sys.argv.remove(arg)
        elif arg == '--show-memory':
            common.show_memory = True
            sys.argv.remove(arg)

    print_versions()
    if not only_versions:
        print_heavy(common.heavy)
        unittest.main(defaultTest='tables.tests.suite')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import
from __future__ import unicode_literals

import io
import six
from six.moves.urllib.request import urlopen

from .. import errors, helpers
from .api import API

# Module API

class Web(API):
    """Loader to load source from the web.
    """

    # Public

    def __init__(self, source, encoding=None, stream=False, **options):
        self.__source = source
        self.__encoding = encoding
        self.__stream = stream
        self.__options = options

    def load(self, mode):

        # Prepare bytes
        if six.PY2:
            response = urlopen(self.__source)
            bytes = io.BufferedRandom(io.BytesIO())
            bytes.write(response.read())
            bytes.seek(0)
        else:
            bytes = _WebStream(self.__source)
            response = bytes.response

        # Prepare encoding
        encoding = self.__encoding
        if encoding is None:
            if six.PY2:
                encoding = response.headers.getparam('charset')
            else:
                encoding = response.headers.get_content_charset()
        if encoding is None:
            encoding = helpers.detect_encoding(bytes)

        # Return or raise
        if mode == 'b':
            return bytes
        elif mode == 't':
            chars = io.TextIOWrapper(bytes, encoding, **self.__options)
            return chars
        else:
            message = 'Mode %s is not supported' % mode
            raise errors.Error(message)

    @property
    def encoding(self):
        return self.__encoding

# Internal

class _WebStream(object):

    # Public

    def __init__(self, source):
        self.__source = source
        self.__response = self.__make_request()

    def __getattr__(self, name):
        return getattr(self.__response, name)

    @property
    def response(self):
        return self.__response

    def seekable(self):
        return True

    def seek(self, offset):
        if offset != 0:
            message = 'Seek support only 0 offset.'
            raise ValueError(message)
        self.__response = self.__make_request()

    # Private

    def __make_request(self):
        return urlopen(self.__source)
<commit_msg>removed stream parameter from loaders.Web<commit_after># -*- coding: utf-8 -*-
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import
from __future__ import unicode_literals

import io
import six
from six.moves.urllib.request import urlopen

from .. import errors, helpers
from .api import API

# Module API

class Web(API):
    """Loader to load source from the web.
    """

    # Public

    def __init__(self, source, encoding=None, **options):
        self.__source = source
        self.__encoding = encoding
        self.__options = options

    def load(self, mode):

        # Prepare bytes
        if six.PY2:
            response = urlopen(self.__source)
            bytes = io.BufferedRandom(io.BytesIO())
            bytes.write(response.read())
            bytes.seek(0)
        else:
            bytes = _WebStream(self.__source)
            response = bytes.response

        # Prepare encoding
        encoding = self.__encoding
        if encoding is None:
            if six.PY2:
                encoding = response.headers.getparam('charset')
            else:
                encoding = response.headers.get_content_charset()
        if encoding is None:
            encoding = helpers.detect_encoding(bytes)

        # Return or raise
        if mode == 'b':
            return bytes
        elif mode == 't':
            chars = io.TextIOWrapper(bytes, encoding, **self.__options)
            return chars
        else:
            message = 'Mode %s is not supported' % mode
            raise errors.Error(message)

    @property
    def encoding(self):
        return self.__encoding

# Internal

class _WebStream(object):

    # Public

    def __init__(self, source):
        self.__source = source
        self.__response = self.__make_request()

    def __getattr__(self, name):
        return getattr(self.__response, name)

    @property
    def response(self):
        return self.__response

    def seekable(self):
        return True

    def seek(self, offset):
        if offset != 0:
            message = 'Seek support only 0 offset.'
            raise ValueError(message)
        self.__response = self.__make_request()

    # Private

    def __make_request(self):
        return urlopen(self.__source)
<|endoftext|>


<commit_before>#-*- python -*-
import difflib
import logging
from pprint import pformat

# Non-stdlib imports
import pkg_resources
from tg import expose, validate, redirect
from pylons import g, c, request
from formencode import validators
from pymongo.bson import ObjectId

from ming.orm.base import mapper

# Pyforge-specific imports
from pyforge.app import Application, ConfigOption, SitemapEntry
from pyforge.lib.helpers import push_config
from pyforge.lib.search import search
from pyforge.lib.decorators import audit, react
from pyforge.lib.security import require, has_artifact_access
from pyforge.model import ProjectRole

# Local imports
from forgewiki import model
from forgewiki import version

log = logging.getLogger(__name__)

# Will not be needed after _dispatch is fixed in tg 2.1
from pyforge.lib.dispatch import _dispatch

class ForgeWikiApp(Application):
    '''This is the Wiki app for PyForge'''
    __version__ = version.__version__
    permissions = [ 'configure', 'read', 'create', 'edit', 'delete', 'comment' ]
    config_options = Application.config_options + [
        ConfigOption('project_name', str, 'pname'),
        ConfigOption('message', str, 'Custom message goes here'),
        ]

    def __init__(self, project, config):
        Application.__init__(self, project, config)
        self.root = RootController()

    @audit('Wiki.#')
    def auditor(self, routing_key, data):
        log.info('JTB13 Auditing data from %s (%s)',
                 routing_key, self.config.options.mount_point)
        try:
            elements = routing_key.split('.')
            count = len(elements)
        except:
            log.info('Audit applies to page Root.')
            p = model.Page.upsert('Root')
        else:
            log.info('JTB13 Audit applies to page ' + elements[1])
            p = model.Page.upsert(elements[1])
            try:
#                p.text = 'This is the JTB13 test page:' + data.body
#                p.text = str(data)
                p.text = str(data['body'])
                p.commit()
#                c = p.reply()
#                c.text = 'This is a test comment.'
#                c.m.save()
            except:
                log.info('JTB13 failed to configure wiki')
            else:
                log.info('JTB13 configured wiki')

    @react('Wiki.#')
    def reactor(self, routing_key, data):
        log.info('Reacting to data from %s (%s)',
                 routing_key, self.config.options.mount_point)

    @property
    def sitemap(self):
        menu_id = 'ForgeWiki (%s)' % self.config.options.mount_point
        with push_config(c, app=self):
            pages = [
                SitemapEntry(p.title, p.url())
                for p in model.Page.query.find(dict(
                        app_config_id=self.config._id)) ]
            return [
                SitemapEntry(menu_id, '.')[SitemapEntry('Pages')[pages]] ]

    def sidebar_menu(self):
        return [
            SitemapEntry(p.title, p.url())
            for p in model.Page.query.find(dict(
                    app_config_id=self.config._id)) ]

    @property
    def templates(self):
         return pkg_resources.resource_filename('forgewiki', 'templates')

    def install(self, project):
        'Set up any default permissions and roles here'

        self.config.options['project_name'] = project._id
        self.uninstall(project)
        # Give the installing user all the permissions
        pr = c.user.project_role()
        for perm in self.permissions:
              self.config.acl[perm] = [ pr._id ]
        self.config.acl['read'].append(
            ProjectRole.query.get(name='*anonymous')._id)
        self.config.acl['comment'].append(
            ProjectRole.query.get(name='*authenticated')._id)
        p = model.Page.upsert('Root')
        p.text = 'This is the root page.'
        p.commit()

    def uninstall(self, project):
        "Remove all the plugin's artifacts from the database"
        mapper(model.Page).remove(dict(project_id=c.project._id))
        mapper(model.Comment).remove(dict(project_id=c.project._id))

class RootController(object):

    @expose('forgewiki.templates.index')
    def index(self):
        redirect('Root/')
        return dict(message=c.app.config.options['message'])

    #Will not be needed after _dispatch is fixed in tg 2.1
    def _dispatch(self, state, remainder):
        return _dispatch(self, state, remainder)

    #Instantiate a Page object, and continue dispatch there
    def _lookup(self, pname, *remainder):
        return PageController(pname), remainder

    @expose()
    def new_page(self, title):
        redirect(title + '/')

    @expose('forgewiki.templates.search')
    @validate(dict(q=validators.UnicodeString(if_empty=None),
                   history=validators.StringBool(if_empty=False)))
    def search(self, q=None, history=None):
        'local wiki search'
        results = []
        count=0
        if not q:
            q = ''
        else:
            search_query = '''%s
            AND is_history_b:%s
            AND project_id_s:%s
            AND mount_point_s:%s''' % (
                q, history, c.project._id, c.app.config.options.mount_point)
            results = search(search_query)
            if results: count=results.hits
        return dict(q=q, history=history, results=results or [], count=count)

class PageController(object):

    def __init__(self, title):
        self.title = title
        self.page = model.Page.upsert(title)
        self.comments = CommentController(self.page)

    def get_version(self, version):
        if not version: return self.page
        try:
            return model.Page.upsert(self.title, version=int(version))
        except ValueError:
            return None

    @expose('forgewiki.templates.page_view')
    @validate(dict(version=validators.Int(if_empty=None)))
    def index(self, version=None):
        require(has_artifact_access('read', self.page))
        page = self.get_version(version)
        if page is None:
            if version: redirect('.?version=%d' % (version-1))
            else: redirect('.')
        cur = page.version
        if cur > 1: prev = cur-1
        else: prev = None
        next = cur+1
        return dict(page=page,
                    cur=cur, prev=prev, next=next)

    @expose('forgewiki.templates.page_edit')
    def edit(self):
        if self.page.version == 1:
            require(has_artifact_access('create', self.page))
        else:
            require(has_artifact_access('edit', self.page))
        return dict(page=self.page,
                    example=MARKDOWN_EXAMPLE)

    @expose('forgewiki.templates.page_history')
    def history(self):
        require(has_artifact_access('read', self.page))
        pages = self.page.history()
        return dict(title=self.title, pages=pages)

    @expose('forgewiki.templates.page_diff')
    def diff(self, v1, v2):
        require(has_artifact_access('read', self.page))
        p1 = self.get_version(int(v1))
        p2 = self.get_version(int(v2))
        t1 = p1.text
        t2 = p2.text
        differ = difflib.SequenceMatcher(None, p1.text, p2.text)
        result = []
        for tag, i1, i2, j1, j2 in differ.get_opcodes():
            if tag in ('delete', 'replace'):
                result += [ '', t1[i1:i2], '' ]
            if tag in ('insert', 'replace'):
                result += [ '', t2[j1:j2], '' ]
            if tag == 'equal':
                result += t1[i1:i2]
        result = ''.join(result).replace('\n', '
\n')
        return dict(p1=p1, p2=p2, edits=result)

    @expose(content_type='text/plain')
    def raw(self):
        require(has_artifact_access('read', self.page))
        return pformat(self.page)

    @expose()
    def revert(self, version):
        require(has_artifact_access('edit', self.page))
        orig = self.get_version(version)
        self.page.text = orig.text
        self.page.commit()
        redirect('.')

    @expose()
    def update(self, text):
        require(has_artifact_access('edit', self.page))
        self.page.text = text
        self.page.commit()
        redirect('.')

class CommentController(object):

    def __init__(self, page, comment_id=None):
        self.page = page
        self.comment_id = comment_id
        self.comment = model.Comment.query.get(_id=self.comment_id)

    @expose()
    def reply(self, text):
        require(has_artifact_access('comment', self.page))
        if self.comment_id:
            c = self.comment.reply()
            c.text = text
        else:
            c = self.page.reply()
            c.text = text
        redirect(request.referer)

    @expose()
    def delete(self):
        require(lambda:c.user._id == self.comment.author()._id)
        self.comment.delete()
        redirect(request.referer)

    def _dispatch(self, state, remainder):
        return _dispatch(self, state, remainder)

    def _lookup(self, next, *remainder):
        if self.comment_id:
            return CommentController(
                self.page,
                self.comment_id + '/' + next), remainder
        else:
            return CommentController(
                self.page, next), remainder

MARKDOWN_EXAMPLE='''
# First-level heading

Some *emphasized* and **strong** text

#### Fourth-level heading

'''
<commit_msg>Stripped some debug information from wiki auditor<commit_after>#-*- python -*-
import difflib
import logging
from pprint import pformat

# Non-stdlib imports
import pkg_resources
from tg import expose, validate, redirect
from pylons import g, c, request
from formencode import validators
from pymongo.bson import ObjectId

from ming.orm.base import mapper

# Pyforge-specific imports
from pyforge.app import Application, ConfigOption, SitemapEntry
from pyforge.lib.helpers import push_config
from pyforge.lib.search import search
from pyforge.lib.decorators import audit, react
from pyforge.lib.security import require, has_artifact_access
from pyforge.model import ProjectRole

# Local imports
from forgewiki import model
from forgewiki import version

log = logging.getLogger(__name__)

# Will not be needed after _dispatch is fixed in tg 2.1
from pyforge.lib.dispatch import _dispatch

class ForgeWikiApp(Application):
    '''This is the Wiki app for PyForge'''
    __version__ = version.__version__
    permissions = [ 'configure', 'read', 'create', 'edit', 'delete', 'comment' ]
    config_options = Application.config_options + [
        ConfigOption('project_name', str, 'pname'),
        ConfigOption('message', str, 'Custom message goes here'),
        ]

    def __init__(self, project, config):
        Application.__init__(self, project, config)
        self.root = RootController()

    @audit('Wiki.#')
    def auditor(self, routing_key, data):
        log.info('Auditing data from %s (%s)',
                 routing_key, self.config.options.mount_point)
        try:
            elements = routing_key.split('.')
            count = len(elements)
        except:
            log.info('Audit applies to page Root.')
            p = model.Page.upsert('Root')
        else:
            log.info('Audit applies to page ' + elements[1])
            p = model.Page.upsert(elements[1])
            try:
                p.text = str(data['body'])
                p.commit()
#                c = p.reply()
#                c.text = 'This is a test comment.'
#                c.m.save()

    @react('Wiki.#')
    def reactor(self, routing_key, data):
        log.info('Reacting to data from %s (%s)',
                 routing_key, self.config.options.mount_point)

    @property
    def sitemap(self):
        menu_id = 'ForgeWiki (%s)' % self.config.options.mount_point
        with push_config(c, app=self):
            pages = [
                SitemapEntry(p.title, p.url())
                for p in model.Page.query.find(dict(
                        app_config_id=self.config._id)) ]
            return [
                SitemapEntry(menu_id, '.')[SitemapEntry('Pages')[pages]] ]

    def sidebar_menu(self):
        return [
            SitemapEntry(p.title, p.url())
            for p in model.Page.query.find(dict(
                    app_config_id=self.config._id)) ]

    @property
    def templates(self):
         return pkg_resources.resource_filename('forgewiki', 'templates')

    def install(self, project):
        'Set up any default permissions and roles here'

        self.config.options['project_name'] = project._id
        self.uninstall(project)
        # Give the installing user all the permissions
        pr = c.user.project_role()
        for perm in self.permissions:
              self.config.acl[perm] = [ pr._id ]
        self.config.acl['read'].append(
            ProjectRole.query.get(name='*anonymous')._id)
        self.config.acl['comment'].append(
            ProjectRole.query.get(name='*authenticated')._id)
        p = model.Page.upsert('Root')
        p.text = 'This is the root page.'
        p.commit()

    def uninstall(self, project):
        "Remove all the plugin's artifacts from the database"
        mapper(model.Page).remove(dict(project_id=c.project._id))
        mapper(model.Comment).remove(dict(project_id=c.project._id))

class RootController(object):

    @expose('forgewiki.templates.index')
    def index(self):
        redirect('Root/')
        return dict(message=c.app.config.options['message'])

    #Will not be needed after _dispatch is fixed in tg 2.1
    def _dispatch(self, state, remainder):
        return _dispatch(self, state, remainder)

    #Instantiate a Page object, and continue dispatch there
    def _lookup(self, pname, *remainder):
        return PageController(pname), remainder

    @expose()
    def new_page(self, title):
        redirect(title + '/')

    @expose('forgewiki.templates.search')
    @validate(dict(q=validators.UnicodeString(if_empty=None),
                   history=validators.StringBool(if_empty=False)))
    def search(self, q=None, history=None):
        'local wiki search'
        results = []
        count=0
        if not q:
            q = ''
        else:
            search_query = '''%s
            AND is_history_b:%s
            AND project_id_s:%s
            AND mount_point_s:%s''' % (
                q, history, c.project._id, c.app.config.options.mount_point)
            results = search(search_query)
            if results: count=results.hits
        return dict(q=q, history=history, results=results or [], count=count)

class PageController(object):

    def __init__(self, title):
        self.title = title
        self.page = model.Page.upsert(title)
        self.comments = CommentController(self.page)

    def get_version(self, version):
        if not version: return self.page
        try:
            return model.Page.upsert(self.title, version=int(version))
        except ValueError:
            return None

    @expose('forgewiki.templates.page_view')
    @validate(dict(version=validators.Int(if_empty=None)))
    def index(self, version=None):
        require(has_artifact_access('read', self.page))
        page = self.get_version(version)
        if page is None:
            if version: redirect('.?version=%d' % (version-1))
            else: redirect('.')
        cur = page.version
        if cur > 1: prev = cur-1
        else: prev = None
        next = cur+1
        return dict(page=page,
                    cur=cur, prev=prev, next=next)

    @expose('forgewiki.templates.page_edit')
    def edit(self):
        if self.page.version == 1:
            require(has_artifact_access('create', self.page))
        else:
            require(has_artifact_access('edit', self.page))
        return dict(page=self.page,
                    example=MARKDOWN_EXAMPLE)

    @expose('forgewiki.templates.page_history')
    def history(self):
        require(has_artifact_access('read', self.page))
        pages = self.page.history()
        return dict(title=self.title, pages=pages)

    @expose('forgewiki.templates.page_diff')
    def diff(self, v1, v2):
        require(has_artifact_access('read', self.page))
        p1 = self.get_version(int(v1))
        p2 = self.get_version(int(v2))
        t1 = p1.text
        t2 = p2.text
        differ = difflib.SequenceMatcher(None, p1.text, p2.text)
        result = []
        for tag, i1, i2, j1, j2 in differ.get_opcodes():
            if tag in ('delete', 'replace'):
                result += [ '', t1[i1:i2], '' ]
            if tag in ('insert', 'replace'):
                result += [ '', t2[j1:j2], '' ]
            if tag == 'equal':
                result += t1[i1:i2]
        result = ''.join(result).replace('\n', '
\n')
        return dict(p1=p1, p2=p2, edits=result)

    @expose(content_type='text/plain')
    def raw(self):
        require(has_artifact_access('read', self.page))
        return pformat(self.page)

    @expose()
    def revert(self, version):
        require(has_artifact_access('edit', self.page))
        orig = self.get_version(version)
        self.page.text = orig.text
        self.page.commit()
        redirect('.')

    @expose()
    def update(self, text):
        require(has_artifact_access('edit', self.page))
        self.page.text = text
        self.page.commit()
        redirect('.')

class CommentController(object):

    def __init__(self, page, comment_id=None):
        self.page = page
        self.comment_id = comment_id
        self.comment = model.Comment.query.get(_id=self.comment_id)

    @expose()
    def reply(self, text):
        require(has_artifact_access('comment', self.page))
        if self.comment_id:
            c = self.comment.reply()
            c.text = text
        else:
            c = self.page.reply()
            c.text = text
        redirect(request.referer)

    @expose()
    def delete(self):
        require(lambda:c.user._id == self.comment.author()._id)
        self.comment.delete()
        redirect(request.referer)

    def _dispatch(self, state, remainder):
        return _dispatch(self, state, remainder)

    def _lookup(self, next, *remainder):
        if self.comment_id:
            return CommentController(
                self.page,
                self.comment_id + '/' + next), remainder
        else:
            return CommentController(
                self.page, next), remainder

MARKDOWN_EXAMPLE='''
# First-level heading

Some *emphasized* and **strong** text

#### Fourth-level heading

'''
<|endoftext|>


<commit_before>"""Version information for Review Board dependencies.

This contains constants that other parts of Review Board (primarily packaging)
can use to look up information on major dependencies of Review Board.

The contents in this file might change substantially between releases. If
you're going to make use of data from this file, code defensively.
"""

from __future__ import unicode_literals

import sys
import textwrap

# NOTE: This file may not import other (non-Python) modules! (Except for
#       the parent reviewboard module, which must be importable anyway) This
#       module is used for packaging and be needed before any dependencies
#       have been installed.

#: The major version of Django we're using for documentation.
django_doc_major_version = '1.6'

#: The major version of Djblets we're using for documentation.
djblets_doc_major_version = '1.0'

#: The version range required for Django.
django_version = '>=1.6.11,<1.6.999'

#: The version range required for Djblets.
djblets_version = '>=1.0.3,<=1.0.999'

#: All dependencies required to install Review Board.
package_dependencies = {
    'cryptography': '>=1.8.1',
    'Django': django_version,
    'django-cors-headers': '>=1.1.0,<1.1.999',
    'django_evolution': '>=0.7.7,<=0.7.999',
    'django-haystack': '>=2.4.0,<=2.4.999',
    'django-multiselectfield': '',
    'django-oauth-toolkit': '>=0.9.0,<0.9.999',
    'Djblets': djblets_version,
    'docutils': '',
    'markdown': '>=2.6.8,<2.6.999',
    'mimeparse': '>=0.1.3',
    'paramiko': '>=1.12',
    'Pygments': '>=2.1',
    'pymdown-extensions': '>=3.4,<3.999',
    'python-dateutil': '>=1.5',
    'python-memcached': '',
    'pytz': '>=2015.2',
    'Whoosh': '>=2.6',
}

#: Dependencies only specified during the packaging process.
#:
#: These dependencies are not used when simply developing Review Board.
#: The dependencies here are generally intended to be those that themselves
#: require Review Board.
package_only_dependencies = {
    'rbintegrations': '>=0.5',
}

_dependency_error_count = 0
_dependency_warning_count = 0

def build_dependency_list(deps, version_prefix=''):
    """Build a list of dependency specifiers from a dependency map.

    This can be used along with :py:data:`package_dependencies`,
    :py:data:`npm_dependencies`, or other dependency dictionaries to build a
    list of dependency specifiers for use on the command line or in
    :file:`setup.py`.

    Args:
        deps (dict):
            A dictionary of dependencies.

    Returns:
        list of unicode:
        A list of dependency specifiers.
    """
    return sorted(
        [
            '%s%s%s' % (dep_name, version_prefix, dep_version)
            for dep_name, dep_version in deps.items()
        ],
        key=lambda s: s.lower())

def _dependency_message(message, prefix=''):
    """Utility function to print and track a dependency-related message.

    This will track that a message was printed, allowing us to determine if
    any messages were shown to the user.

    Args:
        message (unicode):
            The dependency-related message to display. This will be wrapped,
            but long strings (like paths) will not contain line breaks.

        prefix (unicode, optional):
            The prefix for the message. All text will be aligned after this.
    """
    sys.stderr.write('\n%s\n'
                     % textwrap.fill(message,
                                     initial_indent=prefix,
                                     subsequent_indent=' ' * len(prefix),
                                     break_long_words=False,
                                     break_on_hyphens=False))

def dependency_error(message):
    """Print a dependency error.

    This will track that a message was printed, allowing us to determine if
    any messages were shown to the user.

    Args:
        message (unicode):
            The dependency error to display. This will be wrapped, but long
            strings (like paths) will not contain line breaks.
    """
    global _dependency_error_count

    _dependency_message(message, prefix='ERROR: ')
    _dependency_error_count += 1

def dependency_warning(message):
    """Print a dependency warning.

    This will track that a message was printed, allowing us to determine if
    any messages were shown to the user.

    Args:
        message (unicode):
            The dependency warning to display. This will be wrapped, but long
            strings (like paths) will not contain line breaks.
    """
    global _dependency_warning_count

    _dependency_message(message, prefix='WARNING: ')
    _dependency_warning_count += 1

def fail_if_missing_dependencies():
    """Exit the process with an error if dependency messages were shown.

    If :py:func:`dependency_error` or :py:func:`dependency_warning` were
    called, this will print some help information with a link to the manual
    and then exit the process.
    """
    if _dependency_warning_count > 0 or _dependency_error_count > 0:
        from reviewboard import get_manual_url

        _dependency_message('Please see %s for help setting up Review Board.'
                            % get_manual_url())

        if _dependency_error_count > 0:
            sys.exit(1)
<commit_msg>Require Djblets 2.0 dev.<commit_after>"""Version information for Review Board dependencies.

This contains constants that other parts of Review Board (primarily packaging)
can use to look up information on major dependencies of Review Board.

The contents in this file might change substantially between releases. If
you're going to make use of data from this file, code defensively.
"""

from __future__ import unicode_literals

import sys
import textwrap

# NOTE: This file may not import other (non-Python) modules! (Except for
#       the parent reviewboard module, which must be importable anyway) This
#       module is used for packaging and be needed before any dependencies
#       have been installed.

#: The major version of Django we're using for documentation.
django_doc_major_version = '1.6'

#: The major version of Djblets we're using for documentation.
djblets_doc_major_version = '1.0'

#: The version range required for Django.
django_version = '>=1.6.11,<1.6.999'

#: The version range required for Djblets.
djblets_version = '>=2.0.dev,<=2.0.999'

#: All dependencies required to install Review Board.
package_dependencies = {
    'cryptography': '>=1.8.1',
    'Django': django_version,
    'django-cors-headers': '>=1.1.0,<1.1.999',
    'django_evolution': '>=0.7.7,<=0.7.999',
    'django-haystack': '>=2.4.0,<=2.4.999',
    'django-multiselectfield': '',
    'django-oauth-toolkit': '>=0.9.0,<0.9.999',
    'Djblets': djblets_version,
    'docutils': '',
    'markdown': '>=2.6.8,<2.6.999',
    'mimeparse': '>=0.1.3',
    'paramiko': '>=1.12',
    'Pygments': '>=2.1',
    'pymdown-extensions': '>=3.4,<3.999',
    'python-dateutil': '>=1.5',
    'python-memcached': '',
    'pytz': '>=2015.2',
    'Whoosh': '>=2.6',
}

#: Dependencies only specified during the packaging process.
#:
#: These dependencies are not used when simply developing Review Board.
#: The dependencies here are generally intended to be those that themselves
#: require Review Board.
package_only_dependencies = {
    'rbintegrations': '>=0.5',
}

_dependency_error_count = 0
_dependency_warning_count = 0

def build_dependency_list(deps, version_prefix=''):
    """Build a list of dependency specifiers from a dependency map.

    This can be used along with :py:data:`package_dependencies`,
    :py:data:`npm_dependencies`, or other dependency dictionaries to build a
    list of dependency specifiers for use on the command line or in
    :file:`setup.py`.

    Args:
        deps (dict):
            A dictionary of dependencies.

    Returns:
        list of unicode:
        A list of dependency specifiers.
    """
    return sorted(
        [
            '%s%s%s' % (dep_name, version_prefix, dep_version)
            for dep_name, dep_version in deps.items()
        ],
        key=lambda s: s.lower())

def _dependency_message(message, prefix=''):
    """Utility function to print and track a dependency-related message.

    This will track that a message was printed, allowing us to determine if
    any messages were shown to the user.

    Args:
        message (unicode):
            The dependency-related message to display. This will be wrapped,
            but long strings (like paths) will not contain line breaks.

        prefix (unicode, optional):
            The prefix for the message. All text will be aligned after this.
    """
    sys.stderr.write('\n%s\n'
                     % textwrap.fill(message,
                                     initial_indent=prefix,
                                     subsequent_indent=' ' * len(prefix),
                                     break_long_words=False,
                                     break_on_hyphens=False))

def dependency_error(message):
    """Print a dependency error.

    This will track that a message was printed, allowing us to determine if
    any messages were shown to the user.

    Args:
        message (unicode):
            The dependency error to display. This will be wrapped, but long
            strings (like paths) will not contain line breaks.
    """
    global _dependency_error_count

    _dependency_message(message, prefix='ERROR: ')
    _dependency_error_count += 1

def dependency_warning(message):
    """Print a dependency warning.

    This will track that a message was printed, allowing us to determine if
    any messages were shown to the user.

    Args:
        message (unicode):
            The dependency warning to display. This will be wrapped, but long
            strings (like paths) will not contain line breaks.
    """
    global _dependency_warning_count

    _dependency_message(message, prefix='WARNING: ')
    _dependency_warning_count += 1

def fail_if_missing_dependencies():
    """Exit the process with an error if dependency messages were shown.

    If :py:func:`dependency_error` or :py:func:`dependency_warning` were
    called, this will print some help information with a link to the manual
    and then exit the process.
    """
    if _dependency_warning_count > 0 or _dependency_error_count > 0:
        from reviewboard import get_manual_url

        _dependency_message('Please see %s for help setting up Review Board.'
                            % get_manual_url())

        if _dependency_error_count > 0:
            sys.exit(1)
<|endoftext|>


<commit_before>import pdb
import numpy as np
import os
import linecache

class Reader:
    """
    Interface for reader. Must be implemented from all inherited classes.
    """
    def getMatrix(self):
        """
        returns complete matrix
        """
        raise Exception('must be implemented from inherited class.')

    def getRows(self,idx):
        """
        returns the rows M[idx] of the matrix

        input:
        idx   :   row indices
        """
        raise Exception('must be implemented from inherited class.')

class MatrixReader(Reader):
    """
    container for holding matrix or hdf object
    """
    def __init__(self,M):
        """
        constructor

        input:
        M   :   data matrix/hdf object
        """
        self.M = M

    def getMatrix(self):
        """
        returns complete matrix
        """
        return self.M[:]

    def getRows(self,idx):
        """
        returns the rows M[idx] of the matrix

        input:
        idx   :   row indices
        """
        return self.M[idx,:]

class FileReader(Reader):
    """
    data container of the following format
    - data matrix is saved in basefile.matrix
    - column information is saved in basefile.cols
    - row information is saved in basefile.rows
    """

    def __init__(self, basefile, load_rowinfo=True, load_colinfo=True):
        """
        constructor

        input:
        basefile      : name of basefile
        load_rowinfo  : if True, row information is loaded (default: True)
        load_colinfo  : if True, column information is loaded (default: False)
        """
        self.basefile = basefile
        if load_rowinfo: self.row_info = self.getInfo('rows')
        if load_colinfo: self.col_info = self.getInfo('cols')

        if not(os.path.exists(self.basefile + '.cache.npy')):
            self.createCacheFile()

        self.line_offset = np.load(self.basefile + '.cache.npy')

    def createCacheFile(self):
        line_offset = []
        offset = 0
        f = open(self.basefile + '.matrix','r')
        for line in f:
            line_offset.append(offset)
            offset += len(line)

        line_offset = np.array(line_offset)
        np.save(self.basefile + '.cache.npy', line_offset)

    def getMatrix(self):
        """
        returns complete matrix
        """
        M = np.loadtxt(self.basefile + '.matrix')
        return M

    def getRows(self,idx):
        """
        returns the rows M[idx] of the matrix

        input:
        idx   :   row indices
        """

        f = open(self.basefile + '.matrix','r')
        line = f.readline()
        n_cols = len(line.split(' '))
        RV = np.zeros((len(idx),n_cols))

        j=0
        for i in idx:
            f.seek(self.line_offset[i])
            line = f.readline()
            line = line.split(' ')
            RV[j] = np.array(line, dtype=np.float)
            j += 1

        return RV

    def getRowIterator(self):
        f = open(self.basefile + '.matrix', 'r')

        j = 0
        for i,line in enumerate(f):
            line = np.array(line.split(' '), dtype=float)
            yield i, line

    def getInfo(self, which):
        """
        loads the row information into memory
        """
        fn = self.basefile+'.' + which
        if not(os.path.exists(fn)): return None
        M = np.loadtxt(fn,dtype=str)
        if M.ndim==1: M = M[:,np.newaxis]
        header = M[0]
        data = {}
        for ikey,key in enumerate(header):
            arr = M[:,ikey][1:]
            try:
                arr = np.array(arr, dtype=int)
            except:
                pass
            data[key] = arr

        return data

<commit_msg>phenotype matrix can have any delimiter<commit_after>import pdb
import numpy as np
import os
import linecache
import csv

class Reader:
    """
    Interface for reader. Must be implemented from all inherited classes.
    """
    def getMatrix(self):
        """
        returns complete matrix
        """
        raise Exception('must be implemented from inherited class.')

    def getRows(self,idx):
        """
        returns the rows M[idx] of the matrix

        input:
        idx   :   row indices
        """
        raise Exception('must be implemented from inherited class.')

class MatrixReader(Reader):
    """
    container for holding matrix or hdf object
    """
    def __init__(self,M):
        """
        constructor

        input:
        M   :   data matrix/hdf object
        """
        self.M = M

    def getMatrix(self):
        """
        returns complete matrix
        """
        return self.M[:]

    def getRows(self,idx):
        """
        returns the rows M[idx] of the matrix

        input:
        idx   :   row indices
        """
        return self.M[idx,:]

class FileReader(Reader):
    """
    data container of the following format
    - data matrix is saved in basefile.matrix
    - column information is saved in basefile.cols
    - row information is saved in basefile.rows
    """

    def __init__(self, basefile, load_rowinfo=True, load_colinfo=True):
        """
        constructor

        input:
        basefile      : name of basefile
        load_rowinfo  : if True, row information is loaded (default: True)
        load_colinfo  : if True, column information is loaded (default: False)
        """
        self.basefile = basefile
        if load_rowinfo: self.row_info = self.getInfo('rows')
        if load_colinfo: self.col_info = self.getInfo('cols')

        if not(os.path.exists(self.basefile + '.cache.npy')):
            self.createCacheFile()

        self.line_offset = np.load(self.basefile + '.cache.npy')

        f = open(self.basefile + '.matrix','r')
        self.delimiter = csv.Sniffer().sniff(f.read(1024)).delimiter
        f.close()

    def createCacheFile(self):
        line_offset = []
        offset = 0
        f = open(self.basefile + '.matrix','r')
        for line in f:
            line_offset.append(offset)
            offset += len(line)

        line_offset = np.array(line_offset)
        np.save(self.basefile + '.cache.npy', line_offset)

    def getMatrix(self):
        """
        returns complete matrix
        """
        M = np.loadtxt(self.basefile + '.matrix')
        return M

    def getRows(self,idx):
        """
        returns the rows M[idx] of the matrix

        input:
        idx   :   row indices
        """
        f = open(self.basefile + '.matrix','r')
        line = f.readline()
        n_cols = len(line.split(self.delimiter))
        RV = np.zeros((len(idx),n_cols))

        j=0
        for i in idx:
            f.seek(self.line_offset[i])
            line = f.readline()
            line = line.split(self.delimiter)
            RV[j] = np.array(line, dtype=np.float)
            j += 1

        return RV

    def getRowIterator(self):
        f = open(self.basefile + '.matrix', 'r')

        j = 0
        for i,line in enumerate(f):
            line = np.array(line.split(self.delimiter), dtype=float)
            yield i, line

    def getInfo(self, which):
        """
        loads the row information into memory
        """
        fn = self.basefile+'.' + which
        if not(os.path.exists(fn)): return None
        M = np.loadtxt(fn,dtype=str)
        if M.ndim==1: M = M[:,np.newaxis]
        header = M[0]
        data = {}
        for ikey,key in enumerate(header):
            arr = M[:,ikey][1:]
            try:
                arr = np.array(arr, dtype=int)
            except:
                pass
            data[key] = arr

        return data

<|endoftext|>