<commit_before>
from rest_framework import generics
from rest_framework import permissions as drf_permissions
from rest_framework.exceptions import NotAuthenticated, NotFound
from django.contrib.auth.models import AnonymousUser

from modularodm import Q

from framework.auth.core import Auth
from framework.auth.oauth_scopes import CoreScopes

from website.models import User, Node, NodeLog

from api.base import permissions as base_permissions
from api.base.utils import get_object_or_error
from api.base.views import JSONAPIBaseView
from api.base.filters import ODMFilterMixin
from api.nodes.serializers import NodeSerializer
from api.logs.serializers import NodeLogSerializer
from api.registrations.serializers import RegistrationSerializer

from .serializers import UserSerializer, UserDetailSerializer
from .permissions import ReadOnlyOrCurrentUser

class UserMixin(object):
    """Mixin with convenience methods for retrieving the current node based on the
    current URL. By default, fetches the user based on the user_id kwarg.
    """

    serializer_class = UserSerializer
    user_lookup_url_kwarg = 'user_id'

    def get_user(self, check_permissions=True):
        key = self.kwargs[self.user_lookup_url_kwarg]
        current_user = self.request.user

        if key == 'me':
            if isinstance(current_user, AnonymousUser):
                raise NotAuthenticated
            else:
                return self.request.user

        obj = get_object_or_error(User, key, 'user')
        if check_permissions:
            # May raise a permission denied
            self.check_object_permissions(self.request, obj)
        return obj

class UserList(JSONAPIBaseView, generics.ListAPIView, ODMFilterMixin):
    """List of users registered on the OSF. *Read-only*.

    Paginated list of users ordered by the date they registered.  Each resource contains the full representation of the
    user, meaning additional requests to an individual user's detail view are not necessary.

    Note that if an anonymous view_only key is being used, user information will not be serialized, and the id will be
    an empty string. Relationships to a user object will not show in this case, either.

    The subroute [`/me/`](me/) is a special endpoint that always points to the currently logged-in user.

    ##User Attributes

    <!--- Copied Attributes From UserDetail -->

    OSF User entities have the "users" `type`.

        name               type               description
        ----------------------------------------------------------------------------------------
        full_name          string             full name of the user; used for display
        given_name         string             given name of the user; for bibliographic citations
        middle_names       string             middle name of user; for bibliographic citations
        family_name        string             family name of user; for bibliographic citations
        suffix             string             suffix of user's name for bibliographic citations
        date_registered    iso8601 timestamp  timestamp when the user's account was created

    ##Links

    See the [JSON-API spec regarding pagination](http://jsonapi.org/format/1.0/#fetching-pagination).

    ##Actions

    *None*.

    ##Query Params

    + `page=<Int>` -- page number of results to view, default 1

    + `filter[<fieldname>]=<Str>` -- fields and values to filter the search results on.

    Users may be filtered by their `id`, `full_name`, `given_name`, `middle_names`, or `family_name`.

    + `profile_image_size=<Int>` -- Modifies `/links/profile_image_url` of the user entities so that it points to
    the user's profile image scaled to the given size in pixels.  If left blank, the size depends on the image provider.

    #This Request/Response

    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
    )

    required_read_scopes = [CoreScopes.USERS_READ]
    required_write_scopes = [CoreScopes.USERS_WRITE]

    serializer_class = UserSerializer

    ordering = ('-date_registered')
    view_category = 'users'
    view_name = 'user-list'

    # overrides ODMFilterMixin
    def get_default_odm_query(self):
        return (
            Q('is_registered', 'eq', True) &
            Q('is_merged', 'ne', True) &
            Q('date_disabled', 'eq', None)
        )

    # overrides ListAPIView
    def get_queryset(self):
        # TODO: sort
        query = self.get_query_from_request()
        return User.find(query)

class UserDetail(JSONAPIBaseView, generics.RetrieveUpdateAPIView, UserMixin):
    """Details about a specific user. *Writeable*.

    The User Detail endpoint retrieves information about the user whose id is the final part of the path.  If `me`
    is given as the id, the record of the currently logged-in user will be returned.  The returned information includes
    the user's bibliographic information and the date the user registered.

    Note that if an anonymous view_only key is being used, user information will not be serialized, and the id will be
    an empty string. Relationships to a user object will not show in this case, either.

    ##Attributes

    OSF User entities have the "users" `type`.

        name               type               description
        ----------------------------------------------------------------------------------------
        full_name          string             full name of the user; used for display
        given_name         string             given name of the user; for bibliographic citations
        middle_names       string             middle name of user; for bibliographic citations
        family_name        string             family name of user; for bibliographic citations
        suffix             string             suffix of user's name for bibliographic citations
        date_registered    iso8601 timestamp  timestamp when the user's account was created

    ##Relationships

    ###Nodes

    A list of all nodes the user has contributed to.  If the user id in the path is the same as the logged-in user, all
    nodes will be visible.  Otherwise, you will only be able to see the other user's publicly-visible nodes.

    ##Links

        self:               the canonical api endpoint of this user
        html:               this user's page on the OSF website
        profile_image_url:  a url to the user's profile image

    ##Actions

    ###Update

        Method:        PUT / PATCH
        URL:           /links/self
        Query Params:  <none>
        Body (JSON):   {
                         "data": {
                           "type": "users",   # required
                           "id":   {user_id}, # required
                           "attributes": {
                             "full_name":    {full_name},    # mandatory
                             "given_name":   {given_name},   # optional
                             "middle_names": {middle_names}, # optional
                             "family_name":  {family_name},  # optional
                             "suffix":       {suffix}        # optional
                           }
                         }
                       }
        Success:       200 OK + node representation

    To update your user profile, issue a PUT request to either the canonical URL of your user resource (as given in
    `/links/self`) or to `/users/me/`.  Only the `full_name` attribute is required.  Unlike at signup, the given, middle,
    and family names will not be inferred from the `full_name`.  Currently, only `full_name`, `given_name`,
    `middle_names`, `family_name`, and `suffix` are updateable.

    A PATCH request issued to this endpoint will behave the same as a PUT request, but does not require `full_name` to
    be set.

    **NB:** If you PUT/PATCH to the `/users/me/` endpoint, you must still provide your full user id in the `id` field of
    the request.  We do not support using the `me` alias in request bodies at this time.

    ##Query Params

    + `profile_image_size=<Int>` -- Modifies `/links/profile_image_url` so that it points the image scaled to the given
    size in pixels.  If left blank, the size depends on the image provider.

    #This Request/Response

    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        ReadOnlyOrCurrentUser,
        base_permissions.TokenHasScope,
    )

    required_read_scopes = [CoreScopes.USERS_READ]
    required_write_scopes = [CoreScopes.USERS_WRITE]
    view_category = 'users'
    view_name = 'user-detail'

    serializer_class = UserDetailSerializer

    # overrides RetrieveAPIView
    def get_object(self):
        return self.get_user()

    # overrides RetrieveUpdateAPIView
    def get_serializer_context(self):
        # Serializer needs the request in order to make an update to privacy
        context = JSONAPIBaseView.get_serializer_context(self)
        context['request'] = self.request
        return context

class UserNodes(JSONAPIBaseView, generics.ListAPIView, UserMixin, ODMFilterMixin):
    """List of nodes that the user contributes to. *Read-only*.

    Paginated list of nodes that the user contributes to.  Each resource contains the full representation of the node,
    meaning additional requests to an individual node's detail view are not necessary. If the user id in the path is the
    same as the logged-in user, all nodes will be visible.  Otherwise, you will only be able to see the other user's
    publicly-visible nodes.  The special user id `me` can be used to represent the currently logged-in user.

    ##Node Attributes

    <!--- Copied Attributes from NodeDetail -->

    OSF Node entities have the "nodes" `type`.

        name           type               description
        ---------------------------------------------------------------------------------
        title          string             title of project or component
        description    string             description of the node
        category       string             node category, must be one of the allowed values
        date_created   iso8601 timestamp  timestamp that the node was created
        date_modified  iso8601 timestamp  timestamp when the node was last updated
        tags           array of strings   list of tags that describe the node
        fork           boolean            is this project a fork?
        registration   boolean            has this project been registered?
        fork           boolean            is this node a fork of another node?
        dashboard      boolean            is this node visible on the user dashboard?
        public         boolean            has this node been made publicly-visible?

    ##Links

    See the [JSON-API spec regarding pagination](http://jsonapi.org/format/1.0/#fetching-pagination).

    ##Actions

    *None*.

    ##Query Params

    + `page=<Int>` -- page number of results to view, default 1

    + `filter[<fieldname>]=<Str>` -- fields and values to filter the search results on.

    <!--- Copied Query Params from NodeList -->

    Nodes may be filtered by their `title`, `category`, `description`, `public`, `registration`, or `tags`.  `title`,
    `description`, and `category` are string fields and will be filtered using simple substring matching.  `public` and
    `registration` are booleans, and can be filtered using truthy values, such as `true`, `false`, `0`, or `1`.  Note
    that quoting `true` or `false` in the query will cause the match to fail regardless.  `tags` is an array of simple strings.

    #This Request/Response

    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
    )

    required_read_scopes = [CoreScopes.USERS_READ, CoreScopes.NODE_BASE_READ]
    required_write_scopes = [CoreScopes.USERS_WRITE, CoreScopes.NODE_BASE_WRITE]

    serializer_class = NodeSerializer
    view_category = 'users'
    view_name = 'user-nodes'

    # overrides ODMFilterMixin
    def get_default_odm_query(self):
        user = self.get_user()
        return (
            Q('contributors', 'eq', user) &
            Q('is_folder', 'ne', True) &
            Q('is_deleted', 'ne', True)
        )

    # overrides ListAPIView
    def get_queryset(self):
        current_user = self.request.user
        if current_user.is_anonymous():
            auth = Auth(None)
        else:
            auth = Auth(current_user)
        query = self.get_query_from_request()
        raw_nodes = Node.find(self.get_default_odm_query() & query)
        nodes = [each for each in raw_nodes if each.is_public or each.can_view(auth)]
        return nodes

class UserRegistrations(UserNodes):
    """List of registrations that the user contributes to. *Read-only*.

    Paginated list of registrations that the user contributes to.  Each resource contains the full representation of the
    registration, meaning additional requests to an individual registration's detail view are not necessary. If the user
    id in the path is the same as the logged-in user, all nodes will be visible.  Otherwise, you will only be able to
    see the other user's publicly-visible nodes.  The special user id `me` can be used to represent the currently
    logged-in user. Retracted registrations will display a limited number of fields, namely, title, description,
    date_created, registration, retracted, date_registered, retraction_justification, and registration supplement.

    ##Registration Attributes

    <!--- Copied Attributes from RegistrationList -->

    Registrations have the "registrations" `type`.

        name                            type               description
        -------------------------------------------------------------------------------------------------------
        title                           string             title of the registered project or component
        description                     string             description of the registered node
        category                        string             node category, must be one of the allowed values
        date_created                    iso8601 timestamp  timestamp that the node was created
        date_modified                   iso8601 timestamp  timestamp when the node was last updated
        tags                            array of strings   list of tags that describe the registered node
        fork                            boolean            is this project a fork?
        registration                    boolean            has this project been registered?
        dashboard                       boolean            is this registered node visible on the user dashboard?
        public                          boolean            has this registration been made publicly-visible?
        retracted                       boolean            has this registration been retracted?
        date_registered                 iso8601 timestamp  timestamp that the registration was created
        retraction_justification        string             reasons for retracting the registration
        pending_retraction              boolean            is this registration pending retraction?
        pending_registration_approval   boolean            is this registration pending approval?
        pending_embargo                 boolean            is this registration pending an embargo?
        registered_meta                 dictionary         registration supplementary information
        registration_supplement         string             registration template

    ##Relationships

    ###Registered from

    The registration is branched from this node.

    ###Registered by

    The registration was initiated by this user.

    ###Other Relationships

    See documentation on registered_from detail view.  A registration has many of the same properties as a node.

    ##Links

    See the [JSON-API spec regarding pagination](http://jsonapi.org/format/1.0/#fetching-pagination).

    ##Actions

    *None*.

    ##Query Params

    + `page=<Int>` -- page number of results to view, default 1

    + `filter[<fieldname>]=<Str>` -- fields and values to filter the search results on.

    <!--- Copied Query Params from NodeList -->

    Registrations may be filtered by their `title`, `category`, `description`, `public`, or `tags`.  `title`, `description`,
    and `category` are string fields and will be filtered using simple substring matching.  `public` is a boolean and
    can be filtered using truthy values, such as `true`, `false`, `0`, or `1`.  Note that quoting `true` or `false` in
    the query will cause the match to fail regardless.  `tags` is an array of simple strings.

    #This Request/Response

    """
    required_read_scopes = [CoreScopes.USERS_READ, CoreScopes.NODE_REGISTRATIONS_READ]
    required_write_scopes = [CoreScopes.USERS_WRITE, CoreScopes.NODE_REGISTRATIONS_WRITE]

    serializer_class = RegistrationSerializer
    view_category = 'users'
    view_name = 'user-registrations'

    # overrides ODMFilterMixin
    def get_default_odm_query(self):
        user = self.get_user()
        return (
            Q('contributors', 'eq', user) &
            Q('is_folder', 'ne', True) &
            Q('is_deleted', 'ne', True) &
            Q('is_registration', 'eq', True)
        )

class UserNodeLogs(UserNodes):

    serializer_class = NodeLogSerializer
    view_category = 'users'
    view_name = 'node_logs'

    def get_queryset(self):
        current_user = self.request.user
        if current_user.is_anonymous():
            auth = Auth(None)
        else:
            auth = Auth(current_user)
        query = self.get_query_from_request()
        raw_nodes = Node.find(self.get_default_odm_query() & query)
        nodes = [each for each in raw_nodes if each.is_public or each.can_view(auth)]
        if not nodes:
            raise NotFound
        query = [Q('params.node', 'eq', node._id) for node in nodes]
        return NodeLog.find(reduce(lambda x, y: x | y, query))
<commit_msg>Make order right<commit_after>
from rest_framework import generics
from rest_framework import permissions as drf_permissions
from rest_framework.exceptions import NotAuthenticated, NotFound
from django.contrib.auth.models import AnonymousUser

from modularodm import Q

from framework.auth.core import Auth
from framework.auth.oauth_scopes import CoreScopes

from website.models import User, Node, NodeLog

from api.base import permissions as base_permissions
from api.base.utils import get_object_or_error
from api.base.views import JSONAPIBaseView
from api.base.filters import ODMFilterMixin
from api.nodes.serializers import NodeSerializer
from api.logs.serializers import NodeLogSerializer
from api.registrations.serializers import RegistrationSerializer

from .serializers import UserSerializer, UserDetailSerializer
from .permissions import ReadOnlyOrCurrentUser

class UserMixin(object):
    """Mixin with convenience methods for retrieving the current node based on the
    current URL. By default, fetches the user based on the user_id kwarg.
    """

    serializer_class = UserSerializer
    user_lookup_url_kwarg = 'user_id'

    def get_user(self, check_permissions=True):
        key = self.kwargs[self.user_lookup_url_kwarg]
        current_user = self.request.user

        if key == 'me':
            if isinstance(current_user, AnonymousUser):
                raise NotAuthenticated
            else:
                return self.request.user

        obj = get_object_or_error(User, key, 'user')
        if check_permissions:
            # May raise a permission denied
            self.check_object_permissions(self.request, obj)
        return obj

class UserList(JSONAPIBaseView, generics.ListAPIView, ODMFilterMixin):
    """List of users registered on the OSF. *Read-only*.

    Paginated list of users ordered by the date they registered.  Each resource contains the full representation of the
    user, meaning additional requests to an individual user's detail view are not necessary.

    Note that if an anonymous view_only key is being used, user information will not be serialized, and the id will be
    an empty string. Relationships to a user object will not show in this case, either.

    The subroute [`/me/`](me/) is a special endpoint that always points to the currently logged-in user.

    ##User Attributes

    <!--- Copied Attributes From UserDetail -->

    OSF User entities have the "users" `type`.

        name               type               description
        ----------------------------------------------------------------------------------------
        full_name          string             full name of the user; used for display
        given_name         string             given name of the user; for bibliographic citations
        middle_names       string             middle name of user; for bibliographic citations
        family_name        string             family name of user; for bibliographic citations
        suffix             string             suffix of user's name for bibliographic citations
        date_registered    iso8601 timestamp  timestamp when the user's account was created

    ##Links

    See the [JSON-API spec regarding pagination](http://jsonapi.org/format/1.0/#fetching-pagination).

    ##Actions

    *None*.

    ##Query Params

    + `page=<Int>` -- page number of results to view, default 1

    + `filter[<fieldname>]=<Str>` -- fields and values to filter the search results on.

    Users may be filtered by their `id`, `full_name`, `given_name`, `middle_names`, or `family_name`.

    + `profile_image_size=<Int>` -- Modifies `/links/profile_image_url` of the user entities so that it points to
    the user's profile image scaled to the given size in pixels.  If left blank, the size depends on the image provider.

    #This Request/Response

    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
    )

    required_read_scopes = [CoreScopes.USERS_READ]
    required_write_scopes = [CoreScopes.USERS_WRITE]

    serializer_class = UserSerializer

    ordering = ('-date_registered')
    view_category = 'users'
    view_name = 'user-list'

    # overrides ODMFilterMixin
    def get_default_odm_query(self):
        return (
            Q('is_registered', 'eq', True) &
            Q('is_merged', 'ne', True) &
            Q('date_disabled', 'eq', None)
        )

    # overrides ListAPIView
    def get_queryset(self):
        # TODO: sort
        query = self.get_query_from_request()
        return User.find(query)

class UserDetail(JSONAPIBaseView, generics.RetrieveUpdateAPIView, UserMixin):
    """Details about a specific user. *Writeable*.

    The User Detail endpoint retrieves information about the user whose id is the final part of the path.  If `me`
    is given as the id, the record of the currently logged-in user will be returned.  The returned information includes
    the user's bibliographic information and the date the user registered.

    Note that if an anonymous view_only key is being used, user information will not be serialized, and the id will be
    an empty string. Relationships to a user object will not show in this case, either.

    ##Attributes

    OSF User entities have the "users" `type`.

        name               type               description
        ----------------------------------------------------------------------------------------
        full_name          string             full name of the user; used for display
        given_name         string             given name of the user; for bibliographic citations
        middle_names       string             middle name of user; for bibliographic citations
        family_name        string             family name of user; for bibliographic citations
        suffix             string             suffix of user's name for bibliographic citations
        date_registered    iso8601 timestamp  timestamp when the user's account was created

    ##Relationships

    ###Nodes

    A list of all nodes the user has contributed to.  If the user id in the path is the same as the logged-in user, all
    nodes will be visible.  Otherwise, you will only be able to see the other user's publicly-visible nodes.

    ##Links

        self:               the canonical api endpoint of this user
        html:               this user's page on the OSF website
        profile_image_url:  a url to the user's profile image

    ##Actions

    ###Update

        Method:        PUT / PATCH
        URL:           /links/self
        Query Params:  <none>
        Body (JSON):   {
                         "data": {
                           "type": "users",   # required
                           "id":   {user_id}, # required
                           "attributes": {
                             "full_name":    {full_name},    # mandatory
                             "given_name":   {given_name},   # optional
                             "middle_names": {middle_names}, # optional
                             "family_name":  {family_name},  # optional
                             "suffix":       {suffix}        # optional
                           }
                         }
                       }
        Success:       200 OK + node representation

    To update your user profile, issue a PUT request to either the canonical URL of your user resource (as given in
    `/links/self`) or to `/users/me/`.  Only the `full_name` attribute is required.  Unlike at signup, the given, middle,
    and family names will not be inferred from the `full_name`.  Currently, only `full_name`, `given_name`,
    `middle_names`, `family_name`, and `suffix` are updateable.

    A PATCH request issued to this endpoint will behave the same as a PUT request, but does not require `full_name` to
    be set.

    **NB:** If you PUT/PATCH to the `/users/me/` endpoint, you must still provide your full user id in the `id` field of
    the request.  We do not support using the `me` alias in request bodies at this time.

    ##Query Params

    + `profile_image_size=<Int>` -- Modifies `/links/profile_image_url` so that it points the image scaled to the given
    size in pixels.  If left blank, the size depends on the image provider.

    #This Request/Response

    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        ReadOnlyOrCurrentUser,
        base_permissions.TokenHasScope,
    )

    required_read_scopes = [CoreScopes.USERS_READ]
    required_write_scopes = [CoreScopes.USERS_WRITE]
    view_category = 'users'
    view_name = 'user-detail'

    serializer_class = UserDetailSerializer

    # overrides RetrieveAPIView
    def get_object(self):
        return self.get_user()

    # overrides RetrieveUpdateAPIView
    def get_serializer_context(self):
        # Serializer needs the request in order to make an update to privacy
        context = JSONAPIBaseView.get_serializer_context(self)
        context['request'] = self.request
        return context

class UserNodes(JSONAPIBaseView, generics.ListAPIView, UserMixin, ODMFilterMixin):
    """List of nodes that the user contributes to. *Read-only*.

    Paginated list of nodes that the user contributes to.  Each resource contains the full representation of the node,
    meaning additional requests to an individual node's detail view are not necessary. If the user id in the path is the
    same as the logged-in user, all nodes will be visible.  Otherwise, you will only be able to see the other user's
    publicly-visible nodes.  The special user id `me` can be used to represent the currently logged-in user.

    ##Node Attributes

    <!--- Copied Attributes from NodeDetail -->

    OSF Node entities have the "nodes" `type`.

        name           type               description
        ---------------------------------------------------------------------------------
        title          string             title of project or component
        description    string             description of the node
        category       string             node category, must be one of the allowed values
        date_created   iso8601 timestamp  timestamp that the node was created
        date_modified  iso8601 timestamp  timestamp when the node was last updated
        tags           array of strings   list of tags that describe the node
        fork           boolean            is this project a fork?
        registration   boolean            has this project been registered?
        fork           boolean            is this node a fork of another node?
        dashboard      boolean            is this node visible on the user dashboard?
        public         boolean            has this node been made publicly-visible?

    ##Links

    See the [JSON-API spec regarding pagination](http://jsonapi.org/format/1.0/#fetching-pagination).

    ##Actions

    *None*.

    ##Query Params

    + `page=<Int>` -- page number of results to view, default 1

    + `filter[<fieldname>]=<Str>` -- fields and values to filter the search results on.

    <!--- Copied Query Params from NodeList -->

    Nodes may be filtered by their `title`, `category`, `description`, `public`, `registration`, or `tags`.  `title`,
    `description`, and `category` are string fields and will be filtered using simple substring matching.  `public` and
    `registration` are booleans, and can be filtered using truthy values, such as `true`, `false`, `0`, or `1`.  Note
    that quoting `true` or `false` in the query will cause the match to fail regardless.  `tags` is an array of simple strings.

    #This Request/Response

    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
    )

    required_read_scopes = [CoreScopes.USERS_READ, CoreScopes.NODE_BASE_READ]
    required_write_scopes = [CoreScopes.USERS_WRITE, CoreScopes.NODE_BASE_WRITE]

    serializer_class = NodeSerializer
    view_category = 'users'
    view_name = 'user-nodes'

    # overrides ODMFilterMixin
    def get_default_odm_query(self):
        user = self.get_user()
        return (
            Q('contributors', 'eq', user) &
            Q('is_folder', 'ne', True) &
            Q('is_deleted', 'ne', True)
        )

    # overrides ListAPIView
    def get_queryset(self):
        current_user = self.request.user
        if current_user.is_anonymous():
            auth = Auth(None)
        else:
            auth = Auth(current_user)
        query = self.get_query_from_request()
        raw_nodes = Node.find(self.get_default_odm_query() & query)
        nodes = [each for each in raw_nodes if each.is_public or each.can_view(auth)]
        return nodes

class UserRegistrations(UserNodes):
    """List of registrations that the user contributes to. *Read-only*.

    Paginated list of registrations that the user contributes to.  Each resource contains the full representation of the
    registration, meaning additional requests to an individual registration's detail view are not necessary. If the user
    id in the path is the same as the logged-in user, all nodes will be visible.  Otherwise, you will only be able to
    see the other user's publicly-visible nodes.  The special user id `me` can be used to represent the currently
    logged-in user. Retracted registrations will display a limited number of fields, namely, title, description,
    date_created, registration, retracted, date_registered, retraction_justification, and registration supplement.

    ##Registration Attributes

    <!--- Copied Attributes from RegistrationList -->

    Registrations have the "registrations" `type`.

        name                            type               description
        -------------------------------------------------------------------------------------------------------
        title                           string             title of the registered project or component
        description                     string             description of the registered node
        category                        string             node category, must be one of the allowed values
        date_created                    iso8601 timestamp  timestamp that the node was created
        date_modified                   iso8601 timestamp  timestamp when the node was last updated
        tags                            array of strings   list of tags that describe the registered node
        fork                            boolean            is this project a fork?
        registration                    boolean            has this project been registered?
        dashboard                       boolean            is this registered node visible on the user dashboard?
        public                          boolean            has this registration been made publicly-visible?
        retracted                       boolean            has this registration been retracted?
        date_registered                 iso8601 timestamp  timestamp that the registration was created
        retraction_justification        string             reasons for retracting the registration
        pending_retraction              boolean            is this registration pending retraction?
        pending_registration_approval   boolean            is this registration pending approval?
        pending_embargo                 boolean            is this registration pending an embargo?
        registered_meta                 dictionary         registration supplementary information
        registration_supplement         string             registration template

    ##Relationships

    ###Registered from

    The registration is branched from this node.

    ###Registered by

    The registration was initiated by this user.

    ###Other Relationships

    See documentation on registered_from detail view.  A registration has many of the same properties as a node.

    ##Links

    See the [JSON-API spec regarding pagination](http://jsonapi.org/format/1.0/#fetching-pagination).

    ##Actions

    *None*.

    ##Query Params

    + `page=<Int>` -- page number of results to view, default 1

    + `filter[<fieldname>]=<Str>` -- fields and values to filter the search results on.

    <!--- Copied Query Params from NodeList -->

    Registrations may be filtered by their `title`, `category`, `description`, `public`, or `tags`.  `title`, `description`,
    and `category` are string fields and will be filtered using simple substring matching.  `public` is a boolean and
    can be filtered using truthy values, such as `true`, `false`, `0`, or `1`.  Note that quoting `true` or `false` in
    the query will cause the match to fail regardless.  `tags` is an array of simple strings.

    #This Request/Response

    """
    required_read_scopes = [CoreScopes.USERS_READ, CoreScopes.NODE_REGISTRATIONS_READ]
    required_write_scopes = [CoreScopes.USERS_WRITE, CoreScopes.NODE_REGISTRATIONS_WRITE]

    serializer_class = RegistrationSerializer
    view_category = 'users'
    view_name = 'user-registrations'

    # overrides ODMFilterMixin
    def get_default_odm_query(self):
        user = self.get_user()
        return (
            Q('contributors', 'eq', user) &
            Q('is_folder', 'ne', True) &
            Q('is_deleted', 'ne', True) &
            Q('is_registration', 'eq', True)
        )

class UserNodeLogs(UserNodes):

    serializer_class = NodeLogSerializer
    view_category = 'users'
    view_name = 'node_logs'

    def get_queryset(self):
        current_user = self.request.user
        if current_user.is_anonymous():
            auth = Auth(None)
        else:
            auth = Auth(current_user)
        query = self.get_query_from_request()
        raw_nodes = Node.find(self.get_default_odm_query() & query)
        nodes = [each for each in raw_nodes if each.is_public or each.can_view(auth)]
        if not nodes:
            raise NotFound
        query = [Q('params.node', 'eq', node._id) for node in nodes]
        sorted_logs = sorted(list(NodeLog.find(reduce(lambda x, y: x | y, query))), key=lambda x: x.date)
        return sorted_logs
<|endoftext|>


<commit_before>from acq4.analysis.AnalysisModule import AnalysisModule
from acq4.util.DatabaseGui.DatabaseGui import DatabaseGui
from acq4.util.flowchart import *
from collections import OrderedDict
import numpy as np
import acq4.pyqtgraph as pg
from RegionParameter import RegionParameter
import os
#from acq4.pyqtgraph.parametertree import Parameter

class TimecourseAnalyzer(AnalysisModule):

    """A generic module for analyzing features of repeated traces over time."""

    dbIdentity = "TimecourseAnalyzer"

    def __init__(self, host):
        AnalysisModule.__init__(self, host)

        tables = OrderedDict([
            (self.dbIdentity+'.trials', 'TimecourseAnalyzer_Trials')
        ])
        self.dbGui = DatabaseGui(dm=self.dataManager(), tables=tables)

        flowchartDir = os.path.join(os.path.abspath(os.path.split(__file__)[0]), "flowcharts")
        self.flowchart = Flowchart(filePath=flowchartDir)
        self.flowchart.addInput('dataIn')
        self.flowchart.addOutput('results')
        self.flowchart.sigChartLoaded.connect(self.connectPlots)

        self._elements_ = OrderedDict([
            ('Database', {'type':'ctrl', 'object': self.dbGui, 'size':(100,100)}),
            ('Analysis Options', {'type':'ctrl', 'object': self.flowchart.widget(), 'pos':('above', 'Database'),'size': (100, 400)}),
            ('File Loader', {'type':'fileInput', 'size': (100, 100), 'pos':('above', 'Analysis Options'),'host': self}),
            ('Experiment Plot', {'type':'plot', 'pos':('right', 'File Loader'), 'size':(400, 100)}),
            ('Traces Plot', {'type': 'plot', 'pos':('bottom', 'Experiment Plot'), 'size':(400,200)}),
            ('Results Plot', {'type': 'plot', 'pos':('bottom', 'Traces Plot'), 'size':(400,200)}),
            ('Results Table', {'type':'table', 'pos':('bottom', 'Traces Plot'), 'size': (400,200)})
        ])
        self.initializeElements()

        self.exptPlot = self.getElement('Experiment Plot', create=True)
        self.tracesPlot = self.getElement('Traces Plot', create=True)
        self.resultsTable = self.getElement('Results Table', create=True)
        self.resultsTable.setSortingEnabled(False)
        #self.paramTree = self.getElement('Analysis Regions', create=True)

        ### initialize variables
        self.expStart = 0
        self.traces = np.array([], dtype=[('timestamp', float), ('data', object), ('fileHandle', object)])
        self.files = []

        self.traceSelectRgn = pg.LinearRegionItem()
        self.traceSelectRgn.setRegion([0, 300])
        self.exptPlot.addItem(self.traceSelectRgn)
        self.traceSelectRgn.sigRegionChanged.connect(self.updateTracesPlot)
        self.traceSelectRgn.sigRegionChangeFinished.connect(self.updateAnalysis)
        self.flowchart.sigOutputChanged.connect(self.flowchartOutputChanged)

        #self.addRegionParam = pg.parametertree.Parameter.create(name="Add Region", type='action')
        #self.paramTree.addParameters(self.addRegionParam)
        #self.addRegionParam.sigActivated.connect(self.newRegionRequested)

    def connectPlots(self):
        dp = self.getElement('Traces Plot', create=False)
        #fp = self.getElement('Filter Plot', create=False)
        if dp is not None and 'Plot_000' in self.flowchart.nodes().keys():
            self.flowchart.nodes()['Plot_000'].setPlot(dp)
        #if fp is not None and 'Plot_001' in self.flowchart.nodes().keys():
        #    self.flowchart.nodes()['Plot_001'].setPlot(fp)

    def loadFileRequested(self, files):
        """Called by FileLoader when the load EPSP file button is clicked, once for each selected file.
                files - a list of the file currently selected in FileLoader
        """

        if files is None:
            return

        n = len(files[0].ls())

        with pg.ProgressDialog("Loading data..", 0, n) as dlg:
            for f in files:
                arr = np.zeros((len(f.ls())), dtype=[('timestamp', float), ('data', object), ('fileHandle', object)])
                maxi = -1
                for i, protoDir in enumerate(f.ls()):
                    df = self.dataModel.getClampFile(f[protoDir])
                    if df is None:
                        print 'Error in reading data file %s' % f.name()
                        break
                    data = df.read()
                    timestamp = data.infoCopy()[-1]['startTime']
                    arr[i]['fileHandle'] = df
                    arr[i]['timestamp'] = timestamp
                    arr[i]['data'] = data
                    maxi += 1  # keep track of successfully read traces
                    dlg += 1
                    if dlg.wasCanceled():
                        return
                self.traces = np.concatenate((self.traces, arr[:maxi]))  # only concatenate successfully read traces
                #self.lastAverageState = {}
                self.files.append(f)

        self.expStart = self.traces['timestamp'].min()
        #self.averageCtrlChanged()
        self.updateExptPlot()
        self.updateTracesPlot()
        return True

    def updateExptPlot(self):
        """Update the experiment plots with markers for the the timestamps of
        all loaded EPSP traces, and averages (if selected in the UI)."""

        if len(self.traces) == 0:
            return

        self.exptPlot.clear()
        self.exptPlot.addItem(self.traceSelectRgn)

        self.exptPlot.plot(x=self.traces['timestamp']-self.expStart, y=[1]*len(self.traces), pen=None, symbol='o', symbolSize=6)

    def updateTracesPlot(self):
        """Update the Trace display plot to show the traces corresponding to
         the timestamps selected by the region in the experiment plot."""

        rgn = self.traceSelectRgn.getRegion()
        self.tracesPlot.clear()

        ### plot all the traces with timestamps within the selected region (according to self.traceSelectRgn)
        data = self.traces[(self.traces['timestamp'] > rgn[0]+self.expStart)
                          *(self.traces['timestamp'] < rgn[1]+self.expStart)]

        for i, d in enumerate(data['data']):
            self.tracesPlot.plot(d['primary'], pen=pg.intColor(i, len(data)))

    def updateAnalysis(self):
        self.resultsTable.clear()

        rgn = self.traceSelectRgn.getRegion()
        data = self.traces[(self.traces['timestamp'] > rgn[0]+self.expStart)
                          *(self.traces['timestamp'] < rgn[1]+self.expStart)]
        for i, d in enumerate(data):
            self.flowchart.setInput(dataIn=d['fileHandle'])

    def flowchartOutputChanged(self):
        self.resultsTable.appendData([self.flowchart.output()['results']])

<commit_msg>implemented rudimentary saving to DB<commit_after>from acq4.analysis.AnalysisModule import AnalysisModule
from acq4.util.DatabaseGui.DatabaseGui import DatabaseGui
from acq4.util.flowchart import *
from collections import OrderedDict
import numpy as np
import acq4.pyqtgraph as pg
from RegionParameter import RegionParameter
import os
#from acq4.pyqtgraph.parametertree import Parameter

class TimecourseAnalyzer(AnalysisModule):

    """A generic module for analyzing features of repeated traces over time."""

    dbIdentity = "TimecourseAnalyzer"

    def __init__(self, host):
        AnalysisModule.__init__(self, host)

        flowchartDir = os.path.join(os.path.abspath(os.path.split(__file__)[0]), "flowcharts")
        self.flowchart = Flowchart(filePath=flowchartDir)
        self.flowchart.addInput('dataIn')
        self.flowchart.addOutput('results')
        self.flowchart.sigChartLoaded.connect(self.connectPlots)

        tables = OrderedDict([(self.dbIdentity+'.traces', 'TimecourseAnalyzer_traces')])
        self.dbGui = DatabaseGui(dm=host.dataManager(), tables=tables)

        self.ctrl = QtGui.QWidget()
        self.ctrl.setLayout(QtGui.QVBoxLayout())
        self.analyzeBtn = QtGui.QPushButton('Analyze')
        self.storeToDBBtn = QtGui.QPushButton('Store to DB')
        self.ctrl.layout().addWidget(self.analyzeBtn)
        self.ctrl.layout().addWidget(self.storeToDBBtn)

        self._elements_ = OrderedDict([
            ('Database', {'type':'ctrl', 'object': self.dbGui, 'size':(100,100)}),
            ('Analysis Options', {'type':'ctrl', 'object': self.flowchart.widget(), 'pos':('above', 'Database'),'size': (100, 400)}),
            ('File Loader', {'type':'fileInput', 'size': (100, 100), 'pos':('above', 'Analysis Options'),'host': self}),
            ('Experiment Plot', {'type':'plot', 'pos':('right', 'File Loader'), 'size':(400, 100)}),
            ('Traces Plot', {'type': 'plot', 'pos':('bottom', 'Experiment Plot'), 'size':(400,200)}),
            ('Results Plot', {'type': 'plot', 'pos':('bottom', 'Traces Plot'), 'size':(400,200)}),
            ('Results Table', {'type':'table', 'pos':('bottom', 'Traces Plot'), 'size': (400,200)}),
            ('Store Ctrl', {'type': 'ctrl', 'object':self.ctrl, 'size':(100,100), 'pos':('bottom', 'File Loader')})
        ])
        self.initializeElements()

        self.exptPlot = self.getElement('Experiment Plot', create=True)
        self.tracesPlot = self.getElement('Traces Plot', create=True)
        self.resultsTable = self.getElement('Results Table', create=True)
        self.resultsTable.setSortingEnabled(False)
        #self.paramTree = self.getElement('Analysis Regions', create=True)

        ### initialize variables
        self.expStart = 0
        self.traces = np.array([], dtype=[('timestamp', float), ('data', object), ('fileHandle', object), ('results', object)])
        self.files = []

        self.traceSelectRgn = pg.LinearRegionItem()
        self.traceSelectRgn.setRegion([0, 300])
        self.exptPlot.addItem(self.traceSelectRgn)
        self.traceSelectRgn.sigRegionChanged.connect(self.updateTracesPlot)
        self.traceSelectRgn.sigRegionChangeFinished.connect(self.updateAnalysis)
        #self.flowchart.sigOutputChanged.connect(self.flowchartOutputChanged)

        #self.addRegionParam = pg.parametertree.Parameter.create(name="Add Region", type='action')
        #self.paramTree.addParameters(self.addRegionParam)
        #self.addRegionParam.sigActivated.connect(self.newRegionRequested)
        self.analyzeBtn.clicked.connect(self.analyzeBtnClicked)
        self.storeToDBBtn.clicked.connect(self.storeToDBBtnClicked)

    def connectPlots(self):
        dp = self.getElement('Traces Plot', create=False)
        #fp = self.getElement('Filter Plot', create=False)
        if dp is not None and 'Plot_000' in self.flowchart.nodes().keys():
            self.flowchart.nodes()['Plot_000'].setPlot(dp)
        #if fp is not None and 'Plot_001' in self.flowchart.nodes().keys():
        #    self.flowchart.nodes()['Plot_001'].setPlot(fp)

    def loadFileRequested(self, files):
        """Called by FileLoader when the load EPSP file button is clicked, once for each selected file.
                files - a list of the file currently selected in FileLoader
        """

        if files is None:
            return

        n = len(files[0].ls())

        with pg.ProgressDialog("Loading data..", 0, n) as dlg:
            for f in files:
                arr = np.zeros((len(f.ls())), dtype=[('timestamp', float), ('data', object), ('fileHandle', object), ('results', object)])
                maxi = -1
                for i, protoDir in enumerate(f.ls()):
                    df = self.dataModel.getClampFile(f[protoDir])
                    if df is None:
                        print 'Error in reading data file %s' % f.name()
                        break
                    data = df.read()
                    timestamp = data.infoCopy()[-1]['startTime']
                    arr[i]['fileHandle'] = df
                    arr[i]['timestamp'] = timestamp
                    arr[i]['data'] = data
                    maxi += 1  # keep track of successfully read traces
                    dlg += 1
                    if dlg.wasCanceled():
                        return
                self.traces = np.concatenate((self.traces, arr[:maxi]))  # only concatenate successfully read traces
                #self.lastAverageState = {}
                self.files.append(f)

        self.expStart = self.traces['timestamp'].min()
        #self.averageCtrlChanged()
        self.updateExptPlot()
        self.updateTracesPlot()
        return True

    def updateExptPlot(self):
        """Update the experiment plots with markers for the the timestamps of
        all loaded EPSP traces, and averages (if selected in the UI)."""

        if len(self.traces) == 0:
            return

        self.exptPlot.clear()
        self.exptPlot.addItem(self.traceSelectRgn)

        self.exptPlot.plot(x=self.traces['timestamp']-self.expStart, y=[1]*len(self.traces), pen=None, symbol='o', symbolSize=6)

    def updateTracesPlot(self):
        """Update the Trace display plot to show the traces corresponding to
         the timestamps selected by the region in the experiment plot."""

        rgn = self.traceSelectRgn.getRegion()
        self.tracesPlot.clear()

        ### plot all the traces with timestamps within the selected region (according to self.traceSelectRgn)
        data = self.traces[(self.traces['timestamp'] > rgn[0]+self.expStart)
                          *(self.traces['timestamp'] < rgn[1]+self.expStart)]

        for i, d in enumerate(data['data']):
            self.tracesPlot.plot(d['primary'], pen=pg.intColor(i, len(data)))

    def updateAnalysis(self):
        self.resultsTable.clear()

        rgn = self.traceSelectRgn.getRegion()
        data = self.traces[(self.traces['timestamp'] > rgn[0]+self.expStart)
                          *(self.traces['timestamp'] < rgn[1]+self.expStart)]
        for i, d in enumerate(data):
            self.flowchart.setInput(dataIn=d['fileHandle'])

    #def flowchartOutputChanged(self):
    #    self.resultsTable.appendData([self.flowchart.output()['results']])

    def analyzeBtnClicked(self, *args):
        for i, t in enumerate(self.traces):
            results = self.flowchart.process(dataIn=t['fileHandle'])
            t['results'] = results['results']

    def storeToDBBtnClicked(self, *args):
        #if len(self.analysisResults) == 0:
        #    self.analyze()

        db = self.dbGui.getDb()
        if db is None:
            raise Exception("No database loaded.")

        table = self.dbGui.getTableName(self.dbIdentity+'.traces')

        trialFields = OrderedDict([
            ('CellDir', 'directory:Cell'),
            ('ProtocolSequenceDir', 'directory:ProtocolSequence'),
            ('ProtocolDir', 'directory:Protocol'),
            ('timestamp', 'real'),
            ('time', 'real'),
            ('rmp', 'real'),
            ('pspSlope', 'real'),
            ('maxSlopeTime', 'real'),
            ('pspAmplitude', 'real'),
            ('pspPeakTime', 'real'),
            ('condition', 'text'),
            ('APnumber', 'int'),

            ])

        db.checkTable(table, owner=self.dbIdentity+'.traces', columns=trialFields, create=True, addUnknownColumns=True, indexes=[['ProtocolDir'], ['ProtocolSequenceDir'], ['CellDir']])

        data = np.zeros(len(self.traces), dtype=[
                                                            ('CellDir', object),
                                                            ('ProtocolSequenceDir', object),
                                                            ('ProtocolDir', object),
                                                            ('timestamp', float),
                                                            ('time', float),
                                                            ('rmp', float),
                                                            #('inputResistance', float),
                                                            #('tau', float),
                                                            #('bridgeBalance', float)
                                                            ('pspSlope', float),
                                                            #('normalizedPspSlope', float),
                                                            #('slopeFitOffset', float),
                                                            ('maxSlopeTime', float),
                                                            ('pspAmplitude', float),
                                                            ('pspPeakTime', float),
                                                            #('pspRgnStart', float),
                                                            #('pspRgnEnd', float),
                                                            #('analysisCtrlState', object),
                                                            #('averageCtrlState', object),
                                                            #('includedProtocols', object)
                                                            ('condition', str),
                                                            ('APnumber', int)
                                                            ])

        data['CellDir'] = self.dataModel.getParent(self.files[0], 'Cell')
        #data['pspRgnStart'] = self.pspRgn.getRegion()[0]
        #data['pspRgnEnd'] = self.pspRgn.getRegion()[1]
        #data['analysisCtrlState'] = str(self.analysisCtrl.state())
        #data['averageCtrlState'] = str(self.averageCtrl.state())

        #baselineSlope = self.analysisResults[self.analysisResults['time'] < self.expStart+300]['pspSlope'].mean()
        for i in range(len(self.traces)):
            fh = self.traces[i]['fileHandle']
            data[i]['ProtocolDir'] = self.dataModel.getParent(fh, 'Protocol')
            data[i]['ProtocolSequenceDir'] = self.dataModel.getParent(fh, 'ProtocolSequence')
            data[i]['timestamp'] = self.traces[i]['timestamp']
            data[i]['time'] = self.traces[i]['timestamp'] - self.expStart
            data[i]['rmp'] = self.traces[i]['results']['RMP']
            #data[i]['inputResistance'] = self.analysisResults[i]['inputResistance']
            data[i]['pspSlope'] = self.traces[i]['results']['slope']
            #data[i]['normalizedPspSlope'] = self.analysisResults[i]['pspSlope']/baselineSlope
            #data[i]['slopeFitOffset'] = self.analysisResults[i]['slopeFitOffset']
            data[i]['maxSlopeTime'] = self.traces[i]['results']['maxSlopeTime']
            data[i]['pspAmplitude'] = self.traces[i]['results']['amplitude']
            #data[i]['includedProtocols'] = self.averagedTraces[i]['origTimes'] ### TODO: make this protocolDirs instead of timestamps....
            data[i]['pspPeakTime'] = self.traces[i]['results']['pspPeakTime']
            data[i]['condition'] = self.traces[i]['results']['condition']
            data[i]['APnumber'] = self.traces[i]['results']['APnumber']

        old = db.select(table, where={'CellDir':data['CellDir'][0]}, toArray=True)
        if old is not None: ## only do deleting if there is already data stored for this cell
            db.delete(table, where={'CellDir': data['CellDir'][0]})

        db.insert(table, data)

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""pyclangcheck driver code

This code is the driver code for the pyclangcheck tool.

Copyright: (c) 2010, Knut Reinert, FU Berlin
License:   3-clause BSD (see LICENSE)
"""

from __future__ import with_statement

__author__ = 'Manuel Holtgrewe <manuel.holtgrewe@fu-berlin.de>'

import optparse
import os
import pickle
import sys

import clang.cindex as ci

def _hasFileLocation(node):
    """Return True if node has a file lcoation."""
    if not hasattr(node, 'location'):
        return False
    if not hasattr(node.location, 'file'):
        return False
    if not node.location.file:
        return False
    if not hasattr(node.location.file, 'name'):
        return False
    if not node.location.file.name:
        return False
    return True

class CollectCompoundStatementNodeVisitor(object):
    def __init__(self, options):
        self.options = options
        self.stack = []
        self.ranges = []

    def enterNode(self, node):
        self.stack.append(node)
        ## print '  ' * len(self.stack), node.kind,
        num_children = len([x for x in node.get_children()])
        ## if _hasFileLocation(node):
        ##     print node.location.file.name, '%d-%d' % (node.extent.start.line, node.extent.end.line)
        ## else:
        ##     print
        # Only add range for statements that are no compound statements.  Add
        # for empty compounds.
        if not node.kind.is_statement():
            ## print 'skipping, no statement'
            return
        if node.kind == ci.CursorKind.COMPOUND_STMT and num_children > 0:
            ## print 'skipping, non-empty compound statement', num_children
            return
        # Only add if has file location.
        if _hasFileLocation(node):
            self.ranges.append((node.location.file.name, node.extent.start.line, node.extent.end.line))

    def exitNode(self, node):
        self.stack.pop()

class VisitAllowedRule(object):
    def __init__(self, options):
        self.options = options
        self.include_dirs = [os.path.abspath(x) for x in options.include_dirs]
        self.cache = {}

    def visitAllowed(self, node):
        """Return True if visiting is allowed."""
        # TODO(holtgrew): For this application, stopping at compound statements would be enough.
        print node._kind_id
        if node.kind == ci.CursorKind.TRANSLATION_UNIT:
            return True
        if not _hasFileLocation(node):
            return False
        if self.cache.has_key(node.location.file.name):
            return self.cache[node.location.file.name]
        # Check whether node's location is below the include directories.
        filename = os.path.abspath(node.location.file.name)
        result = False
        for x in self.include_dirs:
            if filename.startswith(x):
                # print filename, x
                result = True
                break
        self.cache[node.location.file.name] = result
        return result

class AstTraverser(object):
    def __init__(self, node_visitor, options):
        self.node_visitor = node_visitor
        self.options = options
        self.visit_allowed_rule = VisitAllowedRule(options)

    def _recurse(self, node):
        if not self.visit_allowed_rule.visitAllowed(node):
            return False  # We did not visit this node.
        self.node_visitor.enterNode(node)
        for c in node.get_children():
            self._recurse(c)
        self.node_visitor.exitNode(node)
        return True

    def run(self, filename):
        index = ci.Index.create()
        args = ['-I%s' % s for s in self.options.include_dirs]
        # print args
        tu = index.parse(filename, args=args)
        print 'Translation unit: %s.' % tu.spelling
        return self._recurse(tu.cursor)

    @classmethod
    def visitFile(klass, filename, node_visitor, options):
        traverser = AstTraverser(node_visitor, options)
        res = traverser.run(filename)
        return res == True

def main():
    # Parse command line arguments.
    parser = optparse.OptionParser("USAGE: %prog [options] -s file.cpp")
    parser.add_option('-I', '--include-dir', dest='include_dirs', default=[],
                      type='string', help='Specify include directories',
                      action='append')
    parser.add_option('-s', '--src-file', dest='source_files', default=[],
                      type='string', help='Specify compilation units.',
                      action='append')
    parser.add_option('-l', '--location-file', dest='location_file',
                      default='locations.dat', type='string',
                      help='Path to file with compound statement locations.')
    parser.add_option('-g', '--gcov-file', dest='gcov_files', default=[],
                      type='string', help='Specify gcov files to process.',
                      action='append')
    options, args = parser.parse_args()
    if len(args) != 0:
        parser.error('Incorrect number of arguments!')
        return 1

    options.include_dirs += [os.path.abspath(os.path.dirname(s)) for s in options.source_files]

    if not options.source_files and not options.gcov_files:
        parser.error('Neither source nor gcov file given!')
        return 1

    if options.source_files:
        # If any source file is given, all given source files are parsed and all
        # lines with compound statements in all included files are written to
        # the location file.
        print >>sys.stderr, 'Building Locations'
        print >>sys.stderr, '=================='

        # Fire off AST traversal.
        print >>sys.stderr, 'AST Traversal'
        node_visitor = CollectCompoundStatementNodeVisitor(options)
        for src in options.source_files:
            print >>sys.stderr, '  Compilation Unit', src
            AstTraverser.visitFile(src, node_visitor, options)

        # Convert locations into points.
        locations = {}
        for filename, start, stop in node_visitor.ranges:
            filename = os.path.abspath(filename)
            for i in range(start, stop + 1):
                locations.setdefault(filename, set()).add(i)

        # Write out the source locations.
        print >>sys.stderr, 'Writing out locations to', options.location_file
        with open(options.location_file, 'wb') as f:
            pickle.dump(locations, f)

    if options.gcov_files:
        # If no source files and gcov files are given then
        print >>sys.stderr, 'Updating gcov Results'
        print >>sys.stderr, '====================='

        if not options.source_files:
            print >>sys.stderr, 'Loading locations from', options.location_file
            with open(options.location_file, 'rb') as f:
                locations = pickle.load(f)

        for filename in options.gcov_files:
            filename = os.path.abspath(filename)
            print >>sys.stderr, 'Processing', filename
            with open(filename, 'rb') as f:
                lines = f.readlines()
            pos0 = lines[0].find(':')
            pos1 = lines[0].find(':', pos0 + 1)
            source = None
            result = []
            skip = False
            for i, line in enumerate(lines):
                coverage = line[:pos0]
                lineno = int(line[pos0 + 1:pos1].strip())
                slineno = line[pos0 + 1:pos1]
                txt = line[pos1 + 1:]
                if txt.startswith('Source:'):
                    source = os.path.abspath(txt[len('Source:'):].strip())
                    if not locations.has_key(source):
                        print >>sys.stderr, '  Skipping.'
                        skip = True
                        break
                if not source or lineno == 0:
                    result.append(line)
                    continue  # Proceed only if in file.
                if lineno in locations[source] and coverage.strip() == '-':
                    coverage = ('%%%ds' % pos0) % '#####'
                result.append(':'.join([coverage, slineno, txt]))
            # Write back file if not skipped.
            if skip:
                continue
            print ''.join(result)

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Updates to fixgcov tool.<commit_after>#!/usr/bin/env python
"""pyclangcheck driver code

This code is the driver code for the pyclangcheck tool.

Copyright: (c) 2010, Knut Reinert, FU Berlin
License:   3-clause BSD (see LICENSE)
"""

from __future__ import with_statement

__author__ = 'Manuel Holtgrewe <manuel.holtgrewe@fu-berlin.de>'

import optparse
import os
import pickle
import sys

import clang.cindex as ci

def _hasFileLocation(node):
    """Return True if node has a file lcoation."""
    if not hasattr(node, 'location'):
        return False
    if not hasattr(node.location, 'file'):
        return False
    if not node.location.file:
        return False
    if not hasattr(node.location.file, 'name'):
        return False
    if not node.location.file.name:
        return False
    return True

class CollectCompoundStatementNodeVisitor(object):
    def __init__(self, options):
        self.options = options
        self.stack = []
        self.ranges = []

    def enterNode(self, node):
        self.stack.append(node)
        ## print '  ' * len(self.stack), node.kind,
        num_children = len([x for x in node.get_children()])
        ## if _hasFileLocation(node):
        ##     print node.location.file.name, '%d-%d' % (node.extent.start.line, node.extent.end.line)
        ## else:
        ##     print
        # Only add range for statements that are no compound statements.  Add
        # for empty compounds.
        if not node.kind.is_statement():
            ## print 'skipping, no statement'
            return
        if node.kind == ci.CursorKind.COMPOUND_STMT and num_children > 0:
            ## print 'skipping, non-empty compound statement', num_children
            return
        # Only add if has file location.
        if _hasFileLocation(node):
            self.ranges.append((node.location.file.name, node.extent.start.line, node.extent.end.line))

    def exitNode(self, node):
        self.stack.pop()

class VisitAllowedRule(object):
    def __init__(self, options):
        self.options = options
        self.include_dirs = [os.path.abspath(x) for x in options.include_dirs]
        self.cache = {}

    def visitAllowed(self, node):
        """Return True if visiting is allowed."""
        # TODO(holtgrew): For this application, stopping at compound statements would be enough.
        #print node._kind_id
        if node.kind == ci.CursorKind.TRANSLATION_UNIT:
            return True
        if not _hasFileLocation(node):
            return False
        if self.cache.has_key(node.location.file.name):
            return self.cache[node.location.file.name]
        # Check whether node's location is below the include directories.
        filename = os.path.abspath(node.location.file.name)
        result = False
        for x in self.include_dirs:
            if filename.startswith(x):
                # print filename, x
                result = True
                break
        self.cache[node.location.file.name] = result
        return result

class AstTraverser(object):
    def __init__(self, node_visitor, options):
        self.node_visitor = node_visitor
        self.options = options
        self.visit_allowed_rule = VisitAllowedRule(options)

    def _recurse(self, node):
        if not self.visit_allowed_rule.visitAllowed(node):
            return False  # We did not visit this node.
        self.node_visitor.enterNode(node)
        for c in node.get_children():
            self._recurse(c)
        self.node_visitor.exitNode(node)
        return True

    def run(self, filename):
        index = ci.Index.create()
        args = ['-I%s' % s for s in self.options.include_dirs]
        # print args
        tu = index.parse(filename, args=args)
        if self.options.verbosity >= 1:
            print 'Translation unit: %s.' % tu.spelling
        return self._recurse(tu.cursor)

    @classmethod
    def visitFile(klass, filename, node_visitor, options):
        traverser = AstTraverser(node_visitor, options)
        res = traverser.run(filename)
        return res == True

def main():
    # Parse command line arguments.
    parser = optparse.OptionParser("USAGE: %prog [options] -s file.cpp")
    parser.add_option('-I', '--include-dir', dest='include_dirs', default=[],
                      type='string', help='Specify include directories',
                      action='append')
    parser.add_option('-s', '--src-file', dest='source_files', default=[],
                      type='string', help='Specify compilation units.',
                      action='append')
    parser.add_option('--src-file-files', dest='source_files_files', default=[],
                      type='string', help='Specify file with paths to compilation units.',
                      action='append')
    parser.add_option('-l', '--location-file', dest='location_file',
                      default='locations.dat', type='string',
                      help='Path to file with compound statement locations.')
    parser.add_option('-g', '--gcov-file', dest='gcov_files', default=[],
                      type='string', help='Specify gcov files to process.',
                      action='append')
    parser.add_option('--gcov-files', dest='gcov_files_files', default=[],
                      type='string', help='Specify gcov files to process.',
                      action='append')
    parser.add_option('-q', '--quiet', dest='verbosity', default=1,
                      action='store_const', const=0, help='Fewer message.')
    parser.add_option('-v', '--verbose', dest='verbosity', default=1,
                      action='store_const', const=2, help='More messages.')
    options, args = parser.parse_args()
    if len(args) != 0:
        parser.error('Incorrect number of arguments!')
        return 1

    options.include_dirs += [os.path.abspath(os.path.dirname(s)) for s in options.source_files]

    for path in options.source_files_files:
      with open(path, 'rb') as f:
        options.source_files += [x.strip() for x in f.readlines()]
    for path in options.gcov_files_files:
      with open(path, 'rb') as f:
        options.gcov_files += [x.strip() for x in f.readlines()]

    if not options.source_files and not options.gcov_files:
        parser.error('Neither source nor gcov file given!')
        return 1

    if options.source_files:
        # If any source file is given, all given source files are parsed and all
        # lines with compound statements in all included files are written to
        # the location file.
        if options.verbosity >= 1:
            print >>sys.stderr, 'Building Locations'
        if options.verbosity >= 2:
            print >>sys.stderr, '=================='

        # Fire off AST traversal.
        if options.verbosity >= 1:
            print >>sys.stderr, 'AST Traversal'
        node_visitor = CollectCompoundStatementNodeVisitor(options)
        for src in options.source_files:
            if options.verbosity >= 2:
                print >>sys.stderr, '  Compilation Unit', src
            AstTraverser.visitFile(src, node_visitor, options)

        # Convert locations into points.
        locations = {}
        for filename, start, stop in node_visitor.ranges:
            filename = os.path.abspath(filename)
            for i in range(start, stop + 1):
                locations.setdefault(filename, set()).add(i)

        # Write out the source locations.
        if options.verbosity >= 1:
            print >>sys.stderr, 'Writing out locations to', options.location_file
        with open(options.location_file, 'wb') as f:
            pickle.dump(locations, f)

    if options.gcov_files:
        # If no source files and gcov files are given then
        if options.verbosity >= 1:
            print >>sys.stderr, 'Updating gcov Results'
        if options.verbosity >= 2:
            print >>sys.stderr, '====================='

        if not options.source_files:
            if options.verbosity >= 1:
                print >>sys.stderr, 'Loading locations from', options.location_file
            with open(options.location_file, 'rb') as f:
                locations = pickle.load(f)

        for filename in options.gcov_files:
            filename = os.path.abspath(filename)
            if options.verbosity >= 2:
                print >>sys.stderr, 'Processing', filename
            with open(filename, 'rb') as f:
                lines = f.readlines()
            pos0 = lines[0].find(':')
            pos1 = lines[0].find(':', pos0 + 1)
            source = None
            result = []
            skip = False
            for i, line in enumerate(lines):
                coverage = line[:pos0]
                lineno = int(line[pos0 + 1:pos1].strip())
                slineno = line[pos0 + 1:pos1]
                txt = line[pos1 + 1:]
                if txt.startswith('Source:'):
                    source = os.path.abspath(txt[len('Source:'):].strip())
                    if not locations.has_key(source):
                        if options.verbosity >= 2:
                            print >>sys.stderr, '  Skipping.'
                        skip = True
                        break
                if not source or lineno == 0:
                    result.append(line)
                    continue  # Proceed only if in file.
                if lineno in locations[source] and coverage.strip() == '-':
                    coverage = ('%%%ds' % pos0) % '#####'
                result.append(':'.join([coverage, slineno, txt]))
            # Write back file if not skipped.
            if skip:
                continue
            with open (filename, 'wb') as f:
              f.write(''.join(result))
            #print ''.join(result)

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>#!/usr/bin/env python

from test_common import *
from socket import *

def readline(s):
    buf = ""
    while not buf.endswith("\r\n"):
        buf += s.recv(1)
    return buf

def abbreviate(s):
    if len(s) < 50: return repr(s)
    else: return repr(s[:25]) + "..." + repr(s[-25:])

def expect(s, string):
    msg = ""
    while len(msg) < len(string):
        msg += s.recv(len(string) - len(msg))
    if msg != string:
        raise ValueError("Didn't get what we expected: expected %s, got %s" % (abbreviate(string), abbreviate(msg)));

def test_value_of_size(size, port):

    print "Sending a %d-byte value..." % size,
    sys.stdout.flush()

    s = socket()
    s.connect(("localhost", port))

    s.send("set x 0 0 %d\r\n" % size + "a" * size + "\r\n")
    msg = readline(s)

    if msg == "STORED\r\n":
        print " getting...",
        s.send("get x\r\n");
        print " sent get...",
        expect(s, "VALUE x 0 %d\r\n" % size)
        print " value decl...",
        expect(s, "a" * size + "\r\n")
        print " content...",
        expect(s, "END\r\n")
        too_big = False
        print "ok."

    elif msg.startswith("CLIENT_ERROR"):
        print "too big."
        too_big = True

    else:
        print
        raise ValueError("Something went wrong; sent a value of %d bytes and got %r" % (size, msg))

    s.send("quit")
    s.close()

    return too_big

def test_values_near_size(size, port):

    max_legal_value_size = 1024 * 1024

    for s in xrange(size - 5, size + 5):
        too_big = test_value_of_size(s, port)
        assert too_big == (s > max_legal_value_size)

def test(opts, port):

    # threshold for storing in node vs. storing in large bug
    test_values_near_size(250, port)

    # threshold for buffering vs. streaming
    test_values_near_size(10000, port)

#    test_values_near_size(40000, port)

#    test_values_near_size(2 ** 16, port)

#    test_values_near_size(66000, port)

#    test_values_near_size(70000, port)

    test_values_near_size(71000, port)
    test_values_near_size(73500, port)
    for i in xrange(73700, 73750):
        assert False == test_value_of_size(i, port)

    test_values_near_size(73750, port)
    test_values_near_size(74000, port)
    test_values_near_size(75000, port)

    test_values_near_size(75000, port)

    test_values_near_size(77500, port)

    test_values_near_size(80000, port)

    test_values_near_size(100000, port)

    # threshold for max legal value size
    test_values_near_size(1024 * 1024, port)

if __name__ == "__main__":
    op = make_option_parser()
    opts = op.parse(sys.argv)
    auto_server_test_main(test, opts, timeout = 60)
<commit_msg>Added more values to big_values.py, to show boundary limitations.<commit_after>#!/usr/bin/env python

from test_common import *
from socket import *

def readline(s):
    buf = ""
    while not buf.endswith("\r\n"):
        buf += s.recv(1)
    return buf

def abbreviate(s):
    if len(s) < 50: return repr(s)
    else: return repr(s[:25]) + "..." + repr(s[-25:])

def expect(s, string):
    msg = ""
    while len(msg) < len(string):
        msg += s.recv(len(string) - len(msg))
    if msg != string:
        raise ValueError("Didn't get what we expected: expected %s, got %s" % (abbreviate(string), abbreviate(msg)));

def test_value_of_size(size, port):

    print "Sending a %d-byte value..." % size,
    sys.stdout.flush()

    s = socket()
    s.connect(("localhost", port))

    s.send("set x 0 0 %d\r\n" % size + "a" * size + "\r\n")
    msg = readline(s)

    if msg == "STORED\r\n":
        print " getting...",
        s.send("get x\r\n");
        print " sent get...",
        expect(s, "VALUE x 0 %d\r\n" % size)
        print " value decl...",
        expect(s, "a" * size + "\r\n")
        print " content...",
        expect(s, "END\r\n")
        too_big = False
        print "ok."

    elif msg.startswith("CLIENT_ERROR"):
        print "too big."
        too_big = True

    else:
        print
        raise ValueError("Something went wrong; sent a value of %d bytes and got %r" % (size, msg))

    s.send("quit")
    s.close()

    return too_big

def test_values_near_size(size, port):

    max_legal_value_size = 1024 * 1024

    for s in xrange(size - 5, size + 5):
        too_big = test_value_of_size(s, port)
        assert too_big == (s > max_legal_value_size)

def test(opts, port):

    # threshold for storing in node vs. storing in large bug
    test_values_near_size(250, port)

    # threshold for buffering vs. streaming
    test_values_near_size(10000, port)

    test_values_near_size(2 ** 16, port)

    for i in xrange(73700 - 4096, 73750 - 4096):
        assert False == test_value_of_size(i, port)

    test_values_near_size(71000, port)
    test_values_near_size(73500, port)

    for i in xrange(73700, 73750):
        assert False == test_value_of_size(i, port)

    test_values_near_size(73750, port)
    test_values_near_size(74000, port)
    test_values_near_size(75000, port)

    test_values_near_size(75000, port)

    test_values_near_size(77500, port)

    test_values_near_size(80000, port)

    test_values_near_size(100000, port)

    # threshold for max legal value size
    test_values_near_size(1024 * 1024, port)

if __name__ == "__main__":
    op = make_option_parser()
    opts = op.parse(sys.argv)
    auto_server_test_main(test, opts, timeout = 60)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2014 RethinkDB, all rights reserved.

from __future__ import print_function

import os, pprint, sys, time

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, 'common')))
import driver, scenario_common, utils, vcoptparse

r = utils.import_python_driver()

"""The `interface.table_config` test checks that the special `rethinkdb.table_config` and `rethinkdb.table_status` tables behave as expected."""

op = vcoptparse.OptParser()
scenario_common.prepare_option_parser_mode_flags(op)
opts = op.parse(sys.argv)

with driver.Metacluster() as metacluster:
    cluster1 = driver.Cluster(metacluster)
    _, command_prefix, serve_options = scenario_common.parse_mode_flags(opts)

    print("Spinning up two processes...")
    files1 = driver.Files(metacluster, console_output="create-output-1", server_name="a", command_prefix=command_prefix)
    proc1 = driver.Process(cluster1, files1, console_output="serve-output-1", command_prefix=command_prefix, extra_options=serve_options)
    files2 = driver.Files(metacluster, console_output="create-output-2", server_name="b", command_prefix=command_prefix)
    proc2 = driver.Process(cluster1, files2, console_output="serve-output-2", command_prefix=command_prefix, extra_options=serve_options)
    files3 = driver.Files(metacluster, console_output="create-output-3", server_name="never_used", command_prefix=command_prefix)
    proc3 = driver.Process(cluster1, files3, console_output="serve-output-3", command_prefix=command_prefix, extra_options=serve_options)
    proc1.wait_until_started_up()
    proc2.wait_until_started_up()
    proc3.wait_until_started_up()
    cluster1.check()
    conn = r.connect(proc1.host, proc1.driver_port)

    def check_foo_config_matches(expected):
        config = r.table_config("foo").run(conn)
        assert config["name"] == "foo" and config["db"] == "test"
        found = config["shards"]
        if len(expected) != len(found):
            return False
        for (e_shard, f_shard) in zip(expected, found):
            if set(e_shard["replicas"]) != set(f_shard["replicas"]):
                return False
            if e_shard["director"] != f_shard["director"]:
                return False
        return True

    def check_status_matches_config():
        config = list(r.db("rethinkdb").table("table_config").run(conn))
        status = list(r.db("rethinkdb").table("table_status").run(conn))
        uuids = set(row["id"] for row in config)
        if not (len(uuids) == len(config) == len(status)):
            return False
        if uuids != set(row["id"] for row in status):
            return False
        for c_row in config:
            s_row = [row for row in status if row["id"] == c_row["id"]][0]
            if c_row["db"] != s_row["db"]:
                return False
            if c_row["name"] != s_row["name"]:
                return False
            c_shards = c_row["shards"]
            s_shards = s_row["shards"]
            # Make sure that servers that have never been involved with the table will
            # never appear in `table_status`. (See GitHub issue #3101.)
            for s_shard in s_shards:
                for doc in s_shard["replicas"]:
                    assert doc["server"] != "never_used"
            if len(s_shards) != len(c_shards):
                return False
            for (s_shard, c_shard) in zip(s_shards, c_shards):
                if set(doc["server"] for doc in s_shard["replicas"]) != \
                        set(c_shard["replicas"]):
                    return False
                if s_shard["director"] != c_shard["director"]:
                    return False
                if any(doc["state"] != "ready" for doc in s_shard["replicas"]):
                    return False
            if not s_row["status"]["ready_for_outdated_reads"]:
                return False
            if not s_row["status"]["ready_for_reads"]:
                return False
            if not s_row["status"]["ready_for_writes"]:
                return False
            if not s_row["status"]["all_replicas_ready"]:
                return False
        return True

    def check_tables_named(names):
        config = list(r.db("rethinkdb").table("table_config").run(conn))
        if len(config) != len(names):
            return False
        for row in config:
            if (row["db"], row["name"]) not in names:
                return False
        return True

    def wait_until(condition):
        try:
            start_time = time.time()
            while not condition():
                time.sleep(1)
                if time.time() > start_time + 10:
                    raise RuntimeError("Out of time")
        except:
            config = list(r.db("rethinkdb").table("table_config").run(conn))
            status = list(r.db("rethinkdb").table("table_status").run(conn))
            print("Something went wrong.\nconfig =")
            pprint.pprint(config)
            print("status =")
            pprint.pprint(status)
            raise

    # Make sure that `never_used` will never be picked as a default for a table by
    # removing the `default` tag.
    res = r.db("rethinkdb").table("server_config") \
           .filter({"name": "never_used"}) \
           .update({"tags": []}) \
           .run(conn)
    assert res["replaced"] == 1, res

    print("Creating a table...")
    r.db_create("test").run(conn)
    r.table_create("foo").run(conn)
    r.table_create("bar").run(conn)
    r.db_create("test2").run(conn)
    r.db("test2").table_create("bar2").run(conn)
    r.table("foo").insert([{"i": i} for i in xrange(10)]).run(conn)
    assert set(row["i"] for row in r.table("foo").run(conn)) == set(xrange(10))

    print("Testing that table_config and table_status are sane...")
    wait_until(lambda: check_tables_named(
        [("test", "foo"), ("test", "bar"), ("test2", "bar2")]))
    wait_until(check_status_matches_config)

    print("Testing that we can move around data by writing to table_config...")
    def test_shards(shards):
        print("Reconfiguring:", {"shards": shards})
        res = r.table_config("foo").update({"shards": shards}).run(conn)
        assert res["errors"] == 0, repr(res)
        wait_until(lambda: check_foo_config_matches(shards))
        wait_until(check_status_matches_config)
        assert set(row["i"] for row in r.table("foo").run(conn)) == set(xrange(10))
        print("OK")
    test_shards(
        [{"replicas": ["a"], "director": "a"}])
    test_shards(
        [{"replicas": ["b"], "director": "b"}])
    test_shards(
        [{"replicas": ["a", "b"], "director": "a"}])
    test_shards(
        [{"replicas": ["a"], "director": "a"},
         {"replicas": ["b"], "director": "b"}])
    test_shards(
        [{"replicas": ["a", "b"], "director": "a"},
         {"replicas": ["a", "b"], "director": "b"}])
    test_shards(
        [{"replicas": ["a"], "director": "a"}])

    print("Testing that table_config rejects invalid input...")
    def test_invalid(conf):
        print("Reconfiguring:", conf)
        res = r.db("rethinkdb").table("table_config").filter({"name": "foo"}) \
               .replace(conf).run(conn)
        assert res["errors"] == 1
        print("Error, as expected")
    test_invalid(r.row.merge({"shards": []}))
    test_invalid(r.row.merge({"shards": "this is a string"}))
    test_invalid(r.row.merge({"shards":
        [{"replicas": ["a"], "director": "a", "extra_key": "extra_value"}]}))
    test_invalid(r.row.merge({"shards": [{"replicas": [], "director": None}]}))
    test_invalid(r.row.merge({"shards": [{"replicas": ["a"], "director": "b"}]}))
    test_invalid(r.row.merge(
        {"shards": [{"replicas": ["a"], "director": "b"},
                    {"replicas": ["b"], "director": "a"}]}))
    test_invalid(r.row.merge({"primary_key": "new_primary_key"}))
    test_invalid(r.row.merge({"db": "new_db"}))
    test_invalid(r.row.merge({"extra_key": "extra_value"}))
    test_invalid(r.row.without("name"))
    test_invalid(r.row.without("primary_key"))
    test_invalid(r.row.without("db"))
    test_invalid(r.row.without("shards"))

    print("Testing that we can rename tables through table_config...")
    res = r.table_config("bar").update({"name": "bar2"}).run(conn)
    assert res["errors"] == 0
    wait_until(lambda: check_tables_named(
        [("test", "foo"), ("test", "bar2"), ("test2", "bar2")]))

    print("Testing that we can't rename a table so as to cause a name collision...")
    res = r.table_config("bar2").update({"name": "foo"}).run(conn)
    assert res["errors"] == 1

    print("Testing that we can create a table through table_config...")
    def test_create(doc, pkey):
        res = r.db("rethinkdb").table("table_config") \
               .insert(doc, return_changes=True).run(conn)
        assert res["errors"] == 0, repr(res)
        assert res["inserted"] == 1, repr(res)
        assert doc["name"] in r.table_list().run(conn)
        assert res["changes"][0]["new_val"]["primary_key"] == pkey
        assert "shards" in res["changes"][0]["new_val"]
        for i in xrange(10):
            try:
                r.table(doc["name"]).insert({}).run(conn)
            except r.RqlRuntimeError:
                time.sleep(1)
            else:
                break
        else:
            raise ValueError("Table took too long to become available")
        rows = list(r.table(doc["name"]).run(conn))
        assert len(rows) == 1 and list(rows[0].keys()) == [pkey]
    test_create({
        "name": "baz",
        "db": "test",
        "primary_key": "frob",
        "shards": [{"replicas": ["a"], "director": "a"}]
        }, "frob")
    test_create({
        "name": "baz2",
        "db": "test",
        "shards": [{"replicas": ["a"], "director": "a"}]
        }, "id")
    test_create({
        "name": "baz3",
        "db": "test"
        }, "id")

    print("Testing that we can delete a table through table_config...")
    res = r.table_config("baz").delete().run(conn)
    assert res["errors"] == 0, repr(res)
    assert res["deleted"] == 1, repr(res)
    assert "baz" not in r.table_list().run(conn)

    cluster1.check_and_stop()
print("Done.")
<commit_msg>Make interface.table_config not break now that the table_config() porcelain returns a stream.<commit_after>#!/usr/bin/env python
# Copyright 2014 RethinkDB, all rights reserved.

from __future__ import print_function

import os, pprint, sys, time

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, 'common')))
import driver, scenario_common, utils, vcoptparse

r = utils.import_python_driver()

"""The `interface.table_config` test checks that the special `rethinkdb.table_config` and `rethinkdb.table_status` tables behave as expected."""

op = vcoptparse.OptParser()
scenario_common.prepare_option_parser_mode_flags(op)
opts = op.parse(sys.argv)

with driver.Metacluster() as metacluster:
    cluster1 = driver.Cluster(metacluster)
    _, command_prefix, serve_options = scenario_common.parse_mode_flags(opts)

    print("Spinning up two processes...")
    files1 = driver.Files(metacluster, console_output="create-output-1", server_name="a", command_prefix=command_prefix)
    proc1 = driver.Process(cluster1, files1, console_output="serve-output-1", command_prefix=command_prefix, extra_options=serve_options)
    files2 = driver.Files(metacluster, console_output="create-output-2", server_name="b", command_prefix=command_prefix)
    proc2 = driver.Process(cluster1, files2, console_output="serve-output-2", command_prefix=command_prefix, extra_options=serve_options)
    files3 = driver.Files(metacluster, console_output="create-output-3", server_name="never_used", command_prefix=command_prefix)
    proc3 = driver.Process(cluster1, files3, console_output="serve-output-3", command_prefix=command_prefix, extra_options=serve_options)
    proc1.wait_until_started_up()
    proc2.wait_until_started_up()
    proc3.wait_until_started_up()
    cluster1.check()
    conn = r.connect(proc1.host, proc1.driver_port)

    def check_foo_config_matches(expected):
        config = r.table_config("foo").nth(0).run(conn)
        assert config["name"] == "foo" and config["db"] == "test"
        found = config["shards"]
        if len(expected) != len(found):
            return False
        for (e_shard, f_shard) in zip(expected, found):
            if set(e_shard["replicas"]) != set(f_shard["replicas"]):
                return False
            if e_shard["director"] != f_shard["director"]:
                return False
        return True

    def check_status_matches_config():
        config = list(r.db("rethinkdb").table("table_config").run(conn))
        status = list(r.db("rethinkdb").table("table_status").run(conn))
        uuids = set(row["id"] for row in config)
        if not (len(uuids) == len(config) == len(status)):
            return False
        if uuids != set(row["id"] for row in status):
            return False
        for c_row in config:
            s_row = [row for row in status if row["id"] == c_row["id"]][0]
            if c_row["db"] != s_row["db"]:
                return False
            if c_row["name"] != s_row["name"]:
                return False
            c_shards = c_row["shards"]
            s_shards = s_row["shards"]
            # Make sure that servers that have never been involved with the table will
            # never appear in `table_status`. (See GitHub issue #3101.)
            for s_shard in s_shards:
                for doc in s_shard["replicas"]:
                    assert doc["server"] != "never_used"
            if len(s_shards) != len(c_shards):
                return False
            for (s_shard, c_shard) in zip(s_shards, c_shards):
                if set(doc["server"] for doc in s_shard["replicas"]) != \
                        set(c_shard["replicas"]):
                    return False
                if s_shard["director"] != c_shard["director"]:
                    return False
                if any(doc["state"] != "ready" for doc in s_shard["replicas"]):
                    return False
            if not s_row["status"]["ready_for_outdated_reads"]:
                return False
            if not s_row["status"]["ready_for_reads"]:
                return False
            if not s_row["status"]["ready_for_writes"]:
                return False
            if not s_row["status"]["all_replicas_ready"]:
                return False
        return True

    def check_tables_named(names):
        config = list(r.db("rethinkdb").table("table_config").run(conn))
        if len(config) != len(names):
            return False
        for row in config:
            if (row["db"], row["name"]) not in names:
                return False
        return True

    def wait_until(condition):
        try:
            start_time = time.time()
            while not condition():
                time.sleep(1)
                if time.time() > start_time + 10:
                    raise RuntimeError("Out of time")
        except:
            config = list(r.db("rethinkdb").table("table_config").run(conn))
            status = list(r.db("rethinkdb").table("table_status").run(conn))
            print("Something went wrong.\nconfig =")
            pprint.pprint(config)
            print("status =")
            pprint.pprint(status)
            raise

    # Make sure that `never_used` will never be picked as a default for a table by
    # removing the `default` tag.
    res = r.db("rethinkdb").table("server_config") \
           .filter({"name": "never_used"}) \
           .update({"tags": []}) \
           .run(conn)
    assert res["replaced"] == 1, res

    print("Creating a table...")
    r.db_create("test").run(conn)
    r.table_create("foo").run(conn)
    r.table_create("bar").run(conn)
    r.db_create("test2").run(conn)
    r.db("test2").table_create("bar2").run(conn)
    r.table("foo").insert([{"i": i} for i in xrange(10)]).run(conn)
    assert set(row["i"] for row in r.table("foo").run(conn)) == set(xrange(10))

    print("Testing that table_config and table_status are sane...")
    wait_until(lambda: check_tables_named(
        [("test", "foo"), ("test", "bar"), ("test2", "bar2")]))
    wait_until(check_status_matches_config)

    print("Testing that we can move around data by writing to table_config...")
    def test_shards(shards):
        print("Reconfiguring:", {"shards": shards})
        res = r.table_config("foo").update({"shards": shards}).run(conn)
        assert res["errors"] == 0, repr(res)
        wait_until(lambda: check_foo_config_matches(shards))
        wait_until(check_status_matches_config)
        assert set(row["i"] for row in r.table("foo").run(conn)) == set(xrange(10))
        print("OK")
    test_shards(
        [{"replicas": ["a"], "director": "a"}])
    test_shards(
        [{"replicas": ["b"], "director": "b"}])
    test_shards(
        [{"replicas": ["a", "b"], "director": "a"}])
    test_shards(
        [{"replicas": ["a"], "director": "a"},
         {"replicas": ["b"], "director": "b"}])
    test_shards(
        [{"replicas": ["a", "b"], "director": "a"},
         {"replicas": ["a", "b"], "director": "b"}])
    test_shards(
        [{"replicas": ["a"], "director": "a"}])

    print("Testing that table_config rejects invalid input...")
    def test_invalid(conf):
        print("Reconfiguring:", conf)
        res = r.db("rethinkdb").table("table_config").filter({"name": "foo"}) \
               .replace(conf).run(conn)
        assert res["errors"] == 1
        print("Error, as expected")
    test_invalid(r.row.merge({"shards": []}))
    test_invalid(r.row.merge({"shards": "this is a string"}))
    test_invalid(r.row.merge({"shards":
        [{"replicas": ["a"], "director": "a", "extra_key": "extra_value"}]}))
    test_invalid(r.row.merge({"shards": [{"replicas": [], "director": None}]}))
    test_invalid(r.row.merge({"shards": [{"replicas": ["a"], "director": "b"}]}))
    test_invalid(r.row.merge(
        {"shards": [{"replicas": ["a"], "director": "b"},
                    {"replicas": ["b"], "director": "a"}]}))
    test_invalid(r.row.merge({"primary_key": "new_primary_key"}))
    test_invalid(r.row.merge({"db": "new_db"}))
    test_invalid(r.row.merge({"extra_key": "extra_value"}))
    test_invalid(r.row.without("name"))
    test_invalid(r.row.without("primary_key"))
    test_invalid(r.row.without("db"))
    test_invalid(r.row.without("shards"))

    print("Testing that we can rename tables through table_config...")
    res = r.table_config("bar").update({"name": "bar2"}).run(conn)
    assert res["errors"] == 0
    wait_until(lambda: check_tables_named(
        [("test", "foo"), ("test", "bar2"), ("test2", "bar2")]))

    print("Testing that we can't rename a table so as to cause a name collision...")
    res = r.table_config("bar2").update({"name": "foo"}).run(conn)
    assert res["errors"] == 1

    print("Testing that we can create a table through table_config...")
    def test_create(doc, pkey):
        res = r.db("rethinkdb").table("table_config") \
               .insert(doc, return_changes=True).run(conn)
        assert res["errors"] == 0, repr(res)
        assert res["inserted"] == 1, repr(res)
        assert doc["name"] in r.table_list().run(conn)
        assert res["changes"][0]["new_val"]["primary_key"] == pkey
        assert "shards" in res["changes"][0]["new_val"]
        for i in xrange(10):
            try:
                r.table(doc["name"]).insert({}).run(conn)
            except r.RqlRuntimeError:
                time.sleep(1)
            else:
                break
        else:
            raise ValueError("Table took too long to become available")
        rows = list(r.table(doc["name"]).run(conn))
        assert len(rows) == 1 and list(rows[0].keys()) == [pkey]
    test_create({
        "name": "baz",
        "db": "test",
        "primary_key": "frob",
        "shards": [{"replicas": ["a"], "director": "a"}]
        }, "frob")
    test_create({
        "name": "baz2",
        "db": "test",
        "shards": [{"replicas": ["a"], "director": "a"}]
        }, "id")
    test_create({
        "name": "baz3",
        "db": "test"
        }, "id")

    print("Testing that we can delete a table through table_config...")
    res = r.table_config("baz").delete().run(conn)
    assert res["errors"] == 0, repr(res)
    assert res["deleted"] == 1, repr(res)
    assert "baz" not in r.table_list().run(conn)

    cluster1.check_and_stop()
print("Done.")
<|endoftext|>