<commit_before># Copyright 2013: Mirantis Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Defines interface for DB access.

The underlying driver is loaded as a :class:`LazyPluggable`.

Functions in this module are imported into the rally.common.db namespace.
Call these functions from rally.common.db namespace, not the
rally.common.db.api namespace.

All functions in this module return objects that implement a dictionary-like
interface. Currently, many of these objects are sqlalchemy objects that
implement a dictionary interface. However, a future goal is to have all of
these objects be simple dictionaries.

**Related Flags**

:backend:  string to lookup in the list of LazyPluggable backends.
           `sqlalchemy` is the only supported backend right now.

:connection:  string specifying the sqlalchemy connection to use, like:
              `sqlite:///var/lib/cinder/cinder.sqlite`.

:enable_new_services:  when adding a new service to the database, is it in the
                       pool of available hardware (Default: True)

"""

import datetime as dt

from oslo_config import cfg
from oslo_db import api as db_api
from oslo_db import options as db_options
import six

from rally.common.i18n import _

CONF = cfg.CONF

db_options.set_defaults(CONF, connection="sqlite:////tmp/rally.sqlite")

IMPL = None

def serialize(fn):
    def conv(data):
        if data is None:
            return None
        if isinstance(data, (six.integer_types,
                             six.string_types,
                             six.text_type,
                             dt.date,
                             dt.time,
                             float,
                             )):
            return data
        if isinstance(data, dict):
            return {k: conv(v) for k, v in data.items()}
        if isinstance(data, (list, tuple)):
            return [conv(i) for i in data]
        if hasattr(data, "_as_dict"):
            result = data._as_dict()
            for k, v in result.items():
                result[k] = conv(v)
            return result

        raise ValueError(_("Can not serialize %s") % data)

    def wrapper(*args, **kwargs):
        result = fn(*args, **kwargs)
        return conv(result)
    return wrapper

def get_impl():
    global IMPL

    if not IMPL:
        _BACKEND_MAPPING = {"sqlalchemy": "rally.common.db.sqlalchemy.api"}
        IMPL = db_api.DBAPI.from_config(CONF, backend_mapping=_BACKEND_MAPPING)

    return IMPL

def engine_reset():
    """Reset DB engine."""
    get_impl().engine_reset()

def schema_cleanup():
    """Drop DB schema. This method drops existing database."""
    get_impl().schema_cleanup()

def schema_upgrade(revision=None):
    """Migrate the database to `revision` or the most recent revision."""
    return get_impl().schema_upgrade(revision)

def schema_create():
    """Create database schema from models description."""
    return get_impl().schema_create()

def schema_revision(detailed=False):
    """Return the schema revision."""
    return get_impl().schema_revision(detailed=detailed)

def schema_stamp(revision):
    """Stamps database with provided revision."""
    return get_impl().schema_stamp(revision)

def task_get(uuid):
    """Returns task by uuid.

    :param uuid: UUID of the task.
    :raises TaskNotFound: if the task does not exist.
    :returns: task dict with data on the task.
    """
    return get_impl().task_get(uuid)

def task_get_status(uuid):
    """Returns task by uuid.

    :param uuid: UUID of the task.
    :raises TaskNotFound: if the task does not exist.
    :returns: task dict with data on the task.
    """
    return get_impl().task_get_status(uuid)

def task_get_detailed_last():
    """Returns the most recently created task."""
    return get_impl().task_get_detailed_last()

def task_get_detailed(uuid):
    """Returns task with results by uuid.

    :param uuid: UUID of the task.
    :returns: task dict with data on the task and its results.
    """
    return get_impl().task_get_detailed(uuid)

def task_create(values):
    """Create task record in DB.

    :param values: dict with record values.
    :returns: task dict with data on the task.
    """
    return get_impl().task_create(values)

def task_update(uuid, values):
    """Update task by values.

    :param uuid: UUID of the task.
    :param values: dict with record values.
    :raises TaskNotFound: if the task does not exist.
    :returns: new updated task dict with data on the task.
    """
    return get_impl().task_update(uuid, values)

def task_update_status(task_uuid, status, allowed_statuses):
    """Update task status with specified value.

    :param task_uuid: string with UUID of Task instance.
    :param status: new value to wrote into db instead of status.
    :param allowed_statuses: list of expected statuses to update in db.
    :raises RallyException: if task not found with specified status.
    :returns: the count of rows match as returned by the database's
              "row count" feature
    """
    return get_impl().task_update_status(task_uuid, allowed_statuses,
                                         status)

def task_list(status=None, deployment=None):
    """Get a list of tasks.

    :param status: Task status to filter the returned list on. If set to
                   None, all the tasks will be returned.
    :param deployment: deployment UUID to filter the returned list on.
                      if set to None tasks from all deployments well be
                      returned.
    :returns: A list of dicts with data on the tasks.
    """
    return get_impl().task_list(status=status, deployment=deployment)

def task_delete(uuid, status=None):
    """Delete a task.

    This method removes the task by the uuid, but if the status
    argument is specified, then the task is removed only when these
    statuses are equal otherwise an exception is raised.

    :param uuid: UUID of the task.
    :raises TaskNotFound: if the task does not exist.
    :raises TaskInvalidStatus: if the status of the task does not
                               equal to the status argument.
    """
    return get_impl().task_delete(uuid, status=status)

def task_result_get_all_by_uuid(task_uuid):
    """Get list of task results.

    :param task_uuid: string with UUID of Task instance.
    :returns: list instances of TaskResult.
    """
    return get_impl().task_result_get_all_by_uuid(task_uuid)

def subtask_create(task_uuid, title, description=None, context=None):
    """Create a subtask.

    :param task_uuid: string with UUID of Task instance.
    :param title: subtask title.
    :param description: subtask description.
    :param context: subtask context dict.
    :returns: a dict with data on the subtask.
    """
    return get_impl().subtask_create(task_uuid, title, description, context)

def workload_create(task_uuid, subtask_uuid, key):
    """Create a workload.

    :param task_uuid: string with UUID of Task instance.
    :param subtask_uuid: string with UUID of Subtask instance.
    :param key: dict with record values on the workload.
    :returns: a dict with data on the workload.
    """
    return get_impl().workload_create(task_uuid, subtask_uuid, key)

def workload_data_create(task_uuid, workload_uuid, chunk_order, data):
    """Create a workload data.

    :param task_uuid: string with UUID of Task instance.
    :param workload_uuid: string with UUID of Workload instance.
    :param chunk_order: ordinal index of workload data
    :param data: dict with record values on the workload data.
    :returns: a dict with data on the workload data.
    """
    return get_impl().workload_data_create(task_uuid, workload_uuid,
                                           chunk_order, data)

def workload_set_results(workload_uuid, data):
    """Set workload results.

    :param workload_uuid: string with UUID of Workload instance.
    :param data: dict with workload results.
    :returns: a dict with data on the workload.
    """
    return get_impl().workload_set_results(workload_uuid, data)

def deployment_create(values):
    """Create a deployment from the values dictionary.

    :param values: dict with record values on the deployment.
    :returns: a dict with data on the deployment.
    """
    return get_impl().deployment_create(values)

def deployment_delete(uuid):
    """Delete a deployment by UUID.

    :param uuid: UUID of the deployment.
    :raises DeploymentNotFound: if the deployment does not exist.
    :raises DeploymentIsBusy: if the resource is not enough.
    """
    return get_impl().deployment_delete(uuid)

def deployment_get(deployment):
    """Get a deployment by UUID.

    :param deployment: UUID or name of the deployment.
    :raises DeploymentNotFound: if the deployment does not exist.
    :returns: a dict with data on the deployment.
    """
    return get_impl().deployment_get(deployment)

def deployment_update(uuid, values):
    """Update a deployment by values.

    :param uuid: UUID of the deployment.
    :param values: dict with items to update.
    :raises DeploymentNotFound: if the deployment does not exist.
    :returns: a dict with data on the deployment.
    """
    return get_impl().deployment_update(uuid, values)

def deployment_list(status=None, parent_uuid=None, name=None):
    """Get list of deployments.

    :param status: if None returns any deployments with any status.
    :param parent_uuid: filter by parent. If None, return only "root"
                        deployments.
    :param name: Name of deployment
    :returns: a list of dicts with data on the deployments.
    """
    return get_impl().deployment_list(status=status, parent_uuid=parent_uuid,
                                      name=name)

def resource_create(values):
    """Create a resource from the values dictionary.

    :param values: a dict with data on the resource.
    :returns: a dict with updated data on the resource.
    """
    return get_impl().resource_create(values)

def resource_get_all(deployment_uuid, provider_name=None, type=None):
    """Return resources of a deployment.

    :param deployment_uuid: filter by uuid of a deployment
    :param provider_name: filter by provider_name, if is None, then
                          return all providers
    :param type: filter by type, if is None, then return all types
    :returns: a list of dicts with data on a resource
    """
    return get_impl().resource_get_all(deployment_uuid,
                                       provider_name=provider_name,
                                       type=type)

def resource_delete(id):
    """Delete a resource.

    :param id: ID of a resource.
    :raises ResourceNotFound: if the resource does not exist.
    """
    return get_impl().resource_delete(id)

def verifier_create(name, vtype, namespace, source, version, system_wide,
                    extra_settings=None):
    """Create a verifier record.

    :param name: verifier name
    :param vtype: verifier plugin name
    :param namespace: verifier plugin namespace
    :param source: path or URL to a verifier repo
    :param version: branch, tag or commit ID of a verifier repo
    :param system_wide: whether or not to use the system-wide environment
    :param extra: verifier-specific installation options
    :returns: a dict with verifier data
    """
    return get_impl().verifier_create(name=name, vtype=vtype,
                                      namespace=namespace, source=source,
                                      version=version, system_wide=system_wide,
                                      extra_settings=extra_settings)

def verifier_get(verifier_id):
    """Get a verifier record.

    :param verifier_id: verifier name or UUID
    :raises ResourceNotFound: if verifier does not exist
    :returns: a dict with verifier data
    """
    return get_impl().verifier_get(verifier_id)

def verifier_list(status=None):
    """Get all verifier records.

    :param status: status to filter verifiers by
    :returns: a list of dicts with verifiers data
    """
    return get_impl().verifier_list(status)

def verifier_delete(verifier_id):
    """Delete a verifier record.

    :param verifier_id: verifier name or UUID
    :raises ResourceNotFound: if verifier does not exist
    """
    get_impl().verifier_delete(verifier_id)

def verifier_update(verifier_id, **properties):
    """Update a verifier record.

    :param verifier_id: verifier name or UUID
    :param properties: a dict with new properties to update verifier record
    :raises ResourceNotFound: if verifier does not exist
    :returns: the updated dict with verifier data
    """
    return get_impl().verifier_update(verifier_id, properties)

def verification_create(verifier_uuid, deployment_uuid, tags=None,
                        run_args=None):
    """Create a verification record.

    :param verifier_uuid: verification UUID
    :param deployment_uuid: deployment UUID
    :param tags: a list of tags to assign them to verification
    :param run_args: a dict with run arguments for verification
    :returns: a dict with verification data
    """
    return get_impl().verification_create(verifier_uuid, deployment_uuid,
                                          tags, run_args)

def verification_get(verification_uuid):
    """Get a verification record.

    :param verification_uuid: verification UUID
    :raises ResourceNotFound: if verification does not exist
    :returns: a dict with verification data
    """
    return get_impl().verification_get(verification_uuid)

def verification_list(verifier_id=None, deployment_id=None, tags=None,
                      status=None):
    """List all verification records.

    :param verifier_id: verifier name or UUID to filter verifications by
    :param deployment_id: deployment name or UUID to filter verifications by
    :param tags: tags to filter verifications by
    :param status: status to filter verifications by
    :returns: a list of dicts with verifications data
    """
    return get_impl().verification_list(verifier_id, deployment_id, tags,
                                        status)

def verification_delete(verification_uuid):
    """Delete a verification record.

    :param verification_uuid: verification UUID
    :raises ResourceNotFound: if verification does not exist
    """
    return get_impl().verification_delete(verification_uuid)

def verification_update(uuid, **properties):
    """Update a verification record.

    :param uuid: verification UUID
    :param properties: a dict with new properties to update verification record
    :raises ResourceNotFound: if verification does not exist
    :returns: the updated dict with verification data
    """
    return get_impl().verification_update(uuid, properties)

def register_worker(values):
    """Register a new worker service at the specified hostname.

    :param values: A dict of values which must contain the following:
                   {
                    "hostname": the unique hostname which identifies
                                this worker service.
                   }
    :returns: A worker.
    :raises WorkerAlreadyRegistered: if worker already registered
    """
    return get_impl().register_worker(values)

def get_worker(hostname):
    """Retrieve a worker service record from the database.

    :param hostname: The hostname of the worker service.
    :returns: A worker.
    :raises WorkerNotFound: if worker not found
    """
    return get_impl().get_worker(hostname)

def unregister_worker(hostname):
    """Unregister this worker with the service registry.

    :param hostname: The hostname of the worker service.
    :raises WorkerNotFound: if worker not found
    """
    get_impl().unregister_worker(hostname)

def update_worker(hostname):
    """Mark a worker as active by updating its "updated_at" property.

    :param hostname: The hostname of this worker service.
    :raises WorkerNotFound: if worker not found
    """
    get_impl().update_worker(hostname)
<commit_msg>[Trivial]Correct some type errors<commit_after># Copyright 2013: Mirantis Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Defines interface for DB access.

The underlying driver is loaded as a :class:`LazyPluggable`.

Functions in this module are imported into the rally.common.db namespace.
Call these functions from rally.common.db namespace, not the
rally.common.db.api namespace.

All functions in this module return objects that implement a dictionary-like
interface. Currently, many of these objects are sqlalchemy objects that
implement a dictionary interface. However, a future goal is to have all of
these objects be simple dictionaries.

**Related Flags**

:backend:  string to lookup in the list of LazyPluggable backends.
           `sqlalchemy` is the only supported backend right now.

:connection:  string specifying the sqlalchemy connection to use, like:
              `sqlite:///var/lib/cinder/cinder.sqlite`.

:enable_new_services:  when adding a new service to the database, is it in the
                       pool of available hardware (Default: True)

"""

import datetime as dt

from oslo_config import cfg
from oslo_db import api as db_api
from oslo_db import options as db_options
import six

from rally.common.i18n import _

CONF = cfg.CONF

db_options.set_defaults(CONF, connection="sqlite:////tmp/rally.sqlite")

IMPL = None

def serialize(fn):
    def conv(data):
        if data is None:
            return None
        if isinstance(data, (six.integer_types,
                             six.string_types,
                             six.text_type,
                             dt.date,
                             dt.time,
                             float,
                             )):
            return data
        if isinstance(data, dict):
            return {k: conv(v) for k, v in data.items()}
        if isinstance(data, (list, tuple)):
            return [conv(i) for i in data]
        if hasattr(data, "_as_dict"):
            result = data._as_dict()
            for k, v in result.items():
                result[k] = conv(v)
            return result

        raise ValueError(_("Can not serialize %s") % data)

    def wrapper(*args, **kwargs):
        result = fn(*args, **kwargs)
        return conv(result)
    return wrapper

def get_impl():
    global IMPL

    if not IMPL:
        _BACKEND_MAPPING = {"sqlalchemy": "rally.common.db.sqlalchemy.api"}
        IMPL = db_api.DBAPI.from_config(CONF, backend_mapping=_BACKEND_MAPPING)

    return IMPL

def engine_reset():
    """Reset DB engine."""
    get_impl().engine_reset()

def schema_cleanup():
    """Drop DB schema. This method drops existing database."""
    get_impl().schema_cleanup()

def schema_upgrade(revision=None):
    """Migrate the database to `revision` or the most recent revision."""
    return get_impl().schema_upgrade(revision)

def schema_create():
    """Create database schema from models description."""
    return get_impl().schema_create()

def schema_revision(detailed=False):
    """Return the schema revision."""
    return get_impl().schema_revision(detailed=detailed)

def schema_stamp(revision):
    """Stamps database with provided revision."""
    return get_impl().schema_stamp(revision)

def task_get(uuid):
    """Returns task by uuid.

    :param uuid: UUID of the task.
    :raises TaskNotFound: if the task does not exist.
    :returns: task dict with data on the task.
    """
    return get_impl().task_get(uuid)

def task_get_status(uuid):
    """Returns task by uuid.

    :param uuid: UUID of the task.
    :raises TaskNotFound: if the task does not exist.
    :returns: task dict with data on the task.
    """
    return get_impl().task_get_status(uuid)

def task_get_detailed_last():
    """Returns the most recently created task."""
    return get_impl().task_get_detailed_last()

def task_get_detailed(uuid):
    """Returns task with results by uuid.

    :param uuid: UUID of the task.
    :returns: task dict with data on the task and its results.
    """
    return get_impl().task_get_detailed(uuid)

def task_create(values):
    """Create task record in DB.

    :param values: dict with record values.
    :returns: task dict with data on the task.
    """
    return get_impl().task_create(values)

def task_update(uuid, values):
    """Update task by values.

    :param uuid: UUID of the task.
    :param values: dict with record values.
    :raises TaskNotFound: if the task does not exist.
    :returns: new updated task dict with data on the task.
    """
    return get_impl().task_update(uuid, values)

def task_update_status(task_uuid, status, allowed_statuses):
    """Update task status with specified value.

    :param task_uuid: string with UUID of Task instance.
    :param status: new value to be written into db instead of status.
    :param allowed_statuses: list of expected statuses to update in db.
    :raises RallyException: if task not found with specified status.
    :returns: the count of rows match as returned by the database's
              "row count" feature
    """
    return get_impl().task_update_status(task_uuid, allowed_statuses,
                                         status)

def task_list(status=None, deployment=None):
    """Get a list of tasks.

    :param status: Task status to filter the returned list on. If set to
                   None, all the tasks will be returned.
    :param deployment: Deployment UUID to filter the returned list on.
                      If set to None, tasks from all deployments will be
                      returned.
    :returns: A list of dicts with data on the tasks.
    """
    return get_impl().task_list(status=status, deployment=deployment)

def task_delete(uuid, status=None):
    """Delete a task.

    This method removes the task by the uuid, but if the status
    argument is specified, then the task is removed only when these
    statuses are equal otherwise an exception is raised.

    :param uuid: UUID of the task.
    :raises TaskNotFound: if the task does not exist.
    :raises TaskInvalidStatus: if the status of the task does not
                               equal to the status argument.
    """
    return get_impl().task_delete(uuid, status=status)

def task_result_get_all_by_uuid(task_uuid):
    """Get list of task results.

    :param task_uuid: string with UUID of Task instance.
    :returns: list instances of TaskResult.
    """
    return get_impl().task_result_get_all_by_uuid(task_uuid)

def subtask_create(task_uuid, title, description=None, context=None):
    """Create a subtask.

    :param task_uuid: string with UUID of Task instance.
    :param title: subtask title.
    :param description: subtask description.
    :param context: subtask context dict.
    :returns: a dict with data on the subtask.
    """
    return get_impl().subtask_create(task_uuid, title, description, context)

def workload_create(task_uuid, subtask_uuid, key):
    """Create a workload.

    :param task_uuid: string with UUID of Task instance.
    :param subtask_uuid: string with UUID of Subtask instance.
    :param key: dict with record values on the workload.
    :returns: a dict with data on the workload.
    """
    return get_impl().workload_create(task_uuid, subtask_uuid, key)

def workload_data_create(task_uuid, workload_uuid, chunk_order, data):
    """Create a workload data.

    :param task_uuid: string with UUID of Task instance.
    :param workload_uuid: string with UUID of Workload instance.
    :param chunk_order: ordinal index of workload data.
    :param data: dict with record values on the workload data.
    :returns: a dict with data on the workload data.
    """
    return get_impl().workload_data_create(task_uuid, workload_uuid,
                                           chunk_order, data)

def workload_set_results(workload_uuid, data):
    """Set workload results.

    :param workload_uuid: string with UUID of Workload instance.
    :param data: dict with workload results.
    :returns: a dict with data on the workload.
    """
    return get_impl().workload_set_results(workload_uuid, data)

def deployment_create(values):
    """Create a deployment from the values dictionary.

    :param values: dict with record values on the deployment.
    :returns: a dict with data on the deployment.
    """
    return get_impl().deployment_create(values)

def deployment_delete(uuid):
    """Delete a deployment by UUID.

    :param uuid: UUID of the deployment.
    :raises DeploymentNotFound: if the deployment does not exist.
    :raises DeploymentIsBusy: if the resource is not enough.
    """
    return get_impl().deployment_delete(uuid)

def deployment_get(deployment):
    """Get a deployment by UUID.

    :param deployment: UUID or name of the deployment.
    :raises DeploymentNotFound: if the deployment does not exist.
    :returns: a dict with data on the deployment.
    """
    return get_impl().deployment_get(deployment)

def deployment_update(uuid, values):
    """Update a deployment by values.

    :param uuid: UUID of the deployment.
    :param values: dict with items to update.
    :raises DeploymentNotFound: if the deployment does not exist.
    :returns: a dict with data on the deployment.
    """
    return get_impl().deployment_update(uuid, values)

def deployment_list(status=None, parent_uuid=None, name=None):
    """Get list of deployments.

    :param status: if None returns any deployments with any status.
    :param parent_uuid: filter by parent. If None, return only "root"
                        deployments.
    :param name: name of deployment.
    :returns: a list of dicts with data on the deployments.
    """
    return get_impl().deployment_list(status=status, parent_uuid=parent_uuid,
                                      name=name)

def resource_create(values):
    """Create a resource from the values dictionary.

    :param values: a dict with data on the resource.
    :returns: a dict with updated data on the resource.
    """
    return get_impl().resource_create(values)

def resource_get_all(deployment_uuid, provider_name=None, type=None):
    """Return resources of a deployment.

    :param deployment_uuid: filter by uuid of a deployment
    :param provider_name: filter by provider_name, if is None, then
                          return all providers
    :param type: filter by type, if is None, then return all types
    :returns: a list of dicts with data on a resource
    """
    return get_impl().resource_get_all(deployment_uuid,
                                       provider_name=provider_name,
                                       type=type)

def resource_delete(id):
    """Delete a resource.

    :param id: ID of a resource.
    :raises ResourceNotFound: if the resource does not exist.
    """
    return get_impl().resource_delete(id)

def verifier_create(name, vtype, namespace, source, version, system_wide,
                    extra_settings=None):
    """Create a verifier record.

    :param name: verifier name
    :param vtype: verifier plugin name
    :param namespace: verifier plugin namespace
    :param source: path or URL to a verifier repo
    :param version: branch, tag or commit ID of a verifier repo
    :param system_wide: whether or not to use the system-wide environment
    :param extra: verifier-specific installation options
    :returns: a dict with verifier data
    """
    return get_impl().verifier_create(name=name, vtype=vtype,
                                      namespace=namespace, source=source,
                                      version=version, system_wide=system_wide,
                                      extra_settings=extra_settings)

def verifier_get(verifier_id):
    """Get a verifier record.

    :param verifier_id: verifier name or UUID
    :raises ResourceNotFound: if verifier does not exist
    :returns: a dict with verifier data
    """
    return get_impl().verifier_get(verifier_id)

def verifier_list(status=None):
    """Get all verifier records.

    :param status: status to filter verifiers by
    :returns: a list of dicts with verifiers data
    """
    return get_impl().verifier_list(status)

def verifier_delete(verifier_id):
    """Delete a verifier record.

    :param verifier_id: verifier name or UUID
    :raises ResourceNotFound: if verifier does not exist
    """
    get_impl().verifier_delete(verifier_id)

def verifier_update(verifier_id, **properties):
    """Update a verifier record.

    :param verifier_id: verifier name or UUID
    :param properties: a dict with new properties to update verifier record
    :raises ResourceNotFound: if verifier does not exist
    :returns: the updated dict with verifier data
    """
    return get_impl().verifier_update(verifier_id, properties)

def verification_create(verifier_uuid, deployment_uuid, tags=None,
                        run_args=None):
    """Create a verification record.

    :param verifier_uuid: verification UUID
    :param deployment_uuid: deployment UUID
    :param tags: a list of tags to assign them to verification
    :param run_args: a dict with run arguments for verification
    :returns: a dict with verification data
    """
    return get_impl().verification_create(verifier_uuid, deployment_uuid,
                                          tags, run_args)

def verification_get(verification_uuid):
    """Get a verification record.

    :param verification_uuid: verification UUID
    :raises ResourceNotFound: if verification does not exist
    :returns: a dict with verification data
    """
    return get_impl().verification_get(verification_uuid)

def verification_list(verifier_id=None, deployment_id=None, tags=None,
                      status=None):
    """List all verification records.

    :param verifier_id: verifier name or UUID to filter verifications by
    :param deployment_id: deployment name or UUID to filter verifications by
    :param tags: tags to filter verifications by
    :param status: status to filter verifications by
    :returns: a list of dicts with verifications data
    """
    return get_impl().verification_list(verifier_id, deployment_id, tags,
                                        status)

def verification_delete(verification_uuid):
    """Delete a verification record.

    :param verification_uuid: verification UUID
    :raises ResourceNotFound: if verification does not exist
    """
    return get_impl().verification_delete(verification_uuid)

def verification_update(uuid, **properties):
    """Update a verification record.

    :param uuid: verification UUID
    :param properties: a dict with new properties to update verification record
    :raises ResourceNotFound: if verification does not exist
    :returns: the updated dict with verification data
    """
    return get_impl().verification_update(uuid, properties)

def register_worker(values):
    """Register a new worker service at the specified hostname.

    :param values: A dict of values which must contain the following:
                   {
                    "hostname": the unique hostname which identifies
                                this worker service.
                   }
    :returns: A worker.
    :raises WorkerAlreadyRegistered: if worker already registered
    """
    return get_impl().register_worker(values)

def get_worker(hostname):
    """Retrieve a worker service record from the database.

    :param hostname: The hostname of the worker service.
    :returns: A worker.
    :raises WorkerNotFound: if worker not found
    """
    return get_impl().get_worker(hostname)

def unregister_worker(hostname):
    """Unregister this worker with the service registry.

    :param hostname: The hostname of the worker service.
    :raises WorkerNotFound: if worker not found
    """
    get_impl().unregister_worker(hostname)

def update_worker(hostname):
    """Mark a worker as active by updating its "updated_at" property.

    :param hostname: The hostname of this worker service.
    :raises WorkerNotFound: if worker not found
    """
    get_impl().update_worker(hostname)
<|endoftext|>


<commit_before>"""$ rio shapes"""

import logging
import os

import click
import cligj

from rasterio.rio import options
from rasterio.rio.helpers import coords, write_features
from rasterio.transform import Affine
from rasterio.crs import CRS

logger = logging.getLogger('rio')

# Common options used below

# Unlike the version in cligj, this one doesn't require values.
files_inout_arg = click.argument(
    'files',
    nargs=-1,
    type=click.Path(resolve_path=True),
    metavar="INPUTS... OUTPUT")

all_touched_opt = click.option(
    '-a', '--all', '--all_touched', 'all_touched',
    is_flag=True,
    default=False,
    help='Use all pixels touched by features, otherwise (default) use only '
         'pixels whose center is within the polygon or that are selected by '
         'Bresenhams line algorithm')

@click.command(short_help="Write shapes extracted from bands or masks.")
@options.file_in_arg
@options.output_opt
@cligj.precision_opt
@cligj.indent_opt
@cligj.compact_opt
@cligj.projection_geographic_opt
@cligj.projection_projected_opt
@cligj.sequence_opt
@cligj.use_rs_opt
@cligj.geojson_type_feature_opt(True)
@cligj.geojson_type_bbox_opt(False)
@click.option('--band/--mask', default=True,
              help="Choose to extract from a band (the default) or a mask.")
@click.option('--bidx', 'bandidx', type=int, default=None,
              help="Index of the band or mask that is the source of shapes.")
@click.option('--sampling', type=int, default=1,
              help="Inverse of the sampling fraction; "
                   "a value of 10 decimates.")
@click.option('--with-nodata/--without-nodata', default=False,
              help="Include or do not include (the default) nodata regions.")
@click.option('--as-mask/--not-as-mask', default=False,
              help="Interpret a band as a mask and output only one class of "
                   "valid data shapes.")
@click.pass_context
def shapes(
        ctx, input, output, precision, indent, compact, projection, sequence,
        use_rs, geojson_type, band, bandidx, sampling, with_nodata, as_mask):
    """Extracts shapes from one band or mask of a dataset and writes
    them out as GeoJSON. Unless otherwise specified, the shapes will be
    transformed to WGS 84 coordinates.

    The default action of this command is to extract shapes from the
    first band of the input dataset. The shapes are polygons bounding
    contiguous regions (or features) of the same raster value. This
    command performs poorly for int16 or float type datasets.

    Bands other than the first can be specified using the `--bidx`
    option:

      $ rio shapes --bidx 3 tests/data/RGB.byte.tif

    The valid data footprint of a dataset's i-th band can be extracted
    by using the `--mask` and `--bidx` options:

      $ rio shapes --mask --bidx 1 tests/data/RGB.byte.tif

    Omitting the `--bidx` option results in a footprint extracted from
    the conjunction of all band masks. This is generally smaller than
    any individual band's footprint.

    A dataset band may be analyzed as though it were a binary mask with
    the `--as-mask` option:

      $ rio shapes --as-mask --bidx 1 tests/data/RGB.byte.tif
    """
    # These import numpy, which we don't want to do unless it's needed.
    import numpy as np
    import rasterio.features
    import rasterio.warp

    logger = logging.getLogger('rio')
    dump_kwds = {'sort_keys': True}
    if indent:
        dump_kwds['indent'] = indent
    if compact:
        dump_kwds['separators'] = (',', ':')

    stdout = click.open_file(
        output, 'w') if output else click.get_text_stream('stdout')

    bidx = 1 if bandidx is None and band else bandidx

    # This is the generator for (feature, bbox) pairs.
    class Collection(object):

        def __init__(self, env):
            self._xs = []
            self._ys = []
            self.env = env

        @property
        def bbox(self):
            return min(self._xs), min(self._ys), max(self._xs), max(self._ys)

        def __call__(self):
            with rasterio.open(input) as src:
                if bidx is not None and bidx > src.count:
                    raise ValueError('bidx is out of range for raster')

                img = None
                msk = None

                # Adjust transforms.
                transform = src.transform
                if sampling > 1:
                    # Decimation of the raster produces a georeferencing
                    # shift that we correct with a translation.
                    transform *= Affine.translation(
                        src.width % sampling, src.height % sampling)
                    # And follow by scaling.
                    transform *= Affine.scale(float(sampling))

                # Most of the time, we'll use the valid data mask.
                # We skip reading it if we're extracting every possible
                # feature (even invalid data features) from a band.
                if not band or (band and not as_mask and not with_nodata):
                    if sampling == 1:
                        msk = src.read_masks(bidx)
                    else:
                        msk_shape = (
                            src.height // sampling, src.width // sampling)
                        if bidx is None:
                            msk = np.zeros(
                                (src.count,) + msk_shape, 'uint8')
                        else:
                            msk = np.zeros(msk_shape, 'uint8')
                        msk = src.read_masks(bidx, msk)

                    if bidx is None:
                        msk = np.logical_or.reduce(msk).astype('uint8')

                    # Possibly overridden below.
                    img = msk

                # Read the band data unless the --mask option is given.
                if band:
                    if sampling == 1:
                        img = src.read(bidx, masked=False)
                    else:
                        img = np.zeros(
                            (src.height // sampling, src.width // sampling),
                            dtype=src.dtypes[src.indexes.index(bidx)])
                        img = src.read(bidx, img, masked=False)

                # If --as-mask option was given, convert the image
                # to a binary image. This reduces the number of shape
                # categories to 2 and likely reduces the number of
                # shapes.
                if as_mask:
                    tmp = np.ones_like(img, 'uint8') * 255
                    tmp[img == 0] = 0
                    img = tmp
                    if not with_nodata:
                        msk = tmp

                # Transform the raster bounds.
                bounds = src.bounds
                xs = [bounds[0], bounds[2]]
                ys = [bounds[1], bounds[3]]
                if projection == 'geographic':
                    xs, ys = rasterio.warp.transform(
                        src.crs, CRS({'init': 'epsg:4326'}), xs, ys)
                if precision >= 0:
                    xs = [round(v, precision) for v in xs]
                    ys = [round(v, precision) for v in ys]
                self._xs = xs
                self._ys = ys

                # Prepare keyword arguments for shapes().
                kwargs = {'transform': transform}
                if not with_nodata:
                    kwargs['mask'] = msk

                src_basename = os.path.basename(src.name)

                # Yield GeoJSON features.
                for i, (g, val) in enumerate(
                        rasterio.features.shapes(img, **kwargs)):
                    if projection == 'geographic':
                        g = rasterio.warp.transform_geom(
                            src.crs, 'EPSG:4326', g,
                            antimeridian_cutting=True, precision=precision)
                    xs, ys = zip(*coords(g))
                    yield {
                        'type': 'Feature',
                        'id': "{0}:{1}".format(src_basename, i),
                        'properties': {
                            'val': val, 'filename': src_basename
                        },
                        'bbox': [min(xs), min(ys), max(xs), max(ys)],
                        'geometry': g
                    }

    if not sequence:
        geojson_type = 'collection'

    try:
        with ctx.obj['env'] as env:
            write_features(
                stdout, Collection(env), sequence=sequence,
                geojson_type=geojson_type, use_rs=use_rs,
                **dump_kwds)
    except Exception:
        logger.exception("Exception caught during processing")
        raise click.Abort()
<commit_msg>Avoid generating shapes w/ 0-dimension<commit_after>"""$ rio shapes"""

from __future__ import division

import logging
import math
import os

import click
import cligj

from rasterio.rio import options
from rasterio.rio.helpers import coords, write_features
from rasterio.transform import Affine
from rasterio.crs import CRS

logger = logging.getLogger('rio')

# Common options used below

# Unlike the version in cligj, this one doesn't require values.
files_inout_arg = click.argument(
    'files',
    nargs=-1,
    type=click.Path(resolve_path=True),
    metavar="INPUTS... OUTPUT")

all_touched_opt = click.option(
    '-a', '--all', '--all_touched', 'all_touched',
    is_flag=True,
    default=False,
    help='Use all pixels touched by features, otherwise (default) use only '
         'pixels whose center is within the polygon or that are selected by '
         'Bresenhams line algorithm')

@click.command(short_help="Write shapes extracted from bands or masks.")
@options.file_in_arg
@options.output_opt
@cligj.precision_opt
@cligj.indent_opt
@cligj.compact_opt
@cligj.projection_geographic_opt
@cligj.projection_projected_opt
@cligj.sequence_opt
@cligj.use_rs_opt
@cligj.geojson_type_feature_opt(True)
@cligj.geojson_type_bbox_opt(False)
@click.option('--band/--mask', default=True,
              help="Choose to extract from a band (the default) or a mask.")
@click.option('--bidx', 'bandidx', type=int, default=None,
              help="Index of the band or mask that is the source of shapes.")
@click.option('--sampling', type=int, default=1,
              help="Inverse of the sampling fraction; "
                   "a value of 10 decimates.")
@click.option('--with-nodata/--without-nodata', default=False,
              help="Include or do not include (the default) nodata regions.")
@click.option('--as-mask/--not-as-mask', default=False,
              help="Interpret a band as a mask and output only one class of "
                   "valid data shapes.")
@click.pass_context
def shapes(
        ctx, input, output, precision, indent, compact, projection, sequence,
        use_rs, geojson_type, band, bandidx, sampling, with_nodata, as_mask):
    """Extracts shapes from one band or mask of a dataset and writes
    them out as GeoJSON. Unless otherwise specified, the shapes will be
    transformed to WGS 84 coordinates.

    The default action of this command is to extract shapes from the
    first band of the input dataset. The shapes are polygons bounding
    contiguous regions (or features) of the same raster value. This
    command performs poorly for int16 or float type datasets.

    Bands other than the first can be specified using the `--bidx`
    option:

      $ rio shapes --bidx 3 tests/data/RGB.byte.tif

    The valid data footprint of a dataset's i-th band can be extracted
    by using the `--mask` and `--bidx` options:

      $ rio shapes --mask --bidx 1 tests/data/RGB.byte.tif

    Omitting the `--bidx` option results in a footprint extracted from
    the conjunction of all band masks. This is generally smaller than
    any individual band's footprint.

    A dataset band may be analyzed as though it were a binary mask with
    the `--as-mask` option:

      $ rio shapes --as-mask --bidx 1 tests/data/RGB.byte.tif
    """
    # These import numpy, which we don't want to do unless it's needed.
    import numpy as np
    import rasterio.features
    import rasterio.warp

    logger = logging.getLogger('rio')
    dump_kwds = {'sort_keys': True}
    if indent:
        dump_kwds['indent'] = indent
    if compact:
        dump_kwds['separators'] = (',', ':')

    stdout = click.open_file(
        output, 'w') if output else click.get_text_stream('stdout')

    bidx = 1 if bandidx is None and band else bandidx

    # This is the generator for (feature, bbox) pairs.
    class Collection(object):

        def __init__(self, env):
            self._xs = []
            self._ys = []
            self.env = env

        @property
        def bbox(self):
            return min(self._xs), min(self._ys), max(self._xs), max(self._ys)

        def __call__(self):
            with rasterio.open(input) as src:
                if bidx is not None and bidx > src.count:
                    raise ValueError('bidx is out of range for raster')

                img = None
                msk = None

                # Adjust transforms.
                transform = src.transform
                if sampling > 1:
                    # Decimation of the raster produces a georeferencing
                    # shift that we correct with a translation.
                    transform *= Affine.translation(
                        src.width % sampling, src.height % sampling)
                    # And follow by scaling.
                    transform *= Affine.scale(float(sampling))
                    shape = (int(math.ceil(src.height / sampling)),
                             int(math.ceil(src.width / sampling)))

                # Most of the time, we'll use the valid data mask.
                # We skip reading it if we're extracting every possible
                # feature (even invalid data features) from a band.
                if not band or (band and not as_mask and not with_nodata):
                    if sampling == 1:
                        msk = src.read_masks(bidx)
                    else:
                        msk_shape = shape
                        if bidx is None:
                            msk = np.zeros(
                                (src.count,) + msk_shape, 'uint8')
                        else:
                            msk = np.zeros(msk_shape, 'uint8')
                        msk = src.read_masks(bidx, msk)

                    if bidx is None:
                        msk = np.logical_or.reduce(msk).astype('uint8')

                    # Possibly overridden below.
                    img = msk

                # Read the band data unless the --mask option is given.
                if band:
                    if sampling == 1:
                        img = src.read(bidx, masked=False)
                    else:
                        img = np.zeros(
                            shape,
                            dtype=src.dtypes[src.indexes.index(bidx)])
                        img = src.read(bidx, img, masked=False)

                # If --as-mask option was given, convert the image
                # to a binary image. This reduces the number of shape
                # categories to 2 and likely reduces the number of
                # shapes.
                if as_mask:
                    tmp = np.ones_like(img, 'uint8') * 255
                    tmp[img == 0] = 0
                    img = tmp
                    if not with_nodata:
                        msk = tmp

                # Transform the raster bounds.
                bounds = src.bounds
                xs = [bounds[0], bounds[2]]
                ys = [bounds[1], bounds[3]]
                if projection == 'geographic':
                    xs, ys = rasterio.warp.transform(
                        src.crs, CRS({'init': 'epsg:4326'}), xs, ys)
                if precision >= 0:
                    xs = [round(v, precision) for v in xs]
                    ys = [round(v, precision) for v in ys]
                self._xs = xs
                self._ys = ys

                # Prepare keyword arguments for shapes().
                kwargs = {'transform': transform}
                if not with_nodata:
                    kwargs['mask'] = msk

                src_basename = os.path.basename(src.name)

                # Yield GeoJSON features.
                for i, (g, val) in enumerate(
                        rasterio.features.shapes(img, **kwargs)):
                    if projection == 'geographic':
                        g = rasterio.warp.transform_geom(
                            src.crs, 'EPSG:4326', g,
                            antimeridian_cutting=True, precision=precision)
                    xs, ys = zip(*coords(g))
                    yield {
                        'type': 'Feature',
                        'id': "{0}:{1}".format(src_basename, i),
                        'properties': {
                            'val': val, 'filename': src_basename
                        },
                        'bbox': [min(xs), min(ys), max(xs), max(ys)],
                        'geometry': g
                    }

    if not sequence:
        geojson_type = 'collection'

    try:
        with ctx.obj['env'] as env:
            write_features(
                stdout, Collection(env), sequence=sequence,
                geojson_type=geojson_type, use_rs=use_rs,
                **dump_kwds)
    except Exception:
        logger.exception("Exception caught during processing")
        raise click.Abort()
<|endoftext|>


<commit_before>#
# Copyright (c) 2014 ThoughtWorks, Inc.
#
# Pixelated is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Pixelated is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
from uuid import uuid4
from test.support.integration import MailBuilder
from behave import given
from common import wait_for_condition
from crochet import wait_for

@given('I have a mail in my inbox')
def add_mail_impl(context):
    subject = 'Hi! This the subject %s' % uuid4()

    input_mail = MailBuilder().with_subject(subject).build_input_mail()

    context.single_user_client.add_mail_to_inbox(input_mail)
    wait_for_condition(context, lambda _: context.single_user_client.search_engine.search(subject)[1] > 0, poll_frequency=0.1)

    context.last_subject = subject

@given('I have a mail for {username} in my inbox')
def add_mail_to_user_inbox(context, username):
    subject = 'Hi! This the subject %s' % uuid4()

    input_mail = MailBuilder().with_subject(subject).build_input_mail()

    context.multi_user_client.add_mail_to_user_inbox(input_mail, username)
    wait_for_condition(context, lambda _: context.multi_user_client.account_for(username).search_engine.search(subject)[1] > 0, poll_frequency=0.1)

    context.last_subject = subject

@wait_for(timeout=10.0)
def add_multi_user_account(context, username):
    return context.multi_user_client.create_user('username')

@given(u'Account for user {username} exists')
def add_account(context, username):
    add_multi_user_account(context, username)
<commit_msg>[#845] Fixes absence of email in functional test<commit_after>#
# Copyright (c) 2014 ThoughtWorks, Inc.
#
# Pixelated is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Pixelated is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
from uuid import uuid4
from test.support.integration import MailBuilder
from behave import given
from common import wait_for_condition
from crochet import wait_for

@given('I have a mail in my inbox')
def add_mail_impl(context):
    subject = 'Hi! This the subject %s' % uuid4()

    input_mail = MailBuilder().with_subject(subject).build_input_mail()

    load_mail_into_soledad(context, input_mail)
    wait_for_condition(context, lambda _: context.single_user_client.search_engine.search(subject)[1] > 0, poll_frequency=0.1)

    context.last_subject = subject

@given('I have a mail for {username} in my inbox')
def add_mail_to_user_inbox(context, username):
    subject = 'Hi! This the subject %s' % uuid4()

    input_mail = MailBuilder().with_subject(subject).build_input_mail()

    load_mail_into_user_account(context, input_mail, username)
    wait_for_condition(context, lambda _: context.multi_user_client.account_for(username).search_engine.search(subject)[1] > 0, poll_frequency=0.1)

    context.last_subject = subject

@given(u'Account for user {username} exists')
def add_account(context, username):
    add_multi_user_account(context, username)

@wait_for(timeout=10.0)
def load_mail_into_soledad(context, mail):
    return context.single_user_client.mail_store.add_mail('INBOX', mail.raw)

@wait_for(timeout=10.0)
def load_mail_into_user_account(context, mail, username):
    return context.multi_user_client.add_mail_to_user_inbox(mail, username)

@wait_for(timeout=10.0)
def add_multi_user_account(context, username):
    return context.multi_user_client.create_user('username')
<|endoftext|>


<commit_before>from django.http import HttpResponse
from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response, redirect
from django.views.generic.base import View
from django.core.urlresolvers import reverse
from django.template import RequestContext
from django.core.exceptions import PermissionDenied
from django.contrib import messages

from django.contrib.auth.decorators import login_required
from django.utils.decorators import method_decorator

from explorer import app_settings
from forms import QueryForm
from models import Query
from utils import DatabaseUtils
from rdrf.models import Registry, RegistryForm, Section, CommonDataElement, CDEPermittedValue
from registry.groups.models import WorkingGroup
from rdrf.reporting_table import ReportingTableGenerator

import re
import csv
import json
import urllib2
from bson.json_util import dumps
from bson import json_util
from datetime import datetime
import collections
import logging
from itertools import product
from rdrf.utils import models_from_mongo_key, is_delimited_key, BadKeyError, cached

logger = logging.getLogger("registry_log")

def encode_row(row):
    return [s.encode('utf8') if type(s) is unicode else s for s in row]

class LoginRequiredMixin(object):

    @method_decorator(login_required)
    def dispatch(self, request, *args, **kwargs):
        return super(LoginRequiredMixin, self).dispatch(
            request, *args, **kwargs)

class MainView(LoginRequiredMixin, View):

    def get(self, request):
        user = request.user

        reports = None

        if user.is_superuser:
            reports = Query.objects.all()
        elif user.is_curator:
            reports = Query.objects.filter(
                registry__in=[
                    reg.id for reg in user.get_registries()]).filter(
                access_group__in=[
                    g.id for g in user.get_groups()])

        return render_to_response(
            'explorer/query_list.html',
            {'object_list': reports},
            _get_default_params(request, None))

class NewQueryView(LoginRequiredMixin, View):

    def get(self, request):
        if not request.user.is_superuser:
            raise PermissionDenied()

        params = _get_default_params(request, QueryForm)
        return render_to_response('explorer/query.html', params)

    def post(self, request):
        if not request.user.is_superuser:
            raise PermissionDenied()

        query_form = QueryForm(request.POST)
        if query_form.is_valid():
            m = query_form.save(commit=False)
            m.save()
            query_form.save_m2m()
            return redirect(m)
        return HttpResponse()

class DeleteQueryView(LoginRequiredMixin, View):

    def get(self, request, query_id):
        if not request.user.is_superuser:
            raise PermissionDenied()

        query_model = Query.objects.get(id=query_id)
        query_model.delete()
        return redirect('explorer_main')

class QueryView(LoginRequiredMixin, View):

    def get(self, request, query_id):
        from rdrf.models import Registry

        query_model = Query.objects.get(id=query_id)
        query_form = QueryForm(instance=query_model)
        params = _get_default_params(request, query_form)
        params['edit'] = True
        params['registries'] = Registry.objects.all()
        return render_to_response('explorer/query.html', params)

    def post(self, request, query_id):
        query_model = Query.objects.get(id=query_id)
        registry_model = query_model.registry
        query_form = QueryForm(request.POST, instance=query_model)
        form = QueryForm(request.POST)

        database_utils = DatabaseUtils(form)

        if request.is_ajax():
            # populate temporary table
            from rdrf.reporting_table import ReportingTableGenerator
            humaniser = Humaniser(registry_model)
            multisection_unroller = MultisectionUnRoller({})
            rtg = ReportingTableGenerator(request.user, registry_model, multisection_unroller, humaniser)
            rtg.set_table_name(query_model)
            try:
                database_utils.dump_results_into_reportingdb(reporting_table_generator=rtg)
                return HttpResponse("Temporary Table created")
            except Exception, ex:
                return HttpResponse("Report Error: %s" % ex)
        else:
            if form.is_valid():
                m = query_form.save(commit=False)
                m.save()
                query_form.save_m2m()
                return redirect(m)

class DownloadQueryView(LoginRequiredMixin, View):

    def post(self, request, query_id, action):
        if action not in ["download", "analyse"]:
            raise Exception("bad action")

        query_model = Query.objects.get(id=query_id)
        query_form = QueryForm(instance=query_model)

        query_params = re.findall("%(.*?)%", query_model.sql_query)

        sql_query = query_model.sql_query
        for param in query_params:
            sql_query = sql_query.replace("%%%s%%" % param, request.POST[param])
        query_model.sql_query = sql_query

        if "registry" in query_params:
            query_model.registry = Registry.objects.get(id=request.POST["registry"])
        if "working_group" in query_params:
            query_model.working_group = WorkingGroup.objects.get(
                id=request.POST["working_group"])

        registry_model = query_model.registry

        database_utils = DatabaseUtils(query_model)
        humaniser = Humaniser(registry_model)
        multisection_unrollower = MultisectionUnRoller({})
        rtg = ReportingTableGenerator(request.user, registry_model, multisection_unrollower, humaniser)
        rtg.set_table_name(query_model)
        database_utils.dump_results_into_reportingdb(reporting_table_generator=rtg)
        if action == "analyse":
            return HttpResponseRedirect(reverse("report_datatable", args=[query_model.id]))
        else:
            return self._extract(query_model.title, rtg)

    def get(self, request, query_id, action):
        if action not in ['download', 'analyse']:
            raise Exception("bad action")

        user = request.user
        query_model = Query.objects.get(id=query_id)
        registry_model = query_model.registry
        query_form = QueryForm(instance=query_model)

        query_params = re.findall("%(.*?)%", query_model.sql_query)

        if query_params:
            params = _get_default_params(request, query_form)
            params["action"] = action
            params['query_params'] = query_params
            if "registry" in query_params:
                params["registry"] = Registry.objects.all()
            if "working_group" in query_params:
                if user.is_curator:
                    params["working_group"] = WorkingGroup.objects.filter(
                        id__in=[wg.id for wg in user.get_working_groups()])
                else:
                    params["working_group"] = WorkingGroup.objects.all()
            return render_to_response('explorer/query_download.html', params)

        database_utils = DatabaseUtils(query_model)
        humaniser = Humaniser(registry_model)
        multisection_unrollower = MultisectionUnRoller({})
        rtg = ReportingTableGenerator(request.user, registry_model, multisection_unrollower, humaniser)
        rtg.set_table_name(query_model)
        database_utils.dump_results_into_reportingdb(reporting_table_generator=rtg)
        if action == 'analyse':
            # allow user to view and manipulate
            return HttpResponseRedirect(reverse("report_datatable", args=[query_model.id]))
        else:
            # download csv
            return self._extract(query_model.title, rtg)

    def _extract(self, title, report_table_generator):
        response = HttpResponse(content_type='text/csv')
        response['Content-Disposition'] = 'attachment; filename="query_%s.csv"' % title.lower()
        return report_table_generator.dump_csv(response)

class SqlQueryView(View):

    def post(self, request):
        form = QueryForm(request.POST)
        database_utils = DatabaseUtils(form, True)
        results = database_utils.run_sql().result
        response = HttpResponse(dumps(results, default=json_serial))
        return response

def json_serial(obj):
    """JSON serializer for objects not serializable by default json code"""
    serial = obj.isoformat()
    return serial

def _get_default_params(request, form):
    database_utils = DatabaseUtils()
    status, error = database_utils.connection_status()

    return RequestContext(request, {
        'version': app_settings.APP_VERSION,
        'host': app_settings.VIEWER_MONGO_HOST,
        'status': status,
        'error_msg': error,
        'form': form,
        'csrf_token_name': app_settings.CSRF_NAME
    })

def _get_header(result):
    header = []
    if result:
        for key in result[0].keys():
            header.append(key.encode("utf8"))
        return header

def _get_content(result, header):
    row = []
    for h in header:
        row.append(result.get(h.decode("utf8"), "?"))
    return row

class Humaniser(object):
    """
    If a display name/value is appropriate for a field, return it
    """
    def __init__(self, registry_model):
        self.registry_model = registry_model

    @cached
    def display_name(self, key):
        if is_delimited_key(key):
            try:
                form_model, section_model, cde_model = models_from_mongo_key(self.registry_model, key)
            except BadKeyError:
                logger.error("key %s refers to non-existant models" % key)
                return key

            human_name = "%s/%s/%s" % (form_model.name, section_model.display_name, cde_model.name)
            return human_name
        else:
            return key

    @cached
    def display_value(self, key, mongo_value):
        # return the display value for ranges
        if is_delimited_key(key):
            try:
                form_model, section_model, cde_model = models_from_mongo_key(self.registry_model, key)
            except BadKeyError:
                logger.error("Key %s refers to non-existant models" % key)
                return mongo_value

            if cde_model.pv_group:
                # look up the stored code and return the display value
                range_dict = cde_model.pv_group.as_dict()
                for value_dict in range_dict["values"]:
                    if mongo_value == value_dict["code"]:
                        return value_dict["value"]
        return mongo_value

def _human_friendly(registry_model, result):
    humaniser = Humaniser(registry_model)

    for r in result:
        for key in r.keys():
            mongo_value = r[key]
            cde_value = humaniser.display_value(key, mongo_value)
            if cde_value:
                r[key] = cde_value
            cde_name = humaniser.display_name(key)
            if cde_name:
                r[cde_name] = r[key]
                if cde_name != key:
                    del r[key]
    return result

def _get_non_multiple_mongo_keys(registry_model):
    # return a list of delimited mongo keys for the supplied registry
    # skip the generated questionnaire ( the data from which is copied to the target clinical forms anyway)
    # skip multisections  as these are handled separately
    delimited_keys = []
    from rdrf.utils import mongo_key_from_models
    for form_model in registry_model.forms:
        if not form_model.is_questionnaire:
            for section_model in form_model.section_models:
                if not section_model.allow_multiple:
                    for cde_model in section_model.cde_models:
                        delimited_key = mongo_key_from_models(form_model, section_model, cde_model)
                        delimited_keys.append(delimited_key)
    return delimited_keys

def _get_cdes(registry_obj):
    from rdrf.models import RegistryForm
    from rdrf.models import Section

    cdes = []
    forms = RegistryForm.objects.filter(registry__code=registry_obj.code)

    for form in forms:
        sections = Section.objects.filter(code__in=form.sections.split(","))
        for section in sections:
            for cde in section.get_elements():
                cdes.append("%s____%s____%s" % (form.name, section.code, cde))

    return cdes

def _filler(result, cdes):
    import collections
    munged = []
    for r in result:
        for cde in cdes:
            if cde not in r:
                r[cde] = "?"
        munged.append(collections.OrderedDict(sorted(r.items())))
    return munged

def _final_cleanup(results):
    for res in results:
        for key, value in res.iteritems():
            if key.endswith('timestamp'):
                del res[key]
    return results

class MultisectionUnRoller(object):
    def __init__(self, multisection_column_map):
        # section_code --> column names in the report in that multisection
        # E.g.{ "social": ["friends"], "health": ["drug", "dose"]}
        self.multisection_column_map = multisection_column_map
        self.row_count = 0

    def unroll(self, row):
        """
        Basic idea is to use cartesian product to display all combinations of list elements
        for the multisections:
        if a row originally looks like   normalfield1, normalfield2, [a,b,c], notmalfield4, [d,e,f]
        we need to iterate through the cartesian product of [a,b,c] and [d,e,f] ( a square)
        ( 1 row expands to 9 rows !)
        Hence the use of itertools.product to walk through the generated choices
        ( eg b,e
             a,f
            etc etc)
        for three multisections we iterate through the triple product ( a cube) and so on
        This gets big quick obviously ...

        complication is that multisections can contain more than one field  so the unit we walk through is the section

        :param row:
        :return:
        """
        from itertools import product

        def dl2ld(dl):
            """
            :param dl: A dictionary of lists : e.g. {"drug" : ["aspirin", "neurophen"], "dose": [100,200] }
            ( each list must be same length )
            :return: A list of dictionaries = [ {"drug": "aspirin", "dose": 100}, {"drug": "neurophen", "dose": 200}]
            """
            l = []
            indexes = range(max(map(len, dl.values())))
            for i in indexes:
                d = {}
                for k in dl:
                    d[k] = dl[k][i]
                l.append(d)
            return l

        # e.g. row = {"name": "Lee", "friends": ["fred", "barry"],
        # "drug":["aspirin","neurophen"], "dose" : [20,23], "height": 56}
        new_rows = []  # the extra unrolled rows
        sublists = {}
        for multisection_code in self.multisection_column_map:
            multisection_columns = self.multisection_column_map[multisection_code]
            section_data = {}
            for col in multisection_columns:
                values = row[col]  # each multisection cde will have a list of values
                section_data[col] = values

            sublists[multisection_code] = dl2ld(section_data)

        row_count = 0
        # choice tuple is one choice from each sublist
        for choice_tuple in product(*sublists.values()):
            new_row = row.copy()
            row_count += 1
            for section_dict in choice_tuple:
                for key in section_dict:
                    new_row[key] = section_dict[key]

            new_rows.append(new_row)

        return new_rows
<commit_msg>RDR-1143 changed analyse action name to 'view' ( more accurate)<commit_after>from django.http import HttpResponse
from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response, redirect
from django.views.generic.base import View
from django.core.urlresolvers import reverse
from django.template import RequestContext
from django.core.exceptions import PermissionDenied
from django.contrib import messages

from django.contrib.auth.decorators import login_required
from django.utils.decorators import method_decorator

from explorer import app_settings
from forms import QueryForm
from models import Query
from utils import DatabaseUtils
from rdrf.models import Registry, RegistryForm, Section, CommonDataElement, CDEPermittedValue
from registry.groups.models import WorkingGroup
from rdrf.reporting_table import ReportingTableGenerator

import re
import csv
import json
import urllib2
from bson.json_util import dumps
from bson import json_util
from datetime import datetime
import collections
import logging
from itertools import product
from rdrf.utils import models_from_mongo_key, is_delimited_key, BadKeyError, cached

logger = logging.getLogger("registry_log")

def encode_row(row):
    return [s.encode('utf8') if type(s) is unicode else s for s in row]

class LoginRequiredMixin(object):

    @method_decorator(login_required)
    def dispatch(self, request, *args, **kwargs):
        return super(LoginRequiredMixin, self).dispatch(
            request, *args, **kwargs)

class MainView(LoginRequiredMixin, View):

    def get(self, request):
        user = request.user

        reports = None

        if user.is_superuser:
            reports = Query.objects.all()
        elif user.is_curator:
            reports = Query.objects.filter(
                registry__in=[
                    reg.id for reg in user.get_registries()]).filter(
                access_group__in=[
                    g.id for g in user.get_groups()])

        return render_to_response(
            'explorer/query_list.html',
            {'object_list': reports},
            _get_default_params(request, None))

class NewQueryView(LoginRequiredMixin, View):

    def get(self, request):
        if not request.user.is_superuser:
            raise PermissionDenied()

        params = _get_default_params(request, QueryForm)
        return render_to_response('explorer/query.html', params)

    def post(self, request):
        if not request.user.is_superuser:
            raise PermissionDenied()

        query_form = QueryForm(request.POST)
        if query_form.is_valid():
            m = query_form.save(commit=False)
            m.save()
            query_form.save_m2m()
            return redirect(m)
        return HttpResponse()

class DeleteQueryView(LoginRequiredMixin, View):

    def get(self, request, query_id):
        if not request.user.is_superuser:
            raise PermissionDenied()

        query_model = Query.objects.get(id=query_id)
        query_model.delete()
        return redirect('explorer_main')

class QueryView(LoginRequiredMixin, View):

    def get(self, request, query_id):
        from rdrf.models import Registry

        query_model = Query.objects.get(id=query_id)
        query_form = QueryForm(instance=query_model)
        params = _get_default_params(request, query_form)
        params['edit'] = True
        params['registries'] = Registry.objects.all()
        return render_to_response('explorer/query.html', params)

    def post(self, request, query_id):
        query_model = Query.objects.get(id=query_id)
        registry_model = query_model.registry
        query_form = QueryForm(request.POST, instance=query_model)
        form = QueryForm(request.POST)

        database_utils = DatabaseUtils(form)

        if request.is_ajax():
            # populate temporary table
            from rdrf.reporting_table import ReportingTableGenerator
            humaniser = Humaniser(registry_model)
            multisection_unroller = MultisectionUnRoller({})
            rtg = ReportingTableGenerator(request.user, registry_model, multisection_unroller, humaniser)
            rtg.set_table_name(query_model)
            try:
                database_utils.dump_results_into_reportingdb(reporting_table_generator=rtg)
                return HttpResponse("Temporary Table created")
            except Exception, ex:
                return HttpResponse("Report Error: %s" % ex)
        else:
            if form.is_valid():
                m = query_form.save(commit=False)
                m.save()
                query_form.save_m2m()
                return redirect(m)

class DownloadQueryView(LoginRequiredMixin, View):

    def post(self, request, query_id, action):
        if action not in ["download", "view"]:
            raise Exception("bad action")

        query_model = Query.objects.get(id=query_id)
        query_form = QueryForm(instance=query_model)

        query_params = re.findall("%(.*?)%", query_model.sql_query)

        sql_query = query_model.sql_query
        for param in query_params:
            sql_query = sql_query.replace("%%%s%%" % param, request.POST[param])
        query_model.sql_query = sql_query

        if "registry" in query_params:
            query_model.registry = Registry.objects.get(id=request.POST["registry"])
        if "working_group" in query_params:
            query_model.working_group = WorkingGroup.objects.get(
                id=request.POST["working_group"])

        registry_model = query_model.registry

        database_utils = DatabaseUtils(query_model)
        humaniser = Humaniser(registry_model)
        multisection_unrollower = MultisectionUnRoller({})
        rtg = ReportingTableGenerator(request.user, registry_model, multisection_unrollower, humaniser)
        rtg.set_table_name(query_model)
        database_utils.dump_results_into_reportingdb(reporting_table_generator=rtg)
        if action == "view":
            return HttpResponseRedirect(reverse("report_datatable", args=[query_model.id]))
        else:
            return self._extract(query_model.title, rtg)

    def get(self, request, query_id, action):
        if action not in ['download', 'view']:
            raise Exception("bad action")

        user = request.user
        logger.debug("user = %s" % user)
        query_model = Query.objects.get(id=query_id)
        registry_model = query_model.registry
        query_form = QueryForm(instance=query_model)

        query_params = re.findall("%(.*?)%", query_model.sql_query)

        if query_params:
            params = _get_default_params(request, query_form)
            params["action"] = action
            params['query_params'] = query_params
            if "registry" in query_params:
                params["registry"] = Registry.objects.all()
            if "working_group" in query_params:
                if user.is_superuser:
                    params["working_group"] = WorkingGroup.objects.filter(registry=registry_model)
                elif user.is_curator:
                    params["working_group"] = WorkingGroup.objects.filter(
                        id__in=[wg.id for wg in user.get_working_groups()])
                else:
                    # only curators and admin
                    pass

            return render_to_response('explorer/query_download.html', params)

        database_utils = DatabaseUtils(query_model)
        humaniser = Humaniser(registry_model)
        multisection_unrollower = MultisectionUnRoller({})
        rtg = ReportingTableGenerator(request.user, registry_model, multisection_unrollower, humaniser)
        rtg.set_table_name(query_model)
        database_utils.dump_results_into_reportingdb(reporting_table_generator=rtg)
        if action == 'view':
            # allow user to view and manipulate
            return HttpResponseRedirect(reverse("report_datatable", args=[query_model.id]))
        else:
            # download csv
            return self._extract(query_model.title, rtg)

    def _extract(self, title, report_table_generator):
        response = HttpResponse(content_type='text/csv')
        response['Content-Disposition'] = 'attachment; filename="query_%s.csv"' % title.lower()
        return report_table_generator.dump_csv(response)

class SqlQueryView(View):

    def post(self, request):
        form = QueryForm(request.POST)
        database_utils = DatabaseUtils(form, True)
        results = database_utils.run_sql().result
        response = HttpResponse(dumps(results, default=json_serial))
        return response

def json_serial(obj):
    """JSON serializer for objects not serializable by default json code"""
    serial = obj.isoformat()
    return serial

def _get_default_params(request, form):
    database_utils = DatabaseUtils()
    status, error = database_utils.connection_status()

    return RequestContext(request, {
        'version': app_settings.APP_VERSION,
        'host': app_settings.VIEWER_MONGO_HOST,
        'status': status,
        'error_msg': error,
        'form': form,
        'csrf_token_name': app_settings.CSRF_NAME
    })

def _get_header(result):
    header = []
    if result:
        for key in result[0].keys():
            header.append(key.encode("utf8"))
        return header

def _get_content(result, header):
    row = []
    for h in header:
        row.append(result.get(h.decode("utf8"), "?"))
    return row

class Humaniser(object):
    """
    If a display name/value is appropriate for a field, return it
    """
    def __init__(self, registry_model):
        self.registry_model = registry_model

    @cached
    def display_name(self, key):
        if is_delimited_key(key):
            try:
                form_model, section_model, cde_model = models_from_mongo_key(self.registry_model, key)
            except BadKeyError:
                logger.error("key %s refers to non-existant models" % key)
                return key

            human_name = "%s/%s/%s" % (form_model.name, section_model.display_name, cde_model.name)
            return human_name
        else:
            return key

    @cached
    def display_value(self, key, mongo_value):
        # return the display value for ranges
        if is_delimited_key(key):
            try:
                form_model, section_model, cde_model = models_from_mongo_key(self.registry_model, key)
            except BadKeyError:
                logger.error("Key %s refers to non-existant models" % key)
                return mongo_value

            if cde_model.pv_group:
                # look up the stored code and return the display value
                range_dict = cde_model.pv_group.as_dict()
                for value_dict in range_dict["values"]:
                    if mongo_value == value_dict["code"]:
                        return value_dict["value"]
        return mongo_value

def _human_friendly(registry_model, result):
    humaniser = Humaniser(registry_model)

    for r in result:
        for key in r.keys():
            mongo_value = r[key]
            cde_value = humaniser.display_value(key, mongo_value)
            if cde_value:
                r[key] = cde_value
            cde_name = humaniser.display_name(key)
            if cde_name:
                r[cde_name] = r[key]
                if cde_name != key:
                    del r[key]
    return result

def _get_non_multiple_mongo_keys(registry_model):
    # return a list of delimited mongo keys for the supplied registry
    # skip the generated questionnaire ( the data from which is copied to the target clinical forms anyway)
    # skip multisections  as these are handled separately
    delimited_keys = []
    from rdrf.utils import mongo_key_from_models
    for form_model in registry_model.forms:
        if not form_model.is_questionnaire:
            for section_model in form_model.section_models:
                if not section_model.allow_multiple:
                    for cde_model in section_model.cde_models:
                        delimited_key = mongo_key_from_models(form_model, section_model, cde_model)
                        delimited_keys.append(delimited_key)
    return delimited_keys

def _get_cdes(registry_obj):
    from rdrf.models import RegistryForm
    from rdrf.models import Section

    cdes = []
    forms = RegistryForm.objects.filter(registry__code=registry_obj.code)

    for form in forms:
        sections = Section.objects.filter(code__in=form.sections.split(","))
        for section in sections:
            for cde in section.get_elements():
                cdes.append("%s____%s____%s" % (form.name, section.code, cde))

    return cdes

def _filler(result, cdes):
    import collections
    munged = []
    for r in result:
        for cde in cdes:
            if cde not in r:
                r[cde] = "?"
        munged.append(collections.OrderedDict(sorted(r.items())))
    return munged

def _final_cleanup(results):
    for res in results:
        for key, value in res.iteritems():
            if key.endswith('timestamp'):
                del res[key]
    return results

class MultisectionUnRoller(object):
    def __init__(self, multisection_column_map):
        # section_code --> column names in the report in that multisection
        # E.g.{ "social": ["friends"], "health": ["drug", "dose"]}
        self.multisection_column_map = multisection_column_map
        self.row_count = 0

    def unroll(self, row):
        """
        Basic idea is to use cartesian product to display all combinations of list elements
        for the multisections:
        if a row originally looks like   normalfield1, normalfield2, [a,b,c], notmalfield4, [d,e,f]
        we need to iterate through the cartesian product of [a,b,c] and [d,e,f] ( a square)
        ( 1 row expands to 9 rows !)
        Hence the use of itertools.product to walk through the generated choices
        ( eg b,e
             a,f
            etc etc)
        for three multisections we iterate through the triple product ( a cube) and so on
        This gets big quick obviously ...

        complication is that multisections can contain more than one field  so the unit we walk through is the section

        :param row:
        :return:
        """
        from itertools import product

        def dl2ld(dl):
            """
            :param dl: A dictionary of lists : e.g. {"drug" : ["aspirin", "neurophen"], "dose": [100,200] }
            ( each list must be same length )
            :return: A list of dictionaries = [ {"drug": "aspirin", "dose": 100}, {"drug": "neurophen", "dose": 200}]
            """
            l = []
            indexes = range(max(map(len, dl.values())))
            for i in indexes:
                d = {}
                for k in dl:
                    d[k] = dl[k][i]
                l.append(d)
            return l

        # e.g. row = {"name": "Lee", "friends": ["fred", "barry"],
        # "drug":["aspirin","neurophen"], "dose" : [20,23], "height": 56}
        new_rows = []  # the extra unrolled rows
        sublists = {}
        for multisection_code in self.multisection_column_map:
            multisection_columns = self.multisection_column_map[multisection_code]
            section_data = {}
            for col in multisection_columns:
                values = row[col]  # each multisection cde will have a list of values
                section_data[col] = values

            sublists[multisection_code] = dl2ld(section_data)

        row_count = 0
        # choice tuple is one choice from each sublist
        for choice_tuple in product(*sublists.values()):
            new_row = row.copy()
            row_count += 1
            for section_dict in choice_tuple:
                for key in section_dict:
                    new_row[key] = section_dict[key]

            new_rows.append(new_row)

        return new_rows
<|endoftext|>


<commit_before>__author__ = 'Williams Mendez'
__copyright__ = 'Copyright 2014, Williams Mendez'
__credits__ = ['Williams Mendez, williamsmendez.com, @wmendezt']

__license__ = 'BSD'
__version__ = '1.4.0'
__maintainer__ = 'Williams Mendez'
__email__ = 'wmendez27@gmail.com'
__status__ = 'Production'<commit_msg>Bumped version to 1.5.0<commit_after>__author__ = 'Williams Mendez'
__copyright__ = 'Copyright 2014, Williams Mendez'
__credits__ = ['Williams Mendez, williamsmendez.com, @wmendezt']

__license__ = 'BSD'
__version__ = '1.5.0'
__maintainer__ = 'Williams Mendez'
__email__ = 'wmendez27@gmail.com'
__status__ = 'Production'<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
git-recipe is a small recipe that allows you to use git
repositories

[buildout]
parts = data

[data]
recipe = gitrecipe
repository = git://example.com/my-git-repo.git
rev = origin/redevlop-branch
as_egg = true

"""

import os

from zc.buildout import easy_install
from zc.buildout import UserError

from subprocess import Popen
from subprocess import PIPE
from shutil import rmtree

from re import search
from re import findall
from re import MULTILINE

def get_reponame(url):
    if ":" in url:
        url = '/' + url.rsplit(":", 1)[1]
    match = search('\/(?P<repo_name>[a-zA-Z0-9-_.]*)(.git)$', url)
    if match:
        repo_name = match.groupdict()['repo_name']
        return repo_name
    else:
        raise UserError('Can not find repository name')

class GitRecipe(object):
    '''Simple recipe for fetch code form remote repository, using system git'''
    def __init__(self, buildout, name, options):
        self.options, self.buildout = options, buildout

        if 'repository' not in self.options:
            raise UserError('Repository url must be provided')
        self.url = options['repository']
        self.ref = options.get('ref', 'origin/master')

        self.as_egg = options.get('as_egg', 'false').lower() == 'true'
        self.options['download-directory'] = options.get(
            'download-directory') or buildout['buildout']['parts-directory']

        # determine repository name
        self.repo_name = options.get('repo_name', get_reponame(self.url))
        self.repo_path = os.path.join(
            self.options['download-directory'], self.repo_name)
        self.options['location'] = os.path.join(
            self.options['download-directory'], self.repo_path)

        self.paths = options.get('paths', None)

    def git(self, operation, args, quiet=True):
        if quiet:
            command = ['git'] + [operation] + ['-q'] + args
        else:
            command = ['git'] + [operation] + args

        proc = Popen(' '.join(command), shell=True, stdout=PIPE)
        status = proc.wait()
        if status:
            raise UserError('Error while executing %s' % ' '.join(command))
        return proc.stdout.read()

    def check_same(self):
        old_cwd = os.getcwd()
        existing_repository = None

        if os.path.exists(self.repo_path) and os.path.exists(os.path.join(self.repo_path, '.git')):
            os.chdir(self.repo_path)
            origin = self.git('remote', ['show', 'origin'], quiet=False)
            existing_repository = findall(
                '^\s*Fetch URL:\s*(.*)$', origin, flags=MULTILINE)[0]

        os.chdir(old_cwd)
        if existing_repository == self.url:
            return True

    def install(self):
        '''Clone repository and checkout to version'''
        # go to parts directory
        os.chdir(self.options['download-directory'])
        _installed = False

        try:

            if os.path.exists(self.repo_path):
                if self.check_same():
                    # If the same repository is here, just fetch new data and checkout to revision
                    # aka update ;)
                    _installed = True
                    os.chdir(self.repo_path)
                    self.git('fetch', [self.url, ])

                    os.chdir(self.options['location'])
                    self.git('checkout', [self.ref, ])
                    # return to root directory
                    os.chdir(self.buildout['buildout']['directory'])

                else:
                    # if repository exists but not the same, delete all files
                    # there
                    rmtree(self.repo_path, ignore_errors=True)
                    _installed = False

            # in fact, the install
            if not _installed:
                os.chdir(self.options['download-directory'])
                self.git('clone', [self.url, self.repo_name])

                os.chdir(self.options['location'])
                self.git('checkout', [self.ref, ])

        except UserError:
            # should manually clean files because buildout thinks that no files
            # created
            if os.path.exists(self.options['location']):
                rmtree(self.options['location'])
            raise

        if self.as_egg:
            self._install_as_egg()
        # return to root directory
        os.chdir(self.buildout['buildout']['directory'])
        return self.options['location']

    def update(self):
        '''Update repository rather than download it again'''

        if self.buildout['buildout'].get('offline').lower() == 'true' or \
                self.options.get('newest', 'true').lower() == 'false':
            return

        # go to parts directory
        if self.check_same():
            os.chdir(self.options['location'])
            self.git('fetch', ['origin', ])
            # if revision is given, checkout to revision
            self.git('checkout', [self.ref, ])
            if self.as_egg:
                self._install_as_egg()
        else:
            self.install()

        # return to root directory
        os.chdir(self.buildout['buildout']['directory'])
        return self.options['location']

    def _install_as_egg(self):
        """
        Install clone as development egg.
        """
        def _install(path, target):
            easy_install.develop(path, target)

        target = self.buildout['buildout']['develop-eggs-directory']
        if self.paths:
            for path in self.paths.split():
                path = os.path.join(self.options['location'], path.strip())
                _install(path, target)
        else:
            _install(self.options['location'], target)
<commit_msg>Not all get repositories have .git at the end<commit_after># -*- coding: utf-8 -*-
"""
git-recipe is a small recipe that allows you to use git
repositories

[buildout]
parts = data

[data]
recipe = gitrecipe
repository = git://example.com/my-git-repo.git
rev = origin/redevlop-branch
as_egg = true

"""

import os

from zc.buildout import easy_install
from zc.buildout import UserError

from subprocess import Popen
from subprocess import PIPE
from shutil import rmtree

from re import search
from re import findall
from re import MULTILINE

def get_reponame(url):
    if ":" in url:
        url = '/' + url.rsplit(":", 1)[1]
    match = search('\/(?P<repo_name>[a-zA-Z0-9-_.]*)$', url)
    if match:
        repo_name = match.groupdict()['repo_name']
        return repo_name
    else:
        raise UserError('Can not find repository name')

class GitRecipe(object):
    '''Simple recipe for fetch code form remote repository, using system git'''
    def __init__(self, buildout, name, options):
        self.options, self.buildout = options, buildout

        if 'repository' not in self.options:
            raise UserError('Repository url must be provided')
        self.url = options['repository']
        self.ref = options.get('ref', 'origin/master')

        self.as_egg = options.get('as_egg', 'false').lower() == 'true'
        self.options['download-directory'] = options.get(
            'download-directory') or buildout['buildout']['parts-directory']

        # determine repository name
        self.repo_name = options.get('repo_name', get_reponame(self.url))
        self.repo_path = os.path.join(
            self.options['download-directory'], self.repo_name)
        self.options['location'] = os.path.join(
            self.options['download-directory'], self.repo_path)

        self.paths = options.get('paths', None)

    def git(self, operation, args, quiet=True):
        if quiet:
            command = ['git'] + [operation] + ['-q'] + args
        else:
            command = ['git'] + [operation] + args

        proc = Popen(' '.join(command), shell=True, stdout=PIPE)
        status = proc.wait()
        if status:
            raise UserError('Error while executing %s' % ' '.join(command))
        return proc.stdout.read()

    def check_same(self):
        old_cwd = os.getcwd()
        existing_repository = None

        if os.path.exists(self.repo_path) and os.path.exists(os.path.join(self.repo_path, '.git')):
            os.chdir(self.repo_path)
            origin = self.git('remote', ['show', 'origin'], quiet=False)
            existing_repository = findall(
                '^\s*Fetch URL:\s*(.*)$', origin, flags=MULTILINE)[0]

        os.chdir(old_cwd)
        if existing_repository == self.url:
            return True

    def install(self):
        '''Clone repository and checkout to version'''
        # go to parts directory
        os.chdir(self.options['download-directory'])
        _installed = False

        try:

            if os.path.exists(self.repo_path):
                if self.check_same():
                    # If the same repository is here, just fetch new data and checkout to revision
                    # aka update ;)
                    _installed = True
                    os.chdir(self.repo_path)
                    self.git('fetch', [self.url, ])

                    os.chdir(self.options['location'])
                    self.git('checkout', [self.ref, ])
                    # return to root directory
                    os.chdir(self.buildout['buildout']['directory'])

                else:
                    # if repository exists but not the same, delete all files
                    # there
                    rmtree(self.repo_path, ignore_errors=True)
                    _installed = False

            # in fact, the install
            if not _installed:
                os.chdir(self.options['download-directory'])
                self.git('clone', [self.url, self.repo_name])

                os.chdir(self.options['location'])
                self.git('checkout', [self.ref, ])

        except UserError:
            # should manually clean files because buildout thinks that no files
            # created
            if os.path.exists(self.options['location']):
                rmtree(self.options['location'])
            raise

        if self.as_egg:
            self._install_as_egg()
        # return to root directory
        os.chdir(self.buildout['buildout']['directory'])
        return self.options['location']

    def update(self):
        '''Update repository rather than download it again'''

        if self.buildout['buildout'].get('offline').lower() == 'true' or \
                self.options.get('newest', 'true').lower() == 'false':
            return

        # go to parts directory
        if self.check_same():
            os.chdir(self.options['location'])
            self.git('fetch', ['origin', ])
            # if revision is given, checkout to revision
            self.git('checkout', [self.ref, ])
            if self.as_egg:
                self._install_as_egg()
        else:
            self.install()

        # return to root directory
        os.chdir(self.buildout['buildout']['directory'])
        return self.options['location']

    def _install_as_egg(self):
        """
        Install clone as development egg.
        """
        def _install(path, target):
            easy_install.develop(path, target)

        target = self.buildout['buildout']['develop-eggs-directory']
        if self.paths:
            for path in self.paths.split():
                path = os.path.join(self.options['location'], path.strip())
                _install(path, target)
        else:
            _install(self.options['location'], target)
<|endoftext|>