<commit_before>import logging
from datetime import datetime
from string import Template

import luigi
# from luigi import six

from googleapiclient.errors import HttpError
import time
from luigi_gcloud.gcore import get_default_client, _GCloudTask
from luigi_gcloud.storage import GCSTarget

logger = logging.getLogger('luigi-gcloud')

default_bigquery_udf = None

def _table_with_default(name, client):
    return _split_tablename(name, client.project_id())

def _split_tablename(name, default_project=None):
    cmpt = name.split(':')
    if len(cmpt) == 1:
        project_id = default_project
        rest = cmpt[0]
    elif len(cmpt) == 2:
        project_id = cmpt[0]
        rest = cmpt[1]
    else:
        raise RuntimeError

    cmpt = rest.split('.')
    if len(cmpt) == 2:
        dataset_id = cmpt[0]
        table_id = cmpt[1]
    else:
        raise RuntimeError

    return {
        'projectId': project_id,
        'datasetId': dataset_id,
        'tableId': table_id
    }

class _BqJob:
    def __init__(self, bq, project_id, job):
        self.bq = bq
        self.project_id = project_id
        self.job = self.bq.jobs().insert(projectId=self.project_id, body=job).execute()
        self.job_id = self.job['jobReference']['jobId']
        logger.info('BigQuery job %s is %s', self.job_id, str(self.job['status']))

    def wait_for_done(self):

        while True:
            self.job = self.bq.jobs().get(
                projectId=self.project_id,
                jobId=self.job_id).execute()
            if 'DONE' == self.job['status']['state']:
                if self.job['status'].has_key('errors'):
                    logger.warning('BigQuery job %s has errors', self.job_id)
                    logger.debug(str(self.job))
                    return False
                else:
                    logger.info('BigQuery job %s is DONE', self.job_id)
                    return True
            logger.debug('BigQuery job %s is %s', self.job_id, str(self.job['status']))
            time.sleep(5)

    def raise_error(self, message):
        if self.job['status'].has_key('errors'):
            if message is None:
                message = "Google BigQuery job has error"
            raise Exception(message + ": " + str(self.job['status']['errors']))

    def get(self):
        return self.job

class BigQueryTarget(luigi.Target):
    def __init__(self, table, query=None, client=get_default_client()):
        self.client = client
        self.table = _table_with_default(table, self.client)
        self.query = query
        http = self.client.http_authorized()
        self.bigquery_api = self.client.bigquery_api(http)

    def query(self):
        raise NotImplementedError

    def exists(self):
        tables = self.bigquery_api.tables()
        try:
            table = tables.get(projectId=self.table['projectId'], datasetId=self.table['datasetId'],
                               tableId=self.table['tableId']).execute()
            jobs = self.bigquery_api.jobs()
            job = {
                'projectId': self.table['projectId'],
                'configuration': {
                    'query': {
                        'useQueryCache': 'True',
                        'query': self.query
                    }
                }
            }
            logger.debug(table)
            if self.query is None:
                return True

            insert_job = _BqJob(self.bigquery_api, self.table['projectId'], job=job)
            if insert_job.wait_for_done():
                result = insert_job.get()
                result = jobs.getQueryResults(projectId=self.table['projectId'],
                                              jobId=result['jobReference']['jobId']).execute()
                return result['rows'][0]['f'][0]['v'] in ['true', 'True']
        except HttpError as err:
            if err.resp.status == 404:
                return False
            logger.error("HttpError, when checking existence of " + self.table['projectId'] + ":" + self.table[
                'datasetId'] + "." + self.table['tableId'] + ".")
            logger.debug(str(err))
            return False

class _BqTask(_GCloudTask):
    service_name = 'bigquery'
    client = None
    bigquery_api = None

    def __init__(self, *args, **kwargs):
        self.client = kwargs.get("client") or get_default_client()
        http = self.client.http_authorized()
        self.bigquery_api = self.client.bigquery_api(http)
        super(_BqTask, self).__init__(*args, **kwargs)

class BigQueryLoadTask(_BqTask):
    """ Load/Append data into a BigQuery table """

    def __init__(self, *args, **kwargs):
        super(BigQueryLoadTask, self).__init__(*args, **kwargs)

    def output(self):
        return BigQueryTarget(self.table(), None, self.client)

    def schema(self):
        raise NotImplementedError

    def source(self):
        raise NotImplementedError

    def table(self):
        """
        Format 123456789:DataSet.Table
        :return:
        """
        raise NotImplementedError

    def run(self):
        table = _table_with_default(self.table(), self.client)
        # https://cloud.google.com/bigquery/docs/reference/v2/jobs
        job = {
            'projectId': table['projectId'],
            'configuration': {
                'load': {
                    # 'quote':'',
                    'destinationTable': {
                        'projectId': table['projectId'],
                        'datasetId': table['datasetId'],
                        'tableId': table['tableId']
                    },
                    'sourceFormat': self.get_service_value("sourceFormat", "NEWLINE_DELIMITED_JSON"),
                    'createDisposition': self.get_service_value("createDisposition", "CREATE_IF_NEEDED"),
                    'writeDisposition': self.get_service_value("writeDisposition", "WRITE_APPEND"),
                    'sourceUris': [self.source()],
                    'maxBadRecords': 1000,
                    'schema': {
                        'fields': self.schema()
                    }
                }
            }
        }
        load_job = _BqJob(self.bigquery_api, table['projectId'], job=job)
        if load_job.wait_for_done():
            marker = self.output()
            if hasattr(marker, "touch") and callable(getattr(marker, "touch")):
                logger.info("Writing marker file " + str(marker))
                marker.touch()
            return
        else:
            load_job.raise_error("Error loading data in Google BigQuery")

class BigQueryTask(_BqTask):
    """ Query a BigQuery table into another table or storage file """

    def __init__(self, *args, **kwargs):
        super(BigQueryTask, self).__init__(*args, **kwargs)

    def output(self):
        if self.table() is not None:
            return BigQueryTarget(self.table())
        if self.destination() is not None:
            destination = self.destination()
            if destination.find('*') != -1:
                destination = destination[:destination.rindex('/') + 1]
            return GCSTarget(destination)

    def schema(self):
        return None

    def query(self):
        with open(self.query_file(), 'r') as f:
            content = f.read()
        return content

    def query_file(self):
        return NotImplementedError

    def table(self):
        """
        Format 123456789:DataSet.Table
        :return:
        """
        return None

    def destination(self):
        return None

    def lib_uris(self):
        return []

    def _success(self):
        marker = self.output()
        if hasattr(marker, "touch") and callable(getattr(marker, "touch")):
            logger.info("Writing marker file " + str(marker))
            marker.touch()
        return

    def _js_file_uris(self):
        global default_bigquery_udf
        artifacts = []
        uris = self.lib_uris()
        artifacts.extend(uris)
        if default_bigquery_udf is not None:
            if not default_bigquery_udf.get('append') and len(uris) == 0:
                artifacts.extend(default_bigquery_udf.get('uris'))
            elif default_bigquery_udf.get('append'):
                artifacts.extend(default_bigquery_udf.get('uris'))
        return artifacts

    def run(self):
        query = Template(self.query()).substitute(self.variables())
        job = {
            'projectId': self.client.project_id(),
            "configuration": {
                "query": {
                    "query": query,
                }
            }
        }
        schema = self.schema()
        if schema is not None:
            job["configuration"]["schema"] = {
                'fields': schema
            }

        udf = self._js_file_uris()
        if len(udf) > 0:
            job["configuration"]["query"]["userDefinedFunctionResources"] = []
            for uri in udf:
                job["configuration"]["query"]["userDefinedFunctionResources"].append(
                    {
                        "resourceUri": uri
                    }
                )

        if self.table() is None and self.destination() is None:
            raise RuntimeError("At least table or destination need to be supplied.")
        if self.table() is not None and self.destination() is not None:
            raise RuntimeError("Set either table or destination.")

        if self.table() is not None:
            table = _table_with_default(self.table(), self.client)
            job["configuration"]["query"]["destinationTable"] = {
                'projectId': table['projectId'],
                'datasetId': table['datasetId'],
                'tableId': table['tableId']
            }
            job["configuration"]["query"]["createDisposition"] = \
                self.get_service_value("createDisposition", "CREATE_IF_NEEDED")
            job["configuration"]["query"]["writeDisposition"] = \
                self.get_service_value("writeDisposition", "WRITE_APPEND")
            job["configuration"]["query"]["allowLargeResults"] = \
                self.get_service_value("allowLargeResults", "false")

            insert_job = _BqJob(self.bigquery_api, self.client.project_id(), job=job)
            if insert_job.wait_for_done():
                self._success()
            else:
                insert_job.raise_error("Error executing query in Google BigQuery")

        if self.destination() is not None:
            large = self.get_service_value("allowLargeResults", "false")
            if str(large) in ("true", "1", "True"):
                # This is a workaround for a BigQuery limitation, if the result is to large
                # we need to set a destination table and do allowLargeResults
                # we need a DataSet that has an expiration to make it work properly in production
                job["configuration"]["query"]["destinationTable"] = {
                    'projectId': self.client.project_id(),
                    'datasetId': self.get_service_value("tempDataset", "work_temp"),
                    'tableId': datetime.now().strftime("xlbq_%Y%m%d%H%M%S")
                }
                job["configuration"]["query"]["allowLargeResults"] = "true"

            insert_job = _BqJob(self.bigquery_api, self.client.project_id(), job=job)
            if insert_job.wait_for_done():
                result = insert_job.get()

                job = {
                    'projectId': self.client.project_id(),
                    'configuration': {
                        'extract': {
                            'sourceTable': {
                                'projectId': result["configuration"]["query"]["destinationTable"]["projectId"],
                                'datasetId': result["configuration"]["query"]["destinationTable"]["datasetId"],
                                'tableId': result["configuration"]["query"]["destinationTable"]["tableId"]
                            },
                            'destinationUris': [
                                self.destination()
                            ],
                            'destinationFormat': self.get_service_value("destinationFormat", "NEWLINE_DELIMITED_JSON")
                        }
                    }
                }
                extract_job = _BqJob(self.bigquery_api, self.client.project_id(), job=job)
                if extract_job.wait_for_done():
                    self._success()
                else:
                    extract_job.raise_error("Error extracting query data from Google BigQuery")
            else:
                insert_job.raise_error("Error querying to temp table in Google BigQuery")
<commit_msg>Bug with BigQuery target not taking correct client<commit_after>import logging
from datetime import datetime
from string import Template

import luigi
# from luigi import six

from googleapiclient.errors import HttpError
import time
from luigi_gcloud.gcore import get_default_client, _GCloudTask
from luigi_gcloud.storage import GCSTarget

logger = logging.getLogger('luigi-gcloud')

default_bigquery_udf = None

def _table_with_default(name, client):
    return _split_tablename(name, client.project_id())

def _split_tablename(name, default_project=None):
    cmpt = name.split(':')
    if len(cmpt) == 1:
        project_id = default_project
        rest = cmpt[0]
    elif len(cmpt) == 2:
        project_id = cmpt[0]
        rest = cmpt[1]
    else:
        raise RuntimeError

    cmpt = rest.split('.')
    if len(cmpt) == 2:
        dataset_id = cmpt[0]
        table_id = cmpt[1]
    else:
        raise RuntimeError

    return {
        'projectId': project_id,
        'datasetId': dataset_id,
        'tableId': table_id
    }

class _BqJob:
    def __init__(self, bq, project_id, job):
        self.bq = bq
        self.project_id = project_id
        self.job = self.bq.jobs().insert(projectId=self.project_id, body=job).execute()
        self.job_id = self.job['jobReference']['jobId']
        logger.info('BigQuery job %s is %s', self.job_id, str(self.job['status']))

    def wait_for_done(self):

        while True:
            self.job = self.bq.jobs().get(
                projectId=self.project_id,
                jobId=self.job_id).execute()
            if 'DONE' == self.job['status']['state']:
                if self.job['status'].has_key('errors'):
                    logger.warning('BigQuery job %s has errors', self.job_id)
                    logger.debug(str(self.job))
                    return False
                else:
                    logger.info('BigQuery job %s is DONE', self.job_id)
                    return True
            logger.debug('BigQuery job %s is %s', self.job_id, str(self.job['status']))
            time.sleep(5)

    def raise_error(self, message):
        if self.job['status'].has_key('errors'):
            if message is None:
                message = "Google BigQuery job has error"
            raise Exception(message + ": " + str(self.job['status']['errors']))

    def get(self):
        return self.job

class BigQueryTarget(luigi.Target):
    def __init__(self, table, query=None, client=None):
        self.client = client or get_default_client()
        self.table = _table_with_default(table, self.client)
        self.query = query
        http = self.client.http_authorized()
        self.bigquery_api = self.client.bigquery_api(http)

    def query(self):
        raise NotImplementedError

    def exists(self):
        tables = self.bigquery_api.tables()
        try:
            table = tables.get(projectId=self.table['projectId'], datasetId=self.table['datasetId'],
                               tableId=self.table['tableId']).execute()
            jobs = self.bigquery_api.jobs()
            job = {
                'projectId': self.table['projectId'],
                'configuration': {
                    'query': {
                        'useQueryCache': 'True',
                        'query': self.query
                    }
                }
            }
            logger.debug(table)
            if self.query is None:
                return True

            insert_job = _BqJob(self.bigquery_api, self.table['projectId'], job=job)
            if insert_job.wait_for_done():
                result = insert_job.get()
                result = jobs.getQueryResults(projectId=self.table['projectId'],
                                              jobId=result['jobReference']['jobId']).execute()
                return result['rows'][0]['f'][0]['v'] in ['true', 'True']
        except HttpError as err:
            if err.resp.status == 404:
                return False
            logger.error("HttpError, when checking existence of " + self.table['projectId'] + ":" + self.table[
                'datasetId'] + "." + self.table['tableId'] + ".")
            logger.debug(str(err))
            return False

class _BqTask(_GCloudTask):
    service_name = 'bigquery'
    client = None
    bigquery_api = None

    def __init__(self, *args, **kwargs):
        self.client = kwargs.get("client") or get_default_client()
        http = self.client.http_authorized()
        self.bigquery_api = self.client.bigquery_api(http)
        super(_BqTask, self).__init__(*args, **kwargs)

class BigQueryLoadTask(_BqTask):
    """ Load/Append data into a BigQuery table """

    def __init__(self, *args, **kwargs):
        super(BigQueryLoadTask, self).__init__(*args, **kwargs)

    def output(self):
        return BigQueryTarget(self.table(), None, self.client)

    def schema(self):
        raise NotImplementedError

    def source(self):
        raise NotImplementedError

    def table(self):
        """
        Format 123456789:DataSet.Table
        :return:
        """
        raise NotImplementedError

    def run(self):
        table = _table_with_default(self.table(), self.client)
        # https://cloud.google.com/bigquery/docs/reference/v2/jobs
        job = {
            'projectId': table['projectId'],
            'configuration': {
                'load': {
                    # 'quote':'',
                    'destinationTable': {
                        'projectId': table['projectId'],
                        'datasetId': table['datasetId'],
                        'tableId': table['tableId']
                    },
                    'sourceFormat': self.get_service_value("sourceFormat", "NEWLINE_DELIMITED_JSON"),
                    'createDisposition': self.get_service_value("createDisposition", "CREATE_IF_NEEDED"),
                    'writeDisposition': self.get_service_value("writeDisposition", "WRITE_APPEND"),
                    'sourceUris': [self.source()],
                    'maxBadRecords': 1000,
                    'schema': {
                        'fields': self.schema()
                    }
                }
            }
        }
        load_job = _BqJob(self.bigquery_api, table['projectId'], job=job)
        if load_job.wait_for_done():
            marker = self.output()
            if hasattr(marker, "touch") and callable(getattr(marker, "touch")):
                logger.info("Writing marker file " + str(marker))
                marker.touch()
            return
        else:
            load_job.raise_error("Error loading data in Google BigQuery")

class BigQueryTask(_BqTask):
    """ Query a BigQuery table into another table or storage file """

    def __init__(self, *args, **kwargs):
        super(BigQueryTask, self).__init__(*args, **kwargs)

    def output(self):
        if self.table() is not None:
            return BigQueryTarget(self.table())
        if self.destination() is not None:
            destination = self.destination()
            if destination.find('*') != -1:
                destination = destination[:destination.rindex('/') + 1]
            return GCSTarget(destination)

    def schema(self):
        return None

    def query(self):
        with open(self.query_file(), 'r') as f:
            content = f.read()
        return content

    def query_file(self):
        return NotImplementedError

    def table(self):
        """
        Format 123456789:DataSet.Table
        :return:
        """
        return None

    def destination(self):
        return None

    def lib_uris(self):
        return []

    def _success(self):
        marker = self.output()
        if hasattr(marker, "touch") and callable(getattr(marker, "touch")):
            logger.info("Writing marker file " + str(marker))
            marker.touch()
        return

    def _js_file_uris(self):
        global default_bigquery_udf
        artifacts = []
        uris = self.lib_uris()
        artifacts.extend(uris)
        if default_bigquery_udf is not None:
            if not default_bigquery_udf.get('append') and len(uris) == 0:
                artifacts.extend(default_bigquery_udf.get('uris'))
            elif default_bigquery_udf.get('append'):
                artifacts.extend(default_bigquery_udf.get('uris'))
        return artifacts

    def run(self):
        query = Template(self.query()).substitute(self.variables())
        job = {
            'projectId': self.client.project_id(),
            "configuration": {
                "query": {
                    "query": query,
                }
            }
        }
        schema = self.schema()
        if schema is not None:
            job["configuration"]["schema"] = {
                'fields': schema
            }

        udf = self._js_file_uris()
        if len(udf) > 0:
            job["configuration"]["query"]["userDefinedFunctionResources"] = []
            for uri in udf:
                job["configuration"]["query"]["userDefinedFunctionResources"].append(
                    {
                        "resourceUri": uri
                    }
                )

        if self.table() is None and self.destination() is None:
            raise RuntimeError("At least table or destination need to be supplied.")
        if self.table() is not None and self.destination() is not None:
            raise RuntimeError("Set either table or destination.")

        if self.table() is not None:
            table = _table_with_default(self.table(), self.client)
            job["configuration"]["query"]["destinationTable"] = {
                'projectId': table['projectId'],
                'datasetId': table['datasetId'],
                'tableId': table['tableId']
            }
            job["configuration"]["query"]["createDisposition"] = \
                self.get_service_value("createDisposition", "CREATE_IF_NEEDED")
            job["configuration"]["query"]["writeDisposition"] = \
                self.get_service_value("writeDisposition", "WRITE_APPEND")
            job["configuration"]["query"]["allowLargeResults"] = \
                self.get_service_value("allowLargeResults", "false")

            insert_job = _BqJob(self.bigquery_api, self.client.project_id(), job=job)
            if insert_job.wait_for_done():
                self._success()
            else:
                insert_job.raise_error("Error executing query in Google BigQuery")

        if self.destination() is not None:
            large = self.get_service_value("allowLargeResults", "false")
            if str(large) in ("true", "1", "True"):
                # This is a workaround for a BigQuery limitation, if the result is to large
                # we need to set a destination table and do allowLargeResults
                # we need a DataSet that has an expiration to make it work properly in production
                job["configuration"]["query"]["destinationTable"] = {
                    'projectId': self.client.project_id(),
                    'datasetId': self.get_service_value("tempDataset", "work_temp"),
                    'tableId': datetime.now().strftime("xlbq_%Y%m%d%H%M%S")
                }
                job["configuration"]["query"]["allowLargeResults"] = "true"

            insert_job = _BqJob(self.bigquery_api, self.client.project_id(), job=job)
            if insert_job.wait_for_done():
                result = insert_job.get()

                job = {
                    'projectId': self.client.project_id(),
                    'configuration': {
                        'extract': {
                            'sourceTable': {
                                'projectId': result["configuration"]["query"]["destinationTable"]["projectId"],
                                'datasetId': result["configuration"]["query"]["destinationTable"]["datasetId"],
                                'tableId': result["configuration"]["query"]["destinationTable"]["tableId"]
                            },
                            'destinationUris': [
                                self.destination()
                            ],
                            'destinationFormat': self.get_service_value("destinationFormat", "NEWLINE_DELIMITED_JSON")
                        }
                    }
                }
                extract_job = _BqJob(self.bigquery_api, self.client.project_id(), job=job)
                if extract_job.wait_for_done():
                    self._success()
                else:
                    extract_job.raise_error("Error extracting query data from Google BigQuery")
            else:
                insert_job.raise_error("Error querying to temp table in Google BigQuery")
<|endoftext|>


<commit_before>from __future__ import print_function, unicode_literals

import logging

from cliff.show import ShowOne

from madcore.const import STACK_FOLLOWME
from madcore.libs.cloudformation import StackManagement

class Followme(StackManagement, ShowOne):
    _description = "Update Followme stack with current IP"

    logger = logging.getLogger(__name__)

    def followme_stack_update(self, ipv4):
        parameters = [
            {
                'ParameterKey': 'FollowMeIpAddress',
                'ParameterValue': ipv4,
                'UsePreviousValue': False
            },
            {
                'ParameterKey': 'VpcId',
                'UsePreviousValue': True
            },
        ]

        update_response = self.update_stack(STACK_FOLLOWME, '', parameters, use_prev_template=True)

        return update_response

    def take_action(self, parsed_args):
        stack_details = self.get_stack(STACK_FOLLOWME)

        if stack_details is None:
            self.logger.warn("Stack not created yet, run configuration to setup.")
            self.exit()

        ipv4 = self.get_ipv4()
        self.logger.info('Core Followme: Your public IP detected as: %s', ipv4)
        previous_parameters = stack_details['Parameters']
        ipv4_previous = self.get_param_from_dict(previous_parameters, 'FollowMeIpAddress')
        if ipv4 == ipv4_previous:
            self.logger.info("No need to update.")
            self.exit(code=0)

        self.logger.info("Updating '%s' Stack...", STACK_FOLLOWME)
        self.followme_stack_update(ipv4)

        columns = (
            'New IPv4',
            'Stack ID',
            'Previous IPv4'
        )
        data = (
            ''.join(ipv4.split()),
            stack_details['StackId'],
            ipv4_previous
        )
        return columns, data
<commit_msg>Fixed follow exit status 0 if not exists<commit_after>from __future__ import print_function, unicode_literals

import logging

from cliff.show import ShowOne

from madcore.const import STACK_FOLLOWME
from madcore.libs.cloudformation import StackManagement

class Followme(StackManagement, ShowOne):
    _description = "Update Followme stack with current IP"

    logger = logging.getLogger(__name__)

    def followme_stack_update(self, ipv4):
        parameters = [
            {
                'ParameterKey': 'FollowMeIpAddress',
                'ParameterValue': ipv4,
                'UsePreviousValue': False
            },
            {
                'ParameterKey': 'VpcId',
                'UsePreviousValue': True
            },
        ]

        update_response = self.update_stack(STACK_FOLLOWME, '', parameters, use_prev_template=True)

        return update_response

    def take_action(self, parsed_args):
        stack_details = self.get_stack(STACK_FOLLOWME)

        if stack_details is None:
            self.logger.warn("Stack not created yet, run configuration to setup.")
            self.exit(code=0)

        ipv4 = self.get_ipv4()
        self.logger.info('Core Followme: Your public IP detected as: %s', ipv4)
        previous_parameters = stack_details['Parameters']
        ipv4_previous = self.get_param_from_dict(previous_parameters, 'FollowMeIpAddress')
        if ipv4 == ipv4_previous:
            self.logger.info("No need to update.")
            self.exit(code=0)

        self.logger.info("Updating '%s' Stack...", STACK_FOLLOWME)
        self.followme_stack_update(ipv4)

        columns = (
            'New IPv4',
            'Stack ID',
            'Previous IPv4'
        )
        data = (
            ''.join(ipv4.split()),
            stack_details['StackId'],
            ipv4_previous
        )
        return columns, data
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import json
import requests
from hashlib import md5
from urllib.parse import urlencode

BASE_URL = 'http://www.gravatar.com/avatar/'
SECURE_BASE_URL = 'https://secure.gravatar.com/avatar/'
PROFILE_URL = 'http://www.gravatar.com/'
RATINGS = ('g', 'pg', 'r', 'x')
MAX_SIZE = 512
MIN_SIZE = 1
DEFAULTS = ('404', 'mm', 'identicon', 'monsterid', 'wavatar', 'retro')

def email_hash(string):
    return md5(str(string.strip().lower()).encode('utf-8')).hexdigest()

def get_gravatar(email, secure=False, rating='g', size=80, default='mm'):
    """Generate a link to the users' Gravatar."""
    assert rating.lower() in RATINGS
    assert MIN_SIZE <= size <= MAX_SIZE

    url = SECURE_BASE_URL if secure else BASE_URL

    options = {'s': size, 'r': rating, 'd': default}
    url += email_hash(email) + '?' + urlencode(options)
    return url

def get_profile(email):
    """Retrieves the profile data of the user.

    :return: A dictionary representing the profile or `None` if nothing found.
    """
    profile = None
    url = u'%s%s.json' % (PROFILE_URL, email_hash(email))
    response = requests.get(url)
    if response.status_code == 200:
        profile = json.loads(response.content)['entry'][0]
    return profile
<commit_msg>Make urloptions in gravatar generation sorted, simplifies (and fixes) tests<commit_after># -*- coding: utf-8 -*-
import json
import requests
from hashlib import md5
from urllib.parse import urlencode

BASE_URL = 'http://www.gravatar.com/avatar/'
SECURE_BASE_URL = 'https://secure.gravatar.com/avatar/'
PROFILE_URL = 'http://www.gravatar.com/'
RATINGS = ('g', 'pg', 'r', 'x')
MAX_SIZE = 512
MIN_SIZE = 1
DEFAULTS = ('404', 'mm', 'identicon', 'monsterid', 'wavatar', 'retro')

def email_hash(string):
    return md5(str(string.strip().lower()).encode('utf-8')).hexdigest()

def get_gravatar(email, secure=False, rating='g', size=80, default='mm'):
    """Generate a link to the users' Gravatar."""
    assert rating.lower() in RATINGS
    assert MIN_SIZE <= size <= MAX_SIZE

    url = SECURE_BASE_URL if secure else BASE_URL

    options = (('s', size), ('r', rating), ('d', default))
    url += email_hash(email) + '?' + urlencode(options)
    return url

def get_profile(email):
    """Retrieves the profile data of the user.

    :return: A dictionary representing the profile or `None` if nothing found.
    """
    profile = None
    url = u'%s%s.json' % (PROFILE_URL, email_hash(email))
    response = requests.get(url)
    if response.status_code == 200:
        profile = json.loads(response.content)['entry'][0]
    return profile
<|endoftext|>


<commit_before>from django.http import HttpResponseRedirect
from django.conf import settings
from re import compile

class LoginRequiredMiddleware:
    def __init__(self, get_response):
        self.get_response = get_response
        self.exempt_urls = [compile(settings.LOGIN_URL.lstrip('/')), compile('/api/set-ac'.lstrip('/'))]

    """
    Middleware that requires a user to be authenticated to view any page other
    than LOGIN_URL. Exemptions to this requirement can optionally be specified
    in settings via a list of regular expressions in LOGIN_EXEMPT_URLS (which
    you can copy from your urls.py).

    Requires authentication middleware and template context processors to be
    loaded. You'll get an error if they aren't.
    """

    def __call__(self, request):
        response = self.get_response(request)
        assert hasattr(request, 'user')

        if not request.user.is_authenticated():
            path = request.path_info.lstrip('/')
            if not any(m.match(path) for m in self.exempt_urls):
                return HttpResponseRedirect(settings.LOGIN_URL)

        return response
<commit_msg>add route to exempt urls<commit_after>from django.http import HttpResponseRedirect
from django.conf import settings
from re import compile

class LoginRequiredMiddleware:
    def __init__(self, get_response):
        self.get_response = get_response
        self.exempt_urls = [
            compile(settings.LOGIN_URL.lstrip('/')),
            compile('/api/set-ac'.lstrip('/')),
            compile('/api/bowling-results'.lstrip('/'))
        ]

    """
    Middleware that requires a user to be authenticated to view any page other
    than LOGIN_URL. Exemptions to this requirement can optionally be specified
    in settings via a list of regular expressions in LOGIN_EXEMPT_URLS (which
    you can copy from your urls.py).

    Requires authentication middleware and template context processors to be
    loaded. You'll get an error if they aren't.
    """

    def __call__(self, request):
        response = self.get_response(request)
        assert hasattr(request, 'user')

        if not request.user.is_authenticated():
            path = request.path_info.lstrip('/')
            if not any(m.match(path) for m in self.exempt_urls):
                return HttpResponseRedirect(settings.LOGIN_URL)

        return response
<|endoftext|>


<commit_before>"""
Test file for timers.
"""
import time
import pytest
from osbrain import Agent
from osbrain import run_agent
from osbrain import run_nameserver
from osbrain.common import repeat
from osbrain.helper import wait_agent_attr

from common import nsproxy  # pragma: no flakes
from common import append_received
from common import set_received

@pytest.mark.timeout(1)
def test_repeat_non_blocking():
    """
    A repeated action (i.e. timer) should always be executed in a separate
    thread, even the first execution.
    """
    def foo(x):
        time.sleep(x)

    timer = repeat(1., foo, 2.)
    timer.stop()

def test_repeat_stop():
    """
    Test closing a timer returned by repeat.
    """
    class Bar():
        def __init__(self):
            self.a = 0

        def foo(self):
            self.a += 1

    bar = Bar()
    timer = repeat(0.1, bar.foo)
    time.sleep(1.)
    assert abs(bar.a - 10) <= 1
    timer.stop()
    time.sleep(1.)
    assert abs(bar.a - 10) <= 1

def test_timer_non_blocking_bug(nsproxy):
    """
    A timer call should never block, no matters how long it takes to execute
    the action.
    """
    def long_action(agent):
        time.sleep(1.)
        agent.count += 1

    agent = run_agent('agent')
    agent.set_attr(count=0)
    # Start timer
    t0 = time.time()
    agent.each(0., long_action)
    t1 = time.time()
    assert t1 - t0 < 0.3
    assert agent.get_attr('count') == 0
    time.sleep(1.5)
    assert agent.get_attr('count') > 0

def test_timer_each(nsproxy):
    """
    Test a timer executed periodically.
    """
    def tick(agent):
        agent.send('push', agent.count)
        agent.count += 1

    sender = run_agent('sender')
    sender.set_attr(count=0)
    receiver = run_agent('receiver')
    addr = sender.bind('PUSH', alias='push')
    receiver.connect(addr, handler=set_received)
    # Start timer
    sender.each(0.1, tick)
    time.sleep(2)
    assert abs(receiver.get_attr('received') - 20) <= 1

def test_timer_each_oop(nsproxy):
    """
    Test a timer executed periodically (using OOP).
    """
    class Sender(Agent):
        def on_init(self):
            self.count = 0
            self.bind('PUSH', 'push')

        def tick(self):
            self.send('push', self.count)
            self.count += 1

    sender = run_agent('sender', base=Sender)
    receiver = run_agent('receiver')
    receiver.connect(sender.addr('push'), handler=set_received)
    # Start timer
    sender.each(0.1, 'tick')
    time.sleep(2)
    assert abs(receiver.get_attr('received') - 20) <= 1

def test_timer_each_fall_behind(nsproxy):
    """
    Test a timer executed periodically and falling behind the period.

    If a sequence of events takes longer to run than the time available
    before the next event, the repeater will simply fall behind.
    """
    def tick(agent):
        time.sleep(.2)
        agent.count += 1

    agent = run_agent('agent')
    agent.set_attr(count=0)
    # Start timer
    agent.each(0., tick)
    time.sleep(2.0)
    assert abs(agent.get_attr('count') - 10) <= 1

def test_timer_each_fall_behind_catch_up(nsproxy):
    """
    Test a timer executed periodically and falling behind the period that
    catches-up.

    If a sequence of events takes longer to run than the time available
    before the next event, the repeater will simply fall behind.

    If a timer catches-up, then old executions that fell behind will be lost
    (i.e.: the defined period will be the minimum time between executions).
    """
    def tick(agent):
        if agent.count < 5:
            time.sleep(.2)
        agent.count += 1

    agent = run_agent('agent')
    agent.set_attr(count=0)
    # Start timer
    agent.each(0.1, tick)
    time.sleep(1.0)
    assert abs(agent.get_attr('count') - 5) <= 1
    # Now there will be no delay on tick execution, and old ticks that fell
    # behind will not be executed (the timer goes back to normal operation)
    time.sleep(1.0)
    assert abs(agent.get_attr('count') - 15) <= 1

def test_timer_each_stop_uuid(nsproxy):
    """
    Test a timer executed periodically and stopped by its UUID.
    """
    def tick(agent):
        agent.send('push', agent.count)
        agent.count += 1

    sender = run_agent('sender')
    receiver = run_agent('receiver')
    addr = sender.bind('PUSH', alias='push')
    receiver.connect(addr, handler=set_received)

    sender.set_attr(count=0)
    uuid = sender.each(0.1, tick)
    time.sleep(1)
    assert abs(receiver.get_attr('received') - 10) <= 1
    sender.stop_timer(uuid)
    time.sleep(1)
    assert abs(receiver.get_attr('received') - 10) <= 1
    assert uuid not in sender.list_timers()

def test_timer_each_stop_alias(nsproxy):
    """
    Test a timer executed periodically and stopped by an alias.
    """
    def tick(agent):
        agent.send('push', agent.count)
        agent.count += 1

    sender = run_agent('sender')
    receiver = run_agent('receiver')
    addr = sender.bind('PUSH', alias='push')
    receiver.connect(addr, handler=set_received)

    sender.set_attr(count=0)
    sender.each(0.1, tick, alias='aliased_timer')
    time.sleep(1)
    assert abs(receiver.get_attr('received') - 10) <= 1
    sender.stop_timer('aliased_timer')
    time.sleep(1)
    assert abs(receiver.get_attr('received') - 10) <= 1
    assert 'aliased_timer' not in sender.list_timers()

def test_stop_all_timers(nsproxy):
    """
    Calling `stop_all_timers()` should stop all currently running timers.
    """
    def tick(agent, message):
        agent.send('push', message)

    sender = run_agent('sender')
    receiver = run_agent('receiver', attributes={'received': []})
    addr = sender.bind('PUSH', alias='push')
    receiver.connect(addr, handler=append_received)

    sender.set_attr(count=0)
    sender.each(0.01, tick, 'timer0')
    sender.each(0.01, tick, 'timer1')
    # Make sure both timers are running
    assert wait_agent_attr(receiver, data='timer0')
    assert wait_agent_attr(receiver, data='timer1')
    # Stop all timers
    sender.stop_all_timers()
    assert len(sender.list_timers()) == 0
    # Run a new timer (only this one should be running now)
    sender.each(0.01, tick, 'timer3')
    assert wait_agent_attr(receiver, endswith=['timer3'] * 10)

def test_timer_after(nsproxy):
    """
    Test a timer executed once after a time delay.
    """
    def event(agent, number):
        agent.count += number

    agent = run_agent('agent')
    agent.set_attr(count=0)
    # Start timer
    agent.after(1, event, 2)
    agent.after(2, event, 1)
    time.sleep(0.9)
    assert agent.get_attr('count') == 0
    time.sleep(0.2)
    assert agent.get_attr('count') == 2
    time.sleep(1)
    assert agent.get_attr('count') == 3

def test_timer_after_oop(nsproxy):
    """
    Test a timer executed once after a time delay (using OOP).
    """
    class Foo(Agent):
        def on_init(self):
            self.count = 0

        def event(self, number):
            self.count += number

        def setup_timer(self, delay, number):
            self.after(delay, 'event', number)

    agent = run_agent('agent', base=Foo)
    # Start timer
    agent.setup_timer(0.5, 1)
    agent.setup_timer(1.0, 2)
    time.sleep(0.6)
    assert agent.get_attr('count') == 1
    time.sleep(0.5)
    assert agent.get_attr('count') == 3

def test_timer_after_stop_uuid(nsproxy):
    """
    Test a timer executed once after a time delay and stopped by its UUID.
    """
    def event(agent, number):
        agent.count += number

    agent = run_agent('agent')
    agent.set_attr(count=0)
    # Start timer
    uuid = agent.after(1, event, 2)
    agent.stop_timer(uuid)
    time.sleep(1.1)
    assert agent.get_attr('count') == 0

def test_timer_after_stop_alias(nsproxy):
    """
    Test a timer executed once after a time delay and stopped by its alias.
    """
    def event(agent, number):
        agent.count += number

    agent = run_agent('agent')
    agent.set_attr(count=0)
    # Start timer
    agent.after(1, event, 2, alias='foo')
    agent.stop_timer('foo')
    time.sleep(1.1)
    assert agent.get_attr('count') == 0

def test_timer_delayed_exception_shutdown_before_raising():
    """
    Test a timer that waits for a while and then raises an exception. While
    the timer is executing (during that waiting), the name server calls for
    shutdown.
    """
    def tick(agent):
        agent.count = 1
        time.sleep(2)
        raise RuntimeError()

    ns = run_nameserver()
    agent = run_agent('agent')
    agent.set_attr(count=0)
    # Start timer
    agent.each(0., tick)
    assert wait_agent_attr(agent, name='count', value=1)
    ns.shutdown()

def test_timer_sleep_and_send_shutdown_before_sent():
    """
    Test a timer that waits for a while and then sends a message through a
    socket. While the timer is executing (during that waiting), the name
    server calls for shutdown.

    Basically the sender agent binds a PUSH socket and the receiver connects
    to it. The shutdown call will kill the receiver before the sender sends
    its message. Sender should not block in the send call.
    """
    def tick(agent):
        agent.count += 1
        time.sleep(2)
        agent.send('push', agent.count)

    ns = run_nameserver()
    sender = run_agent('sender', attributes={'count': 0})
    receiver = run_agent('receiver')
    addr = sender.bind('PUSH', alias='push')
    receiver.connect(addr, handler=set_received)
    # Start timer
    sender.each(0., tick)
    assert wait_agent_attr(sender, name='count', value=1)
    ns.shutdown()
<commit_msg>More robust stop-timer-each tests implementation<commit_after>"""
Test file for timers.
"""
import time
import pytest
from osbrain import Agent
from osbrain import run_agent
from osbrain import run_nameserver
from osbrain.common import repeat
from osbrain.helper import wait_agent_attr

from common import nsproxy  # pragma: no flakes
from common import append_received
from common import set_received

@pytest.mark.timeout(1)
def test_repeat_non_blocking():
    """
    A repeated action (i.e. timer) should always be executed in a separate
    thread, even the first execution.
    """
    def foo(x):
        time.sleep(x)

    timer = repeat(1., foo, 2.)
    timer.stop()

def test_repeat_stop():
    """
    Test closing a timer returned by repeat.
    """
    class Bar():
        def __init__(self):
            self.a = 0

        def foo(self):
            self.a += 1

    bar = Bar()
    timer = repeat(0.1, bar.foo)
    time.sleep(1.)
    assert abs(bar.a - 10) <= 1
    timer.stop()
    time.sleep(1.)
    assert abs(bar.a - 10) <= 1

def test_timer_non_blocking_bug(nsproxy):
    """
    A timer call should never block, no matters how long it takes to execute
    the action.
    """
    def long_action(agent):
        time.sleep(1.)
        agent.count += 1

    agent = run_agent('agent')
    agent.set_attr(count=0)
    # Start timer
    t0 = time.time()
    agent.each(0., long_action)
    t1 = time.time()
    assert t1 - t0 < 0.3
    assert agent.get_attr('count') == 0
    time.sleep(1.5)
    assert agent.get_attr('count') > 0

def test_timer_each(nsproxy):
    """
    Test a timer executed periodically.
    """
    def tick(agent):
        agent.send('push', agent.count)
        agent.count += 1

    sender = run_agent('sender')
    sender.set_attr(count=0)
    receiver = run_agent('receiver')
    addr = sender.bind('PUSH', alias='push')
    receiver.connect(addr, handler=set_received)
    # Start timer
    sender.each(0.1, tick)
    time.sleep(2)
    assert abs(receiver.get_attr('received') - 20) <= 1

def test_timer_each_oop(nsproxy):
    """
    Test a timer executed periodically (using OOP).
    """
    class Sender(Agent):
        def on_init(self):
            self.count = 0
            self.bind('PUSH', 'push')

        def tick(self):
            self.send('push', self.count)
            self.count += 1

    sender = run_agent('sender', base=Sender)
    receiver = run_agent('receiver')
    receiver.connect(sender.addr('push'), handler=set_received)
    # Start timer
    sender.each(0.1, 'tick')
    time.sleep(2)
    assert abs(receiver.get_attr('received') - 20) <= 1

def test_timer_each_fall_behind(nsproxy):
    """
    Test a timer executed periodically and falling behind the period.

    If a sequence of events takes longer to run than the time available
    before the next event, the repeater will simply fall behind.
    """
    def tick(agent):
        time.sleep(.2)
        agent.count += 1

    agent = run_agent('agent')
    agent.set_attr(count=0)
    # Start timer
    agent.each(0., tick)
    time.sleep(2.0)
    assert abs(agent.get_attr('count') - 10) <= 1

def test_timer_each_fall_behind_catch_up(nsproxy):
    """
    Test a timer executed periodically and falling behind the period that
    catches-up.

    If a sequence of events takes longer to run than the time available
    before the next event, the repeater will simply fall behind.

    If a timer catches-up, then old executions that fell behind will be lost
    (i.e.: the defined period will be the minimum time between executions).
    """
    def tick(agent):
        if agent.count < 5:
            time.sleep(.2)
        agent.count += 1

    agent = run_agent('agent')
    agent.set_attr(count=0)
    # Start timer
    agent.each(0.1, tick)
    time.sleep(1.0)
    assert abs(agent.get_attr('count') - 5) <= 1
    # Now there will be no delay on tick execution, and old ticks that fell
    # behind will not be executed (the timer goes back to normal operation)
    time.sleep(1.0)
    assert abs(agent.get_attr('count') - 15) <= 1

def test_timer_each_stop_uuid(nsproxy):
    """
    Test a timer executed periodically and stopped by its UUID.
    """
    def tick(agent, message):
        agent.send('push', message)

    sender = run_agent('sender')
    receiver = run_agent('receiver', attributes={'received': []})
    addr = sender.bind('PUSH', alias='push')
    receiver.connect(addr, handler=append_received)

    sender.set_attr(count=0)
    uuid = sender.each(0.01, tick, 'timer0')
    sender.each(0.01, tick, 'timer1')
    # Make sure both timers are running
    assert wait_agent_attr(receiver, data='timer0')
    assert wait_agent_attr(receiver, data='timer1')
    # Stop one timer
    sender.stop_timer(uuid)
    assert len(sender.list_timers()) == 1
    assert uuid not in sender.list_timers()
    assert wait_agent_attr(receiver, endswith=['timer1'] * 10)

def test_timer_each_stop_alias(nsproxy):
    """
    Test a timer executed periodically and stopped by an alias.
    """
    def tick(agent, message):
        agent.send('push', message)

    sender = run_agent('sender')
    receiver = run_agent('receiver', attributes={'received': []})
    addr = sender.bind('PUSH', alias='push')
    receiver.connect(addr, handler=append_received)

    sender.set_attr(count=0)
    sender.each(0.01, tick, 'timer0', alias='aliased_timer')
    sender.each(0.01, tick, 'timer1')
    # Make sure both timers are running
    assert wait_agent_attr(receiver, data='timer0')
    assert wait_agent_attr(receiver, data='timer1')
    # Stop one timer
    sender.stop_timer('aliased_timer')
    assert len(sender.list_timers()) == 1
    assert 'aliased_timer' not in sender.list_timers()
    assert wait_agent_attr(receiver, endswith=['timer1'] * 10)

def test_stop_all_timers(nsproxy):
    """
    Calling `stop_all_timers()` should stop all currently running timers.
    """
    def tick(agent, message):
        agent.send('push', message)

    sender = run_agent('sender')
    receiver = run_agent('receiver', attributes={'received': []})
    addr = sender.bind('PUSH', alias='push')
    receiver.connect(addr, handler=append_received)

    sender.set_attr(count=0)
    sender.each(0.01, tick, 'timer0')
    sender.each(0.01, tick, 'timer1')
    # Make sure both timers are running
    assert wait_agent_attr(receiver, data='timer0')
    assert wait_agent_attr(receiver, data='timer1')
    # Stop all timers
    sender.stop_all_timers()
    assert len(sender.list_timers()) == 0
    # Run a new timer (only this one should be running now)
    sender.each(0.01, tick, 'timer3')
    assert wait_agent_attr(receiver, endswith=['timer3'] * 10)

def test_timer_after(nsproxy):
    """
    Test a timer executed once after a time delay.
    """
    def event(agent, number):
        agent.count += number

    agent = run_agent('agent')
    agent.set_attr(count=0)
    # Start timer
    agent.after(1, event, 2)
    agent.after(2, event, 1)
    time.sleep(0.9)
    assert agent.get_attr('count') == 0
    time.sleep(0.2)
    assert agent.get_attr('count') == 2
    time.sleep(1)
    assert agent.get_attr('count') == 3

def test_timer_after_oop(nsproxy):
    """
    Test a timer executed once after a time delay (using OOP).
    """
    class Foo(Agent):
        def on_init(self):
            self.count = 0

        def event(self, number):
            self.count += number

        def setup_timer(self, delay, number):
            self.after(delay, 'event', number)

    agent = run_agent('agent', base=Foo)
    # Start timer
    agent.setup_timer(0.5, 1)
    agent.setup_timer(1.0, 2)
    time.sleep(0.6)
    assert agent.get_attr('count') == 1
    time.sleep(0.5)
    assert agent.get_attr('count') == 3

def test_timer_after_stop_uuid(nsproxy):
    """
    Test a timer executed once after a time delay and stopped by its UUID.
    """
    def event(agent, number):
        agent.count += number

    agent = run_agent('agent')
    agent.set_attr(count=0)
    # Start timer
    uuid = agent.after(1, event, 2)
    agent.stop_timer(uuid)
    time.sleep(1.1)
    assert agent.get_attr('count') == 0

def test_timer_after_stop_alias(nsproxy):
    """
    Test a timer executed once after a time delay and stopped by its alias.
    """
    def event(agent, number):
        agent.count += number

    agent = run_agent('agent')
    agent.set_attr(count=0)
    # Start timer
    agent.after(1, event, 2, alias='foo')
    agent.stop_timer('foo')
    time.sleep(1.1)
    assert agent.get_attr('count') == 0

def test_timer_delayed_exception_shutdown_before_raising():
    """
    Test a timer that waits for a while and then raises an exception. While
    the timer is executing (during that waiting), the name server calls for
    shutdown.
    """
    def tick(agent):
        agent.count = 1
        time.sleep(2)
        raise RuntimeError()

    ns = run_nameserver()
    agent = run_agent('agent')
    agent.set_attr(count=0)
    # Start timer
    agent.each(0., tick)
    assert wait_agent_attr(agent, name='count', value=1)
    ns.shutdown()

def test_timer_sleep_and_send_shutdown_before_sent():
    """
    Test a timer that waits for a while and then sends a message through a
    socket. While the timer is executing (during that waiting), the name
    server calls for shutdown.

    Basically the sender agent binds a PUSH socket and the receiver connects
    to it. The shutdown call will kill the receiver before the sender sends
    its message. Sender should not block in the send call.
    """
    def tick(agent):
        agent.count += 1
        time.sleep(2)
        agent.send('push', agent.count)

    ns = run_nameserver()
    sender = run_agent('sender', attributes={'count': 0})
    receiver = run_agent('receiver')
    addr = sender.bind('PUSH', alias='push')
    receiver.connect(addr, handler=set_received)
    # Start timer
    sender.each(0., tick)
    assert wait_agent_attr(sender, name='count', value=1)
    ns.shutdown()
<|endoftext|>


<commit_before>__version__ = '0.2.0-dev'
default_app_config = 'oscar_payonline.config.OscarPayonlineConfig'<commit_msg>Bump version. Russian locale added.<commit_after>__version__ = '0.2.1-dev'
default_app_config = 'oscar_payonline.config.OscarPayonlineConfig'<|endoftext|>


<commit_before>from __future__ import print_function, absolute_import, division
import os
import tempfile

from six.moves import cPickle
from sklearn.cluster import KMeans

from osprey.config import Config
from osprey.search_space import IntVariable, FloatVariable, EnumVariable
from osprey.strategies import RandomSearch, HyperoptTPE, MOE

os.environ['OSPREYRC'] = ' '

def test_estimator_pickle():
    with tempfile.NamedTemporaryFile('w+b', 0) as f:

        cPickle.dump(KMeans(), f)

        config = Config.fromdict({
            'estimator': {'pickle': f.name}
        }, check_fields=False)
        assert isinstance(config.estimator(), KMeans)

def test_estimator_eval_1():
    config = Config.fromdict({
        'estimator': {
            'eval': 'KMeans()',
            'eval_scope': 'sklearn',
        }
    }, check_fields=False)
    assert isinstance(config.estimator(), KMeans)

def test_estimator_eval_2():
    config = Config.fromdict({
        'estimator': {
            'eval': 'KMeans()',
            'eval_scope': ['sklearn'],
        }
    }, check_fields=False)
    assert isinstance(config.estimator(), KMeans)

def test_estimator_entry_point():
    config = Config.fromdict({
        'estimator': {
            'entry_point': 'sklearn.cluster.KMeans',
        }
    }, check_fields=False)
    assert isinstance(config.estimator(), KMeans)

def test_search_space():
    config = Config.fromdict({
        'search_space': {
            'intvar': {'type': 'int', 'min': 1, 'max': 2},
            'fvar': {'type': 'float', 'min': 1, 'max': 3.5},
            'logvar': {'type': 'float', 'min': 1, 'max': 2.5, 'warp': 'log'},
            'enumvar': {'type': 'enum', 'choices': [1, False]},
        }}, check_fields=False)
    searchspace = config.search_space()
    assert searchspace['intvar'] == IntVariable('intvar', 1, 2)
    assert searchspace['fvar'] == FloatVariable('fvar', 1, 3.5, warp=None)
    assert searchspace['logvar'] == FloatVariable('logvar', 1, 2.5, warp='log')
    assert searchspace['enumvar'] == EnumVariable('enumvar', [1, False])

def test_strategy_random():
    config = Config.fromdict({
        'strategy': {'name': 'random'}
    }, check_fields=False)
    assert isinstance(config.strategy(), RandomSearch)

def test_search_engine_hyperopt_tpe():
    config = Config.fromdict({
        'strategy': {'name': 'hyperopt_tpe'}
    }, check_fields=False)
    assert isinstance(config.strategy(), HyperoptTPE)

def test_search_engine_moe_1():
    config = Config.fromdict({
        'strategy': {'name': 'moe', 'params': {'url': 'sdfsdf'}}
    }, check_fields=False)
    assert isinstance(config.strategy(), MOE)

def test_search_engine_moe_2():
    config = Config.fromdict({
        'strategy': {'name': 'moe', 'params': {'url': 'abc'}}
    }, check_fields=False)
    strat = config.strategy()
    assert isinstance(strat, MOE)
    assert strat.url == 'abc'

def test_scoring():
    config = Config.fromdict({
        'scoring': 'sdfsfsdf'
    }, check_fields=False)
    assert config.scoring() is 'sdfsfsdf'

def test_cv():
    from sklearn.cross_validation import ShuffleSplit
    config = Config.fromdict({
        'cv': {'name': 'shufflesplit', 'params': {'n_iter': 10}}
    }, check_fields=False)
    cv = config.cv(range(100))
    assert isinstance(cv, ShuffleSplit)
    assert cv.n_iter == 10
<commit_msg>Add test for stratified cv config<commit_after>from __future__ import print_function, absolute_import, division
import numpy as np
import os
import tempfile

from six.moves import cPickle
from sklearn.cluster import KMeans

from osprey.config import Config
from osprey.search_space import IntVariable, FloatVariable, EnumVariable
from osprey.strategies import RandomSearch, HyperoptTPE, MOE

os.environ['OSPREYRC'] = ' '

def test_estimator_pickle():
    with tempfile.NamedTemporaryFile('w+b', 0) as f:

        cPickle.dump(KMeans(), f)

        config = Config.fromdict({
            'estimator': {'pickle': f.name}
        }, check_fields=False)
        assert isinstance(config.estimator(), KMeans)

def test_estimator_eval_1():
    config = Config.fromdict({
        'estimator': {
            'eval': 'KMeans()',
            'eval_scope': 'sklearn',
        }
    }, check_fields=False)
    assert isinstance(config.estimator(), KMeans)

def test_estimator_eval_2():
    config = Config.fromdict({
        'estimator': {
            'eval': 'KMeans()',
            'eval_scope': ['sklearn'],
        }
    }, check_fields=False)
    assert isinstance(config.estimator(), KMeans)

def test_estimator_entry_point():
    config = Config.fromdict({
        'estimator': {
            'entry_point': 'sklearn.cluster.KMeans',
        }
    }, check_fields=False)
    assert isinstance(config.estimator(), KMeans)

def test_search_space():
    config = Config.fromdict({
        'search_space': {
            'intvar': {'type': 'int', 'min': 1, 'max': 2},
            'fvar': {'type': 'float', 'min': 1, 'max': 3.5},
            'logvar': {'type': 'float', 'min': 1, 'max': 2.5, 'warp': 'log'},
            'enumvar': {'type': 'enum', 'choices': [1, False]},
        }}, check_fields=False)
    searchspace = config.search_space()
    assert searchspace['intvar'] == IntVariable('intvar', 1, 2)
    assert searchspace['fvar'] == FloatVariable('fvar', 1, 3.5, warp=None)
    assert searchspace['logvar'] == FloatVariable('logvar', 1, 2.5, warp='log')
    assert searchspace['enumvar'] == EnumVariable('enumvar', [1, False])

def test_strategy_random():
    config = Config.fromdict({
        'strategy': {'name': 'random'}
    }, check_fields=False)
    assert isinstance(config.strategy(), RandomSearch)

def test_search_engine_hyperopt_tpe():
    config = Config.fromdict({
        'strategy': {'name': 'hyperopt_tpe'}
    }, check_fields=False)
    assert isinstance(config.strategy(), HyperoptTPE)

def test_search_engine_moe_1():
    config = Config.fromdict({
        'strategy': {'name': 'moe', 'params': {'url': 'sdfsdf'}}
    }, check_fields=False)
    assert isinstance(config.strategy(), MOE)

def test_search_engine_moe_2():
    config = Config.fromdict({
        'strategy': {'name': 'moe', 'params': {'url': 'abc'}}
    }, check_fields=False)
    strat = config.strategy()
    assert isinstance(strat, MOE)
    assert strat.url == 'abc'

def test_scoring():
    config = Config.fromdict({
        'scoring': 'sdfsfsdf'
    }, check_fields=False)
    assert config.scoring() is 'sdfsfsdf'

def test_cv():
    from sklearn.cross_validation import ShuffleSplit
    config = Config.fromdict({
        'cv': {'name': 'shufflesplit', 'params': {'n_iter': 10}}
    }, check_fields=False)
    cv = config.cv(range(100))
    assert isinstance(cv, ShuffleSplit)
    assert cv.n_iter == 10

def test_stratified_cv():
    from sklearn.cross_validation import StratifiedShuffleSplit
    config = Config.fromdict({
        'cv': {'name': 'stratifiedshufflesplit', 'params': {'n_iter': 10}}
    }, check_fields=False)
    cv = config.cv(range(100), np.random.randint(2, size=100))
    assert isinstance(cv, StratifiedShuffleSplit)
    assert cv.n_iter == 10
<|endoftext|>


<commit_before>"""Collection of routines used to post process (magnetisation)
vector data.

Part of ovf2vtk.

Hans Fangohr, hans.fangohr@physics.org

# Some helping functions:
#
# Set of functions used for ovf2vtk. General data format is a rank-2
# matrix with the row corresponding to the data in the vector field, and
# the 3-component column corresponding to the magnetisation vector at
# that place.
#
# Together with an index function (last component varying fastest,
# C-style) and positions vectors x_vec, y_vec and z_vec (for three
# dimensional domains), these data can be mapped to positions in real
# space.
#
# All this makes only sense for rectilinear grids.
#
# (fangohr 25/08/2003 00:18)

"""

try:
    import numpy as Numeric
except ImportError:
    print "This program needs Numpy. Please download and install. \
(http://sourceforge.net/projects/numpy)."
    print "If you are using Numeric, you can use the older version \
0.1.17 of ovf2vtk."
    raise ImportError("Couldn't import Numpy -- cannot proceed.")

__version__ = "$Revision: 1.3 $"

def magnitude(d):
    """expects d to be a 1xN array,
    returns the magnitude (standard 2-Norm)"""
    # square entries:                  d*d
    # sum over 3-components (axis 1):  add.reduce( d*d, 1)
    # take square root                 sqrt( add.reduce ( d*d, 1 ) )
    return Numeric.sqrt(Numeric.add.reduce(d * d, 1))

def convert_flat_fortran_to_3dmatrix(vf, Nx, Ny, Nz):
    return Numeric.resize(vf, (Nz, Ny, Nx, 3))  # this is in Fortran order

def convert_fortran_3dmatrix_to_flat(M):
    return Numeric.array(M).ravel()

def convert_fortran_3dmatrix_to_flat_vector(M):
    N = len(Numeric.array(M).ravel())
    return Numeric.resize(M, (N/3, 3))

def convert_fortran_to_C(a):
    """assuming Fortran data is stored as M[Nz,Ny,Nx,:,...:] and
    C data as M[Nx,Ny,Nz,:,,...,:] then this function converts from
    one to the other. (fangohr 31/03/2004 23:06)"""
    return Numeric.swapaxes(a, 0, 2)

def convert_C_to_fortran(a):
    return convert_fortran_to_C(a)

def fortranindex(i, j, k, Nx, Ny, Nz):
    """x changes fastest"""
    return i+j*Nx+k*Nx*Ny

def Cindex(i, j, k, Nx, Ny, Nz):
    """z changes fastest """
    return i*Nz*Ny+j*Nz+k

def components(d):
    return (d[:,0], d[:,1], d[:,2])

def plane_angles(d):
    """
    Input is matrix, containing N 3d vectors.

    returns angles in yx, yz and xz plane for all vectors. """
    x, y, z = components(d)

    # if any Ms is smaller that 1e-6, then set to zero to eliminate noise
    cutoff = 1e-6

    Ms = magnitude(d)

    x2 = Numeric.choose(Numeric.less(Ms, cutoff), (x, 0.0))
    y2 = Numeric.choose(Numeric.less(Ms, cutoff), (y, 0.0))
    z2 = Numeric.choose(Numeric.less(Ms, cutoff), (z, 0.0))

    xy = Numeric.arctan2(y2, x2)
    yz = Numeric.arctan2(z2, y2)
    xz = Numeric.arctan2(z2, x2)
    return xy, yz, xz

def clean_surfaces(obs, M, wipe=0, eps=1e-3, zerovalue=0.0):
    """sets all values in obs to zero value for which abs(M)<eps. If wipe > 0,
       then all positions up to 'wipe' indicies away from the M<eps entry are
       set to zerovalue as well.

    Example: hide divergence effects at surface:

       * clean_surfaces( div, M, wipe=1, eps=1e-5)

    This is only of relevance if there is 'vacuum' in the simulation cell
    (i.e. an aero with Ms=0). The computation of the divergence and the curl
    will produce large values across the interface from Ms!=0 to Ms=0. This
    distracts from the more interesting data inside the Ms!=0 volume. We
    therefore simply wipe out the outermost layer of div and curl data that is
    affected by this.

    This process is slow as it is using for-loops.
    """

    assert obs.shape[0:2] == M.shape[0:2], "Internal error"

    Nx, Ny, Nz, dummy = M.shape

    if len(obs.shape) == 3:
        obs_is_scalar = True
        obs_is_vector = False
        obs_rank = 1

    elif len(obs.shape) == 4:
        obs_is_scalar = False
        obs_is_vector = True
        obs_rank = 3
    else:
        raise NotImplementedError("""Can only deal with scalar and vectors\
 (in 3d positional matrix)""")

    if obs_is_scalar:
        big_obs = Numeric.zeros((Nx+2*wipe, Ny+2*wipe, Nz+2*wipe), 'd')
    if obs_is_vector:
        big_obs = Numeric.zeros((Nx+2*wipe, Ny+2*wipe, Nz+2*wipe, 3), 'd')

    offset = wipe

    # copy input values in here
    big_obs[wipe:Nx+wipe, wipe:Ny+wipe, wipe:Nz+wipe] = obs[:, :, :]

    for i in range(Nx):
        for j in range(Ny):
            for k in range(Nz):
                if sum(M[i, j, k]**2) < eps**2:
                    # wipe out matrix entries with indicies +- wipe around...
                    # ...[i,j,k]
                    big_obs[i-wipe+offset:i+wipe+1+offset,
                            j-wipe+offset:j+wipe+1+offset,
                            k-wipe+offset:k+wipe+1+offset] = zerovalue

    obs = big_obs[wipe:Nx+wipe, wipe:Ny+wipe, wipe:Nz+wipe]

    return obs

def divergence_and_curl(vf, surfaceEffects, ovf_run):
    """A new attempt to compute the divergence and the curlfaster using
vector/matrix operations in Numeric rather than for-loops in
Python. (fangohr 31/03/2004 00:25)

Input data is:
    vf             : the VectorField (as created by omfread)
    surfaceEffects : a boolean
    ovf_run        : dictionarry with keyword-value pairs from ovf file header.
    """
    dimensions = (int(ovf_run["xnodes:"]),
                  int(ovf_run["ynodes:"]),
                  int(ovf_run["znodes:"]))

    Nx = dimensions[0]
    Ny = dimensions[1]
    Nz = dimensions[2]

    # get data into array of shape (Nx,Ny,Nz,3) (This is C-style)
    M = convert_flat_fortran_to_3dmatrix(vf, Nx, Ny, Nz)
    M = convert_fortran_to_C(M)

    # print "Nx, Ny, Nz=", Nx, Ny, Nz

    dx = float(ovf_run["xstepsize:"])
    dy = float(ovf_run["ystepsize:"])
    dz = float(ovf_run["zstepsize:"])

    # total magnitude of curl is irrelevant, re-scale dx, dy, dz to...
    # ...avoid overflow:

    scale = min(dx, dy, dz)

    dx /= scale
    dy /= scale
    dz /= scale

    # compute dMx/dx (central differences):

    dMdx = Numeric.array(M[2:, :, :, :]-M[:-2, :, :, :])
    dMxdx, dMydx, dMzdx = \
        dMdx[:, :, :, 0]/dx, dMdx[:, :, :, 1]/dx, dMdx[:, :, :, 2]/dx
    dMdy = Numeric.array(M[:, 2:, :, :]-M[:, :-2, :, :])
    dMxdy, dMydy, dMzdy = \
        dMdy[:, :, :, 0]/dy, dMdy[:, :, :, 1]/dy, dMdy[:, :, :, 2]/dy
    dMdz = Numeric.array(M[:, :, 2:, :]-M[:, :, :-2, :])
    dMxdz, dMydz, dMzdz = \
        dMdz[:, :, :, 0]/dz, dMdz[:, :, :, 1]/dz, dMdz[:, :, :, 2]/dz

    div = Numeric.zeros((Nx, Ny, Nz), 'd')

    div[1:-1, :, :] += dMxdx
    div[:, 1:-1, :] += dMydy
    div[:, :, 1:-1] += dMzdz

    if not surfaceEffects:
        div = clean_surfaces(div, M, wipe=1)

    Fdiv = convert_C_to_fortran(div)
    divflat = convert_fortran_3dmatrix_to_flat(Fdiv)

    rot = Numeric.zeros((Nx, Ny, Nz, 3), 'd')

    # taking the cross product (this excludes the outermost layers as
    # we can't take the central difference for these)
    rot[1:-1, 1:-1, 1:-1, 0] = dMzdy[1:-1, :, 1:-1]-dMydz[1:-1, 1:-1, :]
    rot[1:-1, 1:-1, 1:-1, 1] = dMxdz[1:-1, 1:-1, :]-dMzdx[:, 1:-1, 1:-1]
    rot[1:-1, 1:-1, 1:-1, 2] = dMydx[:, 1:-1, 1:-1]-dMxdy[1:-1, :, 1:-1]

    # special 2d-case (only one layer in z)
    if Nz == 1:
        print "-->Nz==1, special 2d case, will only compute \
z-component of curl"
        rot[1:-1, 1:-1, 0, 2] = dMydx[:, 1:-1, 0]-dMxdy[1:-1, :, 0]

    if not surfaceEffects:
        rot = clean_surfaces(rot, M, wipe=1)

    Frot = convert_C_to_fortran(rot)
    rotflat = convert_fortran_3dmatrix_to_flat_vector(Frot)

    rotmag = magnitude(rotflat)

    return (divflat, rotflat, rotflat[:, 0], rotflat[:, 1],
            rotflat[:, 2], rotmag)
<commit_msg>minor change to fall in line with PEP8 requirements.<commit_after>"""Collection of routines used to post process (magnetisation)
vector data.

Part of ovf2vtk.

Hans Fangohr, hans.fangohr@physics.org

# Some helping functions:
#
# Set of functions used for ovf2vtk. General data format is a rank-2
# matrix with the row corresponding to the data in the vector field, and
# the 3-component column corresponding to the magnetisation vector at
# that place.
#
# Together with an index function (last component varying fastest,
# C-style) and positions vectors x_vec, y_vec and z_vec (for three
# dimensional domains), these data can be mapped to positions in real
# space.
#
# All this makes only sense for rectilinear grids.
#
# (fangohr 25/08/2003 00:18)

"""

try:
    import numpy as Numeric
except ImportError:
    print "This program needs Numpy. Please download and install. \
(http://sourceforge.net/projects/numpy)."
    print "If you are using Numeric, you can use the older version \
0.1.17 of ovf2vtk."
    raise ImportError("Couldn't import Numpy -- cannot proceed.")

__version__ = "$Revision: 1.3 $"

def magnitude(d):
    """expects d to be a 1xN array,
    returns the magnitude (standard 2-Norm)"""
    # square entries:                  d*d
    # sum over 3-components (axis 1):  add.reduce( d*d, 1)
    # take square root                 sqrt( add.reduce ( d*d, 1 ) )
    return Numeric.sqrt(Numeric.add.reduce(d * d, 1))

def convert_flat_fortran_to_3dmatrix(vf, Nx, Ny, Nz):
    return Numeric.resize(vf, (Nz, Ny, Nx, 3))  # this is in Fortran order

def convert_fortran_3dmatrix_to_flat(M):
    return Numeric.array(M).ravel()

def convert_fortran_3dmatrix_to_flat_vector(M):
    N = len(Numeric.array(M).ravel())
    return Numeric.resize(M, (N/3, 3))

def convert_fortran_to_C(a):
    """assuming Fortran data is stored as M[Nz,Ny,Nx,:,...:] and
    C data as M[Nx,Ny,Nz,:,,...,:] then this function converts from
    one to the other. (fangohr 31/03/2004 23:06)"""
    return Numeric.swapaxes(a, 0, 2)

def convert_C_to_fortran(a):
    return convert_fortran_to_C(a)

def fortranindex(i, j, k, Nx, Ny, Nz):
    """x changes fastest"""
    return i+j*Nx+k*Nx*Ny

def Cindex(i, j, k, Nx, Ny, Nz):
    """z changes fastest """
    return i*Nz*Ny+j*Nz+k

def components(d):
    return (d[:, 0], d[:, 1], d[:, 2])

def plane_angles(d):
    """
    Input is matrix, containing N 3d vectors.

    returns angles in yx, yz and xz plane for all vectors. """
    x, y, z = components(d)

    # if any Ms is smaller that 1e-6, then set to zero to eliminate noise
    cutoff = 1e-6

    Ms = magnitude(d)

    x2 = Numeric.choose(Numeric.less(Ms, cutoff), (x, 0.0))
    y2 = Numeric.choose(Numeric.less(Ms, cutoff), (y, 0.0))
    z2 = Numeric.choose(Numeric.less(Ms, cutoff), (z, 0.0))

    xy = Numeric.arctan2(y2, x2)
    yz = Numeric.arctan2(z2, y2)
    xz = Numeric.arctan2(z2, x2)
    return xy, yz, xz

def clean_surfaces(obs, M, wipe=0, eps=1e-3, zerovalue=0.0):
    """sets all values in obs to zero value for which abs(M)<eps. If wipe > 0,
       then all positions up to 'wipe' indicies away from the M<eps entry are
       set to zerovalue as well.

    Example: hide divergence effects at surface:

       * clean_surfaces( div, M, wipe=1, eps=1e-5)

    This is only of relevance if there is 'vacuum' in the simulation cell
    (i.e. an aero with Ms=0). The computation of the divergence and the curl
    will produce large values across the interface from Ms!=0 to Ms=0. This
    distracts from the more interesting data inside the Ms!=0 volume. We
    therefore simply wipe out the outermost layer of div and curl data that is
    affected by this.

    This process is slow as it is using for-loops.
    """

    assert obs.shape[0:2] == M.shape[0:2], "Internal error"

    Nx, Ny, Nz, dummy = M.shape

    if len(obs.shape) == 3:
        obs_is_scalar = True
        obs_is_vector = False
        obs_rank = 1

    elif len(obs.shape) == 4:
        obs_is_scalar = False
        obs_is_vector = True
        obs_rank = 3
    else:
        raise NotImplementedError("""Can only deal with scalar and vectors\
 (in 3d positional matrix)""")

    if obs_is_scalar:
        big_obs = Numeric.zeros((Nx+2*wipe, Ny+2*wipe, Nz+2*wipe), 'd')
    if obs_is_vector:
        big_obs = Numeric.zeros((Nx+2*wipe, Ny+2*wipe, Nz+2*wipe, 3), 'd')

    offset = wipe

    # copy input values in here
    big_obs[wipe:Nx+wipe, wipe:Ny+wipe, wipe:Nz+wipe] = obs[:, :, :]

    for i in range(Nx):
        for j in range(Ny):
            for k in range(Nz):
                if sum(M[i, j, k]**2) < eps**2:
                    # wipe out matrix entries with indicies +- wipe around...
                    # ...[i,j,k]
                    big_obs[i-wipe+offset:i+wipe+1+offset,
                            j-wipe+offset:j+wipe+1+offset,
                            k-wipe+offset:k+wipe+1+offset] = zerovalue

    obs = big_obs[wipe:Nx+wipe, wipe:Ny+wipe, wipe:Nz+wipe]

    return obs

def divergence_and_curl(vf, surfaceEffects, ovf_run):
    """A new attempt to compute the divergence and the curlfaster using
vector/matrix operations in Numeric rather than for-loops in
Python. (fangohr 31/03/2004 00:25)

Input data is:
    vf             : the VectorField (as created by omfread)
    surfaceEffects : a boolean
    ovf_run        : dictionarry with keyword-value pairs from ovf file header.
    """
    dimensions = (int(ovf_run["xnodes:"]),
                  int(ovf_run["ynodes:"]),
                  int(ovf_run["znodes:"]))

    Nx = dimensions[0]
    Ny = dimensions[1]
    Nz = dimensions[2]

    # get data into array of shape (Nx,Ny,Nz,3) (This is C-style)
    M = convert_flat_fortran_to_3dmatrix(vf, Nx, Ny, Nz)
    M = convert_fortran_to_C(M)

    # print "Nx, Ny, Nz=", Nx, Ny, Nz

    dx = float(ovf_run["xstepsize:"])
    dy = float(ovf_run["ystepsize:"])
    dz = float(ovf_run["zstepsize:"])

    # total magnitude of curl is irrelevant, re-scale dx, dy, dz to...
    # ...avoid overflow:

    scale = min(dx, dy, dz)

    dx /= scale
    dy /= scale
    dz /= scale

    # compute dMx/dx (central differences):

    dMdx = Numeric.array(M[2:, :, :, :]-M[:-2, :, :, :])
    dMxdx, dMydx, dMzdx = \
        dMdx[:, :, :, 0]/dx, dMdx[:, :, :, 1]/dx, dMdx[:, :, :, 2]/dx
    dMdy = Numeric.array(M[:, 2:, :, :]-M[:, :-2, :, :])
    dMxdy, dMydy, dMzdy = \
        dMdy[:, :, :, 0]/dy, dMdy[:, :, :, 1]/dy, dMdy[:, :, :, 2]/dy
    dMdz = Numeric.array(M[:, :, 2:, :]-M[:, :, :-2, :])
    dMxdz, dMydz, dMzdz = \
        dMdz[:, :, :, 0]/dz, dMdz[:, :, :, 1]/dz, dMdz[:, :, :, 2]/dz

    div = Numeric.zeros((Nx, Ny, Nz), 'd')

    div[1:-1, :, :] += dMxdx
    div[:, 1:-1, :] += dMydy
    div[:, :, 1:-1] += dMzdz

    if not surfaceEffects:
        div = clean_surfaces(div, M, wipe=1)

    Fdiv = convert_C_to_fortran(div)
    divflat = convert_fortran_3dmatrix_to_flat(Fdiv)

    rot = Numeric.zeros((Nx, Ny, Nz, 3), 'd')

    # taking the cross product (this excludes the outermost layers as
    # we can't take the central difference for these)
    rot[1:-1, 1:-1, 1:-1, 0] = dMzdy[1:-1, :, 1:-1]-dMydz[1:-1, 1:-1, :]
    rot[1:-1, 1:-1, 1:-1, 1] = dMxdz[1:-1, 1:-1, :]-dMzdx[:, 1:-1, 1:-1]
    rot[1:-1, 1:-1, 1:-1, 2] = dMydx[:, 1:-1, 1:-1]-dMxdy[1:-1, :, 1:-1]

    # special 2d-case (only one layer in z)
    if Nz == 1:
        print "-->Nz==1, special 2d case, will only compute \
z-component of curl"
        rot[1:-1, 1:-1, 0, 2] = dMydx[:, 1:-1, 0]-dMxdy[1:-1, :, 0]

    if not surfaceEffects:
        rot = clean_surfaces(rot, M, wipe=1)

    Frot = convert_C_to_fortran(rot)
    rotflat = convert_fortran_3dmatrix_to_flat_vector(Frot)

    rotmag = magnitude(rotflat)

    return (divflat, rotflat, rotflat[:, 0], rotflat[:, 1],
            rotflat[:, 2], rotmag)
<|endoftext|>