<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

from astropy.extern import six

from ..utils import parse_input_data, parse_output_projection
from .core import _reproject_celestial, _reproject_full

__all__ = ['reproject_interp']

ORDER = {}
ORDER['nearest-neighbor'] = 0
ORDER['bilinear'] = 1
ORDER['biquadratic'] = 2
ORDER['bicubic'] = 3

def reproject_interp(input_data, output_projection, shape_out=None, hdu_in=0,
                     order='bilinear', independent_celestial_slices=False):
    """
    Reproject data to a new projection using interpolation (this is typically
    the fastest way to reproject an image).

    Parameters
    ----------
    input_data : str or `~astropy.io.fits.HDUList` or `~astropy.io.fits.PrimaryHDU` or `~astropy.io.fits.ImageHDU` or tuple
        The input data to reproject. This can be:

            * The name of a FITS file
            * An `~astropy.io.fits.HDUList` object
            * An image HDU object such as a `~astropy.io.fits.PrimaryHDU`,
              `~astropy.io.fits.ImageHDU`, or `~astropy.io.fits.CompImageHDU`
              instance
            * A tuple where the first element is a `~numpy.ndarray` and the
              second element is either a `~astropy.wcs.WCS` or a
              `~astropy.io.fits.Header` object

    output_projection : `~astropy.wcs.WCS` or `~astropy.io.fits.Header`
        The output projection, which can be either a `~astropy.wcs.WCS`
        or a `~astropy.io.fits.Header` instance.
    shape_out : tuple, optional
        If ``output_projection`` is a `~astropy.wcs.WCS` instance, the
        shape of the output data should be specified separately.
    hdu_in : int or str, optional
        If ``input_data`` is a FITS file or an `~astropy.io.fits.HDUList`
        instance, specifies the HDU to use.
    order : int or str, optional
        The order of the interpolation (if ``mode`` is set to
        ``'interpolation'``). This can be either one of the following strings:

            * 'nearest-neighbor'
            * 'bilinear'
            * 'biquadratic'
            * 'bicubic'

        or an integer. A value of ``0`` indicates nearest neighbor
        interpolation.
    independent_celestial_slices : bool, optional
        This can be set to `True` for n-dimensional input in the following case
        (all conditions have to be fulfilled):
            * The number of pixels in each non-celestial dimension is the same
              between the input and target header.
            * The WCS coordinates along the non-celestial dimensions are the
              same between the input and target WCS.
            * The celestial WCS component is independent from other WCS
              coordinates.
        In this special case, we can make things a little faster by
        reprojecting each celestial slice independently using the same
        transformation.

    Returns
    -------
    array_new : `~numpy.ndarray`
        The reprojected array
    footprint : `~numpy.ndarray`
        Footprint of the input array in the output array. Values of 0 indicate
        no coverage or valid values in the input image, while values of 1
        indicate valid values.
    """

    array_in, wcs_in = parse_input_data(input_data, hdu_in=hdu_in)
    wcs_out, shape_out = parse_output_projection(output_projection, shape_out=shape_out)

    if isinstance(order, six.string_types):
        order = ORDER[order]

    if (wcs_in.celestial and wcs_in.naxis == 2) or independent_celestial_slices:
        return _reproject_celestial(array_in, wcs_in, wcs_out, shape_out=shape_out, order=order)
    else:
        return _reproject_full(array_in, wcs_in, wcs_out, shape_out=shape_out, order=order)
<commit_msg>got rid of one warning by double backticking.  Still had lots of intersphinx problems locally, though<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

from astropy.extern import six

from ..utils import parse_input_data, parse_output_projection
from .core import _reproject_celestial, _reproject_full

__all__ = ['reproject_interp']

ORDER = {}
ORDER['nearest-neighbor'] = 0
ORDER['bilinear'] = 1
ORDER['biquadratic'] = 2
ORDER['bicubic'] = 3

def reproject_interp(input_data, output_projection, shape_out=None, hdu_in=0,
                     order='bilinear', independent_celestial_slices=False):
    """
    Reproject data to a new projection using interpolation (this is typically
    the fastest way to reproject an image).

    Parameters
    ----------
    input_data : str or `~astropy.io.fits.HDUList` or `~astropy.io.fits.PrimaryHDU` or `~astropy.io.fits.ImageHDU` or tuple
        The input data to reproject. This can be:

            * The name of a FITS file
            * An `~astropy.io.fits.HDUList` object
            * An image HDU object such as a `~astropy.io.fits.PrimaryHDU`,
              `~astropy.io.fits.ImageHDU`, or `~astropy.io.fits.CompImageHDU`
              instance
            * A tuple where the first element is a `~numpy.ndarray` and the
              second element is either a `~astropy.wcs.WCS` or a
              `~astropy.io.fits.Header` object

    output_projection : `~astropy.wcs.WCS` or `~astropy.io.fits.Header`
        The output projection, which can be either a `~astropy.wcs.WCS`
        or a `~astropy.io.fits.Header` instance.
    shape_out : tuple, optional
        If ``output_projection`` is a `~astropy.wcs.WCS` instance, the
        shape of the output data should be specified separately.
    hdu_in : int or str, optional
        If ``input_data`` is a FITS file or an `~astropy.io.fits.HDUList`
        instance, specifies the HDU to use.
    order : int or str, optional
        The order of the interpolation (if ``mode`` is set to
        ``'interpolation'``). This can be either one of the following strings:

            * 'nearest-neighbor'
            * 'bilinear'
            * 'biquadratic'
            * 'bicubic'

        or an integer. A value of ``0`` indicates nearest neighbor
        interpolation.
    independent_celestial_slices : bool, optional
        This can be set to ``True`` for n-dimensional input in the following case
        (all conditions have to be fulfilled):

            * The number of pixels in each non-celestial dimension is the same
              between the input and target header.
            * The WCS coordinates along the non-celestial dimensions are the
              same between the input and target WCS.
            * The celestial WCS component is independent from other WCS
              coordinates.

        In this special case, we can make things a little faster by
        reprojecting each celestial slice independently using the same
        transformation.

    Returns
    -------
    array_new : `~numpy.ndarray`
        The reprojected array
    footprint : `~numpy.ndarray`
        Footprint of the input array in the output array. Values of 0 indicate
        no coverage or valid values in the input image, while values of 1
        indicate valid values.
    """

    array_in, wcs_in = parse_input_data(input_data, hdu_in=hdu_in)
    wcs_out, shape_out = parse_output_projection(output_projection, shape_out=shape_out)

    if isinstance(order, six.string_types):
        order = ORDER[order]

    if (wcs_in.celestial and wcs_in.naxis == 2) or independent_celestial_slices:
        return _reproject_celestial(array_in, wcs_in, wcs_out, shape_out=shape_out, order=order)
    else:
        return _reproject_full(array_in, wcs_in, wcs_out, shape_out=shape_out, order=order)
<|endoftext|>


<commit_before>########
# Copyright (c) 2015 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

# Built-in Imports
import requests
from requests import Request,Session,Response
import json
import constants
import sys
import os
import auth
import utils
from cloudify.exceptions import NonRecoverableError
from cloudify import ctx
from cloudify.decorators import operation

@operation
def creation_validation(**_):
    for property_key in constants.STORAGE_ACCOUNT_REQUIRED_PROPERTIES:
        _validate_node_properties(property_key, ctx.node.properties)

    storage_account = _get_all_storage_account(
        utils.get_storage_account_name())

    if ctx.node.properties['use_external_resource'] and not storage_account:
        raise NonRecoverableError(
        'External resource, but the supplied '
        'storage account does not exist in the account.')
    if not ctx.node.properties['use_external_resource'] and storage_account:
        raise NonRecoverableError(
        'Not external resource, but the supplied '
        'storage account group exists in the account.')

@operation
def create_storage_account(**_):
    if ctx.node.properties['use_external_resource']
        if not resource_group:
            raise NonRecoverableError(
            'External resource, but the supplied '
            'storage account does not exist in the Azure account.')
            sys.exit(1)
        else
            ctx.instance.runtime_properties['existing_storage_account_name']
            return
    else
            location = ctx.node.properties['location']
            vm_name=ctx.node.properties['vm_name']
            RANDOM_SUFFIX_VALUE = utils.random_suffix_generator()
            storage_account_name = vm_name+'storageaccount'+RANDOM_SUFFIX_VALUE
            subscription_id = ctx.node.properties['subscription_id']

            credentials='Bearer '+auth.get_token_from_client_credentials()

            headers = {"Content-Type": "application/json", "Authorization": credentials}

            ctx.logger.info("Checking availability of storage account: " + storage_account_name)
            if 1:
                try:
                    ctx.logger.info("Creating new storage account: " + storage_account_name)
                    storage_account_url= constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/Microsoft.Storage/storageAccounts/'+storage_account_name+'?api-version='+constants.api_version
                    storage_account_params=json.dumps({"properties": {"accountType": constants.storage_account_type,}, "location": location})
                    response_sa = requests.put(url=storage_account_url, data=storage_account_params, headers=headers)
                    print response_sa.text
                    ctx.instance.runtime_properties['storage_account']=storage_account_name
                except:
                    ctx.logger.info("Storage Account " + storage_account_name + "could not be created.")
                    sys.exit(1)
            else:
                ctx.logger.info("Storage Account " + storage_account_name + "has already been provisioned by another user.")

@operation
def delete_storage_account(**_):
    vm_name=ctx.node.properties['vm_name']
    storage_account_name = vm_name+'storageaccount'
    resource_group_name = vm_name+'_resource_group'
    subscription_id = ctx.node.properties['subscription_id']

    credentials='Bearer '+auth.get_token_from_client_credentials()

    headers = {"Content-Type": "application/json", "Authorization": credentials}

    ctx.logger.info("Deleting Storage Account"+storage_account_name)
    if 1:
        try:
            storage_account_url=constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/Microsoft.Storage/storageAccounts/'+storage_account_name+'?api-version='+constants.api_version
            response_sa = requests.delete(url=storage_account_url,headers=headers)
            print response_sa.text

        except:
            ctx.logger.info("Storage Account " + storage_account_name + " could not be deleted.")
            sys.exit(1)
    else:
        ctx.logger.info("Storage Account " + storage_account_name + " does not exist.")

def _validate_node_properties(key, ctx_node_properties):
    if key not in ctx_node_properties:
        raise NonRecoverableError('{0} is a required input. Unable to create.'.format(key))

def _get_all_storage_account(storage_account_name):
    resource_group= ctx.node.properties['exsisting_resource_group_name']
    subscription_id=ctx.node.properties['subscription_id']
    url = constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group+'/providers/Microsoft.Storage/storageAccounts?api-version='+constants.api_version
    headers = {"Content-Type": "application/json", "Authorization": credentials}
    response_list = requests.get(url, headers = headers).json()
    print response_list['value']['name']
<commit_msg>Update storageaccount.py<commit_after>########
# Copyright (c) 2015 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

# Built-in Imports
import requests
from requests import Request,Session,Response
import json
import constants
import sys
import os
import auth
import utils
from cloudify.exceptions import NonRecoverableError
from cloudify import ctx
from cloudify.decorators import operation

@operation
def creation_validation(**_):
    for property_key in constants.STORAGE_ACCOUNT_REQUIRED_PROPERTIES:
        _validate_node_properties(property_key, ctx.node.properties)

    storage_account = _get_all_storage_account(
        utils.get_storage_account_name())

    if ctx.node.properties['use_external_resource'] and not storage_account:
        raise NonRecoverableError(
        'External resource, but the supplied '
        'storage account does not exist in the account.')
    if not ctx.node.properties['use_external_resource'] and storage_account:
        raise NonRecoverableError(
        'Not external resource, but the supplied '
        'storage account group exists in the account.')

@operation
def create_storage_account(**_):
    if ctx.node.properties['use_external_resource']
        if not resource_group:
            raise NonRecoverableError(
            'External resource, but the supplied '
            'storage account does not exist in the Azure account.')
            sys.exit(1)
        else
            ctx.instance.runtime_properties['existing_storage_account_name']
            return
    else
            location = ctx.node.properties['location']
            vm_name=ctx.node.properties['vm_name']
            RANDOM_SUFFIX_VALUE = utils.random_suffix_generator()
            storage_account_name = vm_name+'storageaccount'+RANDOM_SUFFIX_VALUE
            subscription_id = ctx.node.properties['subscription_id']

            credentials='Bearer '+auth.get_token_from_client_credentials()

            headers = {"Content-Type": "application/json", "Authorization": credentials}

            ctx.logger.info("Checking availability of storage account: " + storage_account_name)
            if 1:
                try:
                    ctx.logger.info("Creating new storage account: " + storage_account_name)
                    storage_account_url= constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/Microsoft.Storage/storageAccounts/'+storage_account_name+'?api-version='+constants.api_version
                    storage_account_params=json.dumps({"properties": {"accountType": constants.storage_account_type,}, "location": location})
                    response_sa = requests.put(url=storage_account_url, data=storage_account_params, headers=headers)
                    print response_sa.text
                    ctx.instance.runtime_properties['storage_account']=storage_account_name
                except:
                    ctx.logger.info("Storage Account " + storage_account_name + "could not be created.")
                    sys.exit(1)
            else:
                ctx.logger.info("Storage Account " + storage_account_name + "has already been provisioned by another user.")

@operation
def delete_storage_account(**_):
    vm_name=ctx.node.properties['vm_name']
    storage_account_name = vm_name+'storageaccount'
    resource_group_name = vm_name+'_resource_group'
    subscription_id = ctx.node.properties['subscription_id']

    credentials='Bearer '+auth.get_token_from_client_credentials()

    headers = {"Content-Type": "application/json", "Authorization": credentials}

    ctx.logger.info("Deleting Storage Account"+storage_account_name)
    if 1:
        try:
            storage_account_url=constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/Microsoft.Storage/storageAccounts/'+storage_account_name+'?api-version='+constants.api_version
            response_sa = requests.delete(url=storage_account_url,headers=headers)
            print response_sa.text

        except:
            ctx.logger.info("Storage Account " + storage_account_name + " could not be deleted.")
            sys.exit(1)
    else:
        ctx.logger.info("Storage Account " + storage_account_name + " does not exist.")

def _validate_node_properties(key, ctx_node_properties):
    if key not in ctx_node_properties:
        raise NonRecoverableError('{0} is a required input. Unable to create.'.format(key))

def _get_storage_account(storage_account_name):
    resource_group= ctx.node.properties['exsisting_resource_group_name']
    subscription_id=ctx.node.properties['subscription_id']
    url = constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group+'/providers/Microsoft.Storage/storageAccounts?api-version='+constants.api_version
    headers = {"Content-Type": "application/json", "Authorization": credentials}
    response_list = requests.get(url, headers = headers).json()
    print response_list['value']['name']
<|endoftext|>


<commit_before>"""
Encapsulates task scheduling.

enqueue() submits a task, and tasks are processed sequentially, in the order
in which they are submitted.
The actual task queue implementation is encapsulated (and hidden) in this
module.

Task types are mapped to task implementations by implForName, and those
implementations are executed sequentially.
"""

import datetime
from django.contrib.auth import get_user_model
from django.db import models
from django.utils.timezone import utc
import json
import logging
import signal
import sys
import time
import traceback

from futuschedule.models import (Task, Schedule, CalendarResource, EventTask,
        EventTemplate, SchedulingRequest, ScheduleTemplate, Event,
        DeletionTask, LastApiCall)
from futuschedule import calendar

logging.basicConfig()

def enqueue(taskType, modelId):
    """
    Enqueue a task with the given type and modelId parameter.

    The modelId is passed to the task implementation and should be the ID of a
    model object which tells the task what it must do (i.e. it holds the task's
    parameters).
    """
    # right now delegating to our own implementation because it was the fastest
    # thing to get running. Can easily replace with another task system.
    Task.objects.create(taskType=taskType, modelId=modelId)

def process(taskType, modelId):
    if taskType not in implForName:
        logging.error('Invalid task type ' + str(taskType))
        return
    handler = implForName[taskType]
    handler(modelId)

_exitLoop = False
def loop():
    """
    Loop forever looking for tasks and processing them in order.
    """
    global _exitLoop
    _exitLoop = False

    def sigtermHandler(signum, frame):
        global _exitLoop
        logging.info('Received signal, ' +
                'will end processing after current task completes')
        _exitLoop = True
    signal.signal(signal.SIGTERM, sigtermHandler)
    signal.signal(signal.SIGINT, sigtermHandler)

    while True:
        try:
            if _exitLoop:
                logging.info('Task processing loop exiting')
                return
            if not Task.objects.count():
                time.sleep(3)
                continue
            minId = Task.objects.aggregate(models.Min('id'))['id__min']
            t = Task.objects.get(id=minId)
            t.delete()
            process(t.taskType, t.modelId)
        except:
            # recover from an unexpected exception thrown by the task processor
            logging.error(traceback.format_exc())

def processSchedulingRequest(modelId):
    def makeSchedules():
        schedules = []
        for uid in body['users']:
            try:
                user = UM.objects.get(id=uid)
            except UM.DoesNotExist as e:
                continue
            schedules.append(Schedule.objects.create(
                schedulingRequest=schedReq, forUser=user, template=schedTempl))
        return schedules

    def makeEventTasks():
        for ev in body['events']:
            if ev['meta']['isCollective']:
                evSchedules = schedules
            else:
                evSchedules = [s for s in schedules
                        if s.forUser.id == ev['meta']['forUser']]
            if not evSchedules:
                logging.error('No schedules for event: ' + json.dumps(ev))
                continue

            invitees = list(UM.objects.filter(id__in=ev['data']['invitees']))
            if not invitees:
                logging.error('No invitees for event: ' + json.dumps(ev))
                continue

            rooms = list(CalendarResource.objects.filter(
                    id__in=ev['data']['locations']))
            d = datetime.datetime.strptime(ev['data']['date'],
                    '%Y-%m-%d').date()
            # [:5] drops seconds from 'HH:MM:SS' if present
            sTime = datetime.datetime.strptime(ev['data']['startTime'][:5],
                    '%H:%M').time()
            eTime = datetime.datetime.strptime(ev['data']['endTime'][:5],
                    '%H:%M').time()
            startDt = datetime.datetime.combine(d, sTime).replace(tzinfo=utc)
            endDt = datetime.datetime.combine(d, eTime).replace(tzinfo=utc)

            evTempl = None
            try:
                evTempl = EventTemplate.objects.get(
                        id=ev['data']['eventTemplate'])
            except EventTemplate.DoesNotExist as e:
                pass

            evTask = EventTask.objects.create(summary=ev['data']['summary'],
                    description=ev['data']['description'],
                    startDt=startDt, endDt=endDt, template=evTempl)
            evTask.locations.add(*rooms)
            evTask.attendees.add(*invitees)
            evTask.schedules.add(*evSchedules)
            enqueue(EVENT_TASK, evTask.id)

    try:
        UM = get_user_model()
        schedReq = SchedulingRequest.objects.get(id=modelId)
        body = json.loads(schedReq.json)
        schedTempl = ScheduleTemplate.objects.get(
            id=body['scheduleTemplate'])
        schedules = makeSchedules()
        makeEventTasks()
        enqueue(MARK_SCHED_REQ_SUCCESS, modelId)
    except:
        logging.error(traceback.format_exc())
        failSchedulingRequest(modelId, traceback.format_exc())

def processEventTask(modelId):
    evTask = EventTask.objects.get(id=modelId)
    schedReq = None
    try:
        schedules = evTask.schedules.all()
        if not schedules:
            raise Exception('EventTask to create event for 0 schedules')

        schedReq = schedules[0].schedulingRequest
        if schedReq.status != SchedulingRequest.IN_PROGRESS:
            logging.warning('Drop EventTask: SchedulingRequest status is '
                    + schedReq.status)
            return

        rooms = evTask.locations.all()
        locTxt = ', '.join(r.name for r in rooms)
        attendingEmails = [u.email for u in evTask.attendees.all()]
        for r in rooms:
            attendingEmails.append(r.email)

        # TODO: exception handling, Google API retry logic

        schedTempl = schedules[0].template
        if not schedTempl:
            raise Exception('Drop EventTask because schedule template was ' +
                    'deleted and we can\'t find out the timezone')

        sleepForRateLimit()
        gCalJson = calendar.createEvent(schedTempl.calendar.email, True,
                evTask.summary, evTask.description,
                locTxt, evTask.startDt, evTask.endDt,
                schedTempl.timezone.name, attendingEmails)
        newEv = Event.objects.create(json=json.dumps(gCalJson),
                template=evTask.template)
        newEv.schedules.add(*schedules)
    except:
        logging.error(traceback.format_exc())
        if schedReq:
            failSchedulingRequest(schedReq.id, traceback.format_exc())
    finally:
        evTask.delete()

def processMarkSchedReqSuccess(modelId):
    """
    Mark scheduling request status as SUCCESS if status is IN_PROGRESS.

    This gets submitted after all the event creation tasks. When this runs,
    if the scheduling request's status is still marked as in progress, than
    everything ran successfully so far and didn't mark it as failed, so this
    task sets it as successful. Otherwise this task does nothing.
    """
    schReq = SchedulingRequest.objects.get(id=modelId)
    if schReq.status == SchedulingRequest.IN_PROGRESS:
        schReq.status = SchedulingRequest.SUCCESS
        schReq.save()

def processCleanupSchedulingRequest(modelId):
    """
    Delete what got created in Google Calendar, our Events and Schedules.

    Either something went wrong or the user decided to delete an entire
    scheduling request. In either case, this task is set to roll-back and
    delete objects.
    """
    schReq = SchedulingRequest.objects.get(id=modelId)
    for schedule in schReq.schedule_set.all():
        for event in schedule.event_set.all():
            sleepForRateLimit()
            try:
                evData = json.loads(event.json)
                # Getting the calendar ID like this might be fragile. If so,
                # in the future we can add a foreign key from the Event to a
                # Calendar. But for the way we're making calendar events now,
                # jsonData.organizer.email is the calendar ID.
                calendar.deleteEvent(evData['organizer']['email'], evData['id'])
            except:
                logging.error(traceback.format_exc())
            event.delete()
        schedule.delete()

def failSchedulingRequest(modelId, errTxt=''):
    sr = SchedulingRequest.objects.get(id=modelId)
    # allow this function to be called multiple times
    if sr.status != SchedulingRequest.ERROR:
        sr.status = SchedulingRequest.ERROR
        if errTxt:
            sr.error = errTxt
        sr.save()
        enqueue(CLEANUP_SCHED_REQ, modelId)

def processDeletionTask(modelId):
    """
    Process the user's request to delete a scheduling request.

    This may come in at any point (while still processing the scheduling
    request, after completing with success or after failing with an error).
    """
    deleteTask = DeletionTask.objects.get(id=modelId)
    try:
        if not deleteTask.schedReq:
            logging.warning('Found task to delete a scheduling request ' +
                    'which doesn\'t exist (anymore).')
            return
        user = deleteTask.requestedByUser
        logging.info('Deleting a scheduling request, as requested by ' +
                (user.username if user else 'Unknown User') + ' at ' +
                str(deleteTask.requestedAt))
        failSchedulingRequest(deleteTask.schedReq.id,
                'Deleting, as requested by user')
        enqueue(DELETE_SCHED_REQ, deleteTask.schedReq.id)
    finally:
        deleteTask.delete()

def processDeleteSchedulingRequest(modelId):
    sr = SchedulingRequest.objects.get(id=modelId)
    sr.delete()

def sleepForRateLimit():
    """
    Optionally sleep depending on when we last made an API call.

    Checks when the most recent Google API call was made, optionally sleeps,
    updates the most recent call time to 'right now' then returns.
    Because of this logic, this function must only be called from a single
    thread (e.g. the single thread processing the task queue sequentially).
    """

    if not LastApiCall.objects.count():
        LastApiCall.objects.create()
        return

    now = datetime.datetime.utcnow().replace(tzinfo=utc)
    last = LastApiCall.objects.get()
    minDelta = datetime.timedelta(seconds=1)
    delta = now-last.dt
    if delta < minDelta:
        toSleep = minDelta - delta
        time.sleep(toSleep.seconds + toSleep.microseconds/1000000.)
    last.save()

SCHED_REQ = 'scheduling-request'
EVENT_TASK = 'event-task'
MARK_SCHED_REQ_SUCCESS = 'mark-scheduling-request-successful'
CLEANUP_SCHED_REQ = 'cleanup-scheduling-request'
DELETION_TASK = 'deletion-task'
DELETE_SCHED_REQ = 'delete-scheduling-request'
implForName = {
        SCHED_REQ: processSchedulingRequest,
        EVENT_TASK: processEventTask,
        MARK_SCHED_REQ_SUCCESS: processMarkSchedReqSuccess,
        CLEANUP_SCHED_REQ: processCleanupSchedulingRequest,
        DELETION_TASK: processDeletionTask,
        DELETE_SCHED_REQ: processDeleteSchedulingRequest,
}
<commit_msg>trim EventTask summary<commit_after>"""
Encapsulates task scheduling.

enqueue() submits a task, and tasks are processed sequentially, in the order
in which they are submitted.
The actual task queue implementation is encapsulated (and hidden) in this
module.

Task types are mapped to task implementations by implForName, and those
implementations are executed sequentially.
"""

import datetime
from django.contrib.auth import get_user_model
from django.db import models
from django.utils.timezone import utc
import json
import logging
import signal
import sys
import time
import traceback

from futuschedule.models import (Task, Schedule, CalendarResource, EventTask,
        EventTemplate, SchedulingRequest, ScheduleTemplate, Event,
        DeletionTask, LastApiCall)
from futuschedule import calendar

logging.basicConfig()

def enqueue(taskType, modelId):
    """
    Enqueue a task with the given type and modelId parameter.

    The modelId is passed to the task implementation and should be the ID of a
    model object which tells the task what it must do (i.e. it holds the task's
    parameters).
    """
    # right now delegating to our own implementation because it was the fastest
    # thing to get running. Can easily replace with another task system.
    Task.objects.create(taskType=taskType, modelId=modelId)

def process(taskType, modelId):
    if taskType not in implForName:
        logging.error('Invalid task type ' + str(taskType))
        return
    handler = implForName[taskType]
    handler(modelId)

_exitLoop = False
def loop():
    """
    Loop forever looking for tasks and processing them in order.
    """
    global _exitLoop
    _exitLoop = False

    def sigtermHandler(signum, frame):
        global _exitLoop
        logging.info('Received signal, ' +
                'will end processing after current task completes')
        _exitLoop = True
    signal.signal(signal.SIGTERM, sigtermHandler)
    signal.signal(signal.SIGINT, sigtermHandler)

    while True:
        try:
            if _exitLoop:
                logging.info('Task processing loop exiting')
                return
            if not Task.objects.count():
                time.sleep(3)
                continue
            minId = Task.objects.aggregate(models.Min('id'))['id__min']
            t = Task.objects.get(id=minId)
            t.delete()
            process(t.taskType, t.modelId)
        except:
            # recover from an unexpected exception thrown by the task processor
            logging.error(traceback.format_exc())

def processSchedulingRequest(modelId):
    def makeSchedules():
        schedules = []
        for uid in body['users']:
            try:
                user = UM.objects.get(id=uid)
            except UM.DoesNotExist as e:
                continue
            schedules.append(Schedule.objects.create(
                schedulingRequest=schedReq, forUser=user, template=schedTempl))
        return schedules

    def makeEventTasks():
        for ev in body['events']:
            if ev['meta']['isCollective']:
                evSchedules = schedules
            else:
                evSchedules = [s for s in schedules
                        if s.forUser.id == ev['meta']['forUser']]
            if not evSchedules:
                logging.error('No schedules for event: ' + json.dumps(ev))
                continue

            invitees = list(UM.objects.filter(id__in=ev['data']['invitees']))
            if not invitees:
                logging.error('No invitees for event: ' + json.dumps(ev))
                continue

            rooms = list(CalendarResource.objects.filter(
                    id__in=ev['data']['locations']))
            d = datetime.datetime.strptime(ev['data']['date'],
                    '%Y-%m-%d').date()
            # [:5] drops seconds from 'HH:MM:SS' if present
            sTime = datetime.datetime.strptime(ev['data']['startTime'][:5],
                    '%H:%M').time()
            eTime = datetime.datetime.strptime(ev['data']['endTime'][:5],
                    '%H:%M').time()
            startDt = datetime.datetime.combine(d, sTime).replace(tzinfo=utc)
            endDt = datetime.datetime.combine(d, eTime).replace(tzinfo=utc)

            evTempl = None
            try:
                evTempl = EventTemplate.objects.get(
                        id=ev['data']['eventTemplate'])
            except EventTemplate.DoesNotExist as e:
                pass

            evTask = EventTask.objects.create(
                    summary=ev['data']['summary'][:200],
                    description=ev['data']['description'],
                    startDt=startDt, endDt=endDt, template=evTempl)
            evTask.locations.add(*rooms)
            evTask.attendees.add(*invitees)
            evTask.schedules.add(*evSchedules)
            enqueue(EVENT_TASK, evTask.id)

    try:
        UM = get_user_model()
        schedReq = SchedulingRequest.objects.get(id=modelId)
        body = json.loads(schedReq.json)
        schedTempl = ScheduleTemplate.objects.get(
            id=body['scheduleTemplate'])
        schedules = makeSchedules()
        makeEventTasks()
        enqueue(MARK_SCHED_REQ_SUCCESS, modelId)
    except:
        logging.error(traceback.format_exc())
        failSchedulingRequest(modelId, traceback.format_exc())

def processEventTask(modelId):
    evTask = EventTask.objects.get(id=modelId)
    schedReq = None
    try:
        schedules = evTask.schedules.all()
        if not schedules:
            raise Exception('EventTask to create event for 0 schedules')

        schedReq = schedules[0].schedulingRequest
        if schedReq.status != SchedulingRequest.IN_PROGRESS:
            logging.warning('Drop EventTask: SchedulingRequest status is '
                    + schedReq.status)
            return

        rooms = evTask.locations.all()
        locTxt = ', '.join(r.name for r in rooms)
        attendingEmails = [u.email for u in evTask.attendees.all()]
        for r in rooms:
            attendingEmails.append(r.email)

        # TODO: exception handling, Google API retry logic

        schedTempl = schedules[0].template
        if not schedTempl:
            raise Exception('Drop EventTask because schedule template was ' +
                    'deleted and we can\'t find out the timezone')

        sleepForRateLimit()
        gCalJson = calendar.createEvent(schedTempl.calendar.email, True,
                evTask.summary, evTask.description,
                locTxt, evTask.startDt, evTask.endDt,
                schedTempl.timezone.name, attendingEmails)
        newEv = Event.objects.create(json=json.dumps(gCalJson),
                template=evTask.template)
        newEv.schedules.add(*schedules)
    except:
        logging.error(traceback.format_exc())
        if schedReq:
            failSchedulingRequest(schedReq.id, traceback.format_exc())
    finally:
        evTask.delete()

def processMarkSchedReqSuccess(modelId):
    """
    Mark scheduling request status as SUCCESS if status is IN_PROGRESS.

    This gets submitted after all the event creation tasks. When this runs,
    if the scheduling request's status is still marked as in progress, than
    everything ran successfully so far and didn't mark it as failed, so this
    task sets it as successful. Otherwise this task does nothing.
    """
    schReq = SchedulingRequest.objects.get(id=modelId)
    if schReq.status == SchedulingRequest.IN_PROGRESS:
        schReq.status = SchedulingRequest.SUCCESS
        schReq.save()

def processCleanupSchedulingRequest(modelId):
    """
    Delete what got created in Google Calendar, our Events and Schedules.

    Either something went wrong or the user decided to delete an entire
    scheduling request. In either case, this task is set to roll-back and
    delete objects.
    """
    schReq = SchedulingRequest.objects.get(id=modelId)
    for schedule in schReq.schedule_set.all():
        for event in schedule.event_set.all():
            sleepForRateLimit()
            try:
                evData = json.loads(event.json)
                # Getting the calendar ID like this might be fragile. If so,
                # in the future we can add a foreign key from the Event to a
                # Calendar. But for the way we're making calendar events now,
                # jsonData.organizer.email is the calendar ID.
                calendar.deleteEvent(evData['organizer']['email'], evData['id'])
            except:
                logging.error(traceback.format_exc())
            event.delete()
        schedule.delete()

def failSchedulingRequest(modelId, errTxt=''):
    sr = SchedulingRequest.objects.get(id=modelId)
    # allow this function to be called multiple times
    if sr.status != SchedulingRequest.ERROR:
        sr.status = SchedulingRequest.ERROR
        if errTxt:
            sr.error = errTxt
        sr.save()
        enqueue(CLEANUP_SCHED_REQ, modelId)

def processDeletionTask(modelId):
    """
    Process the user's request to delete a scheduling request.

    This may come in at any point (while still processing the scheduling
    request, after completing with success or after failing with an error).
    """
    deleteTask = DeletionTask.objects.get(id=modelId)
    try:
        if not deleteTask.schedReq:
            logging.warning('Found task to delete a scheduling request ' +
                    'which doesn\'t exist (anymore).')
            return
        user = deleteTask.requestedByUser
        logging.info('Deleting a scheduling request, as requested by ' +
                (user.username if user else 'Unknown User') + ' at ' +
                str(deleteTask.requestedAt))
        failSchedulingRequest(deleteTask.schedReq.id,
                'Deleting, as requested by user')
        enqueue(DELETE_SCHED_REQ, deleteTask.schedReq.id)
    finally:
        deleteTask.delete()

def processDeleteSchedulingRequest(modelId):
    sr = SchedulingRequest.objects.get(id=modelId)
    sr.delete()

def sleepForRateLimit():
    """
    Optionally sleep depending on when we last made an API call.

    Checks when the most recent Google API call was made, optionally sleeps,
    updates the most recent call time to 'right now' then returns.
    Because of this logic, this function must only be called from a single
    thread (e.g. the single thread processing the task queue sequentially).
    """

    if not LastApiCall.objects.count():
        LastApiCall.objects.create()
        return

    now = datetime.datetime.utcnow().replace(tzinfo=utc)
    last = LastApiCall.objects.get()
    minDelta = datetime.timedelta(seconds=1)
    delta = now-last.dt
    if delta < minDelta:
        toSleep = minDelta - delta
        time.sleep(toSleep.seconds + toSleep.microseconds/1000000.)
    last.save()

SCHED_REQ = 'scheduling-request'
EVENT_TASK = 'event-task'
MARK_SCHED_REQ_SUCCESS = 'mark-scheduling-request-successful'
CLEANUP_SCHED_REQ = 'cleanup-scheduling-request'
DELETION_TASK = 'deletion-task'
DELETE_SCHED_REQ = 'delete-scheduling-request'
implForName = {
        SCHED_REQ: processSchedulingRequest,
        EVENT_TASK: processEventTask,
        MARK_SCHED_REQ_SUCCESS: processMarkSchedReqSuccess,
        CLEANUP_SCHED_REQ: processCleanupSchedulingRequest,
        DELETION_TASK: processDeletionTask,
        DELETE_SCHED_REQ: processDeleteSchedulingRequest,
}
<|endoftext|>


<commit_before>from __future__ import print_function
from __future__ import absolute_import
from sfepy import data_dir

filename_mesh = data_dir + '/meshes/2d/special/circle_in_square.mesh'

dim = 2

fields = {
    'scalar_field' : ('real', 'scalar', 'Omega', 1),
    'vector_field' : ('real', 'vector', 'Omega', 1),
}

variables = {
    'us'  : ('unknown field',   'scalar_field', 0),
    'ts'  : ('test field',      'scalar_field', 'us'),
    'ps1' : ('parameter field', 'scalar_field', 'us'),
    'ps2' : ('parameter field', 'scalar_field', 'us'),
    'uv'  : ('unknown field',   'vector_field', 1),
    'tv'  : ('test field',      'vector_field', 'uv'),
    'pv1' : ('parameter field', 'vector_field', 'uv'),
    'pv2' : ('parameter field', 'vector_field', 'uv'),
}

regions = {
    'Omega' : 'all',
    'Left' : ('vertices in (x < -0.499)', 'facet'),
}

integrals = {
    'i' : 2,
}

materials = {
    'm' : 'get_pars',
    'm2' : ({'K' : [[3.0, 0.1], [0.3, 1.0]]},),
}

equations = {
    'eq' : """dw_diffusion.i.Omega( m2.K, ts, us ) = 0"""
}

def get_pars(ts, coor, mode=None, term=None, **kwargs):
    if mode == 'qp':
        n_nod, dim = coor.shape
        sym = (dim + 1) * dim // 2

        if 'biot' in term.name:
            val = nm.zeros((sym, 1), dtype=nm.float64)
            val[:dim] = 0.132
            val[dim:sym] = 0.092
        elif '_dot' in term.name:
            val = 1.0 / nm.array([3.8], dtype=nm.float64)
        elif 'diffusion' in term.name:
            val = nm.eye(dim, dtype=nm.float64)
        else:
            raise ValueError

        return {'val' : nm.tile(val, (coor.shape[0], 1, 1))}

functions = {
    'get_pars' : (get_pars,),
}

# (eval term prefix, parameter corresponding to test variable, 'd' variables,
# 'dw' variables (test must be paired with unknown, which should be at
# index 2!), mat mode)
test_terms = [
    ('%s_biot.i.Omega( m.val, %s, %s )',
     ('dw', 'ps1', ('pv1', 'ps1'), ('pv1', 'ts', 'us', 'uv', 'tv'))),
    ('%s_biot.i.Omega( m.val, %s, %s )',
     ('dw', 'pv1', ('pv1', 'ps1'), ('tv', 'ps1', 'uv', 'us', 'ts'))),
    ('%s_diffusion.i.Omega( m.val, %s, %s )',
     ('dw', 'ps1', ('ps1', 'ps2'), ('ts', 'ps1', 'us'))),
    ('%s_dot.i.Omega( m.val, %s, %s )',
     ('dw', 'ps1', ('ps1', 'ps2'), ('ts', 'ps1', 'us'))),
]

import numpy as nm
from sfepy.base.testing import TestCommon

def _integrate(var, val_qp):
    from sfepy.discrete import Integral
    from sfepy.discrete.common.mappings import get_jacobian

    integral = Integral('i', 2)
    det = get_jacobian(var.field, integral)
    val = (val_qp * det).sum(axis=1) / det.sum(axis=1)

    return val

class Test(TestCommon):

    @staticmethod
    def from_conf(conf, options):
        from sfepy.discrete import Problem

        problem = Problem.from_conf(conf, init_equations=False)
        test = Test(problem=problem,
                    conf=conf, options=options)
        return test

    def test_consistency_d_dw(self):
        from sfepy.discrete import Variables

        ok = True
        pb = self.problem
        for aux in test_terms:
            term_template, (prefix, par_name, d_vars, dw_vars) = aux
            print(term_template, prefix, par_name, d_vars, dw_vars)

            term1 = term_template % ((prefix,) + d_vars)

            variables = Variables.from_conf(self.conf.variables, pb.fields)

            for var_name in d_vars:
                var = variables[var_name]
                n_dof = var.field.n_nod * var.field.shape[0]
                aux = nm.arange(n_dof, dtype=nm.float64)
                var.set_data(aux)

            if prefix == 'd':
                val1 = pb.evaluate(term1, var_dict=variables.as_dict())

            else:
                val1 = pb.evaluate(term1, call_mode='d_eval',
                                   var_dict=variables.as_dict())

            self.report('%s: %s' % (term1, val1))

            term2 = term_template % (('dw',) + dw_vars[:2])

            vec, vv = pb.evaluate(term2, mode='weak',
                                  var_dict=variables.as_dict(),
                                  ret_variables=True)

            pvec = vv.get_vec_part(vec, dw_vars[2])
            val2 = nm.dot(variables[par_name](), pvec)
            self.report('%s: %s' % (term2, val2))

            err = nm.abs(val1 - val2) / nm.abs(val1)
            _ok = err < 1e-12
            self.report('relative difference: %e -> %s' % (err, _ok))

            ok = ok and _ok

        return ok

    def test_eval_matrix(self):
        problem = self.problem

        problem.set_equations()
        problem.time_update(ebcs={}, epbcs={})

        var = problem.get_variables()['us']

        vec = nm.arange(var.n_dof, dtype=var.dtype)

        var.set_data(vec)

        val1 = problem.evaluate('dw_diffusion.i.Omega( m2.K, us, us )',
                                mode='eval', us=var)

        mtx = problem.evaluate('dw_diffusion.i.Omega( m2.K, ts, us )',
                               mode='weak', dw_mode='matrix')

        val2 = nm.dot(vec, mtx * vec)

        ok = (nm.abs(val1 - val2) / nm.abs(val1)) < 1e-14
        self.report('eval: %s, weak: %s, ok: %s' % (val1, val2, ok))

        return ok

    def test_vector_matrix(self):
        problem = self.problem

        problem.set_equations()
        problem.time_update()

        state = problem.create_state()
        state.apply_ebc()

        aux1 = problem.evaluate("dw_diffusion.i.Omega( m2.K, ts, us )",
                                mode='weak', dw_mode='vector')

        problem.time_update(ebcs={}, epbcs={})

        mtx = problem.evaluate("dw_diffusion.i.Omega( m2.K, ts, us )",
                               mode='weak', dw_mode='matrix')
        aux2g = mtx * state()
        problem.time_update(ebcs=self.conf.ebcs,
                            epbcs=self.conf.epbcs)
        aux2 = problem.equations.reduce_vec(aux2g, follow_epbc=True)

        ret = self.compare_vectors(aux1, aux2,
                                   label1='vector mode',
                                   label2='matrix mode')
        if not ret:
            self.report('failed')

        return ret

    def test_surface_evaluate(self):
        from sfepy.discrete import FieldVariable
        problem = self.problem

        us = problem.get_variables()['us']
        vec = nm.empty(us.n_dof, dtype=us.dtype)
        vec[:] = 1.0
        us.set_data(vec)

        expr = 'ev_integrate.i.Left( us )'
        val = problem.evaluate(expr, us=us)
        ok1 = nm.abs(val - 1.0) < 1e-15
        self.report('with unknown: %s, value: %s, ok: %s'
                    % (expr, val, ok1))

        ps1 = FieldVariable('ps1', 'parameter', us.get_field(),
                            primary_var_name='(set-to-None)')
        ps1.set_data(vec)

        expr = 'ev_integrate.i.Left( ps1 )'
        val = problem.evaluate(expr, ps1=ps1)
        ok2 = nm.abs(val - 1.0) < 1e-15
        self.report('with parameter: %s, value: %s, ok: %s'
                    % (expr, val, ok2))
        ok2 = True

        return ok1 and ok2

    def test_ev_grad(self):
        problem = self.problem

        var = problem.create_variables(['us'])['us']
        val = nm.arange(var.n_dof, dtype=var.dtype)
        var.set_data(val)

        val1 = problem.evaluate('ev_grad.i.Omega( us )', us=var, mode='el_avg')
        self.report('ev_grad(el_avg): min, max:', val1.min(), val1.max())

        aux = problem.evaluate('ev_grad.i.Omega( us )', us=var, mode='qp')
        val2 = _integrate(var, aux)
        val2.shape = val1.shape
        self.report('ev_grad(qp): min, max:', val2.min(), val2.max())

        ok = self.compare_vectors(val1, val2,
                                  label1='de mode',
                                  label2='dq mode')
        if not ok:
            self.report('failed')

        return ok

    def test_ev_div(self):
        problem = self.problem

        var = problem.create_variables(['uv'])['uv']
        val = nm.arange(var.n_dof, dtype=var.dtype)
        var.set_data(val)

        val1 = problem.evaluate('ev_div.i.Omega( uv )', uv=var, mode='el_avg')
        self.report('ev_div(el_avg): min, max:', val1.min(), val1.max())

        aux = problem.evaluate('ev_div.i.Omega( uv )', uv=var, mode='qp')
        val2 = _integrate(var, aux)
        val2.shape = val1.shape
        self.report('ev_div(qp): min, max:', val2.min(), val2.max())

        ok = self.compare_vectors(val1, val2,
                                  label1='de mode',
                                  label2='dq mode')
        if not ok:
            self.report('failed')

        return ok
<commit_msg>update test_term_consistency.py for pytest, fix test_surface_evaluate()<commit_after>import pytest

from sfepy import data_dir
import sfepy.base.testing as tst

filename_mesh = data_dir + '/meshes/2d/special/circle_in_square.mesh'

dim = 2

fields = {
    'scalar_field' : ('real', 'scalar', 'Omega', 1),
    'vector_field' : ('real', 'vector', 'Omega', 1),
}

variables = {
    'us'  : ('unknown field',   'scalar_field', 0),
    'ts'  : ('test field',      'scalar_field', 'us'),
    'ps1' : ('parameter field', 'scalar_field', 'us'),
    'ps2' : ('parameter field', 'scalar_field', 'us'),
    'uv'  : ('unknown field',   'vector_field', 1),
    'tv'  : ('test field',      'vector_field', 'uv'),
    'pv1' : ('parameter field', 'vector_field', 'uv'),
    'pv2' : ('parameter field', 'vector_field', 'uv'),
}

regions = {
    'Omega' : 'all',
    'Left' : ('vertices in (x < -0.499)', 'facet'),
}

integrals = {
    'i' : 2,
}

materials = {
    'm' : 'get_pars',
    'm2' : ({'K' : [[3.0, 0.1], [0.3, 1.0]]},),
}

equations = {
    'eq' : """dw_diffusion.i.Omega(m2.K, ts, us) = 0"""
}

def get_pars(ts, coor, mode=None, term=None, **kwargs):
    if mode == 'qp':
        n_nod, dim = coor.shape
        sym = (dim + 1) * dim // 2

        if 'biot' in term.name:
            val = nm.zeros((sym, 1), dtype=nm.float64)
            val[:dim] = 0.132
            val[dim:sym] = 0.092
        elif '_dot' in term.name:
            val = 1.0 / nm.array([3.8], dtype=nm.float64)
        elif 'diffusion' in term.name:
            val = nm.eye(dim, dtype=nm.float64)
        else:
            raise ValueError

        return {'val' : nm.tile(val, (coor.shape[0], 1, 1))}

functions = {
    'get_pars' : (get_pars,),
}

# (eval term prefix, parameter corresponding to test variable, 'd' variables,
# 'dw' variables (test must be paired with unknown, which should be at
# index 2!), mat mode)
test_terms = [
    ('%s_biot.i.Omega(m.val, %s, %s)',
     ('dw', 'ps1', ('pv1', 'ps1'), ('pv1', 'ts', 'us', 'uv', 'tv'))),
    ('%s_biot.i.Omega(m.val, %s, %s)',
     ('dw', 'pv1', ('pv1', 'ps1'), ('tv', 'ps1', 'uv', 'us', 'ts'))),
    ('%s_diffusion.i.Omega(m.val, %s, %s)',
     ('dw', 'ps1', ('ps1', 'ps2'), ('ts', 'ps1', 'us'))),
    ('%s_dot.i.Omega(m.val, %s, %s)',
     ('dw', 'ps1', ('ps1', 'ps2'), ('ts', 'ps1', 'us'))),
]

import numpy as nm

def _integrate(var, val_qp):
    from sfepy.discrete import Integral
    from sfepy.discrete.common.mappings import get_jacobian

    integral = Integral('i', 2)
    det = get_jacobian(var.field, integral)
    val = (val_qp * det).sum(axis=1) / det.sum(axis=1)

    return val

@pytest.fixture(scope='module')
def problem():
    import sys
    from sfepy.discrete import Problem
    from sfepy.base.conf import ProblemConf

    conf = ProblemConf.from_dict(globals(), sys.modules[__name__])

    problem = Problem.from_conf(conf, init_equations=False)
    return problem

def test_consistency_d_dw(problem):
    from sfepy.discrete import Variables

    ok = True
    for aux in test_terms:
        term_template, (prefix, par_name, d_vars, dw_vars) = aux
        tst.report(term_template, prefix, par_name, d_vars, dw_vars)

        term1 = term_template % ((prefix,) + d_vars)

        variables = Variables.from_conf(problem.conf.variables, problem.fields)

        for var_name in d_vars:
            var = variables[var_name]
            n_dof = var.field.n_nod * var.field.shape[0]
            aux = nm.arange(n_dof, dtype=nm.float64)
            var.set_data(aux)

        if prefix == 'd':
            val1 = problem.evaluate(term1, var_dict=variables.as_dict())

        else:
            val1 = problem.evaluate(term1, call_mode='d_eval',
                                    var_dict=variables.as_dict())

        tst.report('%s: %s' % (term1, val1))

        term2 = term_template % (('dw',) + dw_vars[:2])

        vec, vv = problem.evaluate(term2, mode='weak',
                                   var_dict=variables.as_dict(),
                                   ret_variables=True)

        pvec = vv.get_vec_part(vec, dw_vars[2])
        val2 = nm.dot(variables[par_name](), pvec)
        tst.report('%s: %s' % (term2, val2))

        err = nm.abs(val1 - val2) / nm.abs(val1)
        _ok = err < 1e-12
        tst.report('relative difference: %e -> %s' % (err, _ok))

        ok = ok and _ok

    assert ok

def test_eval_matrix(problem):
    problem.set_equations()
    problem.time_update(ebcs={}, epbcs={})

    var = problem.get_variables()['us']

    vec = nm.arange(var.n_dof, dtype=var.dtype)
    var.set_data(vec)

    val1 = problem.evaluate('dw_diffusion.i.Omega(m2.K, us, us)',
                            mode='eval', us=var)

    mtx = problem.evaluate('dw_diffusion.i.Omega(m2.K, ts, us)',
                           mode='weak', dw_mode='matrix')

    val2 = nm.dot(vec, mtx * vec)

    ok = (nm.abs(val1 - val2) / nm.abs(val1)) < 1e-14
    tst.report('eval: %s, weak: %s, ok: %s' % (val1, val2, ok))

    assert ok

def test_vector_matrix(problem):
    problem.set_equations()
    problem.time_update()

    state = problem.create_state()
    state.apply_ebc()

    aux1 = problem.evaluate("dw_diffusion.i.Omega(m2.K, ts, us)",
                            mode='weak', dw_mode='vector')

    problem.time_update(ebcs={}, epbcs={})

    mtx = problem.evaluate("dw_diffusion.i.Omega(m2.K, ts, us)",
                           mode='weak', dw_mode='matrix')
    aux2g = mtx * state()
    problem.time_update(ebcs=problem.conf.ebcs,
                        epbcs=problem.conf.epbcs)
    aux2 = problem.equations.reduce_vec(aux2g, follow_epbc=True)

    ok = tst.compare_vectors(aux1, aux2,
                             label1='vector mode',
                             label2='matrix mode')
    if not ok:
        tst.report('failed')

    assert ok

def test_surface_evaluate(problem):
    from sfepy.discrete import FieldVariable

    us = problem.create_variables(['us'])['us']
    vec = nm.ones(us.n_dof, dtype=us.dtype)
    us.set_data(vec)

    expr = 'ev_integrate.i.Left(us)'
    val = problem.evaluate(expr, us=us)
    ok1 = nm.abs(val - 1.0) < 1e-15
    tst.report('with unknown: %s, value: %s, ok: %s'
               % (expr, val, ok1))

    ps1 = FieldVariable('ps1', 'parameter', us.get_field(),
                        primary_var_name='(set-to-None)')
    ps1.set_data(vec)

    expr = 'ev_integrate.i.Left(ps1)'
    val = problem.evaluate(expr, ps1=ps1)
    ok2 = nm.abs(val - 1.0) < 1e-15
    tst.report('with parameter: %s, value: %s, ok: %s'
               % (expr, val, ok2))

    assert ok1 and ok2

def test_ev_grad(problem):
    var = problem.create_variables(['us'])['us']
    val = nm.arange(var.n_dof, dtype=var.dtype)
    var.set_data(val)

    val1 = problem.evaluate('ev_grad.i.Omega(us)', us=var, mode='el_avg')
    tst.report('ev_grad(el_avg): min, max:', val1.min(), val1.max())

    aux = problem.evaluate('ev_grad.i.Omega(us)', us=var, mode='qp')
    val2 = _integrate(var, aux)
    val2.shape = val1.shape
    tst.report('ev_grad(qp): min, max:', val2.min(), val2.max())

    ok = tst.compare_vectors(val1, val2,
                             label1='de mode',
                             label2='dq mode')
    if not ok:
        tst.report('failed')

    assert ok

def test_ev_div(problem):
    var = problem.create_variables(['uv'])['uv']
    val = nm.arange(var.n_dof, dtype=var.dtype)
    var.set_data(val)

    val1 = problem.evaluate('ev_div.i.Omega(uv)', uv=var, mode='el_avg')
    tst.report('ev_div(el_avg): min, max:', val1.min(), val1.max())

    aux = problem.evaluate('ev_div.i.Omega(uv)', uv=var, mode='qp')
    val2 = _integrate(var, aux)
    val2.shape = val1.shape
    tst.report('ev_div(qp): min, max:', val2.min(), val2.max())

    ok = tst.compare_vectors(val1, val2,
                             label1='de mode',
                             label2='dq mode')
    if not ok:
        tst.report('failed')

    assert ok
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2013 IBM Corp
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
from tempest.test import attr
from tempest.tests.compute import base

class MultipleCreateTestJSON(base.BaseComputeTest):
    _interface = 'json'
    _name = 'multiple-create-test'

    def _get_created_servers(self, name):
        """Get servers created which name match with name param."""
        resp, body = self.servers_client.list_servers()
        servers = body['servers']
        servers_created = []
        for server in servers:
            if server['name'].startswith(name):
                servers_created.append(server)
        return servers_created

    def _generate_name(self):
        return rand_name(self._name)

    def _create_multiple_servers(self, name=None, wait_until=None, **kwargs):
        """
        This is the right way to create_multiple servers and manage to get the
        created servers into the servers list to be cleaned up after all.
        """
        kwargs['name'] = kwargs.get('name', self._generate_name())
        resp, body = self.create_server(**kwargs)
        created_servers = self._get_created_servers(kwargs['name'])
        # NOTE(maurosr): append it to cls.servers list from base.BaseCompute
        # class.
        self.servers.append(created_servers)
        # NOTE(maurosr): get a server list, check status of the ones with names
        # that match and wait for them become active. At a first look, since
        # they are building in parallel, wait inside the for doesn't seem be
        # harmful to the performance
        if wait_until is not None:
            for server in created_servers:
                self.servers_client.wait_for_server_status(server['id'],
                                                           wait_until)

        return resp, body

    @attr(type='positive')
    def test_multiple_create(self):
        resp, body = self._create_multiple_servers(wait_until='ACTIVE',
                                                   min_count=1,
                                                   max_count=2)
        # NOTE(maurosr): do status response check and also make sure that
        # reservation_id is not in the response body when the request send
        # contains return_reservation_id=False
        self.assertEqual('202', resp['status'])
        self.assertFalse('reservation_id' in body)

    @attr(type='negative')
    def test_min_count_less_than_one(self):
        invalid_min_count = 0
        self.assertRaises(exceptions.BadRequest, self._create_multiple_servers,
                          min_count=invalid_min_count)

    @attr(type='negative')
    def test_min_count_non_integer(self):
        invalid_min_count = 2.5
        self.assertRaises(exceptions.BadRequest, self._create_multiple_servers,
                          min_count=invalid_min_count)

    @attr(type='negative')
    def test_max_count_less_than_one(self):
        invalid_max_count = 0
        self.assertRaises(exceptions.BadRequest, self._create_multiple_servers,
                          max_count=invalid_max_count)

    @attr(type='negative')
    def test_max_count_non_integer(self):
        invalid_max_count = 2.5
        self.assertRaises(exceptions.BadRequest, self._create_multiple_servers,
                          max_count=invalid_max_count)

    @attr(type='negative')
    def test_max_count_less_than_min_count(self):
        min_count = 3
        max_count = 2
        self.assertRaises(exceptions.BadRequest, self._create_multiple_servers,
                          min_count=min_count,
                          max_count=max_count)

    @attr(type='positive')
    def test_multiple_create_with_reservation_return(self):
        resp, body = self._create_multiple_servers(wait_until='ACTIVE',
                                                   min_count=1,
                                                   max_count=2,
                                                   return_reservation_id=True)
        self.assertTrue(resp['status'], 202)
        self.assertIn('reservation_id', body)

class MultipleCreateTestXML(MultipleCreateTestJSON):
    _interface = 'xml'
<commit_msg>Clean up servers created in test_multiple_create.py<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2013 IBM Corp
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
from tempest.test import attr
from tempest.tests.compute import base

class MultipleCreateTestJSON(base.BaseComputeTest):
    _interface = 'json'
    _name = 'multiple-create-test'

    def _get_created_servers(self, name):
        """Get servers created which name match with name param."""
        resp, body = self.servers_client.list_servers()
        servers = body['servers']
        servers_created = []
        for server in servers:
            if server['name'].startswith(name):
                servers_created.append(server)
        return servers_created

    def _generate_name(self):
        return rand_name(self._name)

    def _create_multiple_servers(self, name=None, wait_until=None, **kwargs):
        """
        This is the right way to create_multiple servers and manage to get the
        created servers into the servers list to be cleaned up after all.
        """
        kwargs['name'] = kwargs.get('name', self._generate_name())
        resp, body = self.create_server(**kwargs)
        created_servers = self._get_created_servers(kwargs['name'])
        # NOTE(maurosr): append it to cls.servers list from base.BaseCompute
        # class.
        self.servers.extend(created_servers)
        # NOTE(maurosr): get a server list, check status of the ones with names
        # that match and wait for them become active. At a first look, since
        # they are building in parallel, wait inside the for doesn't seem be
        # harmful to the performance
        if wait_until is not None:
            for server in created_servers:
                self.servers_client.wait_for_server_status(server['id'],
                                                           wait_until)

        return resp, body

    @attr(type='positive')
    def test_multiple_create(self):
        resp, body = self._create_multiple_servers(wait_until='ACTIVE',
                                                   min_count=1,
                                                   max_count=2)
        # NOTE(maurosr): do status response check and also make sure that
        # reservation_id is not in the response body when the request send
        # contains return_reservation_id=False
        self.assertEqual('202', resp['status'])
        self.assertFalse('reservation_id' in body)

    @attr(type='negative')
    def test_min_count_less_than_one(self):
        invalid_min_count = 0
        self.assertRaises(exceptions.BadRequest, self._create_multiple_servers,
                          min_count=invalid_min_count)

    @attr(type='negative')
    def test_min_count_non_integer(self):
        invalid_min_count = 2.5
        self.assertRaises(exceptions.BadRequest, self._create_multiple_servers,
                          min_count=invalid_min_count)

    @attr(type='negative')
    def test_max_count_less_than_one(self):
        invalid_max_count = 0
        self.assertRaises(exceptions.BadRequest, self._create_multiple_servers,
                          max_count=invalid_max_count)

    @attr(type='negative')
    def test_max_count_non_integer(self):
        invalid_max_count = 2.5
        self.assertRaises(exceptions.BadRequest, self._create_multiple_servers,
                          max_count=invalid_max_count)

    @attr(type='negative')
    def test_max_count_less_than_min_count(self):
        min_count = 3
        max_count = 2
        self.assertRaises(exceptions.BadRequest, self._create_multiple_servers,
                          min_count=min_count,
                          max_count=max_count)

    @attr(type='positive')
    def test_multiple_create_with_reservation_return(self):
        resp, body = self._create_multiple_servers(wait_until='ACTIVE',
                                                   min_count=1,
                                                   max_count=2,
                                                   return_reservation_id=True)
        self.assertTrue(resp['status'], 202)
        self.assertIn('reservation_id', body)

class MultipleCreateTestXML(MultipleCreateTestJSON):
    _interface = 'xml'
<|endoftext|>


<commit_before>import unittest
import string
from scrapy.contrib_exp.newitem.adaptors import adaptor, ItemAdaptor
from scrapy.contrib_exp.newitem import *

class BaseItem(Item):
    name = StringField()

class TestItem(BaseItem):
    url = StringField()
    summary = StringField()

class BaseAdaptor(ItemAdaptor):
    item_class = TestItem

class TestAdaptor(BaseAdaptor):
    name = lambda v: v.title()

class DefaultedAdaptor(BaseAdaptor):
    default_adaptor = lambda v: v.title()

class InheritDefaultAdaptor(DefaultedAdaptor):
    pass

class MultiValuedTestItem(Item):
    names = MultiValuedField(StringField)

class MultiValuedItemAdaptor(ItemAdaptor):
    item_class = MultiValuedTestItem

    names = adaptor(lambda v: v.title())

class ItemAdaptorTest(unittest.TestCase):

    def test_basic(self):
        ia = TestAdaptor()
        ia.name = 'marta'
        self.assertEqual(ia.item_instance.name, 'Marta')
        self.assertEqual(ia.name, 'Marta')

    def test_defaultadaptor(self):
        dta = DefaultedAdaptor()
        assert dta.default_adaptor
        dta.name = 'marta'
        assert dta.name == 'Marta'

    def test_inheritdefaultadaptor(self):
        ida = InheritDefaultAdaptor()
        ida.name = 'marta'
        assert ida.name == 'Marta'

    def test_inheritance(self):
        class ChildTestAdaptor(TestAdaptor):
            url = lambda v: v.lower()

        ia = ChildTestAdaptor()
        assert 'url' in ia._field_adaptors
        assert 'name' in ia._field_adaptors

        ia.url = 'HTTP://scrapy.ORG'
        self.assertEqual(ia.url, 'http://scrapy.org')

        ia.name = 'marta'
        self.assertEqual(ia.name, 'Marta')

        class ChildChildTestAdaptor(ChildTestAdaptor):
            url = lambda v: v.upper()
            summary = lambda v: v

        ia = ChildChildTestAdaptor()
        assert 'url' in ia._field_adaptors
        assert 'name' in ia._field_adaptors
        assert 'summary' in ia._field_adaptors

        ia.url = 'HTTP://scrapy.ORG'
        self.assertEqual(ia.url, 'HTTP://SCRAPY.ORG')

        ia.name = 'marta'
        self.assertEqual(ia.name, 'Marta')

    def test_staticmethods(self):
        class ChildAdaptor(TestAdaptor):
            name = adaptor(TestAdaptor.name, string.swapcase)

        ia = ChildAdaptor()
        ia.name = 'Marta'
        self.assertEqual(ia.name, 'mARTA')

    def test_staticdefaults(self):
        class ChildAdaptorDefaulted(DefaultedAdaptor):
            name = adaptor(DefaultedAdaptor.name, string.swapcase)

        dia = ChildAdaptorDefaulted()
        dia.name = 'marta'
        self.assertEqual(dia.name, 'mARTA')

    def test_multiplevaluedadaptor(self):
        ma = MultiValuedItemAdaptor()
        ma.names = ['name1', 'name2']
        assert ma.names == ['Name1', 'Name2']

class TreeadaptTest(unittest.TestCase):

    def test_1_passtrough(self):
        ad = adaptor()
        self.assertEqual(ad('string'), ['string'])

    def test_2_composing(self):
        addone = lambda v: v+1
        ad = adaptor(addone)
        self.assertEqual(ad(0), [1])

        addtwo = lambda v: v+2
        ad2 = adaptor(addone, addtwo)
        self.assertEqual(ad2(0), [3])

        ad3 = adaptor(ad, ad2)
        self.assertEqual(ad3(0), [4])

    def test_3_adaptor_args(self):
        addn = lambda v, adaptor_args: v + adaptor_args.get('add_value', 0)

        ad = adaptor(addn)
        self.assertEqual(ad(0), [0])
        self.assertEqual(ad(0, {'add_value': 3}), [3])
        self.assertEqual(ad(0), [0])

        ad = adaptor(addn, add_value=5)
        self.assertEqual(ad(0), [5])
        self.assertEqual(ad(0, {'add_value': 3}), [3])
        self.assertEqual(ad(0), [5])

    def test_4_treelogic(self):
        split1 = lambda v: v.split('&')
        split2 = lambda v: v.split('=')
        ad = adaptor(split1, split2)
        self.assertEqual(ad('name=joe&job=joker'), ['name', 'joe', 'job', 'joker'])

<commit_msg>newitem: change itemadaptor tests to avoid false positives<commit_after>import unittest
import string
from scrapy.contrib_exp.newitem.adaptors import adaptor, ItemAdaptor
from scrapy.contrib_exp.newitem import *

class BaseItem(Item):
    name = StringField()

class TestItem(BaseItem):
    url = StringField()
    summary = StringField()

class BaseAdaptor(ItemAdaptor):
    item_class = TestItem

class TestAdaptor(BaseAdaptor):
    name = lambda v: v.title()

class DefaultedAdaptor(BaseAdaptor):
    default_adaptor = lambda v: v[:-1]

class InheritDefaultAdaptor(DefaultedAdaptor):
    pass

class MultiValuedTestItem(Item):
    names = MultiValuedField(StringField)

class MultiValuedItemAdaptor(ItemAdaptor):
    item_class = MultiValuedTestItem

    names = adaptor(lambda v: v.title())

class ItemAdaptorTest(unittest.TestCase):

    def test_basic(self):
        ia = TestAdaptor()
        ia.name = 'marta'
        self.assertEqual(ia.item_instance.name, 'Marta')
        self.assertEqual(ia.name, 'Marta')

    def test_defaultadaptor(self):
        dta = DefaultedAdaptor()
        assert dta.default_adaptor
        dta.name = 'marta'
        self.assertEqual(dta.name, 'mart')

    def test_inheritdefaultadaptor(self):
        ida = InheritDefaultAdaptor()
        ida.name = 'marta'
        assert ida.name == 'mart'

    def test_inheritance(self):
        class ChildTestAdaptor(TestAdaptor):
            url = lambda v: v.lower()

        ia = ChildTestAdaptor()
        assert 'url' in ia._field_adaptors
        assert 'name' in ia._field_adaptors

        ia.url = 'HTTP://scrapy.ORG'
        self.assertEqual(ia.url, 'http://scrapy.org')

        ia.name = 'marta'
        self.assertEqual(ia.name, 'Marta')

        class ChildChildTestAdaptor(ChildTestAdaptor):
            url = lambda v: v.upper()
            summary = lambda v: v

        ia = ChildChildTestAdaptor()
        assert 'url' in ia._field_adaptors
        assert 'name' in ia._field_adaptors
        assert 'summary' in ia._field_adaptors

        ia.url = 'HTTP://scrapy.ORG'
        self.assertEqual(ia.url, 'HTTP://SCRAPY.ORG')

        ia.name = 'marta'
        self.assertEqual(ia.name, 'Marta')

    def test_staticmethods(self):
        class ChildAdaptor(TestAdaptor):
            name = adaptor(TestAdaptor.name, string.swapcase)

        ia = ChildAdaptor()
        ia.name = 'Marta'
        self.assertEqual(ia.name, 'mARTA')

    def test_staticdefaults(self):
        class ChildAdaptorDefaulted(DefaultedAdaptor):
            name = adaptor(DefaultedAdaptor.name, string.swapcase)

        dia = ChildAdaptorDefaulted()
        dia.name = 'marta'
        self.assertEqual(dia.name, 'MART')

    def test_multiplevaluedadaptor(self):
        ma = MultiValuedItemAdaptor()
        ma.names = ['name1', 'name2']
        assert ma.names == ['Name1', 'Name2']

class TreeadaptTest(unittest.TestCase):

    def test_1_passtrough(self):
        ad = adaptor()
        self.assertEqual(ad('string'), ['string'])

    def test_2_composing(self):
        addone = lambda v: v+1
        ad = adaptor(addone)
        self.assertEqual(ad(0), [1])

        addtwo = lambda v: v+2
        ad2 = adaptor(addone, addtwo)
        self.assertEqual(ad2(0), [3])

        ad3 = adaptor(ad, ad2)
        self.assertEqual(ad3(0), [4])

    def test_3_adaptor_args(self):
        addn = lambda v, adaptor_args: v + adaptor_args.get('add_value', 0)

        ad = adaptor(addn)
        self.assertEqual(ad(0), [0])
        self.assertEqual(ad(0, {'add_value': 3}), [3])
        self.assertEqual(ad(0), [0])

        ad = adaptor(addn, add_value=5)
        self.assertEqual(ad(0), [5])
        self.assertEqual(ad(0, {'add_value': 3}), [3])
        self.assertEqual(ad(0), [5])

    def test_4_treelogic(self):
        split1 = lambda v: v.split('&')
        split2 = lambda v: v.split('=')
        ad = adaptor(split1, split2)
        self.assertEqual(ad('name=joe&job=joker'), ['name', 'joe', 'job', 'joker'])

<|endoftext|>


<commit_before># coding=utf-8
# Author : Jeonghoon Kang, https://github.com/jeonghoonkang

import openpyxl
import argparse
import sys
import pickle
import xlsxwriter

class excell_class :
    __ofile = None

    def __init__(self):
        pass

    #@staticmethod
    def open_exc_doc(self,__file):
        # using unicode file name with u syntax
        __ofile = openpyxl.load_workbook(__file)

        # pout = "ix: %s, mdsid: %s, donecheck: %d \r" %(ix, mds_id, donecheck)
        pout = "   ... file opened \n"
        sys.stdout.write(pout)
        sys.stdout.flush()
        return __ofile

    def read_vertical(self, sheet, __start, __end):
        __vertical = []
        # print " ... Please use column[n]:column[m], vertical read "
        cell_of_col = sheet[__start:__end]
        for row in cell_of_col:
            for cell in row:
                v = cell.value
                if v == None:
                    continue
                __vertical.append(v)
        return __vertical  # __cnt, __cnt_n # 세로 셀 데이터, 데이터 갯수, None 갯수

    def save_list_2_exl(self, _fn, _list):
        workbook = xlsxwriter.Workbook(_fn+'.xlsx')
        worksheet = workbook.add_worksheet()
        row = 0
        col = 0
        for item in (_list):
            worksheet.write(row, col, item)
            row += 1
        workbook.close()

def parse_args():
    story = u"파일명, 데이터 시작셀과 마지막셀 입력"
    usg = u'\n python save_list.py -fname "sample" -s T4 -e T38'
    parser = argparse.ArgumentParser(description=story, usage=usg, formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument("-f", default="sample", help=u"파일명, e.g.) sample")
    parser.add_argument("-s", default='T4', help=u"데이터 시작 셀, e.g.) T4")
    parser.add_argument("-e", default='T38', help=u"데이터 마지막 셀, e.g.) T38")
    parser.add_argument("-n", default='test.py', help=u"생성할 파일명, e.g) test.py")
    args = parser.parse_args()

    # check
    f = args.f
    s = args.s
    e = args.e
    n = args.n
    return f, s, e, n

if __name__ == "__main__":

    filename, start_range, end_range, new_name = parse_args()
    cfile = excell_class()
    _list = [0, 1, 2, 3, 4]
    cfile.save_list_2_exl('test_list_exl', _list)
<commit_msg>Update save_list_2_excel.py<commit_after># -*- coding: utf-8 -*-
# Author : Jeonghoon Kang, https://github.com/jeonghoonkang

import openpyxl
import argparse
import sys
import pickle
import xlsxwriter

class excell_class :
    __ofile = None

    def __init__(self):
        pass

    #@staticmethod
    def open_exc_doc(self,__file):
        # using unicode file name with u syntax
        __ofile = openpyxl.load_workbook(__file)

        # pout = "ix: %s, mdsid: %s, donecheck: %d \r" %(ix, mds_id, donecheck)
        pout = "   ... file opened \n"
        sys.stdout.write(pout)
        sys.stdout.flush()
        return __ofile

    def read_vertical(self, sheet, __start, __end):
        __vertical = []
        # print " ... Please use column[n]:column[m], vertical read "
        cell_of_col = sheet[__start:__end]
        for row in cell_of_col:
            for cell in row:
                v = cell.value
                if v == None:
                    continue
                __vertical.append(v)
        return __vertical  # __cnt, __cnt_n # 세로 셀 데이터, 데이터 갯수, None 갯수

    def save_list_2_exl(self, _fn, _list):
        workbook = xlsxwriter.Workbook(_fn+'.xlsx')
        worksheet = workbook.add_worksheet()
        row = 0
        col = 0
        for item in (_list):
            worksheet.write(row, col, item)
            row += 1
        workbook.close()

def parse_args():
    story = u"파일명, 데이터 시작셀과 마지막셀 입력"
    usg = u'\n python save_list.py -fname "sample" -s T4 -e T38'
    parser = argparse.ArgumentParser(description=story, usage=usg, formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument("-f", default="sample", help=u"파일명, e.g.) sample")
    parser.add_argument("-s", default='T4', help=u"데이터 시작 셀, e.g.) T4")
    parser.add_argument("-e", default='T38', help=u"데이터 마지막 셀, e.g.) T38")
    parser.add_argument("-n", default='test.py', help=u"생성할 파일명, e.g) test.py")
    args = parser.parse_args()

    # check
    f = args.f
    s = args.s
    e = args.e
    n = args.n
    return f, s, e, n

if __name__ == "__main__":

    filename, start_range, end_range, new_name = parse_args()
    cfile = excell_class()
    _list = [0, 1, 2, 3, 4]
    cfile.save_list_2_exl('test_list_exl', _list)
<|endoftext|>


<commit_before>import unittest
import ndio.remote.OCP as OCP
import ndio.ramon
import ndio.utils.autoingest as AutoIngest
import numpy

SERVER_SITE = 'http://openconnecto.me'
DATA_SITE = 'http://54.200.215.161/'

class TestAutoIngest(unittest.TestCase):

    def setUp(self):
        self.ai_1 = AutoIngest.AutoIngest()
        self.ai_1.add_channel('ndio_test', 'uint32', 'image',
                    DATA_SITE, 'SLICE', 'tif')

        self.ai_1.add_project('ndio_test', 'ndio_test')
        self.ai_1.add_dataset('ndio_test', (660, 528, 1), (0, 0, 0))
        self.ai_1.add_metadata('')

        self.ai_1.post_data(SERVER_SITE)

    def test_pull_data(self):

        self.oo = OCP(SERVER_SITE)
        numpy_download = self.oo.get_cutout('ndio_test', 'image',
                                            0, 660,
                                            0, 528,
                                            0, 0,
                                            resolution=0)

        self.assertEqual(type(numpy_download), numpy.ndarray)
        #Verify its the same image?

    def test_post_json(self):
        ai_2 = AutoIngest.AutoIngest()
        ai_2.add_channel('ndio_test_2', 'uint32', 'image',
                    DATA_SITE, 'SLICE', 'tif')

        ai_2.add_project('ndio_test_2', 'ndio_test_2')
        ai_2.add_dataset('ndio_test_2', (660, 528, 1), (0, 0, 0))
        ai_2.add_metadata('')

        ai_2.output_json()

        ai_3 = AutoIngest.AutoIngest()
        ai_3.post_data(SITE_HOST, "/tmp/ND.json")

        numpy_download = self.oo.get_cutout('ndio_test', 'image',
                                            0, 660,
                                            0, 528,
                                            0, 0,
                                            resolution=0)

        self.assertEqual(type(numpy_download), numpy.ndarray)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Made data publicly viewable<commit_after>import unittest
import ndio.remote.OCP as OCP
import ndio.ramon
import ndio.utils.autoingest as AutoIngest
import numpy

SERVER_SITE = 'http://openconnecto.me'
DATA_SITE = 'http://54.200.215.161/'

class TestAutoIngest(unittest.TestCase):

    def setUpClass(self):
        self.ai_1 = AutoIngest.AutoIngest()
        self.ai_1.add_channel('ndio_test', 'uint32', 'image',
                    DATA_SITE, 'SLICE', 'tif')

        self.ai_1.add_project('ndio_test', 'ndio_test', 1)
        self.ai_1.add_dataset('ndio_test', (660, 528, 1), (0, 0, 0))
        self.ai_1.add_metadata('')

        self.ai_1.post_data(SERVER_SITE)

    def test_pull_data(self):

        self.oo = OCP(SERVER_SITE)
        numpy_download = self.oo.get_cutout('ndio_test', 'image',
                                            0, 660,
                                            0, 528,
                                            0, 0,
                                            resolution=0)

        self.assertEqual(type(numpy_download), numpy.ndarray)
        #Verify its the same image?

    def test_post_json(self):
        ai_2 = AutoIngest.AutoIngest()
        ai_2.add_channel('ndio_test_2', 'uint32', 'image',
                    DATA_SITE, 'SLICE', 'tif')

        ai_2.add_project('ndio_test_2', 'ndio_test_2')
        ai_2.add_dataset('ndio_test_2', (660, 528, 1), (0, 0, 0))
        ai_2.add_metadata('')

        ai_2.output_json()

        ai_3 = AutoIngest.AutoIngest()
        ai_3.post_data(SITE_HOST, "/tmp/ND.json")

        numpy_download = self.oo.get_cutout('ndio_test', 'image',
                                            0, 660,
                                            0, 528,
                                            0, 0,
                                            resolution=0)

        self.assertEqual(type(numpy_download), numpy.ndarray)

    def tearDownClass(self):
        self.oo.delete_channel('ndio_test', 'ndio_test')
        self.oo.delete_channel('ndio_test_2', 'ndio_test_2')

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import pytest  # type: ignore
from hypothesis import assume, example, given, note
from hypothesis.strategies import floats
from typing import Type, Union
from utils import vectors

from ppb_vector import Vector2

@given(x=vectors(max_magnitude=1e75), max_length=floats(min_value=0, max_value=1e75))
def test_truncate_length(x: Vector2, max_length: float):
    assert x.truncate(max_length).length <= (1 + 1e-14) * max_length

@given(x=vectors(max_magnitude=1e75), max_length=floats(min_value=0, max_value=1e75))
def test_truncate_invariant(x: Vector2, max_length: float):
    assume(x.length <= max_length)
    assert x.truncate(max_length) == x

@given(x=vectors(max_magnitude=1e75), max_length=floats(min_value=0, max_value=1e75))
@example( # Large example where x.length == max_length but 1 * x != x
    x=Vector2(0.0, 7.387424005855793e+62), max_length=7.387424005855793e+62
)
def test_truncate_equivalent_to_scale(x: Vector2, max_length: float):
    """Vector2.scale_to and truncate are equivalent when max_length <= x.length"""
    assume(max_length <= x.length)
    note(f"x.length = {x.length}")
    if max_length > 0:
        note(f"x.length = {x.length / max_length} * max_length")

    scale    : Union[Vector2, Type[Exception]]
    truncate : Union[Vector2, Type[Exception]]

    try:
        truncate = x.truncate(max_length)
    except Exception as e:
        truncate = type(e)

    try:
        scale = x.scale_to(max_length)
    except Exception as e:
        scale = type(e)

    assert scale == truncate
<commit_msg>test_truncate_equivalent: Loosen test when x.length == max_length<commit_after>import pytest  # type: ignore
from hypothesis import assume, example, given, note
from hypothesis.strategies import floats
from typing import Type, Union
from utils import vectors

from ppb_vector import Vector2

@given(x=vectors(max_magnitude=1e75), max_length=floats(min_value=0, max_value=1e75))
def test_truncate_length(x: Vector2, max_length: float):
    assert x.truncate(max_length).length <= (1 + 1e-14) * max_length

@given(x=vectors(max_magnitude=1e75), max_length=floats(min_value=0, max_value=1e75))
def test_truncate_invariant(x: Vector2, max_length: float):
    assume(x.length <= max_length)
    assert x.truncate(max_length) == x

@given(x=vectors(max_magnitude=1e75), max_length=floats(min_value=0, max_value=1e75))
@example( # Large example where x.length == max_length but 1 * x != x
    x=Vector2(0.0, 7.387424005855793e+62), max_length=7.387424005855793e+62
)
def test_truncate_equivalent_to_scale(x: Vector2, max_length: float):
    """Vector2.scale_to and truncate are equivalent when max_length <= x.length"""
    assume(max_length <= x.length)
    note(f"x.length = {x.length}")
    if max_length > 0:
        note(f"x.length = {x.length / max_length} * max_length")

    scale    : Union[Vector2, Type[Exception]]
    truncate : Union[Vector2, Type[Exception]]

    try:
        truncate = x.truncate(max_length)
    except Exception as e:
        truncate = type(e)

    try:
        scale = x.scale_to(max_length)
    except Exception as e:
        scale = type(e)

    if isinstance(scale, Vector2) and x.length == max_length:
        # Permit some edge-case where truncation and scaling aren't equivalent
        assert scale.isclose(truncate, abs_tol=0, rel_tol=1e-12)

    else:
        assert scale == truncate
<|endoftext|>