<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import eventlet
import functools

from heat.common import exception
from heat.engine import dependencies
from heat.common import identifier
from heat.engine import resource
from heat.engine import template
from heat.engine import timestamp
from heat.engine.parameters import Parameters
from heat.engine.template import Template
from heat.engine.clients import Clients
from heat.db import api as db_api

from heat.openstack.common import log as logging

logger = logging.getLogger(__name__)

(PARAM_STACK_NAME, PARAM_REGION) = ('AWS::StackName', 'AWS::Region')

class Stack(object):
    CREATE_IN_PROGRESS = 'CREATE_IN_PROGRESS'
    CREATE_FAILED = 'CREATE_FAILED'
    CREATE_COMPLETE = 'CREATE_COMPLETE'

    DELETE_IN_PROGRESS = 'DELETE_IN_PROGRESS'
    DELETE_FAILED = 'DELETE_FAILED'
    DELETE_COMPLETE = 'DELETE_COMPLETE'

    UPDATE_IN_PROGRESS = 'UPDATE_IN_PROGRESS'
    UPDATE_COMPLETE = 'UPDATE_COMPLETE'
    UPDATE_FAILED = 'UPDATE_FAILED'

    created_time = timestamp.Timestamp(db_api.stack_get, 'created_at')
    updated_time = timestamp.Timestamp(db_api.stack_get, 'updated_at')

    def __init__(self, context, stack_name, tmpl, parameters=None,
                 stack_id=None, state=None, state_description='',
                 timeout_mins=60, resolve_data=True):
        '''
        Initialise from a context, name, Template object and (optionally)
        Parameters object. The database ID may also be initialised, if the
        stack is already in the database.
        '''

        if '/' in stack_name:
            raise ValueError(_('Stack name may not contain "/"'))

        self.id = stack_id
        self.context = context
        self.clients = Clients(context)
        self.t = tmpl
        self.name = stack_name
        self.state = state
        self.state_description = state_description
        self.timeout_mins = timeout_mins

        if parameters is None:
            parameters = Parameters(self.name, self.t)
        self.parameters = parameters

        if resolve_data:
            self.outputs = self.resolve_static_data(self.t[template.OUTPUTS])
        else:
            self.outputs = {}

        template_resources = self.t[template.RESOURCES]
        self.resources = dict((name,
                               resource.Resource(name, data, self))
                              for (name, data) in template_resources.items())

        self.dependencies = self._get_dependencies(self.resources.itervalues())

    @staticmethod
    def _get_dependencies(resources):
        '''Return the dependency graph for a list of resources'''
        deps = dependencies.Dependencies()
        for resource in resources:
            resource.add_dependencies(deps)

        return deps

    @classmethod
    def load(cls, context, stack_id=None, stack=None, resolve_data=True):
        '''Retrieve a Stack from the database'''
        if stack is None:
            stack = db_api.stack_get(context, stack_id)
        if stack is None:
            message = 'No stack exists with id "%s"' % str(stack_id)
            raise exception.NotFound(message)

        template = Template.load(context, stack.raw_template_id)
        params = Parameters(stack.name, template, stack.parameters)
        stack = cls(context, stack.name, template, params,
                    stack.id, stack.status, stack.status_reason, stack.timeout,
                    resolve_data)

        return stack

    def store(self, owner=None):
        '''
        Store the stack in the database and return its ID
        If self.id is set, we update the existing stack
        '''
        new_creds = db_api.user_creds_create(self.context)

        s = {
            'name': self.name,
            'raw_template_id': self.t.store(self.context),
            'parameters': self.parameters.user_parameters(),
            'owner_id': owner and owner.id,
            'user_creds_id': new_creds.id,
            'username': self.context.username,
            'tenant': self.context.tenant_id,
            'status': self.state,
            'status_reason': self.state_description,
            'timeout': self.timeout_mins,
        }
        if self.id:
            db_api.stack_update(self.context, self.id, s)
        else:
            new_s = db_api.stack_create(self.context, s)
            self.id = new_s.id

        return self.id

    def identifier(self):
        '''
        Return an identifier for this stack.
        '''
        return identifier.HeatIdentifier(self.context.tenant_id,
                                         self.name, self.id)

    def __iter__(self):
        '''
        Return an iterator over this template's resources in the order that
        they should be started.
        '''
        return iter(self.dependencies)

    def __reversed__(self):
        '''
        Return an iterator over this template's resources in the order that
        they should be stopped.
        '''
        return reversed(self.dependencies)

    def __len__(self):
        '''Return the number of resources'''
        return len(self.resources)

    def __getitem__(self, key):
        '''Get the resource with the specified name.'''
        return self.resources[key]

    def __setitem__(self, key, value):
        '''Set the resource with the specified name to a specific value'''
        self.resources[key] = value

    def __contains__(self, key):
        '''Determine whether the stack contains the specified resource'''
        return key in self.resources

    def keys(self):
        '''Return a list of resource keys for the stack'''
        return self.resources.keys()

    def __str__(self):
        '''Return a human-readable string representation of the stack'''
        return 'Stack "%s"' % self.name

    def validate(self):
        '''
        http://docs.amazonwebservices.com/AWSCloudFormation/latest/\
        APIReference/API_ValidateTemplate.html
        '''
        # TODO(sdake) Should return line number of invalid reference

        for res in self:
            try:
                result = res.validate()
            except Exception as ex:
                logger.exception('validate')
                result = str(ex)

            if result:
                return 'Malformed Query Response %s' % result

    def state_set(self, new_status, reason):
        '''Update the stack state in the database'''
        self.state = new_status
        self.state_description = reason

        if self.id is None:
            return

        stack = db_api.stack_get(self.context, self.id)
        stack.update_and_save({'status': new_status,
                               'status_reason': reason})

    def create(self):
        '''
        Create the stack and all of the resources.

        Creation will fail if it exceeds the specified timeout. The default is
        60 minutes, set in the constructor
        '''
        self.state_set(self.CREATE_IN_PROGRESS, 'Stack creation started')

        stack_status = self.CREATE_COMPLETE
        reason = 'Stack successfully created'
        res = None

        with eventlet.Timeout(self.timeout_mins * 60) as tmo:
            try:
                for res in self:
                    if stack_status != self.CREATE_FAILED:
                        result = res.create()
                        if result:
                            stack_status = self.CREATE_FAILED
                            reason = 'Resource %s failed with: %s' % (str(res),
                                                                      result)

                    else:
                        res.state_set(res.CREATE_FAILED,
                                      'Stack creation aborted')

            except eventlet.Timeout as t:
                if t is tmo:
                    stack_status = self.CREATE_FAILED
                    reason = 'Timed out waiting for %s' % str(res)
                else:
                    # not my timeout
                    raise

        self.state_set(stack_status, reason)

    def update(self, newstack):
        '''
        Compare the current stack with newstack,
        and where necessary create/update/delete the resources until
        this stack aligns with newstack.

        Note update of existing stack resources depends on update
        being implemented in the underlying resource types

        Update will fail if it exceeds the specified timeout. The default is
        60 minutes, set in the constructor
        '''
        if self.state not in (self.CREATE_COMPLETE, self.UPDATE_COMPLETE):
            self.state_set(self.UPDATE_FAILED, 'State invalid for update')
            return
        else:
            self.state_set(self.UPDATE_IN_PROGRESS, 'Stack update started')

        # Now make the resources match the new stack definition
        failures = []
        with eventlet.Timeout(self.timeout_mins * 60) as tmo:
            try:
                # First delete any resources which are not in newstack
                for res in reversed(self):
                    if not res.name in newstack.keys():
                        logger.debug("resource %s not found in updated stack"
                                     % res.name + " definition, deleting")
                        result = res.destroy()
                        if result:
                            failures.append('Resource %s delete failed'
                                            % res.name)
                        else:
                            del self.resources[res.name]

                # Then create any which are defined in newstack but not self
                for res in newstack:
                    if not res.name in self.keys():
                        logger.debug("resource %s not found in current stack"
                                     % res.name + " definition, adding")
                        res.stack = self
                        self[res.name] = res
                        result = self[res.name].create()
                        if result:
                            failures.append('Resource %s create failed'
                                            % res.name)

                # Now (the hard part :) update existing resources
                # The Resource base class allows equality-test of resources,
                # based on the parsed template snippet for the resource.
                # If this  test fails, we call the underlying resource.update
                #
                # FIXME : Implement proper update logic for the resources
                # AWS define three update strategies, applied depending
                # on the resource and what is being updated within a
                # resource :
                # - Update with no interruption
                # - Update with some interruption
                # - Update requires replacement
                #
                # Currently all resource have a default handle_update method
                # which returns "requires replacement" (res.UPDATE_REPLACE)
                for res in newstack:
                    if self.resolve_runtime_data(self[res.name].t) !=\
                            self.resolve_runtime_data(res.t):

                        # Can fail if underlying resource class does not
                        # implement update logic or update requires replacement
                        retval = self[res.name].update(res.parsed_template())
                        if retval == self[res.name].UPDATE_REPLACE:
                            logger.info("Resource %s for stack %s" %
                                        (res.name, self.name) +
                                        " update requires replacement")
                            # Resource requires replacement for update
                            result = self[res.name].destroy()
                            if result:
                                failures.append('Resource %s delete failed'
                                                % res.name)
                            else:
                                res.stack = self
                                self[res.name] = res
                                result = self[res.name].create()
                                if result:
                                    failures.append('Resource %s create failed'
                                                    % res.name)
                        else:
                            logger.warning("Cannot update resource %s," %
                                           res.name + " reason %s" % retval)
                            failures.append('Resource %s update failed'
                                            % res.name)

                # Set stack status values
                if not failures:
                    # flip the template & parameters to the newstack values
                    self.t = newstack.t
                    self.parameters = newstack.parameters
                    template_outputs = self.t[template.OUTPUTS]
                    self.outputs = self.resolve_static_data(template_outputs)
                    self.dependencies = self._get_dependencies(
                        self.resources.itervalues())
                    self.store()

                    stack_status = self.UPDATE_COMPLETE
                    reason = 'Stack successfully updated'
                else:
                    stack_status = self.UPDATE_FAILED
                    reason = ",".join(failures)

            except eventlet.Timeout as t:
                if t is tmo:
                    stack_status = self.UPDATE_FAILED
                    reason = 'Timed out waiting for %s' % str(res)
                else:
                    # not my timeout
                    raise

        self.state_set(stack_status, reason)

    def delete(self):
        '''
        Delete all of the resources, and then the stack itself.
        '''
        self.state_set(self.DELETE_IN_PROGRESS, 'Stack deletion started')

        failures = []
        for res in reversed(self):
            result = res.destroy()
            if result:
                logger.error('Failed to delete %s error: %s' % (str(res),
                                                                result))
                failures.append(str(res))

        if failures:
            self.state_set(self.DELETE_FAILED,
                           'Failed to delete ' + ', '.join(failures))
        else:
            self.state_set(self.DELETE_COMPLETE, 'Deleted successfully')
            db_api.stack_delete(self.context, self.id)

    def output(self, key):
        '''
        Get the value of the specified stack output.
        '''
        value = self.outputs[key].get('Value', '')
        return self.resolve_runtime_data(value)

    def restart_resource(self, resource_name):
        '''
        stop resource_name and all that depend on it
        start resource_name and all that depend on it
        '''
        deps = self.dependencies[self[resource_name]]
        failed = False

        for res in reversed(deps):
            try:
                res.destroy()
            except Exception as ex:
                failed = True
                logger.error('delete: %s' % str(ex))

        for res in deps:
            if not failed:
                try:
                    res.create()
                except Exception as ex:
                    logger.exception('create')
                    failed = True
            else:
                res.state_set(res.CREATE_FAILED, 'Resource restart aborted')
        # TODO(asalkeld) if any of this fails we Should
        # restart the whole stack

    def resolve_static_data(self, snippet):
        return resolve_static_data(self.t, self.parameters, snippet)

    def resolve_runtime_data(self, snippet):
        return resolve_runtime_data(self.t, self.resources, snippet)

def resolve_static_data(template, parameters, snippet):
    '''
    Resolve static parameters, map lookups, etc. in a template.

    Example:

    >>> template = Template(template_format.parse(template_path))
    >>> parameters = Parameters('stack', template, {'KeyName': 'my_key'})
    >>> resolve_static_data(template, parameters, {'Ref': 'KeyName'})
    'my_key'
    '''
    return transform(snippet,
                     [functools.partial(template.resolve_param_refs,
                                        parameters=parameters),
                      template.resolve_availability_zones,
                      template.resolve_find_in_map,
                      template.reduce_joins])

def resolve_runtime_data(template, resources, snippet):
    return transform(snippet,
                     [functools.partial(template.resolve_resource_refs,
                                        resources=resources),
                      functools.partial(template.resolve_attributes,
                                        resources=resources),
                      template.resolve_joins,
                      template.resolve_base64])

def transform(data, transformations):
    '''
    Apply each of the transformation functions in the supplied list to the data
    in turn.
    '''
    for t in transformations:
        data = t(data)
    return data
<commit_msg>heat engine : pass fully resolved template to update<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import eventlet
import functools

from heat.common import exception
from heat.engine import dependencies
from heat.common import identifier
from heat.engine import resource
from heat.engine import template
from heat.engine import timestamp
from heat.engine.parameters import Parameters
from heat.engine.template import Template
from heat.engine.clients import Clients
from heat.db import api as db_api

from heat.openstack.common import log as logging

logger = logging.getLogger(__name__)

(PARAM_STACK_NAME, PARAM_REGION) = ('AWS::StackName', 'AWS::Region')

class Stack(object):
    CREATE_IN_PROGRESS = 'CREATE_IN_PROGRESS'
    CREATE_FAILED = 'CREATE_FAILED'
    CREATE_COMPLETE = 'CREATE_COMPLETE'

    DELETE_IN_PROGRESS = 'DELETE_IN_PROGRESS'
    DELETE_FAILED = 'DELETE_FAILED'
    DELETE_COMPLETE = 'DELETE_COMPLETE'

    UPDATE_IN_PROGRESS = 'UPDATE_IN_PROGRESS'
    UPDATE_COMPLETE = 'UPDATE_COMPLETE'
    UPDATE_FAILED = 'UPDATE_FAILED'

    created_time = timestamp.Timestamp(db_api.stack_get, 'created_at')
    updated_time = timestamp.Timestamp(db_api.stack_get, 'updated_at')

    def __init__(self, context, stack_name, tmpl, parameters=None,
                 stack_id=None, state=None, state_description='',
                 timeout_mins=60, resolve_data=True):
        '''
        Initialise from a context, name, Template object and (optionally)
        Parameters object. The database ID may also be initialised, if the
        stack is already in the database.
        '''

        if '/' in stack_name:
            raise ValueError(_('Stack name may not contain "/"'))

        self.id = stack_id
        self.context = context
        self.clients = Clients(context)
        self.t = tmpl
        self.name = stack_name
        self.state = state
        self.state_description = state_description
        self.timeout_mins = timeout_mins

        if parameters is None:
            parameters = Parameters(self.name, self.t)
        self.parameters = parameters

        if resolve_data:
            self.outputs = self.resolve_static_data(self.t[template.OUTPUTS])
        else:
            self.outputs = {}

        template_resources = self.t[template.RESOURCES]
        self.resources = dict((name,
                               resource.Resource(name, data, self))
                              for (name, data) in template_resources.items())

        self.dependencies = self._get_dependencies(self.resources.itervalues())

    @staticmethod
    def _get_dependencies(resources):
        '''Return the dependency graph for a list of resources'''
        deps = dependencies.Dependencies()
        for resource in resources:
            resource.add_dependencies(deps)

        return deps

    @classmethod
    def load(cls, context, stack_id=None, stack=None, resolve_data=True):
        '''Retrieve a Stack from the database'''
        if stack is None:
            stack = db_api.stack_get(context, stack_id)
        if stack is None:
            message = 'No stack exists with id "%s"' % str(stack_id)
            raise exception.NotFound(message)

        template = Template.load(context, stack.raw_template_id)
        params = Parameters(stack.name, template, stack.parameters)
        stack = cls(context, stack.name, template, params,
                    stack.id, stack.status, stack.status_reason, stack.timeout,
                    resolve_data)

        return stack

    def store(self, owner=None):
        '''
        Store the stack in the database and return its ID
        If self.id is set, we update the existing stack
        '''
        new_creds = db_api.user_creds_create(self.context)

        s = {
            'name': self.name,
            'raw_template_id': self.t.store(self.context),
            'parameters': self.parameters.user_parameters(),
            'owner_id': owner and owner.id,
            'user_creds_id': new_creds.id,
            'username': self.context.username,
            'tenant': self.context.tenant_id,
            'status': self.state,
            'status_reason': self.state_description,
            'timeout': self.timeout_mins,
        }
        if self.id:
            db_api.stack_update(self.context, self.id, s)
        else:
            new_s = db_api.stack_create(self.context, s)
            self.id = new_s.id

        return self.id

    def identifier(self):
        '''
        Return an identifier for this stack.
        '''
        return identifier.HeatIdentifier(self.context.tenant_id,
                                         self.name, self.id)

    def __iter__(self):
        '''
        Return an iterator over this template's resources in the order that
        they should be started.
        '''
        return iter(self.dependencies)

    def __reversed__(self):
        '''
        Return an iterator over this template's resources in the order that
        they should be stopped.
        '''
        return reversed(self.dependencies)

    def __len__(self):
        '''Return the number of resources'''
        return len(self.resources)

    def __getitem__(self, key):
        '''Get the resource with the specified name.'''
        return self.resources[key]

    def __setitem__(self, key, value):
        '''Set the resource with the specified name to a specific value'''
        self.resources[key] = value

    def __contains__(self, key):
        '''Determine whether the stack contains the specified resource'''
        return key in self.resources

    def keys(self):
        '''Return a list of resource keys for the stack'''
        return self.resources.keys()

    def __str__(self):
        '''Return a human-readable string representation of the stack'''
        return 'Stack "%s"' % self.name

    def validate(self):
        '''
        http://docs.amazonwebservices.com/AWSCloudFormation/latest/\
        APIReference/API_ValidateTemplate.html
        '''
        # TODO(sdake) Should return line number of invalid reference

        for res in self:
            try:
                result = res.validate()
            except Exception as ex:
                logger.exception('validate')
                result = str(ex)

            if result:
                return 'Malformed Query Response %s' % result

    def state_set(self, new_status, reason):
        '''Update the stack state in the database'''
        self.state = new_status
        self.state_description = reason

        if self.id is None:
            return

        stack = db_api.stack_get(self.context, self.id)
        stack.update_and_save({'status': new_status,
                               'status_reason': reason})

    def create(self):
        '''
        Create the stack and all of the resources.

        Creation will fail if it exceeds the specified timeout. The default is
        60 minutes, set in the constructor
        '''
        self.state_set(self.CREATE_IN_PROGRESS, 'Stack creation started')

        stack_status = self.CREATE_COMPLETE
        reason = 'Stack successfully created'
        res = None

        with eventlet.Timeout(self.timeout_mins * 60) as tmo:
            try:
                for res in self:
                    if stack_status != self.CREATE_FAILED:
                        result = res.create()
                        if result:
                            stack_status = self.CREATE_FAILED
                            reason = 'Resource %s failed with: %s' % (str(res),
                                                                      result)

                    else:
                        res.state_set(res.CREATE_FAILED,
                                      'Stack creation aborted')

            except eventlet.Timeout as t:
                if t is tmo:
                    stack_status = self.CREATE_FAILED
                    reason = 'Timed out waiting for %s' % str(res)
                else:
                    # not my timeout
                    raise

        self.state_set(stack_status, reason)

    def update(self, newstack):
        '''
        Compare the current stack with newstack,
        and where necessary create/update/delete the resources until
        this stack aligns with newstack.

        Note update of existing stack resources depends on update
        being implemented in the underlying resource types

        Update will fail if it exceeds the specified timeout. The default is
        60 minutes, set in the constructor
        '''
        if self.state not in (self.CREATE_COMPLETE, self.UPDATE_COMPLETE):
            self.state_set(self.UPDATE_FAILED, 'State invalid for update')
            return
        else:
            self.state_set(self.UPDATE_IN_PROGRESS, 'Stack update started')

        # Now make the resources match the new stack definition
        failures = []
        with eventlet.Timeout(self.timeout_mins * 60) as tmo:
            try:
                # First delete any resources which are not in newstack
                for res in reversed(self):
                    if not res.name in newstack.keys():
                        logger.debug("resource %s not found in updated stack"
                                     % res.name + " definition, deleting")
                        result = res.destroy()
                        if result:
                            failures.append('Resource %s delete failed'
                                            % res.name)
                        else:
                            del self.resources[res.name]

                # Then create any which are defined in newstack but not self
                for res in newstack:
                    if not res.name in self.keys():
                        logger.debug("resource %s not found in current stack"
                                     % res.name + " definition, adding")
                        res.stack = self
                        self[res.name] = res
                        result = self[res.name].create()
                        if result:
                            failures.append('Resource %s create failed'
                                            % res.name)

                # Now (the hard part :) update existing resources
                # The Resource base class allows equality-test of resources,
                # based on the parsed template snippet for the resource.
                # If this  test fails, we call the underlying resource.update
                #
                # FIXME : Implement proper update logic for the resources
                # AWS define three update strategies, applied depending
                # on the resource and what is being updated within a
                # resource :
                # - Update with no interruption
                # - Update with some interruption
                # - Update requires replacement
                #
                # Currently all resource have a default handle_update method
                # which returns "requires replacement" (res.UPDATE_REPLACE)
                for res in newstack:
                    # Compare resolved pre/post update resource snippets,
                    # note the new resource snippet is resolved in the context
                    # of the existing stack (which is the stack being updated)
                    old_snippet = self.resolve_runtime_data(self[res.name].t)
                    new_snippet = self.resolve_runtime_data(res.t)
                    if old_snippet != new_snippet:

                        # Can fail if underlying resource class does not
                        # implement update logic or update requires replacement
                        retval = self[res.name].update(new_snippet)
                        if retval == self[res.name].UPDATE_REPLACE:
                            logger.info("Resource %s for stack %s" %
                                        (res.name, self.name) +
                                        " update requires replacement")
                            # Resource requires replacement for update
                            result = self[res.name].destroy()
                            if result:
                                failures.append('Resource %s delete failed'
                                                % res.name)
                            else:
                                res.stack = self
                                self[res.name] = res
                                result = self[res.name].create()
                                if result:
                                    failures.append('Resource %s create failed'
                                                    % res.name)
                        else:
                            logger.warning("Cannot update resource %s," %
                                           res.name + " reason %s" % retval)
                            failures.append('Resource %s update failed'
                                            % res.name)

                # Set stack status values
                if not failures:
                    # flip the template & parameters to the newstack values
                    self.t = newstack.t
                    self.parameters = newstack.parameters
                    template_outputs = self.t[template.OUTPUTS]
                    self.outputs = self.resolve_static_data(template_outputs)
                    self.dependencies = self._get_dependencies(
                        self.resources.itervalues())
                    self.store()

                    stack_status = self.UPDATE_COMPLETE
                    reason = 'Stack successfully updated'
                else:
                    stack_status = self.UPDATE_FAILED
                    reason = ",".join(failures)

            except eventlet.Timeout as t:
                if t is tmo:
                    stack_status = self.UPDATE_FAILED
                    reason = 'Timed out waiting for %s' % str(res)
                else:
                    # not my timeout
                    raise

        self.state_set(stack_status, reason)

    def delete(self):
        '''
        Delete all of the resources, and then the stack itself.
        '''
        self.state_set(self.DELETE_IN_PROGRESS, 'Stack deletion started')

        failures = []
        for res in reversed(self):
            result = res.destroy()
            if result:
                logger.error('Failed to delete %s error: %s' % (str(res),
                                                                result))
                failures.append(str(res))

        if failures:
            self.state_set(self.DELETE_FAILED,
                           'Failed to delete ' + ', '.join(failures))
        else:
            self.state_set(self.DELETE_COMPLETE, 'Deleted successfully')
            db_api.stack_delete(self.context, self.id)

    def output(self, key):
        '''
        Get the value of the specified stack output.
        '''
        value = self.outputs[key].get('Value', '')
        return self.resolve_runtime_data(value)

    def restart_resource(self, resource_name):
        '''
        stop resource_name and all that depend on it
        start resource_name and all that depend on it
        '''
        deps = self.dependencies[self[resource_name]]
        failed = False

        for res in reversed(deps):
            try:
                res.destroy()
            except Exception as ex:
                failed = True
                logger.error('delete: %s' % str(ex))

        for res in deps:
            if not failed:
                try:
                    res.create()
                except Exception as ex:
                    logger.exception('create')
                    failed = True
            else:
                res.state_set(res.CREATE_FAILED, 'Resource restart aborted')
        # TODO(asalkeld) if any of this fails we Should
        # restart the whole stack

    def resolve_static_data(self, snippet):
        return resolve_static_data(self.t, self.parameters, snippet)

    def resolve_runtime_data(self, snippet):
        return resolve_runtime_data(self.t, self.resources, snippet)

def resolve_static_data(template, parameters, snippet):
    '''
    Resolve static parameters, map lookups, etc. in a template.

    Example:

    >>> template = Template(template_format.parse(template_path))
    >>> parameters = Parameters('stack', template, {'KeyName': 'my_key'})
    >>> resolve_static_data(template, parameters, {'Ref': 'KeyName'})
    'my_key'
    '''
    return transform(snippet,
                     [functools.partial(template.resolve_param_refs,
                                        parameters=parameters),
                      template.resolve_availability_zones,
                      template.resolve_find_in_map,
                      template.reduce_joins])

def resolve_runtime_data(template, resources, snippet):
    return transform(snippet,
                     [functools.partial(template.resolve_resource_refs,
                                        resources=resources),
                      functools.partial(template.resolve_attributes,
                                        resources=resources),
                      template.resolve_joins,
                      template.resolve_base64])

def transform(data, transformations):
    '''
    Apply each of the transformation functions in the supplied list to the data
    in turn.
    '''
    for t in transformations:
        data = t(data)
    return data
<|endoftext|>


<commit_before>import cPickle as pickle
from rdflib import BNode, Literal, Graph
from modules.turtleGrapher.turtle_utils import generate_uri as gu, generate_hash
from modules.turtleGrapher.turtle_grapher import generate_graph
from modules.blazeUploader.upload_graph import upload_graph
# working with Serotype, Antimicrobial Resistance, & Virulence Factor data
# structures

def parse_serotype(graph, serotyper_dict, uriIsolate):
    if 'O type' in serotyper_dict:
        graph.add((uriIsolate, gu('ge:0001076'),
                   Literal(serotyper_dict['O type'])))
    if 'H type' in serotyper_dict:
        graph.add((uriIsolate, gu('ge:0001077'),
                   Literal(serotyper_dict['H type'])))
    if 'K type' in serotyper_dict:
        graph.add((uriIsolate, gu('ge:0001684'),
                   Literal(serotyper_dict['K type'])))

    return graph

def parse_gene_dict(graph, gene_dict, uriGenome):
    '''
    My intention is to eventually use ECTyper for all of the calls it was meant for.
    Just need to update ECTyper dict format to ref. AMR / VF by contig. as opposed to genome directly.

    These are the common gene related triples to both AMR / VF.
    Note: we are working from uriGenome and assume that the calling functions (
    generate_amr() and generate_vf() are doing the transformations to the
    gene_dict.keys so that they are contig ids (as they differ in return value
    between VF & AMR from ECTyper)
    )

    TODO: offshore rgi calls to ectyper and make it return a dict in the format we need
    -currently, we'll handle ORF_ID to contig id transform in generate_amr()

    Args:
        graph(rdflib.Graph): the running graph with all our triples
        gene_dict({{}}): a dictionary of genes with a assoc info
            ex. {'Some_Contig_ID':[{'START','STOP','ORIENTATION','GENE_NAME'}]}
        uriGenome(rdflib.URIRef): the base uri of the genome
            ex. :4eb02f5676bc808f86c0f014bbce15775adf06ba

    TODO: merge common components with generate_amr()
    '''

    for contig_id in gene_dict.keys():
        for gene_record in gene_dict[contig_id]:

            # uriGenome generation
            file_hash = generate_hash(query_file)
            uriGenome = gu(':' + file_hash)
            # uri for bag of contigs
            # ex. :4eb02f5676bc808f86c0f014bbce15775adf06ba/contigs/
            uriContigs = gu(uriGenome, "/contigs")
            # recreating the contig uri
            uriContig = gu(uriContigs, '/' + contig_id)

            # after this point we switch perspective to the gene and build down to
            # relink the gene with the contig

            bnode_occurrence = BNode()
            bnode_start = BNode()
            bnode_end = BNode()

            # some gene names, esp those which are effectively a description,
            # have spaces
            gene_name = gene_record['GENE_NAME'].replace(' ', '_')

            graph.add((gu(':' + gene_name), gu('faldo:Region'), bnode_occurrence))

            graph.add((bnode_occurrence, gu('faldo:Begin'), bnode_start))
            graph.add((bnode_occurrence, gu('faldo:End'), bnode_end))

            # this is a special case for amr results
            if 'CUT_OFF' in gene_dict.keys():
                graph.add((bnode_start, gu('dc:Description'),
                           Literal(gene_dict['CUT_OFF'])))
                graph.add((bnode_end, gu('dc:Description'),
                           Literal(gene_dict['CUT_OFF'])))

            graph.add((bnode_start, gu('rdf:type'), gu('faldo:Position')))
            graph.add((bnode_start, gu('rdf:type'), gu('faldo:ExactPosition')))
            graph.add((bnode_end, gu('rdf:type'), gu('faldo:Position')))
            graph.add((bnode_end, gu('rdf:type'), gu('faldo:ExactPosition')))

            if gene_record['ORIENTATION'] is '+':
                graph.add((bnode_start, gu('rdf:type'), gu(
                    'faldo:ForwardStrandPosition')))
                graph.add((bnode_end, gu('rdf:type'), gu(
                    'faldo:ForwardStrandPosition')))
            else:
                graph.add((bnode_start, gu('rdf:type'), gu(
                    'faldo:ReverseStrandPosition')))
                graph.add((bnode_end, gu('rdf:type'), gu(
                    'faldo:ReverseStrandPosition')))

            graph.add((bnode_start, gu('faldo:Position'),
                       Literal(gene_record['START'])))
            graph.add((bnode_start, gu('faldo:Reference'), uriContig))

            graph.add((bnode_end, gu('faldo:Position'),
                       Literal(gene_record['STOP'])))
            graph.add((bnode_end, gu('faldo:Reference'), uriContig))

            ####

    return graph

def datastruct_savvy(query_file, id_file, pickled_dictionary):
    """
    Note: we work we base graphs (those generated solely from the fasta file) and result graphs (those generated from analysis modules (RGI/ECtyper) separately - they are only linked once uploaded to blazegraph
    :param args_dict:
    :return:
    """
    # Base graph generation
    graph = generate_graph()

    # uriGenome generation
    file_hash = generate_hash(query_file)
    uriGenome = gu(':' + file_hash)

    # uriIsolate retrieval
    with open(id_file) as f:
        l = f.readline()
        spfyid = int(l)
    uriIsolate = gu(':spfy' + str(spfyid))

    # results dict retrieval
    results_dict = pickle.load(open(pickled_dictionary, 'rb'))

    # graphing functions
    for key in results_dict.keys():
        if key == 'Serotype':
            graph = parse_serotype(graph,results_dict['Serotype'],uriIsolate)
        elif key == 'Virulence Factors':
            graph = parse_gene_dict(graph, results_dict['Virulence Factors'], uriGenome)
        elif key == 'Antimicrobial Resistance':
            graph = parse_gene_dict(graph, results_dict['Antimicrobial Resistance'], uriGenome)

    # upload
    return upload_graph(graph)
<commit_msg>first draft complete of new object struct in gene results as well<commit_after>import cPickle as pickle
from rdflib import BNode, Literal, Graph
from modules.turtleGrapher.turtle_utils import generate_uri as gu, generate_hash
from modules.turtleGrapher.turtle_grapher import generate_graph
from modules.blazeUploader.upload_graph import upload_graph
# working with Serotype, Antimicrobial Resistance, & Virulence Factor data
# structures

def parse_serotype(graph, serotyper_dict, uriIsolate):
    if 'O type' in serotyper_dict:
        graph.add((uriIsolate, gu('ge:0001076'),
                   Literal(serotyper_dict['O type'])))
    if 'H type' in serotyper_dict:
        graph.add((uriIsolate, gu('ge:0001077'),
                   Literal(serotyper_dict['H type'])))
    if 'K type' in serotyper_dict:
        graph.add((uriIsolate, gu('ge:0001684'),
                   Literal(serotyper_dict['K type'])))

    return graph

def parse_gene_dict(graph, gene_dict, uriGenome):
    '''
    My intention is to eventually use ECTyper for all of the calls it was meant for.
    Just need to update ECTyper dict format to ref. AMR / VF by contig. as opposed to genome directly.

    These are the common gene related triples to both AMR / VF.
    Note: we are working from uriGenome and assume that the calling functions (
    generate_amr() and generate_vf() are doing the transformations to the
    gene_dict.keys so that they are contig ids (as they differ in return value
    between VF & AMR from ECTyper)
    )

    TODO: offshore rgi calls to ectyper and make it return a dict in the format we need
    -currently, we'll handle ORF_ID to contig id transform in generate_amr()

    Args:
        graph(rdflib.Graph): the running graph with all our triples
        gene_dict({{}}): a dictionary of genes with a assoc info
            ex. {'Some_Contig_ID':[{'START','STOP','ORIENTATION','GENE_NAME'}]}
        uriGenome(rdflib.URIRef): the base uri of the genome
            ex. :4eb02f5676bc808f86c0f014bbce15775adf06ba

    TODO: merge common components with generate_amr()
    '''

    for contig_id in gene_dict.keys():
        for gene_record in gene_dict[contig_id]:
            # uriGenome generation
            file_hash = generate_hash(query_file)
            uriGenome = gu(':' + file_hash)
            # uri for bag of contigs
            # ex. :4eb02f5676bc808f86c0f014bbce15775adf06ba/contigs/
            uriContigs = gu(uriGenome, "/contigs")
            # recreating the contig uri
            uriContig = gu(uriContigs, '/' + contig_id)

            # after this point we switch perspective to the gene and build down to
            # relink the gene with the contig
            bnode_region = BNode()
            bnode_start = BNode()
            bnode_end = BNode()

            # some gene names, esp those which are effectively a description,
            # have spaces
            gene_name = gene_record['GENE_NAME'].replace(' ', '_')
            # define the object type of bnode_region
            graph.add((bnode_region, gu('rdf:type'), gu('faldo:Region')))
            # link the region (eg. the occurance of the gene in a contig)
            graph.add((gu(':' + gene_name), gu(':hasPart'), bnode_region))

            # define the object type of the start & end bnodes
            graph.add((bnode_start, gu('rdf:type'), gu('faldo:Begin')))
            graph.add((bnode_end, gu('rdf:type'), gu('faldo:End')))
            # link the start and end bnodes to the region
            graph.add((bnode_region, gu(':hasPart'), bnode_start))
            graph.add((bnode_region, gu(':hasPart'), bnode_end))

            # this is a special case for amr results
            if 'CUT_OFF' in gene_dict.keys():
                graph.add((bnode_start, gu('dc:Description'),
                           Literal(gene_dict['CUT_OFF'])))
                graph.add((bnode_end, gu('dc:Description'),
                           Literal(gene_dict['CUT_OFF'])))

            # object types defined in FALDO standard
            graph.add((bnode_start, gu('rdf:type'), gu('faldo:Position')))
            graph.add((bnode_start, gu('rdf:type'), gu('faldo:ExactPosition')))
            graph.add((bnode_end, gu('rdf:type'), gu('faldo:Position')))
            graph.add((bnode_end, gu('rdf:type'), gu('faldo:ExactPosition')))

            if gene_record['ORIENTATION'] is '+':
                graph.add((bnode_start, gu('rdf:type'), gu(
                    'faldo:ForwardStrandPosition')))
                graph.add((bnode_end, gu('rdf:type'), gu(
                    'faldo:ForwardStrandPosition')))
            else:
                graph.add((bnode_start, gu('rdf:type'), gu(
                    'faldo:ReverseStrandPosition')))
                graph.add((bnode_end, gu('rdf:type'), gu(
                    'faldo:ReverseStrandPosition')))

            graph.add((bnode_start, gu('faldo:Position'),
                       Literal(gene_record['START'])))
            graph.add((bnode_end, gu('faldo:Position'),
                       Literal(gene_record['STOP'])))
            # because we've identified a gene, that uriContig is now also a faldo:Reference (note: this isn't how FALDO intended the linkage to be, but we do this to accomadate inferencing in Blazegraph)
            # this also means that if you find (or are querying) a uriContig and it isn't not a faldo:Reference (& only a g:Contig) then there are no genes assoc w it
            graph.add((uriContig, gu('rdf:type'), gu('faldo:Reference')))
            # link up the start/end bnodes to the contig
            graph.add((bnode_start, gu(':hasPart'), uriContig))
            graph.add((bnode_end, gu(':hasPart'), uriContig))
    #### end of nested for
    return graph

def datastruct_savvy(query_file, id_file, pickled_dictionary):
    """
    Note: we work we base graphs (those generated solely from the fasta file) and result graphs (those generated from analysis modules (RGI/ECtyper) separately - they are only linked once uploaded to blazegraph
    :param args_dict:
    :return:
    """
    # Base graph generation
    graph = generate_graph()

    # uriGenome generation
    file_hash = generate_hash(query_file)
    uriGenome = gu(':' + file_hash)

    # uriIsolate retrieval
    with open(id_file) as f:
        l = f.readline()
        spfyid = int(l)
    uriIsolate = gu(':spfy' + str(spfyid))

    # results dict retrieval
    results_dict = pickle.load(open(pickled_dictionary, 'rb'))

    # graphing functions
    for key in results_dict.keys():
        if key == 'Serotype':
            graph = parse_serotype(graph,results_dict['Serotype'],uriIsolate)
        elif key == 'Virulence Factors':
            graph = parse_gene_dict(graph, results_dict['Virulence Factors'], uriGenome)
        elif key == 'Antimicrobial Resistance':
            graph = parse_gene_dict(graph, results_dict['Antimicrobial Resistance'], uriGenome)

    # upload
    return upload_graph(graph)
<|endoftext|>


<commit_before># from db import db,Dock
import random

from flask import abort
from flask import g
from sqlalchemy import text

from db import Kanmusu, db
from . import MemberHelper

def calculate_repair_time(kanmusu: Kanmusu):
    # {Total repair time} = {HP loss} * {base repair time} * {ship type} + {30 seconds}
    # Our modified formula: Total = HP Loss * ((Build Time**0.6) + Level)
    return (kanmusu.ship.max_stats.hp - kanmusu.current_hp) * ((kanmusu.ship.buildtime**0.6) + kanmusu.level)

def get_ship_from_recipe(fuel: int=30, ammo: int=30, steel: int=30, baux: int=30) -> int:

    ship_t = text("""select ship_id, chance from recipe
      join recipe__resources min on recipe.min_resources_id = min.id
      join recipe__resources max on recipe.max_resources_id = max.id
      where
      :fuel between min.fuel and max.fuel and
      :ammo between min.ammo and max.ammo and
      :steel between min.steel and max.steel and
      :baux between min.baux and max.baux;
    """)

    ships = db.session.execute(ship_t, {"fuel": fuel, "ammo": ammo, "steel": steel, "baux": baux})

    if not ships:
        return None

    ship_choices = [[x[0]] * x[1] for x in ships]
    ship_choices = [item for sublist in ship_choices for item in sublist]
    return random.choice(ship_choices)

def get_and_remove_ship_kdock(dockid: int):
    admiral = g.admiral
    try:
        dock = admiral.docks_craft.all()[dockid]
    except IndexError:
        abort(404)
        return None

    dock.resources.fuel, dock.resources.ammo, \
    dock.resources.steel, dock.resources.baux = 0, 0, 0, 0

    dock.kanmusu.active = True

    db.session.add(dock.ship)
    api_data = {
        "api_ship_id": dock.ship.ship.id,
        "api_kdock": MemberHelper.dock_data([dock], False),
        "api_id": dock.kanmusu.number,
        "api_slotitem": [], # TODO: Equipment!
        "api_ship": MemberHelper.kanmusu(kanmusu=dock.kanmusu)
    }
    # Update dock data.
    dock.kanmusu = None
    dock.complete = None
    db.session.add(dock)
    db.session.commit()
    return api_data

def craft_ship(fuel: int, ammo: int, steel: int, baux: int, dockid: int):
    admiral = g.admiral
    ship = get_ship_from_recipe(fuel, ammo, steel, baux)
    print("ship is", ship)
    if ship:
        nship = Kanmusu(ship)
        nship.number = len(g.admiral.kanmusu)
    else:
        print("Something bad happened. Where are your recipes?")
        abort(404)
        return

    print("nship is", nship)

    # Change dock data.
    try:
        dock = admiral.docks_craft[dockid]
    except IndexError:
        abort(404)
        return

    dock = dock.update(fuel, ammo, steel, baux, nship, True)

    admiral.resources.sub(fuel, ammo, steel, baux)

    db.session.add(dock)
    admiral.add_kanmusu(nship)
    db.session.add(admiral)
    api_data = {
        "api_ship_id": ship,
        "api_kdock": MemberHelper.dock_data([dock], False),
        "api_id": nship.number,
        "api_slotitem": [],
        "api_ship": MemberHelper.kanmusu(kanmusu=dock.kanmusu)
    }
    db.session.commit()
    return api_data
<commit_msg>Fix being able to try and create a ship in the same dock, and fix the wrong ship API number being sent down.<commit_after># from db import db,Dock
import random

from flask import abort
from flask import g
from sqlalchemy import text

from db import Kanmusu, db, Ship
from . import MemberHelper

def calculate_repair_time(kanmusu: Kanmusu):
    # {Total repair time} = {HP loss} * {base repair time} * {ship type} + {30 seconds}
    # Our modified formula: Total = HP Loss * ((Build Time**0.6) + Level)
    return (kanmusu.ship.max_stats.hp - kanmusu.current_hp) * ((kanmusu.ship.buildtime**0.6) + kanmusu.level)

def get_ship_from_recipe(fuel: int=30, ammo: int=30, steel: int=30, baux: int=30) -> int:

    ship_t = text("""select ship_id, chance from recipe
      join recipe__resources min on recipe.min_resources_id = min.id
      join recipe__resources max on recipe.max_resources_id = max.id
      where
      :fuel between min.fuel and max.fuel and
      :ammo between min.ammo and max.ammo and
      :steel between min.steel and max.steel and
      :baux between min.baux and max.baux;
    """)

    ships = db.session.execute(ship_t, {"fuel": fuel, "ammo": ammo, "steel": steel, "baux": baux})

    if not ships:
        return None

    ship_choices = [[x[0]] * x[1] for x in ships]
    ship_choices = [item for sublist in ship_choices for item in sublist]
    return random.choice(ship_choices)

def get_and_remove_ship_kdock(dockid: int):
    admiral = g.admiral
    try:
        dock = admiral.docks_craft.all()[dockid]
    except IndexError:
        abort(404)
        return None

    dock.resources.fuel, dock.resources.ammo, \
    dock.resources.steel, dock.resources.baux = 0, 0, 0, 0

    dock.kanmusu.active = True

    db.session.add(dock.ship)
    api_data = {
        "api_ship_id": dock.ship.ship.id,
        "api_kdock": MemberHelper.dock_data([dock], False),
        "api_id": dock.kanmusu.number,
        "api_slotitem": [], # TODO: Equipment!
        "api_ship": MemberHelper.kanmusu(kanmusu=dock.kanmusu)
    }
    # Update dock data.
    dock.kanmusu = None
    dock.complete = None
    db.session.add(dock)
    db.session.commit()
    return api_data

def craft_ship(fuel: int, ammo: int, steel: int, baux: int, dockid: int):
    admiral = g.admiral
    ship = get_ship_from_recipe(fuel, ammo, steel, baux)
    if ship:
        nship = Kanmusu(ship)
        nship.number = len(g.admiral.kanmusu)
    else:
        print("Something bad happened. Where are your recipes?")
        abort(404)
        return

    # Change dock data.
    try:
        dock = admiral.docks_craft[dockid]
    except IndexError:
        abort(404)
        return

    if dock.kanmusu is not None:
        # wat
        abort(400)
        return

    dock = dock.update(fuel, ammo, steel, baux, nship, True)

    admiral.resources.sub(fuel, ammo, steel, baux)

    db.session.add(dock)
    admiral.add_kanmusu(nship)
    db.session.add(admiral)
    api_data = {
        "api_ship_id": nship.ship.api_id,
        "api_kdock": MemberHelper.dock_data([dock], False),
        "api_id": nship.number,
        "api_slotitem": [],
        "api_ship": MemberHelper.kanmusu(kanmusu=dock.kanmusu)
    }
    db.session.commit()
    return api_data
<|endoftext|>


<commit_before>

import numpy as np

from ffthompy import Timer, Struct
from ffthompy.materials import Material
from ffthompy.sparse.homogenisation import (homog_Ga_full_potential, homog_GaNi_full_potential,
                                            homog_Ga_sparse, homog_GaNi_sparse)
from ffthompy.sparse.materials import SparseMaterial

from examples.sparse.material_setting import getMat_conf,recover_Aga,recover_Agani

import matplotlib.pylab as plt
import os
import pickle

#dim=2
material=0

k_list={'2': [2,3,4,5],#[45,81,243,729,2187]
        '3': [1,2,3]}#,4,5,6 ]

kinds = {'2': 0,
         '3': 2,}
#N_list= [15,45,75,125,215]

epsilon=1e-8
kind_list=['cano','tucker','tt']

for dim in [2]:  ## 2,3

    if dim==2:
        N_list = 5*np.power(3, k_list['{}'.format(dim)])
    else:
        N_list = [5, 15,45,135, 155, 175]

    kind =kinds['{}'.format(dim)]

    full_time_list = [None]*len(N_list)
    sparse_time_list = [None]*len(N_list)
    rank_list = [None]*len(N_list)
    memory_list = [None]*len(N_list)

    for i in range(len(N_list)):
        # PARAMETERS ##############################################################
        N=N_list[i]

        pars=Struct(dim=dim, # number of dimensions (works for 2D and 3D)
                    N=dim*(N,), # number of voxels (assumed equal for all directions)
                    Y=np.ones(dim), # size of periodic cell
                    recover_sparse=1, # recalculate full material coefficients from sparse one
                    solver=dict(tol=1e-6,
                                maxiter=50),
                    )
        pars_sparse = pars.copy()
        pars, pars_sparse, mat_conf = getMat_conf( material, pars, pars_sparse)

        # generating material coefficients
        mat=Material(mat_conf)
    #    Agani=mat.get_A_GaNi(pars.N, primaldual='primal')
        Aga=mat.get_A_Ga(pars.Nbar(pars.N), primaldual='primal')

        print('\n== Full solution with potential by CG (Ga) ===========')
        resP_Ga=homog_Ga_full_potential(Aga, pars)
        print('mean of solution={}'.format(resP_Ga.Fu.mean()))
        print('homogenised properties (component 11) = {}'.format(resP_Ga.AH))
        full_time_list[i]=resP_Ga.time

        ###########################
        N=3*N
        ###########################
        pars.update(Struct(N=dim*(N,),)) # number of voxels (assumed equal for all directions)
        pars_sparse.update(Struct(kind=kind_list[kind],  # type of sparse tensor: 'cano', 'tucker', or 'tt'
                                  rank=1,  # rank of solution vector
                                  precond_rank=1,
                                  tol=None,
                                  N=dim * (1 * N,),
                                  solver=dict(method='mr',
                                              # method could be 'Richardson'(r),'minimal_residual'(mr), or 'Chebyshev'(c)
                                              approx_omega=False,  # inner product of tuckers could be so slow
                                              # that using an approximate omega could gain.
                                              eigrange=[0.6, 50],  # for Chebyshev solver
                                              tol=1e-6,
                                              maxiter=50,  # no. of iterations for a solver
                                              divcrit=True),
                                  ))

        # generating material coefficients
        mat=Material(mat_conf)
#        Agani=mat.get_A_GaNi(pars.N, primaldual='primal')
#        Aga=mat.get_A_Ga(pars_sparse.Nbar(pars_sparse.N), primaldual='primal')

        for r in range(5, N+1,5):

            pars_sparse.update(Struct(rank=r, # rank of solution vector
                                      precond_rank=r,
                                      ))

            print('== format={}, N={}, dim={}, material={} ===='.format(pars_sparse.kind,
                                                                        N, dim, material))

            # PROBLEM DEFINITION ######################################################

            # generating material coefficients
            mats=SparseMaterial(mat_conf, pars_sparse.kind)

    #        Aganis=mats.get_A_GaNi(pars_sparse.N, primaldual='primal', k=pars_sparse.matrank)
            Agas=mats.get_A_Ga(pars_sparse.Nbar(pars_sparse.N), primaldual='primal', k=pars_sparse.matrank)
            Agas.set_fft_form()

#            Aga.val = recover_Aga(Aga, Agas)
#            pars_sparse.update(Struct(alpha=0.5*(Aga[0, 0].min()+Aga[0, 0].max())))
            pars_sparse.update(Struct(alpha= 5.5 ))

            print('\n== SPARSE solver with preconditioner (Ga) =======================')
            resS_Ga=homog_Ga_sparse(Agas, pars_sparse)
            print('mean of solution={}'.format(resS_Ga.Fu.mean()))
            print('homogenised properties (component 11) = {}'.format(resS_Ga.AH))
            print(resS_Ga.Fu)
            #print('iterations={}'.format(resS_Ga.solver['kit']))
            #if np.array_equal(pars.N, pars_sparse.N):
            #    print('norm(dif)={}'.format(np.linalg.norm(resP_Ga.Fu.fourier().val-resS_Ga.Fu.fourier().full().val)))
            print('norm(resP)={}'.format(resS_Ga.solver['norm_res']))
            print('memory efficiency = {0}/{1} = {2}'.format(resS_Ga.Fu.memory, resP_Ga.Fu.val.size, resS_Ga.Fu.memory/resP_Ga.Fu.val.size))
            print ("solution discrepancy",resS_Ga.AH - resP_Ga.AH)

            if resS_Ga.AH - resP_Ga.AH <= 0:
                rank_list[i]=r
                sparse_time_list[i]=resS_Ga.time
                memory_list[i]=resS_Ga.Fu.memory/resP_Ga.Fu.val.size # memory efficiency
                print("sparse solver time:",sparse_time_list)
                print("full solver time:",full_time_list)
                print("rank:",rank_list)
                break

    print("sparse solver time:",sparse_time_list)
    print("full solver time:",full_time_list)
    print("rank:",rank_list)

    pickle.dump(N_list, open("data_for_plot/dim_{}/mat_{}/N_list_{}.p".format(dim, material,kind_list[kind]), "wb"))
    pickle.dump(full_time_list, open("data_for_plot/dim_{}/mat_{}/full_time_list_{}.p".format(dim, material,kind_list[kind]), "wb"))
    pickle.dump(sparse_time_list, open("data_for_plot/dim_{}/mat_{}/sparse_time_list_{}.p".format(dim, material,kind_list[kind]), "wb"))

#####################################################
fig, ax2 = plt.subplots()
fig.set_size_inches(5 , 3.5 , forward=True)

ax2.plot(N_list, full_time_list,   linewidth=1 , marker='o' , markersize=2, label="Full")
ax2.plot(N_list, sparse_time_list,   linewidth=1 , marker='o' , markersize=2, label="Sparse")
plt.title('Time cost of full and sparse solvers')
plt.ylabel('Time cost(s)')
plt.xlabel('N')
plt.legend(loc='upper left')
picname = 'time_efficiency_mat0' +'.png'

plt.savefig(picname)
os.system('eog'+' '+picname +' '+ '&')

##figure( 1)
#fig, ax1 = plt.subplots()
#fig.set_size_inches(5 , 3.5 , forward=True)
#
#ax1.plot(N_list, memory_list,   linewidth=1 , marker='o' , markersize=2 )
#plt.title('sparse vs full memory ratio')
#plt.ylabel('Memory ratio S/F')
#picname = 'memory_efficiency_2D' +'.png'
#
#plt.savefig(picname)
#os.system('eog'+' '+picname +' '+ '&')
<commit_msg>cleanup commented lines<commit_after>import numpy as np
import matplotlib.pylab as plt
import os
import pickle

from ffthompy import Timer, Struct
from ffthompy.materials import Material
from ffthompy.sparse.homogenisation import (homog_Ga_full_potential,
                                            homog_Ga_sparse,)
from ffthompy.sparse.materials import SparseMaterial
from examples.sparse.material_setting import getMat_conf

k_list={'2': [2,3,4,5],
        '3': [4,3,2,1]}

kinds = {'2': 0,
         '3': 2,}

W_list = {'2': [45,135,405,1215],
          '3': [5,15,45,135,175]}

epsilon=1e-8
kind_list=['cano','tucker','tt']
material=0

for dim in [2,3]:

    if dim==2:
        N_list =  W_list['{}'.format(dim)]
    else:
        N_list = W_list['{}'.format(dim)]

    kind =kinds['{}'.format(dim)]

    full_time_list = [None]*len(N_list)
    sparse_time_list = [None]*len(N_list)
    rank_list = [None]*len(N_list)
    memory_list = [None]*len(N_list)

    for i in range(len(N_list)):
        # PARAMETERS ##############################################################
        N=N_list[i]

        pars=Struct(dim=dim, # number of dimensions (works for 2D and 3D)
                    N=dim*(N,), # number of voxels (assumed equal for all directions)
                    Y=np.ones(dim), # size of periodic cell
                    recover_sparse=1, # recalculate full material coefficients from sparse one
                    solver=dict(tol=1e-6,
                                maxiter=50),
                    )
        pars_sparse = pars.copy()
        pars, pars_sparse, mat_conf = getMat_conf( material, pars, pars_sparse)

        # generating material coefficients
        mat=Material(mat_conf)
        Aga=mat.get_A_Ga(pars.Nbar(pars.N), primaldual='primal')

        print('\n== Full solution with potential by CG (Ga) ===========')
        resP_Ga=homog_Ga_full_potential(Aga, pars)
        print('mean of solution={}'.format(resP_Ga.Fu.mean()))
        print('homogenised properties (component 11) = {}'.format(resP_Ga.AH))
        full_time_list[i]=resP_Ga.time

        ###########################
        N=3*N
        ###########################
        pars.update(Struct(N=dim*(N,),)) # number of voxels (assumed equal for all directions)
        pars_sparse.update(Struct(kind=kind_list[kind],  # type of sparse tensor: 'cano', 'tucker', or 'tt'
                                  rank=1,  # rank of solution vector
                                  precond_rank=1,
                                  tol=None,
                                  N=dim * (1 * N,),
                                  solver=dict(method='mr',
                                              # method could be 'Richardson'(r),'minimal_residual'(mr), or 'Chebyshev'(c)
                                              approx_omega=False,  # inner product of tuckers could be so slow
                                              # that using an approximate omega could gain.
                                              eigrange=[0.6, 50],  # for Chebyshev solver
                                              tol=1e-6,
                                              maxiter=50,  # no. of iterations for a solver
                                              divcrit=True),
                                  ))

        # generating material coefficients
        mat=Material(mat_conf)

        for r in range(5, N+1,5):
            pars_sparse.update(Struct(rank=r, # rank of solution vector
                                      precond_rank=r,
                                      ))

            print('== format={}, N={}, dim={}, material={} ===='.format(pars_sparse.kind,
                                                                        N, dim, material))

            # PROBLEM DEFINITION ######################################################
            # generating material coefficients
            mats=SparseMaterial(mat_conf, pars_sparse.kind)
            Agas=mats.get_A_Ga(pars_sparse.Nbar(pars_sparse.N), primaldual='primal', k=pars_sparse.matrank)
            Agas.set_fft_form()

            pars_sparse.update(Struct(alpha= 5.5 ))

            print('\n== SPARSE solver with preconditioner (Ga) =======================')
            resS_Ga=homog_Ga_sparse(Agas, pars_sparse)
            print('mean of solution={}'.format(resS_Ga.Fu.mean()))
            print('homogenised properties (component 11) = {}'.format(resS_Ga.AH))
            print(resS_Ga.Fu)
            print('norm(resP)={}'.format(resS_Ga.solver['norm_res']))
            print('memory efficiency = {0}/{1} = {2}'.format(resS_Ga.Fu.memory, resP_Ga.Fu.val.size, resS_Ga.Fu.memory/resP_Ga.Fu.val.size))
            print ("solution discrepancy",resS_Ga.AH - resP_Ga.AH)

            if resS_Ga.AH - resP_Ga.AH <= 0:
                rank_list[i]=r
                sparse_time_list[i]=resS_Ga.time
                memory_list[i]=resS_Ga.Fu.memory/resP_Ga.Fu.val.size # memory efficiency
                print("sparse solver time:",sparse_time_list)
                print("full solver time:",full_time_list)
                print("rank:",rank_list)
                break

    print("sparse solver time:",sparse_time_list)
    print("full solver time:",full_time_list)
    print("rank:",rank_list)

    pickle.dump(N_list, open("data_for_plot/dim_{}/mat_{}/N_list_{}.p".format(dim, material,kind_list[kind]), "wb"))
    pickle.dump(full_time_list, open("data_for_plot/dim_{}/mat_{}/full_time_list_{}.p".format(dim, material,kind_list[kind]), "wb"))
    pickle.dump(sparse_time_list, open("data_for_plot/dim_{}/mat_{}/sparse_time_list_{}.p".format(dim, material,kind_list[kind]), "wb"))

#####################################################
fig, ax2 = plt.subplots()
fig.set_size_inches(5 , 3.5 , forward=True)

ax2.plot(N_list, full_time_list,   linewidth=1 , marker='o' , markersize=2, label="Full")
ax2.plot(N_list, sparse_time_list,   linewidth=1 , marker='o' , markersize=2, label="Sparse")
plt.title('Time cost of full and sparse solvers')
plt.ylabel('Time cost(s)')
plt.xlabel('N')
plt.legend(loc='upper left')
picname = 'time_efficiency_mat0' +'.png'

plt.savefig(picname)
os.system('eog'+' '+picname +' '+ '&')<|endoftext|>


<commit_before># Calculates OPR
# For implementation details, see
# http://www.chiefdelphi.com/forums/showpost.php?p=484220&postcount=19

# M is n x n where n is # of teams
# s is n x 1 where n is # of teams
# solve [M][x]=[s] for [x]

# x is OPR and should be n x 1

import numpy as np
import logging

class OprHelper(object):
    @classmethod
    def calculate_oprs(cls, matches):
        """
        Returns: a dict where
        key: a string representing a team number (Example: "254", "254B", "1114")
        value: a float representing the OPR for that team
        """
        n, parsed_matches, team_list, team_id_map = cls._parse_matches(matches)
        M = np.zeros([n, n])
        s = np.zeros([n, 1])

        # Constructing M and s
        for teams, score in parsed_matches:
            for team1 in teams:
                team1_id = team_id_map[team1]
                for team2 in teams:
                    M[team1_id, team_id_map[team2]] += 1
                s[team1_id] += score

        # Solving M*x = s for x
        x = np.linalg.solve(M, s)

        oprs_dict = {}
        for team, opr in zip(team_list, x):
            oprs_dict[team] = opr[0]

        return oprs_dict

    @classmethod
    def _parse_matches(cls, matches):
        """
        Returns:
        n: # of teams
        m: # of matches
        parsed_matches: list of matches as the tuple ([team, team, team], score)
        team_list: list of strings representing team numbers. Example: "254", "254B", "1114"
        team_id_map: dict that maps a team to a unique integer from 0 to n-1
        """
        team_list = set()
        parsed_matches = []
        for match in matches:
            alliances = match.alliances
            for alliance_color in ['red', 'blue']:
                match_team_list = []
                for team in alliances[alliance_color]['teams']:
                    team = team[3:]  # turns "frc254B" into "254B"
                    team_list.add(team)
                    match_team_list.append(team)
                parsed_matches.append((match_team_list, alliances[alliance_color]['score']))

        team_list = list(team_list)

        team_id_map = {}
        for i, team in enumerate(team_list):
            team_id_map[team] = i

        n = len(team_list)
        return n, parsed_matches, team_list, team_id_map
<commit_msg>only calculate OPRs for quals matches<commit_after># Calculates OPR
# For implementation details, see
# http://www.chiefdelphi.com/forums/showpost.php?p=484220&postcount=19

# M is n x n where n is # of teams
# s is n x 1 where n is # of teams
# solve [M][x]=[s] for [x]

# x is OPR and should be n x 1

import numpy as np
import logging

class OprHelper(object):
    @classmethod
    def calculate_oprs(cls, matches):
        """
        Returns: a dict where
        key: a string representing a team number (Example: "254", "254B", "1114")
        value: a float representing the OPR for that team
        """
        n, parsed_matches, team_list, team_id_map = cls._parse_matches(matches)
        M = np.zeros([n, n])
        s = np.zeros([n, 1])

        # Constructing M and s
        for teams, score in parsed_matches:
            for team1 in teams:
                team1_id = team_id_map[team1]
                for team2 in teams:
                    M[team1_id, team_id_map[team2]] += 1
                s[team1_id] += score

        # Solving M*x = s for x
        x = np.linalg.solve(M, s)

        oprs_dict = {}
        for team, opr in zip(team_list, x):
            oprs_dict[team] = opr[0]

        return oprs_dict

    @classmethod
    def _parse_matches(cls, matches):
        """
        Returns:
        n: # of teams
        m: # of matches
        parsed_matches: list of matches as the tuple ([team, team, team], score)
        team_list: list of strings representing team numbers. Example: "254", "254B", "1114"
        team_id_map: dict that maps a team to a unique integer from 0 to n-1
        """
        team_list = set()
        parsed_matches = []
        for match in matches:
            if match.comp_level != 'qm':  # only calculate OPRs for quals matches
                continue
            alliances = match.alliances
            for alliance_color in ['red', 'blue']:
                match_team_list = []
                for team in alliances[alliance_color]['teams']:
                    team = team[3:]  # turns "frc254B" into "254B"
                    team_list.add(team)
                    match_team_list.append(team)
                parsed_matches.append((match_team_list, alliances[alliance_color]['score']))

        team_list = list(team_list)

        team_id_map = {}
        for i, team in enumerate(team_list):
            team_id_map[team] = i

        n = len(team_list)
        return n, parsed_matches, team_list, team_id_map
<|endoftext|>


<commit_before>"""Analysis of variance (ANOVA)

Author: Yichuan Liu
"""
import pandas as pd
import numpy as np
from statsmodels.regression.linear_model import OLS
from scipy import stats
import patsy
from statsmodels.iolib import summary2

def _ols_exclude_term(y, x, terms):
    """
    Linear regression excluding terms in x

    Parameters
    ----------
    y : array
        Dependent variable
    x : patsy.DesignMatrix
        Independent variables
    terms : a list of strings

    Returns
    -------
    residual sum of squares and degree of freedom

    """
    term_slices = x.design_info.term_name_slices
    ind = np.array([True]*x.shape[1])
    for term in terms:
        s = term_slices[term]
        ind[s] = False
    x = x[:, ind]
    model = OLS(y, x)
    results = model.fit()
    return (results.ssr, results.df_resid)

class ANOVA(object):
    """
    ANOVA using least square regression

    The full model regression residual sum of squared is
    used to compared with reduced model for calculating the
    within subject effect sum of squared [1]

    Between subject effect is not yet supported

    Parameters
    ----------
    data : DataFrame
    dv : string
        Dependent variable
    within : a list of string(s)
        The within-subject factors
    between : a list of string(s)
        The between-subject factors
    subject : string
        Specify the subject id

    Returns
    -------
    ANOVAResults

    .. [1] Rutherford, Andrew. ANOVA and ANCOVA: a GLM approach. John Wiley & Sons, 2011.

    """

    def __init__(self, data, dv, within=None, between=None, subject=None):
        self.data = data
        self.dv = dv
        self.within = within
        for factor in within:
            if factor == 'C':
                raise ValueError('Factor name cannot be "C"! This is '
                                 'conflict with patsy contrast function '
                                 'name')
        self.between = between
        if between is not None:
            raise NotImplementedError('Between subject effect not '
                                      'yet supported!')
        self.subject = subject

    def fit(self):
        factors = self.within + [self.subject]
        factors = ['C(%s, Sum)' % i for i in factors]
        y = self.data[self.dv].values
        x = patsy.dmatrix('*'.join(factors), data=self.data)
        term_last = [':'.join(factors)]
        term_slices = x.design_info.term_name_slices
        ssr, df_resid = _ols_exclude_term(y, x, term_last)

        anova_table = pd.DataFrame(
            {'F Value':[], 'Num DF':[], 'Den DF':[], 'Pr > F':[]})
        for key in term_slices:
            if self.subject not in key and key != 'Intercept':
                ssr1, df_resid1 = _ols_exclude_term(
                    y, x, term_last + [key])
                df1 = df_resid1 - df_resid
                msm = (ssr1 - ssr) / df1
                if key == ':'.join(factors[:-1]):
                    msr = ssr / df_resid
                    df2 = df_resid
                else:
                    ssr1, df_resid1 = _ols_exclude_term(
                        y, x, term_last + [key +
                                           ':C(%s, Sum)' % self.subject])
                    df2 = df_resid1 - df_resid
                    msr = (ssr1 - ssr) / df2

                F = msm / msr
                p = stats.f.sf(F, df1, df2)
                term = key.replace('C(', '').replace(', Sum)', '')
                anova_table.loc[term, 'F Value'] = F
                anova_table.loc[term, 'Num DF'] = df1
                anova_table.loc[term, 'Den DF'] = df2
                anova_table.loc[term, 'Pr > F'] = p
        self.anova_table = anova_table.iloc[:, [1, 2, 0, 3]]
        return ANOVAResults(self)

class ANOVAResults(object):
    """
    ANOVA results class

    Attributes
    ----------
    anova_table : DataFrame
    """
    def __init__(self, anova):
        self.anova_table = anova.anova_table

    def __str__(self):
        return self.summary().__str__()

    def summary(self, contrast_L=False, transform_M=False):
        summ = summary2.Summary()
        summ.add_title('ANOVA')
        summ.add_df(self.anova_table)

        return summ

<commit_msg>Do OLS only once because IV's are orthogonal<commit_after>"""Analysis of variance (ANOVA)

Author: Yichuan Liu
"""
import pandas as pd
import numpy as np
from statsmodels.regression.linear_model import OLS
from scipy import stats
import patsy
from statsmodels.iolib import summary2

def _ols_exclude_term(y, x, terms):
    """
    Linear regression excluding terms in x

    Parameters
    ----------
    y : array
        Dependent variable
    x : patsy.DesignMatrix
        Independent variables
    terms : a list of strings

    Returns
    -------
    residual sum of squares and degree of freedom

    """
    term_slices = x.design_info.term_name_slices
    ind = np.array([True]*x.shape[1])
    for term in terms:
        s = term_slices[term]
        ind[s] = False
    x = x[:, ind]
    model = OLS(y, x)
    results = model.fit()
    return (results.ssr, results.df_resid)

class ANOVA(object):
    """
    ANOVA using least square regression

    The full model regression residual sum of squared is
    used to compared with reduced model for calculating the
    within subject effect sum of squared [1]

    Between subject effect is not yet supported

    Parameters
    ----------
    data : DataFrame
    dv : string
        Dependent variable
    within : a list of string(s)
        The within-subject factors
    between : a list of string(s)
        The between-subject factors
    subject : string
        Specify the subject id

    Returns
    -------
    ANOVAResults

    .. [1] Rutherford, Andrew. ANOVA and ANCOVA: a GLM approach. John Wiley & Sons, 2011.

    """

    def __init__(self, data, dv, within=None, between=None, subject=None):
        self.data = data
        self.dv = dv
        self.within = within
        for factor in within:
            if factor == 'C':
                raise ValueError('Factor name cannot be "C"! This is '
                                 'conflict with patsy contrast function '
                                 'name')
        self.between = between
        if between is not None:
            raise NotImplementedError('Between subject effect not '
                                      'yet supported!')
        self.subject = subject

    def fit(self):
        factors = self.within + [self.subject]
        factors = ['C(%s, Sum)' % i for i in factors]
        y = self.data[self.dv].values
        x = patsy.dmatrix('*'.join(factors), data=self.data)

        term_last = [':'.join(factors)]
        term_slices = x.design_info.term_name_slices

        def _not_slice(slices, slices_to_exclude, n):
            ind = np.array([True]*n)
            for term in slices_to_exclude:
                s = slices[term]
                ind[s] = False
            return ind

        x = x[:, _not_slice(term_slices, term_last, x.shape[1])]
        model = OLS(y, x)
        results = model.fit()
        term_slices.pop(term_last[0])
        params = results.params
        df_resid = results.df_resid
        ssr = results.ssr

        anova_table = pd.DataFrame(
            {'F Value':[], 'Num DF':[], 'Den DF':[], 'Pr > F':[]})

        def _ssr_reduced_model(y, x, term_slices, params, keys):
            ind = _not_slice(term_slices, keys, x.shape[1])
            params1 = params[ind]
            ssr = np.subtract(y, x[:, ind].dot(params1))
            ssr = ssr.T.dot(ssr)
            df_resid = len(y) - len(params1)
            return ssr, df_resid

        for key in term_slices:
            if self.subject not in key and key != 'Intercept':
                #  Independen variables are orthogonal
                ssr1, df_resid1 = _ssr_reduced_model(
                    y, x, term_slices, params, [key])
                df1 = df_resid1 - df_resid
                msm = (ssr1 - ssr) / df1
                if key == ':'.join(factors[:-1]):
                    msr = ssr / df_resid
                    df2 = df_resid
                else:
                    ssr1, df_resid1 = _ssr_reduced_model(
                        y, x, term_slices, params,
                        [key+ ':C(%s, Sum)' % self.subject])
                    df2 = df_resid1 - df_resid
                    msr = (ssr1 - ssr) / df2

                F = msm / msr
                p = stats.f.sf(F, df1, df2)
                term = key.replace('C(', '').replace(', Sum)', '')
                anova_table.loc[term, 'F Value'] = F
                anova_table.loc[term, 'Num DF'] = df1
                anova_table.loc[term, 'Den DF'] = df2
                anova_table.loc[term, 'Pr > F'] = p
        self.anova_table = anova_table.iloc[:, [1, 2, 0, 3]]
        return ANOVAResults(self)

class ANOVAResults(object):
    """
    ANOVA results class

    Attributes
    ----------
    anova_table : DataFrame
    """
    def __init__(self, anova):
        self.anova_table = anova.anova_table

    def __str__(self):
        return self.summary().__str__()

    def summary(self, contrast_L=False, transform_M=False):
        summ = summary2.Summary()
        summ.add_title('ANOVA')
        summ.add_df(self.anova_table)

        return summ

<|endoftext|>