<commit_before>'''
Created on Jun 23, 2011

@package: support plugin
@copyright: 2012 Sourcefabric o.p.s.
@license: http://www.gnu.org/licenses/gpl-3.0.txt
@author: Gabriel Nistor

SQL alchemy implementation for the generic entities API.
'''

from ally.api.operator.type import TypeModel, TypeQuery
from ally.api.type import typeFor
from ally.exception import InputError, Ref
from ally.internationalization import _
from ally.support.api import entity as api
from ally.support.api.util_service import copy
from ally.support.sqlalchemy.session import SessionSupport
from ally.support.sqlalchemy.util_service import buildQuery, buildLimits, handle
from inspect import isclass
from sqlalchemy.exc import SQLAlchemyError, OperationalError
import logging
from ally.support.sqlalchemy.mapper import MappedSupport
from ally.api.extension import IterPart

# --------------------------------------------------------------------

log = logging.getLogger(__name__)

# --------------------------------------------------------------------

class EntitySupportAlchemy(SessionSupport):
    '''
    Provides support generic entity handling.
    '''

    def __init__(self, Entity, QEntity=None):
        '''
        Construct the entity support for the provided model class and query class.

        @param Entity: class
            The mapped entity model class.
        @param QEntity: class|None
            The query mapped class if there is one.
        '''
        assert isclass(Entity), 'Invalid class %s' % Entity
        assert issubclass(Entity, api.Entity), 'Invalid entity class %s' % Entity
        assert isinstance(Entity, MappedSupport), 'Invalid mapped class %s' % Entity
        self.modelType = typeFor(Entity)
        assert isinstance(self.modelType, TypeModel), 'Invalid model class %s' % Entity
        SessionSupport.__init__(self)

        self.model = self.modelType.container
        self.Entity = Entity

        if QEntity is not None:
            assert isclass(QEntity), 'Invalid class %s' % QEntity
            assert issubclass(QEntity, api.QEntity), 'Invalid query entity class %s' % QEntity
            self.queryType = typeFor(QEntity)
            assert isinstance(self.queryType, TypeQuery), 'Invalid query class %s' % QEntity
            self.query = self.queryType.query
        else:
            self.query = self.queryType = None
        self.QEntity = QEntity

    def _getAll(self, filter=None, query=None, offset=None, limit=None, sqlQuery=None):
        '''
        Provides all the entities for the provided filter, with offset and limit. Also if query is known to the
        service then also a query can be provided.

        @param filter: SQL alchemy filtering|None
            The sql alchemy conditions to filter by.
        @param query: query
            The REST query object to provide filtering on.
        @param offset: integer|None
            The offset to fetch elements from.
        @param limit: integer|None
            The limit of elements to get.
        @param sqlQuery: SQL alchemy|None
            The sql alchemy query to use.
        @return: list
            The list of all filtered and limited elements.
        '''
        if limit == 0: return []
        sqlQuery = sqlQuery or self.session().query(self.Entity)
        if filter is not None: sqlQuery = sqlQuery.filter(filter)
        if query:
            assert self.QEntity, 'No query provided for the entity service'
            assert self.queryType.isValid(query), 'Invalid query %s, expected %s' % (query, self.QEntity)
            sqlQuery = buildQuery(sqlQuery, query, self.Entity)
        sqlQuery = buildLimits(sqlQuery, offset, limit)
        return sqlQuery.all()

    def _getAllWithCount(self, filter=None, query=None, offset=None, limit=None, sql=None):
        '''
        Provides all the entities for the provided filter, with offset and limit and the total count. Also if query is
        known to the service then also a query can be provided.

        @param filter: SQL alchemy filtering|None
            The sql alchemy conditions to filter by.
        @param query: query
            The REST query object to provide filtering on.
        @param offset: integer|None
            The offset to fetch elements from.
        @param limit: integer|None
            The limit of elements to get.
        @param sql: SQL alchemy|None
            The sql alchemy query to use.
        @return: tuple(list, integer)
            The list of all filtered and limited elements and the count of the total elements.
        '''
        sql = sql or self.session().query(self.Entity)
        if filter is not None: sql = sql.filter(filter)
        if query:
            assert self.QEntity, 'No query provided for the entity service'
            assert self.queryType.isValid(query), 'Invalid query %s, expected %s' % (query, self.QEntity)
            sql = buildQuery(sql, query, self.Entity)
        sqlLimit = buildLimits(sql, offset, limit)
        if limit == 0: return [], sql.count()
        return sqlLimit.all(), sql.count()

# --------------------------------------------------------------------

class EntityGetServiceAlchemy(EntitySupportAlchemy):
    '''
    Generic implementation for @see: IEntityGetService
    '''

    def getById(self, id):
        '''
        @see: IEntityGetService.getById
        '''
        entity = self.session().query(self.Entity).get(id)
        if not entity: raise InputError(Ref(_('Unknown id'), ref=self.Entity.Id))
        return entity

class EntityFindServiceAlchemy(EntitySupportAlchemy):
    '''
    Generic implementation for @see: IEntityFindService
    '''

    def getAll(self, offset=None, limit=None, detailed=False):
        '''
        @see: IEntityQueryService.getAll
        '''
        if detailed:
            total, entities = self._getAllWithCount(None, None, offset, limit)
            return IterPart(entities, total, offset, limit)
        return self._getAll(None, None, offset, limit)

class EntityQueryServiceAlchemy(EntitySupportAlchemy):
    '''
    Generic implementation for @see: IEntityQueryService
    '''

    def getAll(self, offset=None, limit=None, detailed=False, q=None):
        '''
        @see: IEntityQueryService.getAll
        '''
        if detailed:
            entities, total = self._getAllWithCount(None, q, offset, limit)
            return IterPart(entities, total, offset, limit)
        return self._getAll(None, q, offset, limit)

class EntityCRUDServiceAlchemy(EntitySupportAlchemy):
    '''
    Generic implementation for @see: IEntityCRUDService
    '''

    def insert(self, entity):
        '''
        @see: IEntityCRUDService.insert
        '''
        assert self.modelType.isValid(entity), 'Invalid entity %s, expected %s' % (entity, self.Entity)
        entityDb = copy(entity, self.Entity())
        try:
            self.session().add(entityDb)
            self.session().flush((entityDb,))
        except SQLAlchemyError as e: handle(e, entityDb)
        entity.Id = entityDb.Id
        return entityDb.Id

    def update(self, entity):
        '''
        @see: IEntityCRUDService.update
        '''
        assert self.modelType.isValid(entity), 'Invalid entity %s, expected %s' % (entity, self.Entity)
        assert isinstance(entity.Id, int), 'Invalid entity %s, with id %s' % (entity, entity.Id)
        entityDb = self.session().query(self.Entity).get(entity.Id)
        if not entityDb: raise InputError(Ref(_('Unknown id'), ref=self.Entity.Id))
        try: self.session().flush((copy(entity, entityDb),))
        except SQLAlchemyError as e: handle(e, self.Entity)

    def delete(self, id):
        '''
        @see: IEntityCRUDService.delete
        '''
        try:
            return self.session().query(self.Entity).filter(self.Entity.Id == id).delete() > 0
        except OperationalError:
            assert log.debug('Could not delete entity %s with id \'%s\'', self.Entity, id, exc_info=True) or True
            raise InputError(Ref(_('Cannot delete because is in use'), model=self.model))

class EntityGetCRUDServiceAlchemy(EntityGetServiceAlchemy, EntityCRUDServiceAlchemy):
    '''
    Generic implementation for @see: IEntityGetCRUDService
    '''

class EntityNQServiceAlchemy(EntityGetServiceAlchemy, EntityFindServiceAlchemy, EntityCRUDServiceAlchemy):
    '''
    Generic implementation for @see: IEntityService
    '''

    def __init__(self, Entity):
        '''
        @see: EntitySupportAlchemy.__init__
        '''
        EntitySupportAlchemy.__init__(self, Entity)

class EntityServiceAlchemy(EntityGetServiceAlchemy, EntityQueryServiceAlchemy, EntityCRUDServiceAlchemy):
    '''
    Generic implementation for @see: IEntityService
    '''

<commit_msg>A fix for some entity service<commit_after>'''
Created on Jun 23, 2011

@package: support plugin
@copyright: 2012 Sourcefabric o.p.s.
@license: http://www.gnu.org/licenses/gpl-3.0.txt
@author: Gabriel Nistor

SQL alchemy implementation for the generic entities API.
'''

from ally.api.operator.type import TypeModel, TypeQuery
from ally.api.type import typeFor
from ally.exception import InputError, Ref
from ally.internationalization import _
from ally.support.api import entity as api
from ally.support.api.util_service import copy
from ally.support.sqlalchemy.session import SessionSupport
from ally.support.sqlalchemy.util_service import buildQuery, buildLimits, handle
from inspect import isclass
from sqlalchemy.exc import SQLAlchemyError, OperationalError
import logging
from ally.support.sqlalchemy.mapper import MappedSupport
from ally.api.extension import IterPart

# --------------------------------------------------------------------

log = logging.getLogger(__name__)

# --------------------------------------------------------------------

class EntitySupportAlchemy(SessionSupport):
    '''
    Provides support generic entity handling.
    '''

    def __init__(self, Entity, QEntity=None):
        '''
        Construct the entity support for the provided model class and query class.

        @param Entity: class
            The mapped entity model class.
        @param QEntity: class|None
            The query mapped class if there is one.
        '''
        assert isclass(Entity), 'Invalid class %s' % Entity
        assert issubclass(Entity, api.Entity), 'Invalid entity class %s' % Entity
        assert isinstance(Entity, MappedSupport), 'Invalid mapped class %s' % Entity
        self.modelType = typeFor(Entity)
        assert isinstance(self.modelType, TypeModel), 'Invalid model class %s' % Entity
        SessionSupport.__init__(self)

        self.model = self.modelType.container
        self.Entity = Entity

        if QEntity is not None:
            assert isclass(QEntity), 'Invalid class %s' % QEntity
            assert issubclass(QEntity, api.QEntity), 'Invalid query entity class %s' % QEntity
            self.queryType = typeFor(QEntity)
            assert isinstance(self.queryType, TypeQuery), 'Invalid query class %s' % QEntity
            self.query = self.queryType.query
        else:
            self.query = self.queryType = None
        self.QEntity = QEntity

    def _getAll(self, filter=None, query=None, offset=None, limit=None, sqlQuery=None):
        '''
        Provides all the entities for the provided filter, with offset and limit. Also if query is known to the
        service then also a query can be provided.

        @param filter: SQL alchemy filtering|None
            The sql alchemy conditions to filter by.
        @param query: query
            The REST query object to provide filtering on.
        @param offset: integer|None
            The offset to fetch elements from.
        @param limit: integer|None
            The limit of elements to get.
        @param sqlQuery: SQL alchemy|None
            The sql alchemy query to use.
        @return: list
            The list of all filtered and limited elements.
        '''
        if limit == 0: return []
        sqlQuery = sqlQuery or self.session().query(self.Entity)
        if filter is not None: sqlQuery = sqlQuery.filter(filter)
        if query:
            assert self.QEntity, 'No query provided for the entity service'
            assert self.queryType.isValid(query), 'Invalid query %s, expected %s' % (query, self.QEntity)
            sqlQuery = buildQuery(sqlQuery, query, self.Entity)
        sqlQuery = buildLimits(sqlQuery, offset, limit)
        return sqlQuery.all()

    def _getAllWithCount(self, filter=None, query=None, offset=None, limit=None, sql=None):
        '''
        Provides all the entities for the provided filter, with offset and limit and the total count. Also if query is
        known to the service then also a query can be provided.

        @param filter: SQL alchemy filtering|None
            The sql alchemy conditions to filter by.
        @param query: query
            The REST query object to provide filtering on.
        @param offset: integer|None
            The offset to fetch elements from.
        @param limit: integer|None
            The limit of elements to get.
        @param sql: SQL alchemy|None
            The sql alchemy query to use.
        @return: tuple(list, integer)
            The list of all filtered and limited elements and the count of the total elements.
        '''
        sql = sql or self.session().query(self.Entity)
        if filter is not None: sql = sql.filter(filter)
        if query:
            assert self.QEntity, 'No query provided for the entity service'
            assert self.queryType.isValid(query), 'Invalid query %s, expected %s' % (query, self.QEntity)
            sql = buildQuery(sql, query, self.Entity)
        sqlLimit = buildLimits(sql, offset, limit)
        if limit == 0: return [], sql.count()
        return sqlLimit.all(), sql.count()

# --------------------------------------------------------------------

class EntityGetServiceAlchemy(EntitySupportAlchemy):
    '''
    Generic implementation for @see: IEntityGetService
    '''

    def getById(self, id):
        '''
        @see: IEntityGetService.getById
        '''
        entity = self.session().query(self.Entity).get(id)
        if not entity: raise InputError(Ref(_('Unknown id'), ref=self.Entity.Id))
        return entity

class EntityFindServiceAlchemy(EntitySupportAlchemy):
    '''
    Generic implementation for @see: IEntityFindService
    '''

    def getAll(self, offset=None, limit=None, detailed=False):
        '''
        @see: IEntityQueryService.getAll
        '''
        if detailed:
            entities, total = self._getAllWithCount(None, None, offset, limit)
            return IterPart(entities, total, offset, limit)
        return self._getAll(None, None, offset, limit)

class EntityQueryServiceAlchemy(EntitySupportAlchemy):
    '''
    Generic implementation for @see: IEntityQueryService
    '''

    def getAll(self, offset=None, limit=None, detailed=False, q=None):
        '''
        @see: IEntityQueryService.getAll
        '''
        if detailed:
            entities, total = self._getAllWithCount(None, q, offset, limit)
            return IterPart(entities, total, offset, limit)
        return self._getAll(None, q, offset, limit)

class EntityCRUDServiceAlchemy(EntitySupportAlchemy):
    '''
    Generic implementation for @see: IEntityCRUDService
    '''

    def insert(self, entity):
        '''
        @see: IEntityCRUDService.insert
        '''
        assert self.modelType.isValid(entity), 'Invalid entity %s, expected %s' % (entity, self.Entity)
        entityDb = copy(entity, self.Entity())
        try:
            self.session().add(entityDb)
            self.session().flush((entityDb,))
        except SQLAlchemyError as e: handle(e, entityDb)
        entity.Id = entityDb.Id
        return entityDb.Id

    def update(self, entity):
        '''
        @see: IEntityCRUDService.update
        '''
        assert self.modelType.isValid(entity), 'Invalid entity %s, expected %s' % (entity, self.Entity)
        assert isinstance(entity.Id, int), 'Invalid entity %s, with id %s' % (entity, entity.Id)
        entityDb = self.session().query(self.Entity).get(entity.Id)
        if not entityDb: raise InputError(Ref(_('Unknown id'), ref=self.Entity.Id))
        try: self.session().flush((copy(entity, entityDb),))
        except SQLAlchemyError as e: handle(e, self.Entity)

    def delete(self, id):
        '''
        @see: IEntityCRUDService.delete
        '''
        try:
            return self.session().query(self.Entity).filter(self.Entity.Id == id).delete() > 0
        except OperationalError:
            assert log.debug('Could not delete entity %s with id \'%s\'', self.Entity, id, exc_info=True) or True
            raise InputError(Ref(_('Cannot delete because is in use'), model=self.model))

class EntityGetCRUDServiceAlchemy(EntityGetServiceAlchemy, EntityCRUDServiceAlchemy):
    '''
    Generic implementation for @see: IEntityGetCRUDService
    '''

class EntityNQServiceAlchemy(EntityGetServiceAlchemy, EntityFindServiceAlchemy, EntityCRUDServiceAlchemy):
    '''
    Generic implementation for @see: IEntityService
    '''

    def __init__(self, Entity):
        '''
        @see: EntitySupportAlchemy.__init__
        '''
        EntitySupportAlchemy.__init__(self, Entity)

class EntityServiceAlchemy(EntityGetServiceAlchemy, EntityQueryServiceAlchemy, EntityCRUDServiceAlchemy):
    '''
    Generic implementation for @see: IEntityService
    '''

<|endoftext|>


<commit_before>from datetime import datetime, timedelta
import demistomock as demisto
import dateparser

server_url = 'https://vigilant.alienvault.cloud/api/2.0/alarms?page=1&size=1' \
             '&sort=timestamp_occured%2Casc&timestamp_occured_gte=1547560049000'

def approximate_compare(time1, time2):
    if isinstance(time1, int):
        time1 = datetime.fromtimestamp(time1 / 1000)
    if isinstance(time2, int):
        time2 = datetime.fromtimestamp(time2 / 1000)

    return timedelta(seconds=-30) <= time1 - time2 <= timedelta(seconds=3)

def test_fetch_incidents(mocker, requests_mock):
    mocker.patch.object(demisto, 'params', return_value={
        'fetch_limit': '1',
        'url': 'https://vigilant.alienvault.cloud/'
    })
    mocker.patch.object(demisto, 'getLastRun', return_value={'time': '2019-01-15T15:47:29.959Z'})
    mocker.patch.object(demisto, 'setLastRun')
    mocker.patch.object(demisto, 'incidents')
    from AlienVault_USM_Anywhere import fetch_incidents
    requests_mock.get(
        server_url,
        json={
            '_embedded': {
                'alarms': [
                    {
                        'uuid': '444444444',
                        'timestamp_occured_iso8601': '2019-07-12T06:00:38.000Z',
                    }
                ]
            },
            'page': {
                'totalElements': 1861
            }
        }
    )
    fetch_incidents()

def test_get_time_range():
    from AlienVault_USM_Anywhere import get_time_range
    from CommonServerPython import date_to_timestamp

    assert get_time_range(None, None, None) == (None, None)

    dt = datetime.now()
    start, end = get_time_range('Today', None, None)
    assert datetime.fromtimestamp(start / 1000).date() == dt.date() and approximate_compare(dt, end)

    dt = datetime.now()
    # should ignore the start/end time values
    start, end = get_time_range('Today', 'asfd', 'asdf')
    assert datetime.fromtimestamp(start / 1000).date() == dt.date() and approximate_compare(dt, end)

    dt = datetime.now()
    start, end = get_time_range('Yesterday', None, None)
    assert datetime.fromtimestamp(start / 1000).date() == (dt.date() - timedelta(days=1)) and approximate_compare(dt, end)

    start, end = get_time_range('Custom', '2019-12-30T01:02:03Z', '2019-12-30T04:05:06Z')
    assert ((start, end) == (date_to_timestamp(dateparser.parse('2019-12-30T01:02:03Z')),
                             date_to_timestamp(dateparser.parse('2019-12-30T04:05:06Z'))))

    start, end = get_time_range('Custom', '2019-12-30T01:02:03Z', None)
    assert (start == date_to_timestamp(dateparser.parse('2019-12-30T01:02:03Z'))
            and approximate_compare(end, datetime.now()))
<commit_msg>updated server url<commit_after>import demistomock as demisto
from CommonServerPython import *
from CommonServerUserPython import *

''' IMPORTS '''
import dateparser
from datetime import datetime, timedelta

server_url = 'https://vigilant.alienvault.cloud/api/2.0/alarms?page=1&size=1' \
             '&sort=timestamp_occured%2Casc&timestamp_occured_gte=1547567249000'

def approximate_compare(time1, time2):
    if isinstance(time1, int):
        time1 = datetime.fromtimestamp(time1 / 1000)
    if isinstance(time2, int):
        time2 = datetime.fromtimestamp(time2 / 1000)

    return timedelta(seconds=-30) <= time1 - time2 <= timedelta(seconds=3)

def test_fetch_incidents(mocker, requests_mock):
    mocker.patch.object(demisto, 'params', return_value={
        'fetch_limit': '1',
        'url': 'https://vigilant.alienvault.cloud/'
    })
    mocker.patch.object(demisto, 'getLastRun', return_value={'time': '2019-01-15T15:47:29.959Z'})
    mocker.patch.object(demisto, 'setLastRun')
    mocker.patch.object(demisto, 'incidents')
    from AlienVault_USM_Anywhere import fetch_incidents
    requests_mock.get(
        server_url,
        json={
            '_embedded': {
                'alarms': [
                    {
                        'uuid': '444444444',
                        'timestamp_occured_iso8601': '2019-07-12T06:00:38.000Z',
                    }
                ]
            },
            'page': {
                'totalElements': 1861
            }
        }
    )
    fetch_incidents()

def test_get_time_range():
    from AlienVault_USM_Anywhere import get_time_range
    from CommonServerPython import date_to_timestamp

    assert get_time_range(None, None, None) == (None, None)

    dt = datetime.now()
    start, end = get_time_range('Today', None, None)
    assert datetime.fromtimestamp(start / 1000).date() == dt.date() and approximate_compare(dt, end)

    dt = datetime.now()
    # should ignore the start/end time values
    start, end = get_time_range('Today', 'asfd', 'asdf')
    assert datetime.fromtimestamp(start / 1000).date() == dt.date() and approximate_compare(dt, end)

    dt = datetime.now()
    start, end = get_time_range('Yesterday', None, None)
    assert datetime.fromtimestamp(start / 1000).date() == (dt.date() - timedelta(days=1)) and approximate_compare(dt, end)

    start, end = get_time_range('Custom', '2019-12-30T01:02:03Z', '2019-12-30T04:05:06Z')
    assert ((start, end) == (date_to_timestamp(dateparser.parse('2019-12-30T01:02:03Z')),
                             date_to_timestamp(dateparser.parse('2019-12-30T04:05:06Z'))))

    start, end = get_time_range('Custom', '2019-12-30T01:02:03Z', None)
    assert (start == date_to_timestamp(dateparser.parse('2019-12-30T01:02:03Z'))
            and approximate_compare(end, datetime.now()))
<|endoftext|>


<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: See license.txt

from __future__ import unicode_literals

import frappe, json, os
from frappe.utils import strip, cint
from frappe.translate import (set_default_language, get_dict, send_translations)
from frappe.geo.country_info import get_country_info
from frappe.utils.file_manager import save_file
from frappe.utils.password import update_password
from werkzeug.useragents import UserAgent
from . import install_fixtures
from six import string_types

def get_setup_stages(args):

	# App setup stage functions should not include frappe.db.commit
	# That is done by frappe after successful completion of all stages
	stages = [
		{
			'status': 'Updating global settings',
			'fail_msg': 'Failed to update global settings',
			'tasks': [
				{
					'fn': update_global_settings,
					'args': args,
					'fail_msg': 'Failed to update global settings'
				}
			]
		}
	]

	stages += get_stages_hooks(args) + get_setup_complete_hooks(args)

	stages.append({
		# post executing hooks
		'status': 'Wrapping up',
		'fail_msg': 'Failed to complete setup',
		'tasks': [
			{
				'fn': run_post_setup_complete,
				'args': args,
				'fail_msg': 'Failed to complete setup'
			}
		]
	})

	return stages

@frappe.whitelist()
def setup_complete(args):
	"""Calls hooks for `setup_wizard_complete`, sets home page as `desktop`
	and clears cache. If wizard breaks, calls `setup_wizard_exception` hook"""

	# Setup complete: do not throw an exception, let the user continue to desk
	if (frappe.cache().hget("setup_wizard", "in_setup") or
		cint(frappe.db.get_single_value('System Settings', 'setup_complete'))):
		return

	frappe.cache().hset("setup_wizard", "in_setup", True)

	args = parse_args(args)

	stages = get_setup_stages(args)

	try:
		current_task = None
		for idx, stage in enumerate(stages):
			frappe.publish_realtime('setup_task', {"progress": [idx, len(stages)],
				"stage_status": stage.get('status')}, user=frappe.session.user)

			for task in stage.get('tasks'):
				current_task = task
				task.get('fn')(task.get('args'))
	except Exception:
		handle_setup_exception(args)
		return {'status': 'fail', 'fail': current_task.get('fail_msg')}
	else:
		run_setup_success(args)
		return {'status': 'ok'}
	finally:
		frappe.cache().hdel("setup_wizard", "in_setup")

def update_global_settings(args):
	if args.language and args.language != "English":
		set_default_language(get_language_code(args.lang))
		frappe.db.commit()
	frappe.clear_cache()

	update_system_settings(args)
	update_user_name(args)

def run_post_setup_complete(args):
	disable_future_access()
	frappe.db.commit()
	frappe.clear_cache()

def run_setup_success(args):
	for hook in frappe.get_hooks("setup_wizard_success"):
		frappe.get_attr(hook)(args)
	install_fixtures.install()

def get_stages_hooks(args):
	stages = []
	for method in frappe.get_hooks("setup_wizard_stages"):
		stages += frappe.get_attr(method)(args)
	return stages

def get_setup_complete_hooks(args):
	stages = []
	for method in frappe.get_hooks("setup_wizard_complete"):
		stages.append({
			'status': 'Executing method',
			'fail_msg': 'Failed to execute method',
			'tasks': [
				{
					'fn': frappe.get_attr(method),
					'args': args,
					'fail_msg': 'Failed to execute method'
				}
			]
		})
	return stages

def handle_setup_exception(args):
	frappe.db.rollback()
	if args:
		traceback = frappe.get_traceback()
		for hook in frappe.get_hooks("setup_wizard_exception"):
			frappe.get_attr(hook)(traceback, args)

def update_system_settings(args):
	number_format = get_country_info(args.get("country")).get("number_format", "#,###.##")

	# replace these as float number formats, as they have 0 precision
	# and are currency number formats and not for floats
	if number_format=="#.###":
		number_format = "#.###,##"
	elif number_format=="#,###":
		number_format = "#,###.##"

	system_settings = frappe.get_doc("System Settings", "System Settings")
	system_settings.update({
		"country": args.get("country"),
		"language": get_language_code(args.get("language")),
		"time_zone": args.get("timezone"),
		"float_precision": 3,
		'date_format': frappe.db.get_value("Country", args.get("country"), "date_format"),
		'number_format': number_format,
		'enable_scheduler': 1 if not frappe.flags.in_test else 0,
		'backup_limit': 3 # Default for downloadable backups
	})
	system_settings.save()

def update_user_name(args):
	first_name, last_name = args.get('full_name', ''), ''
	if ' ' in first_name:
		first_name, last_name = first_name.split(' ', 1)

	if args.get("email"):
		if frappe.db.exists('User', args.get('email')):
			# running again
			return

		args['name'] = args.get("email")

		_mute_emails, frappe.flags.mute_emails = frappe.flags.mute_emails, True
		doc = frappe.get_doc({
			"doctype":"User",
			"email": args.get("email"),
			"first_name": first_name,
			"last_name": last_name
		})
		doc.flags.no_welcome_mail = True
		doc.insert()
		frappe.flags.mute_emails = _mute_emails
		update_password(args.get("email"), args.get("password"))

	elif first_name:
		args.update({
			"name": frappe.session.user,
			"first_name": first_name,
			"last_name": last_name
		})

		frappe.db.sql("""update `tabUser` SET first_name=%(first_name)s,
			last_name=%(last_name)s WHERE name=%(name)s""", args)

	if args.get("attach_user"):
		attach_user = args.get("attach_user").split(",")
		if len(attach_user)==3:
			filename, filetype, content = attach_user
			fileurl = save_file(filename, content, "User", args.get("name"), decode=True).file_url
			frappe.db.set_value("User", args.get("name"), "user_image", fileurl)

	if args.get('name'):
		add_all_roles_to(args.get("name"))

def parse_args(args):
	if not args:
		args = frappe.local.form_dict
	if isinstance(args, string_types):
		args = json.loads(args)

	args = frappe._dict(args)

	# strip the whitespace
	for key, value in args.items():
		if isinstance(value, string_types):
			args[key] = strip(value)

	return args

def add_all_roles_to(name):
	user = frappe.get_doc("User", name)
	for role in frappe.db.sql("""select name from tabRole"""):
		if role[0] not in ["Administrator", "Guest", "All", "Customer", "Supplier", "Partner", "Employee"]:
			d = user.append("roles")
			d.role = role[0]
	user.save()

def disable_future_access():
	frappe.db.set_default('desktop:home_page', 'desktop')
	frappe.db.set_value('System Settings', 'System Settings', 'setup_complete', 1)
	frappe.db.set_value('System Settings', 'System Settings', 'is_first_startup', 1)

	if not frappe.flags.in_test:
		# remove all roles and add 'Administrator' to prevent future access
		page = frappe.get_doc('Page', 'setup-wizard')
		page.roles = []
		page.append('roles', {'role': 'Administrator'})
		page.flags.do_not_update_json = True
		page.flags.ignore_permissions = True
		page.save()

@frappe.whitelist()
def load_messages(language):
	"""Load translation messages for given language from all `setup_wizard_requires`
	javascript files"""
	frappe.clear_cache()
	set_default_language(get_language_code(language))
	frappe.db.commit()
	m = get_dict("page", "setup-wizard")

	for path in frappe.get_hooks("setup_wizard_requires"):
		# common folder `assets` served from `sites/`
		js_file_path = os.path.abspath(frappe.get_site_path("..", *path.strip("/").split("/")))
		m.update(get_dict("jsfile", js_file_path))

	m.update(get_dict("boot"))
	send_translations(m)
	return frappe.local.lang

@frappe.whitelist()
def load_languages():
	language_codes = frappe.db.sql('select language_code, language_name from tabLanguage order by name', as_dict=True)
	codes_to_names = {}
	for d in language_codes:
		codes_to_names[d.language_code] = d.language_name
	return {
		"default_language": frappe.db.get_value('Language', frappe.local.lang, 'language_name') or frappe.local.lang,
		"languages": sorted(frappe.db.sql_list('select language_name from tabLanguage order by name')),
		"codes_to_names": codes_to_names
	}

@frappe.whitelist()
def load_country():
	from frappe.sessions import get_geo_ip_country
	return get_geo_ip_country(frappe.local.request_ip) if frappe.local.request_ip else None

@frappe.whitelist()
def load_user_details():
	return {
		"full_name": frappe.cache().hget("full_name", "signup"),
		"email": frappe.cache().hget("email", "signup")
	}

@frappe.whitelist()
def reset_is_first_startup():
	frappe.db.set_value('System Settings', 'System Settings', 'is_first_startup', 0)

def prettify_args(args):
	# remove attachments
	for key, val in args.items():
		if isinstance(val, string_types) and "data:image" in val:
			filename = val.split("data:image", 1)[0].strip(", ")
			size = round((len(val) * 3 / 4) / 1048576.0, 2)
			args[key] = "Image Attached: '{0}' of size {1} MB".format(filename, size)

	pretty_args = []
	for key in sorted(args):
		pretty_args.append("{} = {}".format(key, args[key]))
	return pretty_args

def email_setup_wizard_exception(traceback, args):
	if not frappe.local.conf.setup_wizard_exception_email:
		return

	pretty_args = prettify_args(args)

	if frappe.local.request:
		user_agent = UserAgent(frappe.local.request.headers.get('User-Agent', ''))

	else:
		user_agent = frappe._dict()

	message = """

#### Traceback

{traceback}

---

#### Setup Wizard Arguments

{args}

---

#### Request Headers

{headers}

---

#### Basic Information

- **Site:** {site}
- **User:** {user}
- **Browser:** {user_agent.platform} {user_agent.browser} version: {user_agent.version} language: {user_agent.language}
- **Browser Languages**: `{accept_languages}`""".format(
		site=frappe.local.site,
		traceback=traceback,
		args="\n".join(pretty_args),
		user=frappe.session.user,
		user_agent=user_agent,
		headers=frappe.local.request.headers,
		accept_languages=", ".join(frappe.local.request.accept_languages.values()))

	frappe.sendmail(recipients=frappe.local.conf.setup_wizard_exception_email,
		sender=frappe.session.user,
		subject="Setup failed: {}".format(frappe.local.site),
		message=message,
		delayed=False)

def get_language_code(lang):
	return frappe.db.get_value('Language', {'language_name':lang})

def enable_twofactor_all_roles():
	all_role = frappe.get_doc('Role',{'role_name':'All'})
	all_role.two_factor_auth = True
	all_role.save(ignore_permissions=True)

<commit_msg>fix: setup wizard<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: See license.txt

from __future__ import unicode_literals

import frappe, json, os
from frappe.utils import strip, cint
from frappe.translate import (set_default_language, get_dict, send_translations)
from frappe.geo.country_info import get_country_info
from frappe.utils.file_manager import save_file
from frappe.utils.password import update_password
from werkzeug.useragents import UserAgent
from . import install_fixtures
from six import string_types

def get_setup_stages(args):

	# App setup stage functions should not include frappe.db.commit
	# That is done by frappe after successful completion of all stages
	stages = [
		{
			'status': 'Updating global settings',
			'fail_msg': 'Failed to update global settings',
			'tasks': [
				{
					'fn': update_global_settings,
					'args': args,
					'fail_msg': 'Failed to update global settings'
				}
			]
		}
	]

	stages += get_stages_hooks(args) + get_setup_complete_hooks(args)

	stages.append({
		# post executing hooks
		'status': 'Wrapping up',
		'fail_msg': 'Failed to complete setup',
		'tasks': [
			{
				'fn': run_post_setup_complete,
				'args': args,
				'fail_msg': 'Failed to complete setup'
			}
		]
	})

	return stages

@frappe.whitelist()
def setup_complete(args):
	"""Calls hooks for `setup_wizard_complete`, sets home page as `desktop`
	and clears cache. If wizard breaks, calls `setup_wizard_exception` hook"""

	# Setup complete: do not throw an exception, let the user continue to desk
	if cint(frappe.db.get_single_value('System Settings', 'setup_complete')):
		return

	args = parse_args(args)

	stages = get_setup_stages(args)

	try:
		current_task = None
		for idx, stage in enumerate(stages):
			frappe.publish_realtime('setup_task', {"progress": [idx, len(stages)],
				"stage_status": stage.get('status')}, user=frappe.session.user)

			for task in stage.get('tasks'):
				current_task = task
				task.get('fn')(task.get('args'))
	except Exception:
		handle_setup_exception(args)
		return {'status': 'fail', 'fail': current_task.get('fail_msg')}
	else:
		run_setup_success(args)
		return {'status': 'ok'}

def update_global_settings(args):
	if args.language and args.language != "English":
		set_default_language(get_language_code(args.lang))
		frappe.db.commit()
	frappe.clear_cache()

	update_system_settings(args)
	update_user_name(args)

def run_post_setup_complete(args):
	disable_future_access()
	frappe.db.commit()
	frappe.clear_cache()

def run_setup_success(args):
	for hook in frappe.get_hooks("setup_wizard_success"):
		frappe.get_attr(hook)(args)
	install_fixtures.install()

def get_stages_hooks(args):
	stages = []
	for method in frappe.get_hooks("setup_wizard_stages"):
		stages += frappe.get_attr(method)(args)
	return stages

def get_setup_complete_hooks(args):
	stages = []
	for method in frappe.get_hooks("setup_wizard_complete"):
		stages.append({
			'status': 'Executing method',
			'fail_msg': 'Failed to execute method',
			'tasks': [
				{
					'fn': frappe.get_attr(method),
					'args': args,
					'fail_msg': 'Failed to execute method'
				}
			]
		})
	return stages

def handle_setup_exception(args):
	frappe.db.rollback()
	if args:
		traceback = frappe.get_traceback()
		for hook in frappe.get_hooks("setup_wizard_exception"):
			frappe.get_attr(hook)(traceback, args)

def update_system_settings(args):
	number_format = get_country_info(args.get("country")).get("number_format", "#,###.##")

	# replace these as float number formats, as they have 0 precision
	# and are currency number formats and not for floats
	if number_format=="#.###":
		number_format = "#.###,##"
	elif number_format=="#,###":
		number_format = "#,###.##"

	system_settings = frappe.get_doc("System Settings", "System Settings")
	system_settings.update({
		"country": args.get("country"),
		"language": get_language_code(args.get("language")),
		"time_zone": args.get("timezone"),
		"float_precision": 3,
		'date_format': frappe.db.get_value("Country", args.get("country"), "date_format"),
		'number_format': number_format,
		'enable_scheduler': 1 if not frappe.flags.in_test else 0,
		'backup_limit': 3 # Default for downloadable backups
	})
	system_settings.save()

def update_user_name(args):
	first_name, last_name = args.get('full_name', ''), ''
	if ' ' in first_name:
		first_name, last_name = first_name.split(' ', 1)

	if args.get("email"):
		if frappe.db.exists('User', args.get('email')):
			# running again
			return

		args['name'] = args.get("email")

		_mute_emails, frappe.flags.mute_emails = frappe.flags.mute_emails, True
		doc = frappe.get_doc({
			"doctype":"User",
			"email": args.get("email"),
			"first_name": first_name,
			"last_name": last_name
		})
		doc.flags.no_welcome_mail = True
		doc.insert()
		frappe.flags.mute_emails = _mute_emails
		update_password(args.get("email"), args.get("password"))

	elif first_name:
		args.update({
			"name": frappe.session.user,
			"first_name": first_name,
			"last_name": last_name
		})

		frappe.db.sql("""update `tabUser` SET first_name=%(first_name)s,
			last_name=%(last_name)s WHERE name=%(name)s""", args)

	if args.get("attach_user"):
		attach_user = args.get("attach_user").split(",")
		if len(attach_user)==3:
			filename, filetype, content = attach_user
			fileurl = save_file(filename, content, "User", args.get("name"), decode=True).file_url
			frappe.db.set_value("User", args.get("name"), "user_image", fileurl)

	if args.get('name'):
		add_all_roles_to(args.get("name"))

def parse_args(args):
	if not args:
		args = frappe.local.form_dict
	if isinstance(args, string_types):
		args = json.loads(args)

	args = frappe._dict(args)

	# strip the whitespace
	for key, value in args.items():
		if isinstance(value, string_types):
			args[key] = strip(value)

	return args

def add_all_roles_to(name):
	user = frappe.get_doc("User", name)
	for role in frappe.db.sql("""select name from tabRole"""):
		if role[0] not in ["Administrator", "Guest", "All", "Customer", "Supplier", "Partner", "Employee"]:
			d = user.append("roles")
			d.role = role[0]
	user.save()

def disable_future_access():
	frappe.db.set_default('desktop:home_page', 'desktop')
	frappe.db.set_value('System Settings', 'System Settings', 'setup_complete', 1)
	frappe.db.set_value('System Settings', 'System Settings', 'is_first_startup', 1)

	if not frappe.flags.in_test:
		# remove all roles and add 'Administrator' to prevent future access
		page = frappe.get_doc('Page', 'setup-wizard')
		page.roles = []
		page.append('roles', {'role': 'Administrator'})
		page.flags.do_not_update_json = True
		page.flags.ignore_permissions = True
		page.save()

@frappe.whitelist()
def load_messages(language):
	"""Load translation messages for given language from all `setup_wizard_requires`
	javascript files"""
	frappe.clear_cache()
	set_default_language(get_language_code(language))
	frappe.db.commit()
	m = get_dict("page", "setup-wizard")

	for path in frappe.get_hooks("setup_wizard_requires"):
		# common folder `assets` served from `sites/`
		js_file_path = os.path.abspath(frappe.get_site_path("..", *path.strip("/").split("/")))
		m.update(get_dict("jsfile", js_file_path))

	m.update(get_dict("boot"))
	send_translations(m)
	return frappe.local.lang

@frappe.whitelist()
def load_languages():
	language_codes = frappe.db.sql('select language_code, language_name from tabLanguage order by name', as_dict=True)
	codes_to_names = {}
	for d in language_codes:
		codes_to_names[d.language_code] = d.language_name
	return {
		"default_language": frappe.db.get_value('Language', frappe.local.lang, 'language_name') or frappe.local.lang,
		"languages": sorted(frappe.db.sql_list('select language_name from tabLanguage order by name')),
		"codes_to_names": codes_to_names
	}

@frappe.whitelist()
def load_country():
	from frappe.sessions import get_geo_ip_country
	return get_geo_ip_country(frappe.local.request_ip) if frappe.local.request_ip else None

@frappe.whitelist()
def load_user_details():
	return {
		"full_name": frappe.cache().hget("full_name", "signup"),
		"email": frappe.cache().hget("email", "signup")
	}

@frappe.whitelist()
def reset_is_first_startup():
	frappe.db.set_value('System Settings', 'System Settings', 'is_first_startup', 0)

def prettify_args(args):
	# remove attachments
	for key, val in args.items():
		if isinstance(val, string_types) and "data:image" in val:
			filename = val.split("data:image", 1)[0].strip(", ")
			size = round((len(val) * 3 / 4) / 1048576.0, 2)
			args[key] = "Image Attached: '{0}' of size {1} MB".format(filename, size)

	pretty_args = []
	for key in sorted(args):
		pretty_args.append("{} = {}".format(key, args[key]))
	return pretty_args

def email_setup_wizard_exception(traceback, args):
	if not frappe.local.conf.setup_wizard_exception_email:
		return

	pretty_args = prettify_args(args)

	if frappe.local.request:
		user_agent = UserAgent(frappe.local.request.headers.get('User-Agent', ''))

	else:
		user_agent = frappe._dict()

	message = """

#### Traceback

{traceback}

---

#### Setup Wizard Arguments

{args}

---

#### Request Headers

{headers}

---

#### Basic Information

- **Site:** {site}
- **User:** {user}
- **Browser:** {user_agent.platform} {user_agent.browser} version: {user_agent.version} language: {user_agent.language}
- **Browser Languages**: `{accept_languages}`""".format(
		site=frappe.local.site,
		traceback=traceback,
		args="\n".join(pretty_args),
		user=frappe.session.user,
		user_agent=user_agent,
		headers=frappe.local.request.headers,
		accept_languages=", ".join(frappe.local.request.accept_languages.values()))

	frappe.sendmail(recipients=frappe.local.conf.setup_wizard_exception_email,
		sender=frappe.session.user,
		subject="Setup failed: {}".format(frappe.local.site),
		message=message,
		delayed=False)

def get_language_code(lang):
	return frappe.db.get_value('Language', {'language_name':lang})

def enable_twofactor_all_roles():
	all_role = frappe.get_doc('Role',{'role_name':'All'})
	all_role.two_factor_auth = True
	all_role.save(ignore_permissions=True)

<|endoftext|>


<commit_before>from collections import OrderedDict

import numpy as np

from symfit.core.objectives import (
    LeastSquares, VectorLeastSquares, LogLikelihood
)
from symfit.core.support import keywordonly

class FitResults(object):
    """
    Class to display the results of a fit in a nice and unambiguous way.
    All things related to the fit are available on this class, e.g.
    - parameter values + stdev
    - R squared (Regression coefficient.) or other fit quality qualifiers.
    - fitting status message
    - covariance matrix
    - objective and minimizer used.

    Contains the attribute `params`, which is an
    :class:`~collections.OrderedDict` containing all the parameter names and
    their optimized values. Can be `**` unpacked when evaluating
    :class:`~symfit.core.fit.Model`'s.
    """
    @keywordonly(constraints=None)
    def __init__(self, model, popt, covariance_matrix, minimizer, objective, message, **minimizer_output):
        """
        :param model: :class:`~symfit.core.fit.Model` that was fit to.
        :param popt: best fit parameters, same ordering as in model.params.
        :param covariance_matrix: covariance matrix.
        :param minimizer: Minimizer instance used.
        :param objective: Objective function which was optimized.
        :param message: Status message returned by the minimizer.
        :param \**minimizer_output: Raw output as given by the minimizer.
        """
        constraints = minimizer_output.pop('constraints')
        self.constraints = constraints if constraints is not None else []
        self.minimizer_output = minimizer_output
        self.model = model
        self.minimizer = minimizer
        self.objective = objective
        self.status_message = message

        self._popt = popt
        self.params = OrderedDict(
            [(p.name, value) for p, value in zip(self.model.params, popt)]
        )
        self.covariance_matrix = covariance_matrix
        self.gof_qualifiers = self._gof_qualifiers()

    @property
    def iterations(self):
        if 'nit' in self.minimizer_output:
            return self.minimizer_output['nit']
        else:
            return None

    def __str__(self):
        """
        Pretty print the results as a table.
        """
        res = '\nParameter Value        Standard Deviation\n'
        for p in self.model.params:
            value = self.value(p)
            value_str = '{:e}'.format(value) if value is not None else 'None'
            stdev = self.stdev(p)
            stdev_str = '{:e}'.format(stdev) if stdev is not None else 'None'
            res += '{:10}{} {}\n'.format(p.name, value_str, stdev_str, width=20)

        res += '{:<22} {}\n'.format('Status message', self.status_message)
        res += '{:<22} {}\n'.format('Number of iterations', self.iterations)
        res += '{:<22} {}\n'.format('Objective', self.objective)
        res += '{:<22} {}\n'.format('Minimizer', self.minimizer)

        res += '\nGoodness of fit qualifiers:\n'
        res += '\n'.join('{:<22} {}'.format(gof, value)
                         for gof, value in sorted(self.gof_qualifiers.items()))

        if self.constraints:
            res += '\n\nConstraints:\n'
            res += 20 * '-' + '\n'
            # res += '{:<22} {}\n'.format('Constraint', 'Value')
            for constraint in self.constraints:
                # Print the component and the value of the constraint
                res += 'Question: {} {} 0?\n'.format(
                    constraint.model[constraint.model.dependent_vars[0]],
                    constraint.model.constraint_type.rel_op
                )
                res += 'Answer:   {}\n\n'.format(constraint(**self.params)[0])

        return res

    def __getattr__(self, item):
        """
        Return the requested `item` if it can be found in the gof_qualifiers
        dict.

        :param item: Name of Goodness of Fit qualifier.
        :return: Goodness of Fit qualifier if present.
        """
        if 'gof_qualifiers' in vars(self):
            if item in self.gof_qualifiers:
                return self.gof_qualifiers[item]
        raise AttributeError

    def stdev(self, param):
        """
        Return the standard deviation in a given parameter as found by the fit.

        :param param: ``Parameter`` Instance.
        :return: Standard deviation of ``param``.
        """
        try:
            return np.sqrt(self.variance(param))
        except AttributeError:
            # This happens when variance returns None.
            return None

    def value(self, param):
        """
        Return the value in a given parameter as found by the fit.

        :param param: ``Parameter`` Instance.
        :return: Value of ``param``.
        """
        return self.params[param.name]

    def variance(self, param):
        """
        Return the variance in a given parameter as found by the fit.

        :param param: ``Parameter`` Instance.
        :return: Variance of ``param``.
        """
        param_number = self.model.params.index(param)
        try:
            return self.covariance_matrix[param_number, param_number]
        except TypeError:
            # covariance_matrix can be None
            return None

    def covariance(self, param_1, param_2):
        """
        Return the covariance between param_1 and param_2.

        :param param_1: ``Parameter`` Instance.
        :param param_2: ``Parameter`` Instance.
        :return: Covariance of the two params.
        """
        param_1_number = self.model.params.index(param_1)
        param_2_number = self.model.params.index(param_2)
        return self.covariance_matrix[param_1_number, param_2_number]

    @staticmethod
    def _array_safe_dict_eq(one_dict, other_dict):
        """
        Dicts containing arrays are hard to compare. This function uses
        numpy.allclose to compare arrays, and does normal comparison for all
        other types.

        This pretty mich defines FitResult equality, but because there are
        still some questions on how and if that should be defined, __eq__ has
        not been implemented.

        :param one_dict: __dict__ of a FitResults object
        :param other_dict: __dict__ of a FitResults object
        :return: bool
        """
        for key in one_dict:
            try:
                if key == 'minimizer':
                    assert one_dict[key].__class__ == other_dict[key].__class__
                elif key == 'minimizer_output':
                    # Ignore this, because it can contain unexpected terms and
                    # if all the derived attributes are correct I see no reason
                    # why this term shouldn't be at least close enough.
                    pass
                else:
                    assert one_dict[key] == other_dict[key]
            except ValueError as err:
                # When dealing with arrays, we need to use numpy for comparison
                if isinstance(one_dict[key], dict):
                    assert FitResults._array_safe_dict_eq(one_dict[key], other_dict[key])
                else:
                    assert np.allclose(one_dict[key], other_dict[key])
            except AssertionError:
                return False
        else: return True

    def __getstate__(self):
        state = self.__dict__.copy()

        if hasattr(state['minimizer'], 'minimizers'):
            # ChainedMinimizer pickles well, just use the instance.
            minimizer_cls = state['minimizer']
        else:
            minimizer_cls = type(state['minimizer'])
        state['minimizer'] = (minimizer_cls,
                              state['minimizer'].objective,
                              state['minimizer'].parameters)
        return state

    def __setstate__(self, state):
        min_class, objective, parameters = state['minimizer']
        try:
            state['minimizer'] = min_class(objective, parameters)
        except TypeError:  # assume min_class is already an instance.
            state['minimizer'] = min_class
        self.__dict__.update(state)

    def _gof_qualifiers(self):
        """
        Based on the objective used, we can infer certain goodness of fit
        (g.o.f.) qualifiers.

        The ``objective_value`` itself always exists, and then depending on the
        objective we also get the following:

        - The coefficient of determination :math:`R^2` and :math:`\\chi^2` for
          :class:`~symfit.core.objectives.LeastSquares` and
          :class:`~symfit.core.objectives.VectorLeastSquares`.
        - Likelihood and log-likelihood for
          :class:`~symfit.core.objectives.LogLikelihood`.

        :return: ``dict`` containing goodness of fit qualifiers.
        """
        gof_qualifiers = {}

        gof_qualifiers['objective_value'] = self.minimizer_output['fun']
        if isinstance(self.objective, (LeastSquares, VectorLeastSquares)):
            R2 = r_squared(self.objective.model, fit_result=self,
                           data=self.objective.data)
            gof_qualifiers['r_squared'] = R2

        if isinstance(self.objective, VectorLeastSquares):
            # In this case the objective value is the residuals
            chi_squared = np.sum(gof_qualifiers['objective_value'] ** 2)
            gof_qualifiers['chi_squared'] = chi_squared
        elif isinstance(self.objective, LeastSquares):
            # Undo the normalization to get back chi^2.
            gof_qualifiers['chi_squared'] = 2 * gof_qualifiers['objective_value']
        elif isinstance(self.objective, LogLikelihood):
            # We undo the minus sign we have included to maximize likelihood
            gof_qualifiers['log_likelihood'] = - gof_qualifiers['objective_value']
            gof_qualifiers['likelihood'] = np.exp(gof_qualifiers['log_likelihood'])
        return gof_qualifiers

def r_squared(model, fit_result, data):
    """
    Calculates the coefficient of determination, R^2, for the fit.

    (Is not defined properly for vector valued functions.)

    :param model: Model instance
    :param fit_result: FitResults instance
    :param data: data with which the fit was performed.
    """
    # First filter out the dependent vars
    y_is = [data[var] for var in model if var in data]
    x_is = [value for var, value in data.items() if var.name in model.__signature__.parameters]
    y_bars = [np.mean(y_i) if y_i is not None else None for y_i in y_is]
    f_is = model(*x_is, **fit_result.params)
    SS_res = np.sum([np.sum((y_i - f_i)**2) for y_i, f_i in zip(y_is, f_is) if y_i is not None])
    SS_tot = np.sum([np.sum((y_i - y_bar)**2) for y_i, y_bar in zip(y_is, y_bars) if y_i is not None])
    return 1 - SS_res/SS_tot<commit_msg>Made pickling more complicated again, for py27 reasons.<commit_after>from collections import OrderedDict

import numpy as np

from symfit.core.objectives import (
    LeastSquares, VectorLeastSquares, LogLikelihood
)
from symfit.core.support import keywordonly

class FitResults(object):
    """
    Class to display the results of a fit in a nice and unambiguous way.
    All things related to the fit are available on this class, e.g.
    - parameter values + stdev
    - R squared (Regression coefficient.) or other fit quality qualifiers.
    - fitting status message
    - covariance matrix
    - objective and minimizer used.

    Contains the attribute `params`, which is an
    :class:`~collections.OrderedDict` containing all the parameter names and
    their optimized values. Can be `**` unpacked when evaluating
    :class:`~symfit.core.fit.Model`'s.
    """
    @keywordonly(constraints=None)
    def __init__(self, model, popt, covariance_matrix, minimizer, objective, message, **minimizer_output):
        """
        :param model: :class:`~symfit.core.fit.Model` that was fit to.
        :param popt: best fit parameters, same ordering as in model.params.
        :param covariance_matrix: covariance matrix.
        :param minimizer: Minimizer instance used.
        :param objective: Objective function which was optimized.
        :param message: Status message returned by the minimizer.
        :param \**minimizer_output: Raw output as given by the minimizer.
        """
        constraints = minimizer_output.pop('constraints')
        self.constraints = constraints if constraints is not None else []
        self.minimizer_output = minimizer_output
        self.model = model
        self.minimizer = minimizer
        self.objective = objective
        self.status_message = message

        self._popt = popt
        self.params = OrderedDict(
            [(p.name, value) for p, value in zip(self.model.params, popt)]
        )
        self.covariance_matrix = covariance_matrix
        self.gof_qualifiers = self._gof_qualifiers()

    @property
    def iterations(self):
        if 'nit' in self.minimizer_output:
            return self.minimizer_output['nit']
        else:
            return None

    def __str__(self):
        """
        Pretty print the results as a table.
        """
        res = '\nParameter Value        Standard Deviation\n'
        for p in self.model.params:
            value = self.value(p)
            value_str = '{:e}'.format(value) if value is not None else 'None'
            stdev = self.stdev(p)
            stdev_str = '{:e}'.format(stdev) if stdev is not None else 'None'
            res += '{:10}{} {}\n'.format(p.name, value_str, stdev_str, width=20)

        res += '{:<22} {}\n'.format('Status message', self.status_message)
        res += '{:<22} {}\n'.format('Number of iterations', self.iterations)
        res += '{:<22} {}\n'.format('Objective', self.objective)
        res += '{:<22} {}\n'.format('Minimizer', self.minimizer)

        res += '\nGoodness of fit qualifiers:\n'
        res += '\n'.join('{:<22} {}'.format(gof, value)
                         for gof, value in sorted(self.gof_qualifiers.items()))

        if self.constraints:
            res += '\n\nConstraints:\n'
            res += 20 * '-' + '\n'
            # res += '{:<22} {}\n'.format('Constraint', 'Value')
            for constraint in self.constraints:
                # Print the component and the value of the constraint
                res += 'Question: {} {} 0?\n'.format(
                    constraint.model[constraint.model.dependent_vars[0]],
                    constraint.model.constraint_type.rel_op
                )
                res += 'Answer:   {}\n\n'.format(constraint(**self.params)[0])

        return res

    def __getattr__(self, item):
        """
        Return the requested `item` if it can be found in the gof_qualifiers
        dict.

        :param item: Name of Goodness of Fit qualifier.
        :return: Goodness of Fit qualifier if present.
        """
        if 'gof_qualifiers' in vars(self):
            if item in self.gof_qualifiers:
                return self.gof_qualifiers[item]
        raise AttributeError

    def stdev(self, param):
        """
        Return the standard deviation in a given parameter as found by the fit.

        :param param: ``Parameter`` Instance.
        :return: Standard deviation of ``param``.
        """
        try:
            return np.sqrt(self.variance(param))
        except AttributeError:
            # This happens when variance returns None.
            return None

    def value(self, param):
        """
        Return the value in a given parameter as found by the fit.

        :param param: ``Parameter`` Instance.
        :return: Value of ``param``.
        """
        return self.params[param.name]

    def variance(self, param):
        """
        Return the variance in a given parameter as found by the fit.

        :param param: ``Parameter`` Instance.
        :return: Variance of ``param``.
        """
        param_number = self.model.params.index(param)
        try:
            return self.covariance_matrix[param_number, param_number]
        except TypeError:
            # covariance_matrix can be None
            return None

    def covariance(self, param_1, param_2):
        """
        Return the covariance between param_1 and param_2.

        :param param_1: ``Parameter`` Instance.
        :param param_2: ``Parameter`` Instance.
        :return: Covariance of the two params.
        """
        param_1_number = self.model.params.index(param_1)
        param_2_number = self.model.params.index(param_2)
        return self.covariance_matrix[param_1_number, param_2_number]

    @staticmethod
    def _array_safe_dict_eq(one_dict, other_dict):
        """
        Dicts containing arrays are hard to compare. This function uses
        numpy.allclose to compare arrays, and does normal comparison for all
        other types.

        This pretty mich defines FitResult equality, but because there are
        still some questions on how and if that should be defined, __eq__ has
        not been implemented.

        :param one_dict: __dict__ of a FitResults object
        :param other_dict: __dict__ of a FitResults object
        :return: bool
        """
        for key in one_dict:
            try:
                if key == 'minimizer':
                    assert one_dict[key].__class__ == other_dict[key].__class__
                elif key == 'minimizer_output':
                    # Ignore this, because it can contain unexpected terms and
                    # if all the derived attributes are correct I see no reason
                    # why this term shouldn't be at least close enough.
                    pass
                else:
                    assert one_dict[key] == other_dict[key]
            except ValueError as err:
                # When dealing with arrays, we need to use numpy for comparison
                if isinstance(one_dict[key], dict):
                    assert FitResults._array_safe_dict_eq(one_dict[key], other_dict[key])
                else:
                    assert np.allclose(one_dict[key], other_dict[key])
            except AssertionError:
                return False
        else: return True

    def __getstate__(self):
        state = self.__dict__.copy()

        if hasattr(state['minimizer'], 'minimizers'):  # ChainedMinimizer
            # ToDo: when py27 support is droppend at least this can be replaced
            #       with just pickling the instance, perhaps also for other
            #       minimizers.
            minimizer_cls = [type(state['minimizer'])]
            minimizer_cls.extend(
                [type(minimizer) for minimizer in state['minimizer'].minimizers]
            )
        else:
            minimizer_cls = type(state['minimizer'])
        state['minimizer'] = (minimizer_cls,
                              state['minimizer'].objective,
                              state['minimizer'].parameters)
        return state

    def __setstate__(self, state):
        min_class, objective, parameters = state['minimizer']
        try:
            # If min_class is iterable, initiate a ChainedMinimizer.
            minimizers = [cls(objective, parameters) for cls in min_class[1:]]
        except TypeError:
            state['minimizer'] = min_class(objective, parameters)
        else:
            state['minimizer'] = min_class[0](objective, parameters,
                                              minimizers=minimizers)
        self.__dict__.update(state)

    def _gof_qualifiers(self):
        """
        Based on the objective used, we can infer certain goodness of fit
        (g.o.f.) qualifiers.

        The ``objective_value`` itself always exists, and then depending on the
        objective we also get the following:

        - The coefficient of determination :math:`R^2` and :math:`\\chi^2` for
          :class:`~symfit.core.objectives.LeastSquares` and
          :class:`~symfit.core.objectives.VectorLeastSquares`.
        - Likelihood and log-likelihood for
          :class:`~symfit.core.objectives.LogLikelihood`.

        :return: ``dict`` containing goodness of fit qualifiers.
        """
        gof_qualifiers = {}

        gof_qualifiers['objective_value'] = self.minimizer_output['fun']
        if isinstance(self.objective, (LeastSquares, VectorLeastSquares)):
            R2 = r_squared(self.objective.model, fit_result=self,
                           data=self.objective.data)
            gof_qualifiers['r_squared'] = R2

        if isinstance(self.objective, VectorLeastSquares):
            # In this case the objective value is the residuals
            chi_squared = np.sum(gof_qualifiers['objective_value'] ** 2)
            gof_qualifiers['chi_squared'] = chi_squared
        elif isinstance(self.objective, LeastSquares):
            # Undo the normalization to get back chi^2.
            gof_qualifiers['chi_squared'] = 2 * gof_qualifiers['objective_value']
        elif isinstance(self.objective, LogLikelihood):
            # We undo the minus sign we have included to maximize likelihood
            gof_qualifiers['log_likelihood'] = - gof_qualifiers['objective_value']
            gof_qualifiers['likelihood'] = np.exp(gof_qualifiers['log_likelihood'])
        return gof_qualifiers

def r_squared(model, fit_result, data):
    """
    Calculates the coefficient of determination, R^2, for the fit.

    (Is not defined properly for vector valued functions.)

    :param model: Model instance
    :param fit_result: FitResults instance
    :param data: data with which the fit was performed.
    """
    # First filter out the dependent vars
    y_is = [data[var] for var in model if var in data]
    x_is = [value for var, value in data.items() if var.name in model.__signature__.parameters]
    y_bars = [np.mean(y_i) if y_i is not None else None for y_i in y_is]
    f_is = model(*x_is, **fit_result.params)
    SS_res = np.sum([np.sum((y_i - f_i)**2) for y_i, f_i in zip(y_is, f_is) if y_i is not None])
    SS_tot = np.sum([np.sum((y_i - y_bar)**2) for y_i, y_bar in zip(y_is, y_bars) if y_i is not None])
    return 1 - SS_res/SS_tot<|endoftext|>


<commit_before>from tastypie.authentication import Authentication
from tastypie.authorization import Authorization
from spa.api.v1.BackboneCompatibleResource import BackboneCompatibleResource
from spa.models import UserProfile
from  spa.models._Activity import _Activity

class ActivityResource(BackboneCompatibleResource):
    class Meta:
        queryset = _Activity.objects.all().order_by('-date')
        resource_name = 'activity'
        authorization = Authorization()
        authentication = Authentication()
        always_return_data = True

    def get_object_list(self, request):
        return self._meta.queryset.select_subclasses()

    def dehydrate(self, bundle):
        try:
            if bundle.obj.user is not None:
                user_name = bundle.obj.user.get_full_name()
                user_image = bundle.obj.user.get_profile().get_small_profile_image()
                user_profile = bundle.obj.user.get_profile().get_profile_url()
            else:
                user_name = "Anonymous user"
                user_image = UserProfile.get_default_avatar_image()
                user_profile = ""

            bundle.data["verb"] = bundle.obj.get_verb_passed(),
            bundle.data["object"] = bundle.obj.get_object_singular(),
            bundle.data["item_name"] = bundle.obj.get_object_name(),
            bundle.data["item_url"] = bundle.obj.get_object_url(),
            bundle.data["user_name"] = user_name,
            bundle.data["user_profile"] = user_profile,
            bundle.data["user_image"] = user_image
            return bundle

        except AttributeError, ae:
            self.logger.debug("AttributeError: Error dehydrating activity, %s" % ae.message)
        except TypeError, te:
            self.logger.debug("TypeError: Error dehydrating activity, %s" % te.message)
        except Exception, ee:
            self.logger.debug("Exception: Error dehydrating activity, %s" % ee.message)
        return None

    """"
    def alter_list_data_to_serialize(self, request, data):
        return [i for i in data['objects'] if i is not None and i.obj.user is not None and i.obj.get_object_name is not None and i.obj.get_object_url is not None]
    """

    def dehydrate_date(self, bundle):
        return self.humanize_date(bundle.obj.date)
<commit_msg>Updated activity to use nice name<commit_after>from tastypie.authentication import Authentication
from tastypie.authorization import Authorization
from spa.api.v1.BackboneCompatibleResource import BackboneCompatibleResource
from spa.models import UserProfile
from  spa.models._Activity import _Activity

class ActivityResource(BackboneCompatibleResource):
    class Meta:
        queryset = _Activity.objects.all().order_by('-date')
        resource_name = 'activity'
        authorization = Authorization()
        authentication = Authentication()
        always_return_data = True

    def get_object_list(self, request):
        return self._meta.queryset.select_subclasses()

    def dehydrate(self, bundle):
        try:
            if bundle.obj.user is not None:
                user_name = bundle.obj.user.get_profile().nice_name()
                user_image = bundle.obj.user.get_profile().get_small_profile_image()
                user_profile = bundle.obj.user.get_profile().get_profile_url()
            else:
                user_name = "Anonymous user"
                user_image = UserProfile.get_default_avatar_image()
                user_profile = ""

            bundle.data["verb"] = bundle.obj.get_verb_passed(),
            bundle.data["object"] = bundle.obj.get_object_singular(),
            bundle.data["item_name"] = bundle.obj.get_object_name(),
            bundle.data["item_url"] = bundle.obj.get_object_url(),
            bundle.data["user_name"] = user_name,
            bundle.data["user_profile"] = user_profile,
            bundle.data["user_image"] = user_image
            return bundle

        except AttributeError, ae:
            self.logger.debug("AttributeError: Error dehydrating activity, %s" % ae.message)
        except TypeError, te:
            self.logger.debug("TypeError: Error dehydrating activity, %s" % te.message)
        except Exception, ee:
            self.logger.debug("Exception: Error dehydrating activity, %s" % ee.message)
        return None

    """"
    def alter_list_data_to_serialize(self, request, data):
        return [i for i in data['objects'] if i is not None and i.obj.user is not None and i.obj.get_object_name is not None and i.obj.get_object_url is not None]
    """

    def dehydrate_date(self, bundle):
        return self.humanize_date(bundle.obj.date)
<|endoftext|>


<commit_before>__author__ = "John Kirkham <kirkhamj@janelia.hhmi.org>"
__date__ = "$Jun 20, 2014 12:07:48 EDT$"

#import nanshe.advanced_debugging

#logger = nanshe.advanced_debugging.logging.getLogger(__name__)

class SPAMSException(Exception):
    pass

#@nanshe.advanced_debugging.log_call(logger)
def run_multiprocessing_queue_spams_trainDL(out_queue, *args, **kwargs):
    """
        Designed to run spams.trainDL in a separate process.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Queue to return the resulting dictionary.

        Args:
            out_queue(multiprocessing.Queue):       what will take the returned dictionary from spams.trainDL.
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            *kwargs(dict):                          a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            Todo
            Look into having the raw data for input for spams.trainDL copied in.
    """

    # It is not needed outside of calling spams.trainDL.
    # Also, it takes a long time to load this module.
    import spams

    result = spams.trainDL(*args, **kwargs)
    out_queue.put(result)

#@nanshe.advanced_debugging.log_call(logger)
def call_multiprocessing_queue_spams_trainDL(*args, **kwargs):
    """
        Designed to start spams.trainDL in a separate process and handle the result in an unnoticeably different way.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Queue to return the resulting dictionary.

        Args:
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            *kwargs(dict):                          a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            Todo
            Look into having the raw data for input for spams.trainDL copied in.

        Returns:
            result(numpy.matrix): the dictionary found
    """

    # Only necessary for dealing with SPAMS
    import multiprocessing

    out_queue = multiprocessing.Queue()

    p = multiprocessing.Process(target = run_multiprocessing_queue_spams_trainDL, args = (out_queue,) + args, kwargs = kwargs)
    p.start()
    result = out_queue.get()
    result = result.copy()
    p.join()

    if p.exitcode != 0:
        raise SPAMSException("SPAMS has terminated with exitcode \"" + repr(p.exitcode) + "\".")

    return(result)

#@nanshe.advanced_debugging.log_call(logger)
def run_multiprocessing_array_spams_trainDL(output_array, *args, **kwargs):
    """
        Designed to start spams.trainDL in a separate process and handle the result in an unnoticeably different way.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Array to share memory to return the resulting dictionary.

        Args:
            output_array(multiprocessing.Array):    shared memory array to store results in.
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            *kwargs(dict):                          a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            This is somewhat faster than using multiprocessing.Queue.

            Todo
            Need to deal with return_model case.
            Look into having the raw data for input for spams.trainDL copied in.
    """

    # Only necessary for dealing with multiprocessing.Array for SPAMS
    import ctypes
    # Just to make sure this exists in the new process. Shouldn't be necessary.
    import numpy
    # Just to make sure this exists in the new process. Shouldn't be necessary.
    # Also, it is not needed outside of calling this function.
    import spams

    # Create a numpy.ndarray that uses the shared buffer.
    result = numpy.frombuffer(output_array.get_obj(), dtype = ctypes.c_double).reshape((-1, kwargs["K"]))

    result[:] = spams.trainDL(*args, **kwargs)

#@nanshe.advanced_debugging.log_call(logger)
def call_multiprocessing_array_spams_trainDL(X, *args, **kwargs):
    """
        Designed to start spams.trainDL in a separate process and handle the result in an unnoticeably different way.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Array to share memory to return the resulting dictionary.

        Args:
            X(numpy.matrix):                        a Fortran order NumPy Matrix with the same name as used by spams.trainDL (so if someone tries to use it as a keyword argument...).
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            **kwargs(dict):                         a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            This is somewhat faster than using multiprocessing.Queue.

            Todo
            Need to deal with return_model case.
            Look into having the raw data for input for spams.trainDL copied in.
    """

    # Only necessary for dealing with SPAMS
    import multiprocessing

    # Only necessary for dealing with multiprocessing.Array for SPAMS
    import ctypes
    # Just to make sure this exists in the new process. Shouldn't be necessary.
    import numpy

    output_array_size = X.shape[0] * kwargs["K"]
    output_array = multiprocessing.Array(ctypes.c_double, output_array_size)

    p = multiprocessing.Process(target = run_multiprocessing_array_spams_trainDL, args = (output_array, X,) + args, kwargs = kwargs)
    p.start()
    p.join()

    if p.exitcode != 0:
        raise SPAMSException("SPAMS has terminated with exitcode \"" + repr(p.exitcode) + "\".")

    result = numpy.frombuffer(output_array.get_obj(), dtype = ctypes.c_double).reshape((-1, kwargs["K"]))
    result = result.copy()

    return(result)

#@nanshe.advanced_debugging.log_call(logger)
def call_spams_trainDL(*args, **kwargs):
    """
        Encapsulates call to spams.trainDL. Ensures copy of results occur just in case.
        Designed to be like the multiprocessing calls.

        Args:
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            **kwargs(dict):                         a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            For legacy.
    """

    # It is not needed outside of calling spams.trainDL.
    # Also, it takes a long time to load this module.
    import spams

    result = spams.trainDL(*args, **kwargs)
    result = result.copy()

    return(result)<commit_msg>spams_sandbox/spams_sandbox.py: Rename output_array to result_array to be more consistent.<commit_after>__author__ = "John Kirkham <kirkhamj@janelia.hhmi.org>"
__date__ = "$Jun 20, 2014 12:07:48 EDT$"

#import nanshe.advanced_debugging

#logger = nanshe.advanced_debugging.logging.getLogger(__name__)

class SPAMSException(Exception):
    pass

#@nanshe.advanced_debugging.log_call(logger)
def run_multiprocessing_queue_spams_trainDL(out_queue, *args, **kwargs):
    """
        Designed to run spams.trainDL in a separate process.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Queue to return the resulting dictionary.

        Args:
            out_queue(multiprocessing.Queue):       what will take the returned dictionary from spams.trainDL.
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            *kwargs(dict):                          a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            Todo
            Look into having the raw data for input for spams.trainDL copied in.
    """

    # It is not needed outside of calling spams.trainDL.
    # Also, it takes a long time to load this module.
    import spams

    result = spams.trainDL(*args, **kwargs)
    out_queue.put(result)

#@nanshe.advanced_debugging.log_call(logger)
def call_multiprocessing_queue_spams_trainDL(*args, **kwargs):
    """
        Designed to start spams.trainDL in a separate process and handle the result in an unnoticeably different way.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Queue to return the resulting dictionary.

        Args:
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            *kwargs(dict):                          a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            Todo
            Look into having the raw data for input for spams.trainDL copied in.

        Returns:
            result(numpy.matrix): the dictionary found
    """

    # Only necessary for dealing with SPAMS
    import multiprocessing

    out_queue = multiprocessing.Queue()

    p = multiprocessing.Process(target = run_multiprocessing_queue_spams_trainDL, args = (out_queue,) + args, kwargs = kwargs)
    p.start()
    result = out_queue.get()
    result = result.copy()
    p.join()

    if p.exitcode != 0:
        raise SPAMSException("SPAMS has terminated with exitcode \"" + repr(p.exitcode) + "\".")

    return(result)

#@nanshe.advanced_debugging.log_call(logger)
def run_multiprocessing_array_spams_trainDL(result_array, *args, **kwargs):
    """
        Designed to start spams.trainDL in a separate process and handle the result in an unnoticeably different way.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Array to share memory to return the resulting dictionary.

        Args:
            result_array(multiprocessing.Array):    shared memory array to store results in.
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            *kwargs(dict):                          a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            This is somewhat faster than using multiprocessing.Queue.

            Todo
            Need to deal with return_model case.
            Look into having the raw data for input for spams.trainDL copied in.
    """

    # Only necessary for dealing with multiprocessing.Array for SPAMS
    import ctypes
    # Just to make sure this exists in the new process. Shouldn't be necessary.
    import numpy
    # Just to make sure this exists in the new process. Shouldn't be necessary.
    # Also, it is not needed outside of calling this function.
    import spams

    # Create a numpy.ndarray that uses the shared buffer.
    result = numpy.frombuffer(result_array.get_obj(), dtype = ctypes.c_double).reshape((-1, kwargs["K"]))

    result[:] = spams.trainDL(*args, **kwargs)

#@nanshe.advanced_debugging.log_call(logger)
def call_multiprocessing_array_spams_trainDL(X, *args, **kwargs):
    """
        Designed to start spams.trainDL in a separate process and handle the result in an unnoticeably different way.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Array to share memory to return the resulting dictionary.

        Args:
            X(numpy.matrix):                        a Fortran order NumPy Matrix with the same name as used by spams.trainDL (so if someone tries to use it as a keyword argument...).
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            **kwargs(dict):                         a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            This is somewhat faster than using multiprocessing.Queue.

            Todo
            Need to deal with return_model case.
            Look into having the raw data for input for spams.trainDL copied in.
    """

    # Only necessary for dealing with SPAMS
    import multiprocessing

    # Only necessary for dealing with multiprocessing.Array for SPAMS
    import ctypes
    # Just to make sure this exists in the new process. Shouldn't be necessary.
    import numpy

    result_array_size = X.shape[0] * kwargs["K"]
    result_array = multiprocessing.Array(ctypes.c_double, result_array_size)

    p = multiprocessing.Process(target = run_multiprocessing_array_spams_trainDL, args = (result_array, X,) + args, kwargs = kwargs)
    p.start()
    p.join()

    if p.exitcode != 0:
        raise SPAMSException("SPAMS has terminated with exitcode \"" + repr(p.exitcode) + "\".")

    result = numpy.frombuffer(result_array.get_obj(), dtype = ctypes.c_double).reshape((-1, kwargs["K"]))
    result = result.copy()

    return(result)

#@nanshe.advanced_debugging.log_call(logger)
def call_spams_trainDL(*args, **kwargs):
    """
        Encapsulates call to spams.trainDL. Ensures copy of results occur just in case.
        Designed to be like the multiprocessing calls.

        Args:
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            **kwargs(dict):                         a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            For legacy.
    """

    # It is not needed outside of calling spams.trainDL.
    # Also, it takes a long time to load this module.
    import spams

    result = spams.trainDL(*args, **kwargs)
    result = result.copy()

    return(result)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    static_file

    :copyright: (c) 2015 by Fulfil.IO Inc.
    :license: see LICENSE for details.
"""
import os
from trytond.pool import PoolMeta

__all__ = ['NereidStaticFile']
__metaclass__ = PoolMeta

class NereidStaticFile:
    "Static files for Nereid"
    __name__ = "nereid.static.file"

    def _set_file_binary(self, value):
        """
        Setter for static file that stores file in file system
        :param value: The value to set
        """
        if self.name == 'no-product-image.png' and \
                value == 'binary-default-image':
            # XXX: This is default webshop image that need to be replaced
            # by binary content of product no image. Ugly hack ;)
            # Can't find provision to store binary file in xml.
            value = open(os.path.join(
                os.path.dirname(__file__), 'static/images/no-product-image.png'
            )).read()
        super(NereidStaticFile, self)._set_file_binary(value)
<commit_msg>fetch default image from webshop static<commit_after># -*- coding: utf-8 -*-
"""
    static_file

    :copyright: (c) 2015 by Fulfil.IO Inc.
    :license: see LICENSE for details.
"""
import os
from trytond.pool import PoolMeta, Pool

__all__ = ['NereidStaticFile']
__metaclass__ = PoolMeta

class NereidStaticFile:
    "Static files for Nereid"
    __name__ = "nereid.static.file"

    def get_file_binary(self, name):
        '''
        Getter for the binary_file field. This fetches the file from the
        file system, coverts it to buffer and returns it.
        :param name: Field name
        :return: File buffer
        '''
        ModelData = Pool().get('ir.model.data')
        try:
            mystery_box_id = ModelData.get_id("nereid_webshop", "mystery_box")
        except Exception:
            mystery_box_id = None

        if self.id == mystery_box_id:
            # XXX: This is default webshop image that need to be replaced
            # by binary content of product no image. Ugly hack ;)
            # Can't find provision to store binary file in xml.
            location = os.path.join(
                os.path.dirname(__file__), 'static/images/no-product-image.png'
            )
            with open(location, 'rb') as file_reader:
                return buffer(file_reader.read())
        return super(NereidStaticFile, self).get_file_binary(name)
<|endoftext|>


<commit_before>"""
Test calling an overriden method.

Note:
  This verifies that LLDB is correctly building the method overrides table.
  If this table is not built correctly then calls to overridden methods in
  derived classes may generate references to non-existant vtable entries,
  as the compiler treats the overridden method as a totally new virtual
  method definition.
  <rdar://problem/14205774>

"""

import lldb
from lldbsuite.test.decorators import *
from lldbsuite.test.lldbtest import *
from lldbsuite.test import lldbutil

class ExprCommandCallOverriddenMethod(TestBase):

    mydir = TestBase.compute_mydir(__file__)

    def setUp(self):
        # Call super's setUp().
        TestBase.setUp(self)
        # Find the line number to break for main.c.
        self.line = line_number('main.cpp', '// Set breakpoint here')

    def test(self):
        """Test calls to overridden methods in derived classes."""
        self.build()

        # Set breakpoint in main and run exe
        self.runCmd("file " + self.getBuildArtifact("a.out"),
                    CURRENT_EXECUTABLE_SET)
        lldbutil.run_break_set_by_file_and_line(
            self, "main.cpp", self.line, num_expected_locations=-1, loc_exact=True)

        self.runCmd("run", RUN_SUCCEEDED)

        # Test call to method in base class (this should always work as the base
        # class method is never an override).
        self.expect("expr b->foo()", substrs=["2"])

        # Test call to overridden method in derived class (this will fail if the
        # overrides table is not correctly set up, as Derived::foo will be assigned
        # a vtable entry that does not exist in the compiled program).
        self.expect("expr d.foo()", substrs=["2"])

        # Test calling the base class.
        self.expect("expr realbase.foo()", substrs=["1"])

        # Test with locally constructed instances.
        self.expect("expr Base().foo()", substrs=["1"])
        self.expect("expr Derived().foo()", substrs=["2"])

    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr43707")
    def test_call_on_temporary(self):
        """Test calls to overridden methods in derived classes."""
        self.build()

        # Set breakpoint in main and run exe
        self.runCmd("file " + self.getBuildArtifact("a.out"),
                    CURRENT_EXECUTABLE_SET)
        lldbutil.run_break_set_by_file_and_line(
            self, "main.cpp", self.line, num_expected_locations=-1, loc_exact=True)

        self.runCmd("run", RUN_SUCCEEDED)

        # Test with locally constructed instances.
        self.expect("expr Base().foo()", substrs=["1"])
        self.expect("expr Derived().foo()", substrs=["2"])
<commit_msg>[lldb][NFC] Remove wrong tests in TestCallOverriddenMethod<commit_after>"""
Test calling an overriden method.

Note:
  This verifies that LLDB is correctly building the method overrides table.
  If this table is not built correctly then calls to overridden methods in
  derived classes may generate references to non-existant vtable entries,
  as the compiler treats the overridden method as a totally new virtual
  method definition.
  <rdar://problem/14205774>

"""

import lldb
from lldbsuite.test.decorators import *
from lldbsuite.test.lldbtest import *
from lldbsuite.test import lldbutil

class ExprCommandCallOverriddenMethod(TestBase):

    mydir = TestBase.compute_mydir(__file__)

    def setUp(self):
        # Call super's setUp().
        TestBase.setUp(self)
        # Find the line number to break for main.c.
        self.line = line_number('main.cpp', '// Set breakpoint here')

    def test(self):
        """Test calls to overridden methods in derived classes."""
        self.build()

        # Set breakpoint in main and run exe
        self.runCmd("file " + self.getBuildArtifact("a.out"),
                    CURRENT_EXECUTABLE_SET)
        lldbutil.run_break_set_by_file_and_line(
            self, "main.cpp", self.line, num_expected_locations=-1, loc_exact=True)

        self.runCmd("run", RUN_SUCCEEDED)

        # Test call to method in base class (this should always work as the base
        # class method is never an override).
        self.expect("expr b->foo()", substrs=["2"])

        # Test call to overridden method in derived class (this will fail if the
        # overrides table is not correctly set up, as Derived::foo will be assigned
        # a vtable entry that does not exist in the compiled program).
        self.expect("expr d.foo()", substrs=["2"])

        # Test calling the base class.
        self.expect("expr realbase.foo()", substrs=["1"])

    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr43707")
    def test_call_on_temporary(self):
        """Test calls to overridden methods in derived classes."""
        self.build()

        # Set breakpoint in main and run exe
        self.runCmd("file " + self.getBuildArtifact("a.out"),
                    CURRENT_EXECUTABLE_SET)
        lldbutil.run_break_set_by_file_and_line(
            self, "main.cpp", self.line, num_expected_locations=-1, loc_exact=True)

        self.runCmd("run", RUN_SUCCEEDED)

        # Test with locally constructed instances.
        self.expect("expr Base().foo()", substrs=["1"])
        self.expect("expr Derived().foo()", substrs=["2"])
<|endoftext|>