<commit_before>"""
SurfacePicker

:Authors:
	Berend Klein Haneveld
"""
import math
from Picker import Picker
from ui.Interactor import Interactor
from vtk import vtkVolumePicker
from vtk import vtkActor
from vtk import vtkConeSource
from vtk import vtkDataSetMapper
from PySide.QtCore import Signal
from core.vtkDrawing import TransformWithMatrix

class SurfacePicker(Picker, Interactor):
	"""
	SurfacePicker is a dedicated picker for points in 3D datasets.
	"""

	pickedLocation = Signal(list)

	def __init__(self):
		super(SurfacePicker, self).__init__()
		self.props = []
		self.picker = vtkVolumePicker()
		self.picker.SetTolerance(1e-6)
		self.picker.SetVolumeOpacityIsovalue(0.1)
		self.widget = None

	def setWidget(self, widget):
		self.widget = widget

		# TODO: make size relative to image data size
		bounds = self.widget.imageData.GetBounds()
		mean = reduce(lambda x, y: x + y, bounds) / 3.0
		multiplier = mean / 100.0
		coneSource = vtkConeSource()
		coneSource.CappingOn()
		coneSource.SetHeight(12*multiplier)
		coneSource.SetRadius(5*multiplier)
		coneSource.SetResolution(6)
		coneSource.SetCenter(6*multiplier, 0, 0)
		coneSource.SetDirection(-1, 0, 0)

		coneMapper = vtkDataSetMapper()
		coneMapper.SetInputConnection(coneSource.GetOutputPort())

		self.redCone = vtkActor()
		self.redCone.PickableOff()
		self.redCone.SetMapper(coneMapper)
		self.redCone.GetProperty().SetColor(1, 0, 0)

		self.greenCone = vtkActor()
		self.greenCone.PickableOff()
		self.greenCone.SetMapper(coneMapper)
		self.greenCone.GetProperty().SetColor(0, 1, 0)

		self.widget.renderer.AddViewProp(self.redCone)
		self.widget.renderer.AddViewProp(self.greenCone)

		self.AddObserver(self.widget.rwi, "MouseMoveEvent", self.mouseMove)
		self.AddObserver(self.widget.rwi, "KeyPressEvent", self.keyPress)

	def cleanUp(self):
		if self.widget:
			self.widget.renderer.RemoveViewProp(self.greenCone)
			self.widget.renderer.RemoveViewProp(self.redCone)
			self.widget.rwi.ShowCursor()

		self.cleanUpCallbacks()

	# Callbacks

	def mouseMove(self, iren, event=""):
		self.widget.rwi.HideCursor()
		p, n = PickPoint(iren, self.widget.renderer, self.picker)

		self.redCone.SetPosition(p[0], p[1], p[2])
		self.greenCone.SetPosition(p[0], p[1], p[2])
		PointCone(self.redCone, n[0], n[1], n[2])
		PointCone(self.greenCone, -n[0], -n[1], -n[2])

		self.widget.render()

	def keyPress(self, iren, event=""):
		key = iren.GetKeyCode()
		if key != "a":
			return
		point, normal = PickPoint(iren, self.widget.renderer, self.picker)
		# point in world coordinates
		transform = TransformWithMatrix(self.widget.volume.GetMatrix())
		transform.Inverse()
		# transformedPoint in local coordinates
		tranformedPoint = transform.TransformPoint(point)
		point = list(tranformedPoint)
		self.pickedLocation.emit(point)

def PointCone(actor, nx, ny, nz):
	"""
	A function to point an actor along a vector.
	"""
	actor.SetOrientation(0.0, 0.0, 0.0)
	n = math.sqrt(nx**2 + ny**2 + nz**2)
	if (nx < 0.0):
		actor.RotateWXYZ(180, 0, 1, 0)
		n = -n
	actor.RotateWXYZ(180, (nx+n)*0.5, ny*0.5, nz*0.5)

def PickPoint(iren, renderer, picker):
	"""
	Returns the picked position and normal.
	"""
	x, y = iren.GetEventPosition()
	picker.Pick(x, y, 0, renderer)
	return picker.GetPickPosition(), picker.GetPickNormal()
<commit_msg>Make call to super instead of manual cleanup implementation.<commit_after>"""
SurfacePicker

:Authors:
	Berend Klein Haneveld
"""
import math
from Picker import Picker
from ui.Interactor import Interactor
from vtk import vtkVolumePicker
from vtk import vtkActor
from vtk import vtkConeSource
from vtk import vtkDataSetMapper
from PySide.QtCore import Signal
from core.vtkDrawing import TransformWithMatrix

class SurfacePicker(Picker, Interactor):
	"""
	SurfacePicker is a dedicated picker for points in 3D datasets.
	"""

	pickedLocation = Signal(list)

	def __init__(self):
		super(SurfacePicker, self).__init__()
		self.props = []
		self.picker = vtkVolumePicker()
		self.picker.SetTolerance(1e-6)
		self.picker.SetVolumeOpacityIsovalue(0.1)
		self.widget = None

	def setWidget(self, widget):
		self.widget = widget

		# TODO: make size relative to image data size
		bounds = self.widget.imageData.GetBounds()
		mean = reduce(lambda x, y: x + y, bounds) / 3.0
		multiplier = mean / 100.0
		coneSource = vtkConeSource()
		coneSource.CappingOn()
		coneSource.SetHeight(12*multiplier)
		coneSource.SetRadius(5*multiplier)
		coneSource.SetResolution(6)
		coneSource.SetCenter(6*multiplier, 0, 0)
		coneSource.SetDirection(-1, 0, 0)

		coneMapper = vtkDataSetMapper()
		coneMapper.SetInputConnection(coneSource.GetOutputPort())

		self.redCone = vtkActor()
		self.redCone.PickableOff()
		self.redCone.SetMapper(coneMapper)
		self.redCone.GetProperty().SetColor(1, 0, 0)

		self.greenCone = vtkActor()
		self.greenCone.PickableOff()
		self.greenCone.SetMapper(coneMapper)
		self.greenCone.GetProperty().SetColor(0, 1, 0)

		self.widget.renderer.AddViewProp(self.redCone)
		self.widget.renderer.AddViewProp(self.greenCone)

		self.AddObserver(self.widget.rwi, "MouseMoveEvent", self.mouseMove)
		self.AddObserver(self.widget.rwi, "KeyPressEvent", self.keyPress)

	def cleanUp(self):
		super(SurfacePicker, self).cleanUp()
		if self.widget:
			self.widget.renderer.RemoveViewProp(self.greenCone)
			self.widget.renderer.RemoveViewProp(self.redCone)
			self.widget.rwi.ShowCursor()

	# Callbacks

	def mouseMove(self, iren, event=""):
		self.widget.rwi.HideCursor()
		p, n = PickPoint(iren, self.widget.renderer, self.picker)

		self.redCone.SetPosition(p[0], p[1], p[2])
		self.greenCone.SetPosition(p[0], p[1], p[2])
		PointCone(self.redCone, n[0], n[1], n[2])
		PointCone(self.greenCone, -n[0], -n[1], -n[2])

		self.widget.render()

	def keyPress(self, iren, event=""):
		key = iren.GetKeyCode()
		if key != "a":
			return
		point, normal = PickPoint(iren, self.widget.renderer, self.picker)
		# point in world coordinates
		transform = TransformWithMatrix(self.widget.volume.GetMatrix())
		transform.Inverse()
		# transformedPoint in local coordinates
		tranformedPoint = transform.TransformPoint(point)
		point = list(tranformedPoint)
		self.pickedLocation.emit(point)

def PointCone(actor, nx, ny, nz):
	"""
	A function to point an actor along a vector.
	"""
	actor.SetOrientation(0.0, 0.0, 0.0)
	n = math.sqrt(nx**2 + ny**2 + nz**2)
	if (nx < 0.0):
		actor.RotateWXYZ(180, 0, 1, 0)
		n = -n
	actor.RotateWXYZ(180, (nx+n)*0.5, ny*0.5, nz*0.5)

def PickPoint(iren, renderer, picker):
	"""
	Returns the picked position and normal.
	"""
	x, y = iren.GetEventPosition()
	picker.Pick(x, y, 0, renderer)
	return picker.GetPickPosition(), picker.GetPickNormal()
<|endoftext|>


<commit_before># Django settings for dbpatterns project.
import os

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    os.path.join(os.path.dirname(__file__), "static"),
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'dbpatterns.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'dbpatterns.wsgi.application'

TEMPLATE_DIRS = (
    os.path.join(os.path.dirname(__file__), "templates")
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',

    'gravatar',
    'social_auth',

    'documents',
)

TEMPLATE_CONTEXT_PROCESSORS = (
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.static',
    'django.core.context_processors.tz',
    'django.core.context_processors.csrf',
    'django.contrib.messages.context_processors.messages',
    'dbpatterns.context_processors.assets_version',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

AUTHENTICATION_BACKENDS = (
    'social_auth.backends.contrib.github.GithubBackend',
    'django.contrib.auth.backends.ModelBackend',
)

LOGIN_REDIRECT_URL = '/'

SOCIAL_AUTH_COMPLETE_URL_NAME  = 'socialauth_complete'
SOCIAL_AUTH_ASSOCIATE_URL_NAME = 'socialauth_associate_complete'

SOCIAL_AUTH_PIPELINE = (
    'social_auth.backends.pipeline.social.social_auth_user',
    'social_auth.backends.pipeline.associate.associate_by_email',
    'social_auth.backends.pipeline.user.get_username',
    'social_auth.backends.pipeline.user.create_user',
    'social_auth.backends.pipeline.social.associate_user',
    'social_auth.backends.pipeline.social.load_extra_data',
    'social_auth.backends.pipeline.user.update_user_details',
)

MONGODB_DATABASE = "dbpatterns"

ASSETS_VERSION = "1.0.2"

try:
    from settings_local import *
except ImportError:
    pass<commit_msg>changed asset version for new updates.<commit_after># Django settings for dbpatterns project.
import os

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    os.path.join(os.path.dirname(__file__), "static"),
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'dbpatterns.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'dbpatterns.wsgi.application'

TEMPLATE_DIRS = (
    os.path.join(os.path.dirname(__file__), "templates")
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',

    'gravatar',
    'social_auth',

    'documents',
)

TEMPLATE_CONTEXT_PROCESSORS = (
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.static',
    'django.core.context_processors.tz',
    'django.core.context_processors.csrf',
    'django.contrib.messages.context_processors.messages',
    'dbpatterns.context_processors.assets_version',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

AUTHENTICATION_BACKENDS = (
    'social_auth.backends.contrib.github.GithubBackend',
    'django.contrib.auth.backends.ModelBackend',
)

LOGIN_REDIRECT_URL = '/'

SOCIAL_AUTH_COMPLETE_URL_NAME  = 'socialauth_complete'
SOCIAL_AUTH_ASSOCIATE_URL_NAME = 'socialauth_associate_complete'

SOCIAL_AUTH_PIPELINE = (
    'social_auth.backends.pipeline.social.social_auth_user',
    'social_auth.backends.pipeline.associate.associate_by_email',
    'social_auth.backends.pipeline.user.get_username',
    'social_auth.backends.pipeline.user.create_user',
    'social_auth.backends.pipeline.social.associate_user',
    'social_auth.backends.pipeline.social.load_extra_data',
    'social_auth.backends.pipeline.user.update_user_details',
)

MONGODB_DATABASE = "dbpatterns"

ASSETS_VERSION = "1.0.3"

try:
    from settings_local import *
except ImportError:
    pass<|endoftext|>


<commit_before># encoding: utf-8

#   Copyright 2013 Red Hat, Inc.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import logging
import uuid
import zope
import inspect
from imgfac.CloudDelegate import CloudDelegate
from imgfac.PersistentImageManager import PersistentImageManager
from imgfac.TargetImage import TargetImage
from imagefactory_plugins.ovfcommon.ovfcommon import RHEVOVFPackage, VsphereOVFPackage
from imgfac.ImageFactoryException import ImageFactoryException
from oz.ozutil import copyfile_sparse

class OVA(object):
    zope.interface.implements(CloudDelegate)

    def __init__(self):
        self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))

    def builder_should_create_target_image(self, builder, target, image_id, template, parameters):
        retval = False

        if isinstance(builder.base_image, TargetImage):
            if builder.base_image.target in ('vsphere', 'rhevm'):
                retval = True

        self.log.info('builder_should_create_target_image() called on OVA plugin - returning %s' % retval)

        return retval

    def builder_did_create_target_image(self, builder, target, image_id, template, parameters):
        self.log.info('builder_did_create_target_image() called in OVA plugin')
        self.status="BUILDING"

        self.target_image = builder.base_image
        self.base_image = PersistentImageManager.default_manager().image_with_id(self.target_image.base_image_id)
        self.image = builder.target_image
        self.parameters = parameters

        # This lets our logging helper know what image is being operated on
        self.active_image = self.image

        self.generate_ova()

        self.percent_complete=100
        self.status="COMPLETED"

    def generate_ova(self):
        if self.target_image.target == 'rhevm':
            klass = RHEVOVFPackage
        elif self.target_image.target == 'vsphere':
            klass = VsphereOVFPackage
        else:
            raise ImageFactoryException("OVA plugin only support rhevm and vsphere images")

        pkg = klass(disk=self.image.data, base_image=self.base_image.data)
        ova = pkg.make_ova_package()
        copyfile_sparse(ova, self.image.data)
        pkg.delete()
<commit_msg>Add parameters support to OVA plugin<commit_after># encoding: utf-8

#   Copyright 2013 Red Hat, Inc.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import logging
import uuid
import zope
import inspect
from imgfac.CloudDelegate import CloudDelegate
from imgfac.PersistentImageManager import PersistentImageManager
from imgfac.TargetImage import TargetImage
from imagefactory_plugins.ovfcommon.ovfcommon import RHEVOVFPackage, VsphereOVFPackage
from imgfac.ImageFactoryException import ImageFactoryException
from oz.ozutil import copyfile_sparse

class OVA(object):
    zope.interface.implements(CloudDelegate)

    def __init__(self):
        self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))

    def builder_should_create_target_image(self, builder, target, image_id, template, parameters):
        retval = False

        if isinstance(builder.base_image, TargetImage):
            if builder.base_image.target in ('vsphere', 'rhevm'):
                retval = True

        self.log.info('builder_should_create_target_image() called on OVA plugin - returning %s' % retval)

        return retval

    def builder_did_create_target_image(self, builder, target, image_id, template, parameters):
        self.log.info('builder_did_create_target_image() called in OVA plugin')
        self.status="BUILDING"

        self.target_image = builder.base_image
        self.base_image = PersistentImageManager.default_manager().image_with_id(self.target_image.base_image_id)
        self.image = builder.target_image
        self.parameters = parameters

        # This lets our logging helper know what image is being operated on
        self.active_image = self.image

        self.generate_ova()

        self.percent_complete=100
        self.status="COMPLETED"

    def generate_ova(self):
        if self.target_image.target == 'rhevm':
            klass = RHEVOVFPackage
        elif self.target_image.target == 'vsphere':
            klass = VsphereOVFPackage
        else:
            raise ImageFactoryException("OVA plugin only support rhevm and vsphere images")

        klass_parameters = dict()

        if self.parameters:
            params = ['ovf_cpu_count','ovf_memory_mb',
                      'rhevm_default_display_type','rhevm_description','rhevm_os_descriptor',
                      'vsphere_product_name','vsphere_product_vendor_name','vsphere_product_version']

            for param in params:
                if (self.parameters.get(param) and
                    klass.__init__.func_code.co_varnames.__contains__(param)):
                    klass_parameters[param] = self.parameters.get(param)

        pkg = klass(disk=self.image.data, base_image=self.base_image.data,
                    **klass_parameters)
        ova = pkg.make_ova_package()
        copyfile_sparse(ova, self.image.data)
        pkg.delete()
<|endoftext|>


<commit_before>from django.contrib import admin
from django.core import urlresolvers
from imagine.models import Collection, Directory, Image, ImageMeta, PhotoSize, ExifItem, Comment, Event
from imagine.actions import update_collection
from django.core.urlresolvers import reverse
from django.utils.safestring import mark_safe

class CollectionAdmin(admin.ModelAdmin):
    list_display = ('title', 'base_dir', 'archive_dir', 'is_public', 'passwordprotected', 'nr_directories', 'nr_images', )
    search_fields = ('title', 'slug', 'base_dir', 'archive_dir', 'description', )
    prepopulated_fields = {'slug': ('title',), }

    actions = ['refresh']

    def refresh(self, request, queryset):
        for collection in queryset:
            update_collection(collection)
        self.message_user(request, 'Successfully updated {} collection(s)'.format(queryset.count()))

class DirectoryAdmin(admin.ModelAdmin):
    list_display = ('directory', 'relative_path', 'parent_directory', 'collection_link', 'title', 'nr_images', )

    def collection_link(self, obj):
        return mark_safe('{}'.format(
            reverse("admin:imagine_collection_change", args=(obj.collection.pk,)),
            obj.collection.title
        ))
    collection_link.short_description = 'collection'

class ImageAdmin(admin.ModelAdmin):
    #list_display = ('get_filepath', 'filename', 'width', 'height', 'megapixel', 'filesize', 'image_hash', )
    list_display = ('filename', 'collection_link', 'file_path', 'width', 'height', 'megapixel', 'filter_modified', 'filesize', 'image_hash', 'meta_link', )
    search_fields = ('filename', )
    #readonly_fields = ('imageinegallery_collection_link',)

    def collection_link(self, obj):
        return mark_safe('{}'.format(
            reverse("admin:imagine_collection_change", args=(obj.directory.collection.pk,)),
            obj.directory.collection.title
        ))
    collection_link.short_description = 'collection'

    def collection_path(self, instance):
        return instance.directory.collection.base_dir

    def meta_link(self, obj):
        return mark_safe('{}'.format(
            reverse("admin:imagine_imagemeta_change", args=(obj.image_hash,)),
            obj.image_hash
        ))
    meta_link.short_description = 'image meta'

class ImageMetaAdmin(admin.ModelAdmin):
    search_fields = ('title', 'description', 'image_hash', )
    list_display = ('image_hash', 'title', )

class PhotoSizeAdmin(admin.ModelAdmin):
    list_display = ('name', 'width', 'height', 'crop_to_fit', )

class ExifItemAdmin(admin.ModelAdmin):
    list_display = ('key', 'get_value', 'from_image', )
    search_fields = ('key', 'value_int', 'value_str', 'value_float', )

    def from_image(self, obj):
        link=urlresolvers.reverse("admin:imagine_image_change", args=[obj.image.id]) #model name has to be lowercase
        return u'%s' % (link,obj.image.filename)
    from_image.allow_tags=True

class CommentAdmin(admin.ModelAdmin):
    list_display = ('from_image', 'name', 'email', )
    search_fields = ('name', 'email', 'comment', )

    def from_image(self, obj):
        link=urlresolvers.reverse("admin:imagine_image_change", args=[obj.image.id]) #model name has to be lowercase
        return u'%s' % (link,obj.image.filename)
    from_image.allow_tags=True

class EventAdmin(admin.ModelAdmin):
    list_display = ('title', 'start_datetime', 'end_datetime', )
    search_fields = ('title', 'slug', 'description', )
    prepopulated_fields = {'slug': ('title',), }

admin.site.register(Collection, CollectionAdmin)
admin.site.register(Directory, DirectoryAdmin)
admin.site.register(Image, ImageAdmin)
admin.site.register(ImageMeta, ImageMetaAdmin)
admin.site.register(PhotoSize, PhotoSizeAdmin)
admin.site.register(ExifItem, ExifItemAdmin)
admin.site.register(Comment, CommentAdmin)
admin.site.register(Event, EventAdmin)
<commit_msg>Filter Images, also search on Image in ExifItem<commit_after>from django.contrib import admin
from django.core import urlresolvers
from imagine.models import Collection, Directory, Image, ImageMeta, PhotoSize, ExifItem, Comment, Event
from imagine.actions import update_collection
from django.core.urlresolvers import reverse
from django.utils.safestring import mark_safe

class CollectionAdmin(admin.ModelAdmin):
    list_display = ('title', 'base_dir', 'archive_dir', 'is_public', 'passwordprotected', 'nr_directories', 'nr_images', )
    search_fields = ('title', 'slug', 'base_dir', 'archive_dir', 'description', )
    prepopulated_fields = {'slug': ('title',), }

    actions = ['refresh']

    def refresh(self, request, queryset):
        for collection in queryset:
            update_collection(collection)
        self.message_user(request, 'Successfully updated {} collection(s)'.format(queryset.count()))

class DirectoryAdmin(admin.ModelAdmin):
    list_display = ('directory', 'relative_path', 'parent_directory', 'collection_link', 'title', 'nr_images', )

    def collection_link(self, obj):
        return mark_safe('{}'.format(
            reverse("admin:imagine_collection_change", args=(obj.collection.pk,)),
            obj.collection.title
        ))
    collection_link.short_description = 'collection'

class ImageAdmin(admin.ModelAdmin):
    #list_display = ('get_filepath', 'filename', 'width', 'height', 'megapixel', 'filesize', 'image_hash', )
    list_display = ('filename', 'collection_link', 'file_path', 'width', 'height', 'megapixel', 'filter_modified', 'filesize', 'image_hash', 'meta_link', )
    search_fields = ('filename', )
    #readonly_fields = ('imageinegallery_collection_link',)
    list_filter = ('geo_country', 'geo_city', 'is_photosphere', 'is_visible', 'collection', )

    # TODO: use fieldsets to make the form more usable
    # https://docs.djangoproject.com/en/1.10/ref/contrib/admin/#django.contrib.admin.ModelAdmin.fieldsets

    def collection_link(self, obj):
        return mark_safe('{}'.format(
            reverse("admin:imagine_collection_change", args=(obj.directory.collection.pk,)),
            obj.directory.collection.title
        ))
    collection_link.short_description = 'collection'

    def collection_path(self, instance):
        return instance.directory.collection.base_dir

    def meta_link(self, obj):
        return mark_safe('{}'.format(
            reverse("admin:imagine_imagemeta_change", args=(obj.image_hash,)),
            obj.image_hash
        ))
    meta_link.short_description = 'image meta'

class ImageMetaAdmin(admin.ModelAdmin):
    search_fields = ('title', 'description', 'image_hash', )
    list_display = ('image_hash', 'title', )

class PhotoSizeAdmin(admin.ModelAdmin):
    list_display = ('name', 'width', 'height', 'crop_to_fit', )

class ExifItemAdmin(admin.ModelAdmin):
    list_display = ('key', 'get_value', 'from_image', )
    search_fields = ('key', 'value_int', 'value_str', 'value_float', 'image__filename', )

    def from_image(self, obj):
        link=urlresolvers.reverse("admin:imagine_image_change", args=[obj.image.id]) #model name has to be lowercase
        return u'%s' % (link,obj.image.filename)
    from_image.allow_tags=True

class CommentAdmin(admin.ModelAdmin):
    list_display = ('from_image', 'name', 'email', )
    search_fields = ('name', 'email', 'comment', )

    def from_image(self, obj):
        link=urlresolvers.reverse("admin:imagine_image_change", args=[obj.image.id]) #model name has to be lowercase
        return u'%s' % (link,obj.image.filename)
    from_image.allow_tags=True

class EventAdmin(admin.ModelAdmin):
    list_display = ('title', 'start_datetime', 'end_datetime', )
    search_fields = ('title', 'slug', 'description', )
    prepopulated_fields = {'slug': ('title',), }

admin.site.register(Collection, CollectionAdmin)
admin.site.register(Directory, DirectoryAdmin)
admin.site.register(Image, ImageAdmin)
admin.site.register(ImageMeta, ImageMetaAdmin)
admin.site.register(PhotoSize, PhotoSizeAdmin)
admin.site.register(ExifItem, ExifItemAdmin)
admin.site.register(Comment, CommentAdmin)
admin.site.register(Event, EventAdmin)
<|endoftext|>


<commit_before><commit_msg>bug 1612569: fix SignatureIPCChannelError docstring<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.http.response import JsonResponse
from 臺灣言語資料庫.資料模型 import 外語表
import json

_自己json字串=json.dumps('自己')
def 加外語請教條(request):
	欄位表 = ['來源',
				'種類',
				'語言腔口',
				'著作所在地',
				'著作年',
				'屬性',
				'外語語言',
				'外語資料', ]
	內容 = {}
	for 欄位 in 欄位表:
		內容[欄位] = request.POST[欄位]
# 	內容['屬性']=json.loads(request.POST['屬性'])
	內容['版權'] = '會使公開'
	內容['收錄者'] = 1
	if 內容['來源'] == _自己json字串:
		內容['來源'] = 內容['收錄者']
	print(內容)
	外語 = 外語表.加資料(內容)
	平臺項目 = 外語.平臺項目.create(是資料源頭=True)
	return JsonResponse({
		'結果':'成功',
		'平臺項目編號':str(平臺項目.編號()),
		})
<commit_msg>賟仝款資料佮屬性無轉json的試驗猶未過 仝款資料愛佇資料庫彼爿檢查 屬性無轉，愛予資料料還原rollback 丞宏<commit_after># -*- coding: utf-8 -*-
from django.http.response import JsonResponse
from 臺灣言語資料庫.資料模型 import 外語表
import json
from django.utils.datastructures import MultiValueDictKeyError
from 臺灣言語資料庫.資料模型 import 種類表

_自己json字串=json.dumps('自己')
def 加外語請教條(request):
	欄位表 = ['來源',
				'種類',
				'語言腔口',
				'著作所在地',
				'著作年',
				'屬性',
				'外語語言',
				'外語資料', ]
	內容 = {}
	try:
		for 欄位 in 欄位表:
			內容[欄位] = request.POST[欄位]
			if not isinstance(內容[欄位],str):
				return JsonResponse({
					'結果':'失敗',
					'原因':'資料全部一般欄位必須都是字串',
				})
	# 	內容['屬性']=json.loads(request.POST['屬性'])
		內容['版權'] = '會使公開'
		內容['收錄者'] = 1
		if 內容['來源'] == _自己json字串:
			內容['來源'] = 內容['收錄者']
	# 	print(type(內容),內容)
		try:
			外語 = 外語表.加資料(內容)
		except:
			raise
		平臺項目 = 外語.平臺項目.create(是資料源頭=True)
	except ValueError:
		return JsonResponse({
			'結果':'失敗',
			'原因':'來源抑是屬性無轉json字串',
		})
	except MultiValueDictKeyError:
		return JsonResponse({
			'結果':'失敗',
			'原因':'資料欄位有缺',
		})
	except KeyError:
		return JsonResponse({
			'結果':'失敗',
			'原因':'來源沒有「名」的欄位',
		})
	except 種類表.DoesNotExist:
		return JsonResponse({
			'結果':'失敗',
			'原因':'種類欄位不符規範',
		})
	else:
		return JsonResponse({
			'結果':'成功',
			'平臺項目編號':str(平臺項目.編號()),
		})
	<|endoftext|>


<commit_before>__author__ = 'Frederik Diehl'

from apsis.optimizers.bayesian_optimization import SimpleBayesianOptimizer
from nose.tools import assert_is_none, assert_equal, assert_dict_equal, \
    assert_true, assert_false
from apsis.optimizers.bayesian.acquisition_functions import ExpectedImprovement, ProbabilityOfImprovement
from apsis.models.experiment import Experiment
from apsis.models.parameter_definition import MinMaxNumericParamDef
from apsis.models.candidate import Candidate

class testSimpleBayesianOptimization(object):

    def test_init(self):
        #test default parameters
        opt = SimpleBayesianOptimizer(None)
        assert_equal(opt.initial_random_runs, 10)
        assert_is_none(opt.acquisition_hyperparams)
        assert_equal(opt.num_gp_restarts, 10)
        assert_true(isinstance(opt.acquisition_function, ExpectedImprovement))
        assert_dict_equal(opt.kernel_params, {})
        assert_equal(opt.kernel, "matern52")
        assert_false(opt.mcmc)
        assert_equal(opt.num_precomputed, 10)

        #test correct initialization
        opt_arguments = {
            "initial_random_runs": 5,
            "acquisition_hyperparams": {},
            "num_gp_restarts": 5,
            "acquisition": ProbabilityOfImprovement,
            "kernel_params": {},
            "kernel": "matern52",
            "mcmc": True,
            "num_precomputed": 5
        }
        opt = SimpleBayesianOptimizer(opt_arguments)

        assert_equal(opt.initial_random_runs, 5)
        assert_dict_equal(opt.acquisition_hyperparams, {})
        assert_equal(opt.num_gp_restarts, 5)
        assert_true(isinstance(opt.acquisition_function, ProbabilityOfImprovement))
        assert_dict_equal(opt.kernel_params, {})
        assert_equal(opt.kernel, "matern52")
        assert_true(opt.mcmc)
        assert_equal(opt.num_precomputed, 5)

    def test_get_next_candidate(self):
        opt = SimpleBayesianOptimizer({"initial_random_runs": 3})
        exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)})
        for i in range(5):
            cand = opt.get_next_candidates(exp)[0]
            assert_true(isinstance(cand, Candidate))
            cand.result = 2
            exp.add_finished(cand)
        cands = opt.get_next_candidates(exp, num_candidates=3)
        assert_equal(len(cands), 3)<commit_msg>Fixed test failing when pymcmc was not installed.<commit_after>__author__ = 'Frederik Diehl'

from apsis.optimizers.bayesian_optimization import SimpleBayesianOptimizer
from nose.tools import assert_is_none, assert_equal, assert_dict_equal, \
    assert_true, assert_false
from apsis.optimizers.bayesian.acquisition_functions import ExpectedImprovement, ProbabilityOfImprovement
from apsis.models.experiment import Experiment
from apsis.models.parameter_definition import MinMaxNumericParamDef
from apsis.models.candidate import Candidate
from apsis.utilities.import_utils import import_if_exists

class testSimpleBayesianOptimization(object):

    def test_init(self):
        #test default parameters
        opt = SimpleBayesianOptimizer(None)
        assert_equal(opt.initial_random_runs, 10)
        assert_is_none(opt.acquisition_hyperparams)
        assert_equal(opt.num_gp_restarts, 10)
        assert_true(isinstance(opt.acquisition_function, ExpectedImprovement))
        assert_dict_equal(opt.kernel_params, {})
        assert_equal(opt.kernel, "matern52")
        assert_false(opt.mcmc)
        assert_equal(opt.num_precomputed, 10)

        #test correct initialization
        opt_arguments = {
            "initial_random_runs": 5,
            "acquisition_hyperparams": {},
            "num_gp_restarts": 5,
            "acquisition": ProbabilityOfImprovement,
            "kernel_params": {},
            "kernel": "matern52",
            "mcmc": True,
            "num_precomputed": 5
        }
        opt = SimpleBayesianOptimizer(opt_arguments)

        assert_equal(opt.initial_random_runs, 5)
        assert_dict_equal(opt.acquisition_hyperparams, {})
        assert_equal(opt.num_gp_restarts, 5)
        assert_true(isinstance(opt.acquisition_function, ProbabilityOfImprovement))
        assert_dict_equal(opt.kernel_params, {})
        assert_equal(opt.kernel, "matern52")

        if import_if_exists("pymcmc")[0]:
            assert_true(opt.mcmc)
        else:
            assert_false(opt.mcmc)
        assert_equal(opt.num_precomputed, 5)

    def test_get_next_candidate(self):
        opt = SimpleBayesianOptimizer({"initial_random_runs": 3})
        exp = Experiment("test", {"x": MinMaxNumericParamDef(0, 1)})
        for i in range(5):
            cand = opt.get_next_candidates(exp)[0]
            assert_true(isinstance(cand, Candidate))
            cand.result = 2
            exp.add_finished(cand)
        cands = opt.get_next_candidates(exp, num_candidates=3)
        assert_equal(len(cands), 3)<|endoftext|>


<commit_before>"""
A form action adapter that sends the form submission to a web service.
"""

__author__ = 'Paul Rentschler <par117@psu.edu>'
__docformat__ = 'plaintext'

from AccessControl import ClassSecurityInfo
from Acquisition import aq_parent

from Products.Archetypes.atapi import *
from Products.ATContentTypes.content.base import registerATCT
from Products.CMFCore.permissions import View, ModifyPortalContent
from Products.CMFCore.utils import getToolByName

from Products.PloneFormGen.config import *
from Products.PloneFormGen.content.actionAdapter import \
    FormActionAdapter, FormAdapterSchema
from Products.PloneFormGen.content.formMailerAdapter import FormMailerAdapter
from Products.PloneFormGen.content.likertField import LikertField
from Products.PloneFormGen.content.saveDataAdapter import FormSaveDataAdapter
from Products.PloneFormGen.interfaces import \
    IPloneFormGenActionAdapter, IPloneFormGenFieldset, IPloneFormGenForm

from collective.webservicespfgadapter.config import *

from collections import OrderedDict
from types import StringTypes

import json, logging, requests, sys, traceback

logger = logging.getLogger("PloneFormGen")

formWebServiceAdapterSchema = FormAdapterSchema.copy() + Schema((
    StringField('url',
        required=1,
        searchable=0,
        write_permission=EDIT_URL_PERMISSION,
        read_permission=ModifyPortalContent,
        widget=StringWidget(
            label=u'Web Services Address',
            description=u"""
                Specify the web address of the service that will receive
                the form submission information. It's highly recommended
                that the web address start with 'https://' to ensure the
                form data is transmitted securely.
                """,
            ),
        ),
    LinesField('showFields',
        required=0,
        searchable=0,
        read_permission=ModifyPortalContent,
        vocabulary='allFieldDisplayList',
        widget=PicklistWidget(
            label=u'Include selected fields',
            description=u"""
                Pick the fields whose inputs you would like to include in
                the web service submission. If empty, all fields will be sent.
                """,
            ),
        ),
    StringField('fieldset_separator',
        required=0,
        searchable=0,
        default=' / ',
        read_permission=ModifyPortalContent,
        widget=StringWidget(
            label=u'Fieldset separator',
            description=u"""
                For fields contained in a fieldset, prefix the field name
                with the fieldset title and separate them with these
                characters. If blank, the fieldset title is NOT prepended.
                """
            ),
        ),
    LinesField('extraData',
        required=0,
        searchable=0,
        read_permission=ModifyPortalContent,
        vocabulary='extraDataDisplayList',
        widget=MultiSelectionWidget(
            label=u'Extra Data',
            description=u"""
                Pick any extra data you would like included with the
                web service submission.
                """,
            format='checkbox',
            ),
        ),
    BooleanField('failSilently',
        required=0,
        searchable=0,
        default='0',
        write_permission=EDIT_FAILURE_SETTINGS_PERMISSION,
        read_permission=ModifyPortalContent,
        widget=BooleanWidget(
            label=u'Fail silently',
            description=u"""
                If an error occurs while submitting the data to the web
                service all warnings and error messages will be suppressed.
                ONLY enable this option if you have configured another action
                adapter, otherwise the form data WILL BE LOST!
                """,
            ),
        ),
    StringField('notifyOnFailure',
        required=0,
        searchable=0,
        write_permission=EDIT_FAILURE_SETTINGS_PERMISSION,
        read_permission=ModifyPortalContent,
        widget=StringWidget(
            label=u'Notify on failure',
            description=u"""
                Comma separated list of email addresses that will be notified
                if an error occurs while submitting the form data to the web
                service and 'Fail Silently' is checked.
                """,
            ),
        ),
    BooleanField('runDisabledAdapters',
        required=0,
        searchable=0,
        default='0',
        write_permission=EDIT_FAILURE_SETTINGS_PERMISSION,
        read_permission=ModifyPortalContent,
        widget=BooleanWidget(
            label=u'Run disabled adapters',
            description=u"""
                If an error occurs while submitting the form data to the web
                service and 'Fail Silently' is checked, should the disabled
                action adapters be run?
                This allows you to setup other action adapters (e.g. Save
                Data Adapter) that are ONLY run when an error occurs while
                submitting to the web service.
                """,
            ),
        ),
))

class FormWebServiceAdapter(FormActionAdapter):
    """
    A form action adapter that sends the form submission to a web service.
    """
    schema = formWebServiceAdapterSchema
    portal_type = meta_type = 'FormWebServiceAdapter'
    archetype_name = 'Web Service Adapter'
    content_icon = 'FormAction.gif'

    security = ClassSecurityInfo()

    security.declareProtected(View, 'allFieldDisplayList')

    def allFieldDisplayList(self):
        """
        returns a DisplayList of all fields
        """
        return self.fgFieldsDisplayList()

    def __bobo_traverse__(self, REQUEST, name):
        # prevent traversal to attributes we want to protect
        if name == 'submission_pt':
            raise AttributeError
        return super(FormWebServiceAdapter, self).__bobo_traverse__(REQUEST, name)

    security.declareProtected(View, 'extraDataDisplayList')
    def extraDataDisplayList(self):
        """
        returns a DisplayList of the extra data options
        """
        dl = DisplayList()
        for key, value in extra_data.iteritems():
            dl.add(key, value)
        return dl

    security.declarePrivate('_getParentForm')
    def _getParentForm(self):
        """
        Gets the IPloneFormGenForm parent of this object.
        """
        parent = self.aq_parent
        while not IPloneFormGenForm.providedBy(parent):
            try:
                parent = parent.aq_parent
            except AttributeError:
                parent = None
                break;
        return parent

    def _onSuccess(self, fields, REQUEST=None):
        """
        Submits the form data to the web service.
        """
        data = OrderedDict()
        fieldset = ''
        showFields = getattr(self, 'showFields', [])
        for field in fields:
            if showFields and field.id not in showFields:
                continue
            if field.isLabel() and field.portal_type == 'FieldsetStart':
                fieldset = field.title
            elif field.isLabel() and field.portal_type == 'FieldsetEnd':
                fieldset = ''
            if not field.isLabel():
                val = REQUEST.form.get(field.fgField.getName(), '')
                if isinstance(val, list):
                    pass
                elif isinstance(field.fgField, LikertField):
                    likert_vals = OrderedDict()
                    for index, question in enumerate(field.getLikertQuestions()):
                        try:
                            likert_vals[question] = val[str(index + 1)]
                        except (KeyError, TypeError):
                            likert_vals[question] = 'No answer'
                    val = likert_vals
                elif not type(val) in StringTypes:
                    # Zope has marshalled the field into
                    # something other than a string
                    val = str(val)
                title = field.title
                if IPloneFormGenFieldset.providedBy(field.aq_parent) \
                and self.fieldset_separator != '':
                    title = "%s%s%s" % (
                        field.aq_parent.title,
                        self.fieldset_separator,
                        field.title
                        )
                elif fieldset != '' and self.fieldset_separator != '':
                    title = "%s%s%s" % (
                        fieldset,
                        self.fieldset_separator,
                        field.title
                        )
                if title in data.keys():
                    # start at 2 since the title without a number is 1
                    increment = 2
                    while "%s %s" % (title, increment) in data.keys():
                        increment += 1
                    title = "%s %s" % (title, increment)
                data[title] = val

        if self.extraData:
            for field in self.extraData:
                data[extra_data[field]] = getattr(REQUEST, field, '')

        pfg = self._getParentForm()
        submission = {
            'form-id': pfg.id,
            'name': pfg.title,
            'url': pfg.absolute_url(),
            'owner': pfg.Creator(),
            'data': json.dumps(data),
            }
        try:
            # timeout is set for 5 seconds which is an eternity on the web
            response = requests.post(
                self.url,
                data=submission,
                timeout=60,
                )
        except:
            raise
        else:
            if response.status_code != 201:
                msg = "Web service submission failed by returning status " \
                    + "code: %s. Was expecting status code 201." % response.status_code
                raise Exception(msg)

    security.declareProtected(View, 'onSuccess')
    def onSuccess(self, fields, REQUEST=None):
        """
        Wrap _onSuccess so fallback behavior can be implemented when calls
        to the web service fail.
        """
        # heavily borrowed from: https://plone.org/products/salesforcepfgadapter

        message = None
        try:
            self._onSuccess(fields, REQUEST)
        except Exception as e:
            # swallow the exception
            t, v = sys.exc_info()[:2]

            # generate a log message regarding the exception
            log_msg = 'Unable to save form data to the web service at (%s). '
            logger.exception(log_msg % '/'.join(self.getPhysicalPath()))

            if not self.failSilently:
                raise

            else:
                # get all the active and inactive save data and mailer adapters
                formFolder = self._getParentForm()
                enabled_adapters = formFolder.getActionAdapter()
                adapters = [o for o in formFolder.objectValues() if IPloneFormGenActionAdapter.providedBy(o)]
                active_savedata = [o for o in adapters if isinstance(o, FormSaveDataAdapter)
                                                       and o.id in enabled_adapters]
                inactive_savedata = [o for o in adapters if isinstance(o, FormSaveDataAdapter)
                                                         and o.id not in enabled_adapters]
                active_mailer = [o for o in adapters if isinstance(o, FormMailerAdapter)
                                                     and o.id in enabled_adapters]
                inactive_mailer = [o for o in adapters if isinstance(o, FormMailerAdapter)
                                                       and o.id not in enabled_adapters]

                # start the failure email message
                message = "Someone submitted this form (%s), but the data " \
                    + "couldn't be saved to the web service due to an exception." \
                    + "\n\n" \
                    + "The data was saved in the following locations:\n"
                message = message % (
                        formFolder.absolute_url(),
                        )

                # add a list of where the data was stored to the email message
                for adapter in active_savedata:
                    message += "  - Save Data Adapter (%s)\n" % adapter.absolute_url()

                if self.runDisabledAdapters:
                    for adapter in inactive_savedata:
                        message += "  - Save Data Adapter (%s)\n" % adapter.absolute_url()
                        # Trigger the adapter since it's disabled.
                        # This can be used to record data *only* when submitting to
                        #   the web service fails.
                        adapter.onSuccess(fields, REQUEST)

                for adapter in active_mailer:
                    message += "  - Mailer Adapter (%s)\n" % adapter.absolute_url()

                if self.runDisabledAdapters:
                    for adapter in inactive_mailer:
                        message += "  - Mailer Adapter (%s)\n" % adapter.absolute_url()
                        # Trigger the adapter since it's disabled.
                        # This can be used to record data *only* when submitting to
                        #   the web service fails.
                        adapter.onSuccess(fields, REQUEST)

                if not active_savedata and not inactive_savedata and \
                   not active_mailer and not inactive_mailer:
                    message += "  - NO WHERE! The data was lost.\n"

                message += "\nTechnical details on the exception:\n"
                message += ''.join(traceback.format_exception_only(t, v))

                # send an email if an address is provided
                if self.notifyOnFailure:
                    # get email configuration from Plone
                    pprops = getToolByName(self, 'portal_properties')
                    site_props = getToolByName(pprops, 'site_properties')
                    portal = getToolByName(self, 'portal_url').getPortalObject()

                    from_addr = site_props.getProperty('email_from_address') or \
                        portal.getProperty('email_from_address')
                    mailer = getToolByName(self, 'MailHost')
                    try:
                        mailer.send(
                            message,
                            mto=self.notifyOnFailure,
                            mfrom=from_addr,
                            subject='Form submission to web service failed',
                            immediate=False,
                            )
                    except Exception as e:
                        logger.exception(e)

    security.declareProtected(ModifyPortalContent, 'setShowFields')
    def setShowFields(self, value, **kwargs):
        """
        Reorder form inputs to match field order.
        """
        # This wouldn't be necessary if the PickWidget retained order.
        self.showFields = []
        for field in self.fgFields(excludeServerSide=False):
            id = field.getName()
            if id in value:
                self.showFields.append(id)

registerATCT(FormWebServiceAdapter, 'WebServicesPFGAdapter')
<commit_msg>improve the JSON encoding by converting unknown things to a dictionary<commit_after>"""
A form action adapter that sends the form submission to a web service.
"""

__author__ = 'Paul Rentschler <par117@psu.edu>'
__docformat__ = 'plaintext'

from AccessControl import ClassSecurityInfo
from Acquisition import aq_parent

from Products.Archetypes.atapi import *
from Products.ATContentTypes.content.base import registerATCT
from Products.CMFCore.permissions import View, ModifyPortalContent
from Products.CMFCore.utils import getToolByName

from Products.PloneFormGen.config import *
from Products.PloneFormGen.content.actionAdapter import \
    FormActionAdapter, FormAdapterSchema
from Products.PloneFormGen.content.formMailerAdapter import FormMailerAdapter
from Products.PloneFormGen.content.likertField import LikertField
from Products.PloneFormGen.content.saveDataAdapter import FormSaveDataAdapter
from Products.PloneFormGen.interfaces import \
    IPloneFormGenActionAdapter, IPloneFormGenFieldset, IPloneFormGenForm

from collective.webservicespfgadapter.config import *

from collections import OrderedDict
from types import StringTypes

import json, logging, requests, sys, traceback

logger = logging.getLogger("PloneFormGen")

formWebServiceAdapterSchema = FormAdapterSchema.copy() + Schema((
    StringField('url',
        required=1,
        searchable=0,
        write_permission=EDIT_URL_PERMISSION,
        read_permission=ModifyPortalContent,
        widget=StringWidget(
            label=u'Web Services Address',
            description=u"""
                Specify the web address of the service that will receive
                the form submission information. It's highly recommended
                that the web address start with 'https://' to ensure the
                form data is transmitted securely.
                """,
            ),
        ),
    LinesField('showFields',
        required=0,
        searchable=0,
        read_permission=ModifyPortalContent,
        vocabulary='allFieldDisplayList',
        widget=PicklistWidget(
            label=u'Include selected fields',
            description=u"""
                Pick the fields whose inputs you would like to include in
                the web service submission. If empty, all fields will be sent.
                """,
            ),
        ),
    StringField('fieldset_separator',
        required=0,
        searchable=0,
        default=' / ',
        read_permission=ModifyPortalContent,
        widget=StringWidget(
            label=u'Fieldset separator',
            description=u"""
                For fields contained in a fieldset, prefix the field name
                with the fieldset title and separate them with these
                characters. If blank, the fieldset title is NOT prepended.
                """
            ),
        ),
    LinesField('extraData',
        required=0,
        searchable=0,
        read_permission=ModifyPortalContent,
        vocabulary='extraDataDisplayList',
        widget=MultiSelectionWidget(
            label=u'Extra Data',
            description=u"""
                Pick any extra data you would like included with the
                web service submission.
                """,
            format='checkbox',
            ),
        ),
    BooleanField('failSilently',
        required=0,
        searchable=0,
        default='0',
        write_permission=EDIT_FAILURE_SETTINGS_PERMISSION,
        read_permission=ModifyPortalContent,
        widget=BooleanWidget(
            label=u'Fail silently',
            description=u"""
                If an error occurs while submitting the data to the web
                service all warnings and error messages will be suppressed.
                ONLY enable this option if you have configured another action
                adapter, otherwise the form data WILL BE LOST!
                """,
            ),
        ),
    StringField('notifyOnFailure',
        required=0,
        searchable=0,
        write_permission=EDIT_FAILURE_SETTINGS_PERMISSION,
        read_permission=ModifyPortalContent,
        widget=StringWidget(
            label=u'Notify on failure',
            description=u"""
                Comma separated list of email addresses that will be notified
                if an error occurs while submitting the form data to the web
                service and 'Fail Silently' is checked.
                """,
            ),
        ),
    BooleanField('runDisabledAdapters',
        required=0,
        searchable=0,
        default='0',
        write_permission=EDIT_FAILURE_SETTINGS_PERMISSION,
        read_permission=ModifyPortalContent,
        widget=BooleanWidget(
            label=u'Run disabled adapters',
            description=u"""
                If an error occurs while submitting the form data to the web
                service and 'Fail Silently' is checked, should the disabled
                action adapters be run?
                This allows you to setup other action adapters (e.g. Save
                Data Adapter) that are ONLY run when an error occurs while
                submitting to the web service.
                """,
            ),
        ),
))

class FormWebServiceAdapter(FormActionAdapter):
    """
    A form action adapter that sends the form submission to a web service.
    """
    schema = formWebServiceAdapterSchema
    portal_type = meta_type = 'FormWebServiceAdapter'
    archetype_name = 'Web Service Adapter'
    content_icon = 'FormAction.gif'

    security = ClassSecurityInfo()

    security.declareProtected(View, 'allFieldDisplayList')

    def allFieldDisplayList(self):
        """
        returns a DisplayList of all fields
        """
        return self.fgFieldsDisplayList()

    def __bobo_traverse__(self, REQUEST, name):
        # prevent traversal to attributes we want to protect
        if name == 'submission_pt':
            raise AttributeError
        return super(FormWebServiceAdapter, self).__bobo_traverse__(REQUEST, name)

    security.declareProtected(View, 'extraDataDisplayList')
    def extraDataDisplayList(self):
        """
        returns a DisplayList of the extra data options
        """
        dl = DisplayList()
        for key, value in extra_data.iteritems():
            dl.add(key, value)
        return dl

    security.declarePrivate('_getParentForm')
    def _getParentForm(self):
        """
        Gets the IPloneFormGenForm parent of this object.
        """
        parent = self.aq_parent
        while not IPloneFormGenForm.providedBy(parent):
            try:
                parent = parent.aq_parent
            except AttributeError:
                parent = None
                break;
        return parent

    def _onSuccess(self, fields, REQUEST=None):
        """
        Submits the form data to the web service.
        """
        data = OrderedDict()
        fieldset = ''
        showFields = getattr(self, 'showFields', [])
        for field in fields:
            if showFields and field.id not in showFields:
                continue
            if field.isLabel() and field.portal_type == 'FieldsetStart':
                fieldset = field.title
            elif field.isLabel() and field.portal_type == 'FieldsetEnd':
                fieldset = ''
            if not field.isLabel():
                val = REQUEST.form.get(field.fgField.getName(), '')
                if isinstance(val, list):
                    pass
                elif isinstance(field.fgField, LikertField):
                    likert_vals = OrderedDict()
                    for index, question in enumerate(field.getLikertQuestions()):
                        try:
                            likert_vals[question] = val[str(index + 1)]
                        except (KeyError, TypeError):
                            likert_vals[question] = 'No answer'
                    val = likert_vals
                elif not type(val) in StringTypes:
                    # Zope has marshalled the field into
                    # something other than a string
                    val = str(val)
                title = field.title
                if IPloneFormGenFieldset.providedBy(field.aq_parent) \
                and self.fieldset_separator != '':
                    title = "%s%s%s" % (
                        field.aq_parent.title,
                        self.fieldset_separator,
                        field.title
                        )
                elif fieldset != '' and self.fieldset_separator != '':
                    title = "%s%s%s" % (
                        fieldset,
                        self.fieldset_separator,
                        field.title
                        )
                if title in data.keys():
                    # start at 2 since the title without a number is 1
                    increment = 2
                    while "%s %s" % (title, increment) in data.keys():
                        increment += 1
                    title = "%s %s" % (title, increment)
                data[title] = val

        if self.extraData:
            for field in self.extraData:
                data[extra_data[field]] = getattr(REQUEST, field, '')

        pfg = self._getParentForm()
        submission = {
            'form-id': pfg.id,
            'name': pfg.title,
            'url': pfg.absolute_url(),
            'owner': pfg.Creator(),
            'data': json.dumps(data, default=lambda obj: obj.__dict__),
            }
        try:
            # timeout is set for 5 seconds which is an eternity on the web
            response = requests.post(
                self.url,
                data=submission,
                timeout=60,
                )
        except:
            raise
        else:
            if response.status_code != 201:
                msg = "Web service submission failed by returning status " \
                    + "code: %s. Was expecting status code 201." % response.status_code
                raise Exception(msg)

    security.declareProtected(View, 'onSuccess')
    def onSuccess(self, fields, REQUEST=None):
        """
        Wrap _onSuccess so fallback behavior can be implemented when calls
        to the web service fail.
        """
        # heavily borrowed from: https://plone.org/products/salesforcepfgadapter

        message = None
        try:
            self._onSuccess(fields, REQUEST)
        except Exception as e:
            # swallow the exception
            t, v = sys.exc_info()[:2]

            # generate a log message regarding the exception
            log_msg = 'Unable to save form data to the web service at (%s). '
            logger.exception(log_msg % '/'.join(self.getPhysicalPath()))

            if not self.failSilently:
                raise

            else:
                # get all the active and inactive save data and mailer adapters
                formFolder = self._getParentForm()
                enabled_adapters = formFolder.getActionAdapter()
                adapters = [o for o in formFolder.objectValues() if IPloneFormGenActionAdapter.providedBy(o)]
                active_savedata = [o for o in adapters if isinstance(o, FormSaveDataAdapter)
                                                       and o.id in enabled_adapters]
                inactive_savedata = [o for o in adapters if isinstance(o, FormSaveDataAdapter)
                                                         and o.id not in enabled_adapters]
                active_mailer = [o for o in adapters if isinstance(o, FormMailerAdapter)
                                                     and o.id in enabled_adapters]
                inactive_mailer = [o for o in adapters if isinstance(o, FormMailerAdapter)
                                                       and o.id not in enabled_adapters]

                # start the failure email message
                message = "Someone submitted this form (%s), but the data " \
                    + "couldn't be saved to the web service due to an exception." \
                    + "\n\n" \
                    + "The data was saved in the following locations:\n"
                message = message % (
                        formFolder.absolute_url(),
                        )

                # add a list of where the data was stored to the email message
                for adapter in active_savedata:
                    message += "  - Save Data Adapter (%s)\n" % adapter.absolute_url()

                if self.runDisabledAdapters:
                    for adapter in inactive_savedata:
                        message += "  - Save Data Adapter (%s)\n" % adapter.absolute_url()
                        # Trigger the adapter since it's disabled.
                        # This can be used to record data *only* when submitting to
                        #   the web service fails.
                        adapter.onSuccess(fields, REQUEST)

                for adapter in active_mailer:
                    message += "  - Mailer Adapter (%s)\n" % adapter.absolute_url()

                if self.runDisabledAdapters:
                    for adapter in inactive_mailer:
                        message += "  - Mailer Adapter (%s)\n" % adapter.absolute_url()
                        # Trigger the adapter since it's disabled.
                        # This can be used to record data *only* when submitting to
                        #   the web service fails.
                        adapter.onSuccess(fields, REQUEST)

                if not active_savedata and not inactive_savedata and \
                   not active_mailer and not inactive_mailer:
                    message += "  - NO WHERE! The data was lost.\n"

                message += "\nTechnical details on the exception:\n"
                message += ''.join(traceback.format_exception_only(t, v))

                # send an email if an address is provided
                if self.notifyOnFailure:
                    # get email configuration from Plone
                    pprops = getToolByName(self, 'portal_properties')
                    site_props = getToolByName(pprops, 'site_properties')
                    portal = getToolByName(self, 'portal_url').getPortalObject()

                    from_addr = site_props.getProperty('email_from_address') or \
                        portal.getProperty('email_from_address')
                    mailer = getToolByName(self, 'MailHost')
                    try:
                        mailer.send(
                            message,
                            mto=self.notifyOnFailure,
                            mfrom=from_addr,
                            subject='Form submission to web service failed',
                            immediate=False,
                            )
                    except Exception as e:
                        logger.exception(e)

    security.declareProtected(ModifyPortalContent, 'setShowFields')
    def setShowFields(self, value, **kwargs):
        """
        Reorder form inputs to match field order.
        """
        # This wouldn't be necessary if the PickWidget retained order.
        self.showFields = []
        for field in self.fgFields(excludeServerSide=False):
            id = field.getName()
            if id in value:
                self.showFields.append(id)

registerATCT(FormWebServiceAdapter, 'WebServicesPFGAdapter')
<|endoftext|>


<commit_before>'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
''This script is used to download content from webpages
It Download the content from provided url.

It imports:
  -urllib2
  -wx
  -time
  -os
  -Thread from threading
  -setupkwargs from wx.lib.pubsub
  -pub from wx.lib.pubsub
  -pygauge from wx.lib.agw
  -opj from utils
  -utils

It defines:
  -read_config
  -Thread
    -__init__
    -run
  -main
  -update_progress
'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
'''Required modules'''
import urllib2
import wx
import time
import os
from threading import Thread
from wx.lib.pubsub import setupkwargs
from wx.lib.pubsub import pub
from wx.lib.agw import pygauge as PG

from utils import opj
import utils
#---------------------------CONSTANTS---------------------------------------
#to check if file already exist
FILE_EXIST = None
#If partial file exist
PART_EXIST = None
#for reading rename option
RENAME = ''
#the progress bar to be used in update_progress
PROGRESS = None
#dictionary for files sizes
SIZE_DICT = {}
#To stop downloading
STOP = False
#segment option
SEGMENT = 1

#--------------------------------------------------------------------------
#Reading configuration file
def read_config():
    '''
    function to read config_file for rename option
    '''
    global RENAME, SEGMENT

    with open(opj('config.txt')) as config_file:
        data = config_file.read()

    #rename option
    radio_point = data.find('RENAME')
    if radio_point >= 0:
        end_point = data.find('\n',radio_point+1)
        RENAME = data[radio_point+9:end_point].strip()

    #segment option
    segment_point = data.find('SEGMENT')
    if segment_point >= 0:
        end_point = data.find('\n',segment_point+1)
        SEGMENT = data[segment_point+10:end_point].strip()

    var = [RENAME, SEGMENT]

    #Trailing extra whitespaces
    RENAME, SEGMENT = utils.sanitize_string(var)

#---------------------------------------------------------------------------------------------------------------
class TestThread(Thread):
    '''
    TestThread class to run the thread of downloading.
    '''
    def __init__(self,urls,path):
        self.urls = urls
        self.path = path
        Thread.__init__(self)
        self.setDaemon(True)
        self.start()

    #-----------------------------------------------------------------------------------------------------------
    def run(self):
        try:
            time.sleep(1)
            global RENAME, FILE_EXIST, PART_EXIST, PROGRESS, STOP, SEGMENT

            #--------------------------------reading configurations----------------------------------------
            read_config()
            print "Downloading into "+self.path+"..."

            for url in self.urls:

                dl_size = 0
                cancel = False

                f_name = os.path.basename(url)
                disk_file = self.path+"/"+f_name

                #Check if file exists and want to rename
                FILE_EXIST = os.path.exists(disk_file)

                #----------------IF default option is to cancel download, when already exist---------------
                if FILE_EXIST and RENAME == 'Cancel':
                    dlg = wx.MessageDialog(PROGRESS.Parent,f_name+" already exists, Canceling download",
                                           'Oops!', wx.OK|wx.ICON_INFORMATION)
                    dlg.ShowModal()
                    dlg.Destroy()

                    cancel = True

                #---------------If default option is to rename new download, when already exist------------
                if FILE_EXIST and RENAME == 'Rename':
                    count = 1
                    old_f_name = f_name
                    while True:
                        if os.path.exists(disk_file):
                               tmp, ext = os.path.splitext(f_name)
                               cnt = "(%s)" % count
                               if count > 1:
                                tmp = tmp[:-3]
                               f_name = tmp+cnt+ext
                               count += 1
                               print f_name
                               disk_file = self.path+"/"+f_name
                        else:
                            break
                    dlg = wx.MessageDialog(PROGRESS.Parent,old_f_name+" already exist, renaming to "+f_name,
                                           'Oops!', wx.OK|wx.ICON_INFORMATION)
                    dlg.ShowModal()
                    dlg.Destroy()

                #----------------If default option is to remove old download, when already exist-----------
                elif FILE_EXIST and RENAME == 'Replace':
                    os.remove(disk_file)
                    dlg = wx.MessageDialog(PROGRESS.Parent,f_name+" already exists, removing older one.",
                                           'Oops!', wx.OK|wx.ICON_INFORMATION)
                    dlg.ShowModal()
                    dlg.Destroy()

                PART_EXIST = os.path.exists(disk_file+"_part")

                #-----------------------Begin Process-------------------------------------------------------
                if not cancel:

                    seek_point = 0

                    #----------------If a partial file exists, remove `_part` string------------------------
                    if PART_EXIST:
                        size_of_file = utils.file_size(disk_file+"_part")
                        if size_of_file < SIZE_DICT[url]:
                            seek_point = size_of_file+1
                            print 'seek_point: ',seek_point

                    #--------------------Downloading the file-----------------------------------------------
                    try:
                        #------------------Setup connection-------------------------------------------------
                        req = urllib2.Request(url)
                        req.headers["Range"]="bytes=%s-%s" % (seek_point, int(SIZE_DICT[url]))
                        connection = urllib2.urlopen(req)

                        #------------------If server does not accept range-header---------------------------
                        status_code = connection.getcode()
                        if  status_code== 200 and PART_EXIST:
                            dlg = wx.MessageDialog(PROGRESS.Parent,"Partial content not allowed, starting download from beginning",
                                           'Oops!', wx.OK|wx.ICON_INFORMATION)
                            dlg.ShowModal()
                            dlg.Destroy()
                            print "Partial content not allowed, starting download from beginning"

                        print connection.getcode()

                        #--------------------open disk file for writing-------------------------------------
                        if PART_EXIST and not (status_code == 200):
                            save_file = open(disk_file+"_part", 'ab+')
                        else:
                            save_file = open(disk_file, 'wb')

                        if SEGMENT == 1 or SEGMENT == "Default":
                            block_sz = 1024
                        else:
                            block_sz = int(SIZE_DICT[url] / int(SEGMENT))

                        print block_sz

                        #-------------------------Setting the progress bar ---------------------------------
                        PROGRESS.SetRange(SIZE_DICT[url] - seek_point)
                        PROGRESS.SetValue(0)
                        PROGRESS.Parent.Refresh()

                        #--------------------------Saving Content-------------------------------------------
                        while True:
                            buffer = connection.read(block_sz)
                            if not buffer:
                                break
                            if STOP:
                                STOP = False
                                raise IOError
                            dl_size += len(buffer)
                            save_file.write(buffer)
                            save_file.flush()
                            wx.CallAfter(pub.sendMessage,"update",msg=dl_size)

                        #-------------------------Close the file--------------------------------------------
                        save_file.close()
                        print f_name+" downloaded"

                        #----------------------If it was a part file, rename it to original-----------------
                        if PART_EXIST:
                            old_name = disk_file+"_part"

                            #need to reformat the name to avooid the rename error:
                                # The name of file is incorrect in windows
                            new_name = old_name.split("/")[0]+"\\"+old_name.split("/")[1][:-5]

                            os.rename(old_name,new_name)

                    except IOError:
                        #--------------If file was not downloaded completely, mark it as part---------------
                        if dl_size < SIZE_DICT[url]:
                            try:
                                save_file.close()
                            except:
                                pass
                            if not PART_EXIST:
                                os.rename(disk_file,disk_file+"_part")
                            print disk_file

                        dlg = wx.MessageDialog(PROGRESS.Parent,"The connection was broken.",
                                           'Oops!', wx.OK|wx.ICON_INFORMATION)
                        dlg.ShowModal()
                        dlg.Destroy()

                    except Exception as e:
                        print e
                        dlg = wx.MessageDialog(PROGRESS.Parent,str(e),
                                           'Oops!', wx.OK|wx.ICON_INFORMATION)
                        dlg.ShowModal()
                        dlg.Destroy()

        except Exception as e:
            print 'Exception:',e

#---------------------------------------------------------------------------------------------------------------
def main(urls, path,progress_bar):
    '''
    Main module to call the TestThread for downloading and
    to set attributes of the progress bar. It also calculates the size
    of file before downloading.
    '''
    global PROGRESS, SIZE_DICT
    PROGRESS = progress_bar

    #When called with direct url
    if (not(urls[0].startswith("http"))
        and ".html" not in urls[0]):
        return "Invalid url"
    #-------------------Fetching file size before download------------------------
    try:
        total_size = 0
        for url in urls:
            connection = urllib2.urlopen(url)
            meta = connection.info()
            try:
                size = float(meta.getheaders("content-length")[0])
                total_size += size
                SIZE_DICT[url] = size
            except IndexError:
                size = len(connection.read())
                total_size += size
                SIZE_DICT[url] = size
    except IOError:
        return "The connection could not establish"
    except Exception as e:
        return e

    #-----------------------------------------------------------------------------
    print_size = total_size/float(1024**2)
    print str(print_size) + "Mb selected for download"
    #-----------------------------------------------------------------------------

    #------------------------progress bar attributes------------------------------
    progress_bar.SetBarColor([wx.Colour(162,255,178),wx.Colour(159,176,255)])
    progress_bar.SetBackgroundColour(wx.CYAN)
    progress_bar.SetBorderColor(wx.BLACK)
    progress_bar.SetBorderPadding(2)
    print 'total_size:',total_size

    #updating bar
    pub.subscribe(update_progress, "update")
    wx.Yield()
    #--------------------------Start thread---------------------------------------
    TestThread(urls, path)

#---------------------------------------------------------------------------------------------------------------
def update_progress(msg):
    '''
    updates the progress bar.
    '''
    global PROGRESS
    print 'msg:',msg
    PROGRESS.SetValue(msg)
    PROGRESS.Parent.Refresh()

#--------------------------------------------------------------------------------------------------
if __name__ == "__main__":
    main(url,dir_)
<commit_msg>updated<commit_after>'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
''This script is used to download content from webpages
It Download the content from provided url.

It imports:
  -urllib2
  -wx
  -time
  -os
  -Thread from threading
  -setupkwargs from wx.lib.pubsub
  -pub from wx.lib.pubsub
  -pygauge from wx.lib.agw
  -opj from utils
  -utils

It defines:
  -read_config
  -Thread
    -__init__
    -run
  -main
  -update_progress
'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
'''Required modules'''
import urllib2
import wx
import time
import os
from threading import Thread
from wx.lib.pubsub import setupkwargs
from wx.lib.pubsub import pub
from wx.lib.agw import pygauge as PG

from utils import opj
import utils
#---------------------------CONSTANTS---------------------------------------
#to check if file already exist
FILE_EXIST = None
#If partial file exist
PART_EXIST = None
#for reading rename option
RENAME = ''
#the progress bar to be used in update_progress
PROGRESS = None
#dictionary for files sizes
SIZE_DICT = {}
#To stop downloading
STOP = False
#segment option
SEGMENT = '1'

#--------------------------------------------------------------------------
#Reading configuration file
def read_config():
    '''
    function to read config_file for rename option
    '''
    global RENAME, SEGMENT

    with open(opj('config/config.txt')) as config_file:
        data = config_file.read()

    #rename option
    radio_point = data.find('RENAME')
    if radio_point >= 0:
        end_point = data.find('\n',radio_point+1)
        RENAME = data[radio_point+9:end_point].strip()

    #segment option
    segment_point = data.find('SEGMENT')
    if segment_point >= 0:
        end_point = data.find('\n',segment_point+1)
        SEGMENT = data[segment_point+10:end_point].strip()

    var = [RENAME, SEGMENT]

    #Trailing extra whitespaces
    RENAME, SEGMENT = utils.sanitize_string(var)

#---------------------------------------------------------------------------------------------------------------
class TestThread(Thread):
    '''
    TestThread class to run the thread of downloading.
    '''
    def __init__(self,urls,path):
        self.urls = urls
        self.path = path
        Thread.__init__(self)
        self.setDaemon(True)
        self.start()

    #-----------------------------------------------------------------------------------------------------------
    def run(self):

        try:
            time.sleep(1)
            global RENAME, FILE_EXIST, PART_EXIST, PROGRESS, STOP, SEGMENT

            #--------------------------------reading configurations----------------------------------------
            read_config()
            print "Downloading into "+self.path+"..."

            for url in self.urls:

                dl_size = 0
                cancel = False

                f_name = os.path.basename(url)
                disk_file = self.path+"/"+f_name

                #Check if file exists and want to rename
                FILE_EXIST = os.path.exists(disk_file)

                #----------------IF default option is to cancel download, when already exist---------------
                if FILE_EXIST and RENAME == 'Cancel':
                    dlg = wx.MessageDialog(PROGRESS.Parent,f_name+" already exists, Canceling download",
                                           'Oops!', wx.OK|wx.ICON_INFORMATION)
                    dlg.ShowModal()
                    dlg.Destroy()

                    cancel = True

                #---------------If default option is to rename new download, when already exist------------
                if FILE_EXIST and RENAME == 'Rename':
                    count = 1
                    old_f_name = f_name
                    while True:
                        if os.path.exists(disk_file):
                               tmp, ext = os.path.splitext(f_name)
                               cnt = "(%s)" % count
                               if count > 1:
                                tmp = tmp[:-3]
                               f_name = tmp+cnt+ext
                               count += 1
                               print f_name
                               disk_file = self.path+"/"+f_name
                        else:
                            break
                    dlg = wx.MessageDialog(PROGRESS.Parent,old_f_name+" already exist, renaming to "+f_name,
                                           'Oops!', wx.OK|wx.ICON_INFORMATION)
                    dlg.ShowModal()
                    dlg.Destroy()

                #----------------If default option is to remove old download, when already exist-----------
                elif FILE_EXIST and RENAME == 'Replace':
                    os.remove(disk_file)
                    dlg = wx.MessageDialog(PROGRESS.Parent,f_name+" already exists, removing older one.",
                                           'Oops!', wx.OK|wx.ICON_INFORMATION)
                    dlg.ShowModal()
                    dlg.Destroy()

                PART_EXIST = os.path.exists(disk_file+"_part")

                #-----------------------Begin Process-------------------------------------------------------
                if not cancel:

                    seek_point = 0

                    #----------------If a partial file exists, remove `_part` string------------------------
                    if PART_EXIST:
                        size_of_file = utils.file_size(disk_file+"_part")
                        if size_of_file < SIZE_DICT[url]:
                            seek_point = size_of_file+1
                            print 'seek_point: ',seek_point

                    #--------------------Downloading the file-----------------------------------------------
                    try:
                        #------------------Setup connection-------------------------------------------------
                        req = urllib2.Request(url)
                        req.headers["Range"]="bytes=%s-%s" % (seek_point, int(SIZE_DICT[url]))
                        connection = urllib2.urlopen(req)

                        #------------------If server does not accept range-header---------------------------
                        status_code = connection.getcode()
                        if  status_code== 200 and PART_EXIST:
                            dlg = wx.MessageDialog(PROGRESS.Parent,"Partial content not allowed, starting download from beginning",
                                           'Oops!', wx.OK|wx.ICON_INFORMATION)
                            dlg.ShowModal()
                            dlg.Destroy()
                            print "Partial content not allowed, starting download from beginning"

                        print connection.getcode()

                        #--------------------open disk file for writing-------------------------------------
                        if PART_EXIST and not (status_code == 200):
                            save_file = open(disk_file+"_part", 'ab+')
                        else:
                            save_file = open(disk_file, 'wb')

                        if SEGMENT == '1' or SEGMENT == "Default":
                            block_sz = 1024
                        else:
                            block_sz = int(SIZE_DICT[url] / int(SEGMENT))

                        print block_sz

                        #-------------------------Setting the progress bar ---------------------------------
                        PROGRESS.SetRange(SIZE_DICT[url] - seek_point)
                        PROGRESS.SetValue(0)
                        PROGRESS.Parent.Refresh()

                        #--------------------------Saving Content-------------------------------------------
                        while True:
                            buffer = connection.read(block_sz)
                            if not buffer:
                                break
                            if STOP:
                                STOP = False
                                raise IOError
                            dl_size += len(buffer)
                            save_file.write(buffer)
                            save_file.flush()
                            wx.CallAfter(pub.sendMessage,"update",msg=dl_size)

                        #-------------------------Close the file--------------------------------------------
                        save_file.close()
                        print f_name+" downloaded"

                        #----------------------If it was a part file, rename it to original-----------------
                        if PART_EXIST:
                            old_name = disk_file+"_part"

                            #need to reformat the name to avooid the rename error:
                                # The name of file is incorrect in windows
                            new_name = old_name.split("/")[0]+"\\"+old_name.split("/")[1][:-5]

                            os.rename(old_name,new_name)

                    except IOError:
                        #--------------If file was not downloaded completely, mark it as part---------------
                        if dl_size < SIZE_DICT[url]:
                            try:
                                save_file.close()
                            except:
                                pass
                            if not PART_EXIST:
                                os.rename(disk_file,disk_file+"_part")
                            print disk_file

                        dlg = wx.MessageDialog(PROGRESS.Parent,"The connection was broken.",
                                           'Oops!', wx.OK|wx.ICON_INFORMATION)
                        dlg.ShowModal()
                        dlg.Destroy()

                    except Exception as e:
                        print e
                        dlg = wx.MessageDialog(PROGRESS.Parent,str(e),
                                           'Oops!', wx.OK|wx.ICON_INFORMATION)
                        dlg.ShowModal()
                        dlg.Destroy()

        except Exception as e:
            print 'Exception:',e

#---------------------------------------------------------------------------------------------------------------
def main(urls, path,progress_bar):
    '''
    Main module to call the TestThread for downloading and
    to set attributes of the progress bar. It also calculates the size
    of file before downloading.
    '''
    global PROGRESS, SIZE_DICT
    PROGRESS = progress_bar

    #When called with direct url
    if (not(urls[0].startswith("http"))
        and ".html" not in urls[0]):
        return "Invalid url"
    #-------------------Fetching file size before download------------------------
    try:
        total_size = 0
        for url in urls:
            connection = urllib2.urlopen(url)
            meta = connection.info()
            try:
                size = float(meta.getheaders("content-length")[0])
                total_size += size
                SIZE_DICT[url] = size
            except IndexError:
                size = len(connection.read())
                total_size += size
                SIZE_DICT[url] = size
    except IOError:
        return "The connection could not establish"
    except Exception as e:
        return e

    #-----------------------------------------------------------------------------
    print_size = total_size/float(1024**2)
    print str(print_size) + "Mb selected for download"
    #-----------------------------------------------------------------------------

    #------------------------progress bar attributes------------------------------
    progress_bar.SetBarColor([wx.Colour(162,255,178),wx.Colour(159,176,255)])
    progress_bar.SetBackgroundColour(wx.CYAN)
    progress_bar.SetBorderColor(wx.BLACK)
    progress_bar.SetBorderPadding(2)
    print 'total_size:',total_size

    #updating bar
    pub.subscribe(update_progress, "update")
    wx.Yield()
    #--------------------------Start thread---------------------------------------
    TestThread(urls, path)

#---------------------------------------------------------------------------------------------------------------
def update_progress(msg):
    '''
    updates the progress bar.
    '''
    global PROGRESS
    print 'msg:',msg
    PROGRESS.SetValue(msg)
    PROGRESS.Parent.Refresh()

#--------------------------------------------------------------------------------------------------
if __name__ == "__main__":
    main(url,dir_)
<|endoftext|>