<commit_before>from django.core.urlresolvers import reverse
from django.shortcuts import render_to_response, get_object_or_404
from django.http import Http404, HttpResponseRedirect, HttpResponse, HttpResponseNotAllowed
from django.template import RequestContext#,Template
from django.utils import simplejson
from django.contrib.auth.decorators import login_required

from base.shortcuts import get_object_or_create

from jetpack.models import Jet, JetVersion, Cap, CapVersion
from jetpack.default_settings import settings

def gallery(r, page=None):
	"""
	Display mixed list (Jetpacks with Capabilities)
	"""
	items = list(Jet.objects.all()[0:20])
	items.extend(list(Cap.objects.all()[0:20]))
	items = filter(lambda i: i.base_version, items)
	items.sort(lambda i, j: (j.base_version.last_update - i.base_version.last_update).seconds)
	page = "packages"

	return render_to_response(
		'gallery.html',
		locals(),
		context_instance=RequestContext(r))

@login_required
def jetpack_edit(r, slug):
	"""
	Get jetpack and send it to item_edit
	"""
	jetpack = get_object_or_404(Jet, slug=slug)
	return item_edit(r, jetpack, "jetpack")

@login_required
def capability_edit(r, slug):
	"""
	Get capability and send it to item_edit
	"""
	capability = get_object_or_404(Cap, slug=slug)
	return item_edit(r, capability, "capability")

def item_edit(r, item, type):
	"""
	retrieve item and (if possible) version
	Render the right edit page for the given type
	"""
	try:
		version = item.base_version
	except:
		#valid, as newly created item has no version yet
		pass
	if type == "jetpack":
		other_versions = JetVersion.objects.filter_by_slug(slug=item.slug)
	elif type == "capability":
		other_versions = CapVersion.objects.filter_by_slug(slug=item.slug)

	item_page = True
	page = "editor"
	jetpack_create_url = Jet.get_create_url()
	capability_create_url = Cap.get_create_url()
	return render_to_response("edit_item.html", locals(),
				context_instance=RequestContext(r))

@login_required
def jetpack_version_edit(r, slug, version, counter):
	version = get_object_or_404(JetVersion, jetpack__slug=slug, name=version, counter=counter)
	item = version.jetpack
	type = "jetpack"
	page = "editor"
	other_versions = JetVersion.objects.filter_by_slug(slug=slug)
	return render_to_response('edit_item.html', locals(),
				context_instance=RequestContext(r))

@login_required
def capability_version_edit(r, slug, version, counter):
	version = get_object_or_404(CapVersion, capability__slug=slug, name=version, counter=counter)
	item = version.capability
	other_versions = CapVersion.objects.filter_by_slug(slug=slug)
	type = "capability"
	page = "editor"
	return render_to_response('edit_item.html', locals(),
				context_instance=RequestContext(r))

@login_required
def item_create(r, type):
	"""
	Create new item (Jetpack/Capability)
	This is a result of a popup window with just name and description
	Version will be saved in the item_version_create
	"""
	Klass = Jet if type=="jetpack" else Cap
	item = Klass(
		creator=r.user,
		name=r.POST.get("%s_name" % type),
		description=r.POST.get("%s_description" % type)
	)
	# TODO: validate
	item.save()
	return render_to_response("json/%s_created.json" % type, {type: item},
				context_instance=RequestContext(r),
				mimetype='application/json')

@login_required
def item_update(r, slug, type):
	"""
	Update the existing item's metadata only
	"""
	Klass = Jet if type=="jetpack" else Cap
	item = get_object_or_404(Klass, slug=slug)
	if not item.can_be_updated_by(r.user):
		return HttpResponseNotAllowed(HttpResponse(""))

	if '%s_description' % type in r.POST:
		item.description = r.POST.get('%s_description' % type)
	if '%s_public_permission' % type in r.POST:
		item.public_permission = r.POST.get('%s_public_permission' % type)
	if '%s_group_permission' % type in r.POST:
		item.group_permission = r.POST.get('%s_group_permission' % type)

	item.save()
	return render_to_response('json/%s_updated.json' % type, {type: item},
				context_instance=RequestContext(r),
				mimetype='application/json')

@login_required
def item_version_create(r, slug, type):
	"""
	Save new version for the item, get data from POST
	"""
	if type == "jetpack":
		Klass = Jet
		KlassVersion = JetVersion
	elif type == "capability":
		Klass = Cap
		KlassVersion = CapVersion

	item = get_object_or_404(Klass, slug=slug)
	version_data = {
		type: item,
		"author": r.user,
		"name": r.POST.get("version_name"),
		"content": r.POST.get("version_content"),
		"description": r.POST.get("version_description"),
	}
	if type == "jetpack":
		version_data["manifest"] = r.POST.get("version_manifest")

	if "version_status" in r.POST:
		version_data["status"] = r.POST.get("version_status")
	if "version_published" in r.POST:
		version_data["published"] = r.POST.get("version_published")
	if "version_is_base" in r.POST:
		version_data["is_base"] = r.POST.get("version_is_base")

	version = KlassVersion(**version_data)
	version.save()
	dep_capabilities = simplejson.loads(r.POST.get('capabilities','[]'));

	for c in dep_capabilities:
		dep_cap = CapVersion.objects.get(
						capability__slug=c['slug'],
						name=c['version'],
						counter=c['counter'])
		version.capabilities.add(dep_cap)

	return render_to_response('json/version_absolute_url.json', {'version': version},
				context_instance=RequestContext(r),
				mimetype='application/json')

@login_required
def item_version_update(r, slug, version, counter, type):
	"""
	Update the given version - no counter change
	"""
	if type == "jetpack":
		version = get_object_or_404(JetVersion,
						jetpack__slug=slug, name=version, counter=counter)
	elif type == "capability":
		version = get_object_or_404(CapVersion,
						capability__slug=slug, name=version, counter=counter)

	# permission check
	if not version.author == r.user:
		return HttpResponseNotAllowed(HttpResponse(""))

	version.author = r.user
	version.name = r.POST.get("version_name", version.name)
	if type == "jetpack":
		version.manifest = r.POST.get("version_manifest", version.manifest)
		version.published =  r.POST.get("version_published", version.published)
	version.content = r.POST.get("version_content", version.content)
	version.description = r.POST.get("version_description", version.description)
	version.status = r.POST.get("version_status", version.status)
	version.is_base = r.POST.get("version_is_base", version.is_base)
	version.save()
	return render_to_response('json/version_updated.json', {'version': version},
				context_instance=RequestContext(r),
				mimetype='application/json')

@login_required
def item_version_save_as_base(r, slug, version, counter, type):
	"""
	Update the given version - no counter change
	"""
	if type == "jetpack":
		version = get_object_or_404(JetVersion,
						jetpack__slug=slug, name=version, counter=counter)
		item = version.jetpack
	elif type == "capability":
		version = get_object_or_404(CapVersion,
						capability__slug=slug, name=version, counter=counter)
		item = version.capability

	# permission check
	if not item.can_be_updated_by(r.user):
		return HttpResponseNotAllowed(HttpResponse(""))

	version.is_base = True
	version.save()
	return render_to_response('json/version_saved_as_base.json', {'version': version},
				context_instance=RequestContext(r),
				mimetype='application/json')

@login_required
def capabilities_autocomplete(r):
	"""
	Display names of the modules (capabilities) which mark the pattern
	"""

@login_required
def add_dependency(r, slug, type, version=None, counter=None):
	"""
	Add dependency to the item represented by slug
	"""
	# TODO: add more protection (do not allow two versions of the same Cap)
	if type == 'jetpack':
		item_version = get_object_or_404(JetVersion,
					jetpack__slug=slug, name=version, counter=counter)
		item = item_version.jetpack
	elif type == 'capability':
		item_version = get_object_or_404(CapVersion,
					capability__slug=slug, name=version, counter=counter)
		item = item_version.capability

	dependency_slug = r.POST.get("dependency_slug")
	dependency_version = r.POST.get("dependency_version", None)
	dependency_counter = r.POST.get("dependency_counter", None)
	if dependency_version:
		dependency = CapVersion.objects.get(
						capability__slug=dependency_slug,
						name=dependency_version,
						counter=dependency_counter)
		cap = dependency.capability
	else:
		cap = Cap.objects.get(slug=dependency_slug)
		dependency = cap.base_version

	item_version.capabilities.add(dependency)
	item_version.save()

	dependency_remove_url = reverse("jp_%s_remove_dependency" % type, args=[
		item.slug, item_version.name, item_version.counter,
		cap.slug, dependency.name, dependency.counter])

	return render_to_response('json/dependency_added.json', {
					'item': item_version,
					'version': dependency,
					'cap': dependency.capability,
					'dependency_remove_url': dependency_remove_url
				},
				context_instance=RequestContext(r),
				mimetype='application/json')

@login_required
def remove_dependency(r, slug, version, counter, type, d_slug, d_version, d_counter):
	"""
	Remove dependency from item
	"""
	if type == 'jetpack':
		item_version = get_object_or_404(JetVersion,
					jetpack__slug=slug, name=version, counter=counter)
	elif type == 'capability':
		item_version = get_object_or_404(CapVersion,
					capability__slug=slug, name=version, counter=counter)

	dependency = get_object_or_404(CapVersion,
					capability__slug=d_slug, name=d_version, counter=d_counter)

	item_version.capabilities.remove(dependency)
	item_version.save()
	return render_to_response('json/dependency_removed.json', locals(),
				context_instance=RequestContext(r),
				mimetype='application/json')
<commit_msg>gallery limit removed - it will be added with paging<commit_after>from django.core.urlresolvers import reverse
from django.shortcuts import render_to_response, get_object_or_404
from django.http import Http404, HttpResponseRedirect, HttpResponse, HttpResponseNotAllowed
from django.template import RequestContext#,Template
from django.utils import simplejson
from django.contrib.auth.decorators import login_required

from base.shortcuts import get_object_or_create

from jetpack.models import Jet, JetVersion, Cap, CapVersion
from jetpack.default_settings import settings

def gallery(r, page=None, with_new=False):
	"""
	Display mixed list (Jetpacks with Capabilities)
	"""
	items = list(Jet.objects.all())
	items.extend(list(Cap.objects.all()))
	if not with_new:
		items = filter(lambda i: i.base_version, items)
	items.sort(lambda i, j: (j.base_version.last_update - i.base_version.last_update).seconds)
	page = "packages"

	return render_to_response(
		'gallery.html',
		locals(),
		context_instance=RequestContext(r))

@login_required
def jetpack_edit(r, slug):
	"""
	Get jetpack and send it to item_edit
	"""
	jetpack = get_object_or_404(Jet, slug=slug)
	return item_edit(r, jetpack, "jetpack")

@login_required
def capability_edit(r, slug):
	"""
	Get capability and send it to item_edit
	"""
	capability = get_object_or_404(Cap, slug=slug)
	return item_edit(r, capability, "capability")

def item_edit(r, item, type):
	"""
	retrieve item and (if possible) version
	Render the right edit page for the given type
	"""
	try:
		version = item.base_version
	except:
		#valid, as newly created item has no version yet
		pass
	if type == "jetpack":
		other_versions = JetVersion.objects.filter_by_slug(slug=item.slug)
	elif type == "capability":
		other_versions = CapVersion.objects.filter_by_slug(slug=item.slug)

	item_page = True
	page = "editor"
	jetpack_create_url = Jet.get_create_url()
	capability_create_url = Cap.get_create_url()
	return render_to_response("edit_item.html", locals(),
				context_instance=RequestContext(r))

@login_required
def jetpack_version_edit(r, slug, version, counter):
	version = get_object_or_404(JetVersion, jetpack__slug=slug, name=version, counter=counter)
	item = version.jetpack
	type = "jetpack"
	page = "editor"
	other_versions = JetVersion.objects.filter_by_slug(slug=slug)
	return render_to_response('edit_item.html', locals(),
				context_instance=RequestContext(r))

@login_required
def capability_version_edit(r, slug, version, counter):
	version = get_object_or_404(CapVersion, capability__slug=slug, name=version, counter=counter)
	item = version.capability
	other_versions = CapVersion.objects.filter_by_slug(slug=slug)
	type = "capability"
	page = "editor"
	return render_to_response('edit_item.html', locals(),
				context_instance=RequestContext(r))

@login_required
def item_create(r, type):
	"""
	Create new item (Jetpack/Capability)
	This is a result of a popup window with just name and description
	Version will be saved in the item_version_create
	"""
	Klass = Jet if type=="jetpack" else Cap
	item = Klass(
		creator=r.user,
		name=r.POST.get("%s_name" % type),
		description=r.POST.get("%s_description" % type)
	)
	# TODO: validate
	item.save()
	return render_to_response("json/%s_created.json" % type, {type: item},
				context_instance=RequestContext(r),
				mimetype='application/json')

@login_required
def item_update(r, slug, type):
	"""
	Update the existing item's metadata only
	"""
	Klass = Jet if type=="jetpack" else Cap
	item = get_object_or_404(Klass, slug=slug)
	if not item.can_be_updated_by(r.user):
		return HttpResponseNotAllowed(HttpResponse(""))

	if '%s_description' % type in r.POST:
		item.description = r.POST.get('%s_description' % type)
	if '%s_public_permission' % type in r.POST:
		item.public_permission = r.POST.get('%s_public_permission' % type)
	if '%s_group_permission' % type in r.POST:
		item.group_permission = r.POST.get('%s_group_permission' % type)

	item.save()
	return render_to_response('json/%s_updated.json' % type, {type: item},
				context_instance=RequestContext(r),
				mimetype='application/json')

@login_required
def item_version_create(r, slug, type):
	"""
	Save new version for the item, get data from POST
	"""
	if type == "jetpack":
		Klass = Jet
		KlassVersion = JetVersion
	elif type == "capability":
		Klass = Cap
		KlassVersion = CapVersion

	item = get_object_or_404(Klass, slug=slug)
	version_data = {
		type: item,
		"author": r.user,
		"name": r.POST.get("version_name"),
		"content": r.POST.get("version_content"),
		"description": r.POST.get("version_description"),
	}
	if type == "jetpack":
		version_data["manifest"] = r.POST.get("version_manifest")

	if "version_status" in r.POST:
		version_data["status"] = r.POST.get("version_status")
	if "version_published" in r.POST:
		version_data["published"] = r.POST.get("version_published")
	if "version_is_base" in r.POST:
		version_data["is_base"] = r.POST.get("version_is_base")

	version = KlassVersion(**version_data)
	version.save()
	dep_capabilities = simplejson.loads(r.POST.get('capabilities','[]'));

	for c in dep_capabilities:
		dep_cap = CapVersion.objects.get(
						capability__slug=c['slug'],
						name=c['version'],
						counter=c['counter'])
		version.capabilities.add(dep_cap)

	return render_to_response('json/version_absolute_url.json', {'version': version},
				context_instance=RequestContext(r),
				mimetype='application/json')

@login_required
def item_version_update(r, slug, version, counter, type):
	"""
	Update the given version - no counter change
	"""
	if type == "jetpack":
		version = get_object_or_404(JetVersion,
						jetpack__slug=slug, name=version, counter=counter)
	elif type == "capability":
		version = get_object_or_404(CapVersion,
						capability__slug=slug, name=version, counter=counter)

	# permission check
	if not version.author == r.user:
		return HttpResponseNotAllowed(HttpResponse(""))

	version.author = r.user
	version.name = r.POST.get("version_name", version.name)
	if type == "jetpack":
		version.manifest = r.POST.get("version_manifest", version.manifest)
		version.published =  r.POST.get("version_published", version.published)
	version.content = r.POST.get("version_content", version.content)
	version.description = r.POST.get("version_description", version.description)
	version.status = r.POST.get("version_status", version.status)
	version.is_base = r.POST.get("version_is_base", version.is_base)
	version.save()
	return render_to_response('json/version_updated.json', {'version': version},
				context_instance=RequestContext(r),
				mimetype='application/json')

@login_required
def item_version_save_as_base(r, slug, version, counter, type):
	"""
	Update the given version - no counter change
	"""
	if type == "jetpack":
		version = get_object_or_404(JetVersion,
						jetpack__slug=slug, name=version, counter=counter)
		item = version.jetpack
	elif type == "capability":
		version = get_object_or_404(CapVersion,
						capability__slug=slug, name=version, counter=counter)
		item = version.capability

	# permission check
	if not item.can_be_updated_by(r.user):
		return HttpResponseNotAllowed(HttpResponse(""))

	version.is_base = True
	version.save()
	return render_to_response('json/version_saved_as_base.json', {'version': version},
				context_instance=RequestContext(r),
				mimetype='application/json')

@login_required
def capabilities_autocomplete(r):
	"""
	Display names of the modules (capabilities) which mark the pattern
	"""

@login_required
def add_dependency(r, slug, type, version=None, counter=None):
	"""
	Add dependency to the item represented by slug
	"""
	# TODO: add more protection (do not allow two versions of the same Cap)
	if type == 'jetpack':
		item_version = get_object_or_404(JetVersion,
					jetpack__slug=slug, name=version, counter=counter)
		item = item_version.jetpack
	elif type == 'capability':
		item_version = get_object_or_404(CapVersion,
					capability__slug=slug, name=version, counter=counter)
		item = item_version.capability

	dependency_slug = r.POST.get("dependency_slug")
	dependency_version = r.POST.get("dependency_version", None)
	dependency_counter = r.POST.get("dependency_counter", None)
	if dependency_version:
		dependency = CapVersion.objects.get(
						capability__slug=dependency_slug,
						name=dependency_version,
						counter=dependency_counter)
		cap = dependency.capability
	else:
		cap = Cap.objects.get(slug=dependency_slug)
		dependency = cap.base_version

	item_version.capabilities.add(dependency)
	item_version.save()

	dependency_remove_url = reverse("jp_%s_remove_dependency" % type, args=[
		item.slug, item_version.name, item_version.counter,
		cap.slug, dependency.name, dependency.counter])

	return render_to_response('json/dependency_added.json', {
					'item': item_version,
					'version': dependency,
					'cap': dependency.capability,
					'dependency_remove_url': dependency_remove_url
				},
				context_instance=RequestContext(r),
				mimetype='application/json')

@login_required
def remove_dependency(r, slug, version, counter, type, d_slug, d_version, d_counter):
	"""
	Remove dependency from item
	"""
	if type == 'jetpack':
		item_version = get_object_or_404(JetVersion,
					jetpack__slug=slug, name=version, counter=counter)
	elif type == 'capability':
		item_version = get_object_or_404(CapVersion,
					capability__slug=slug, name=version, counter=counter)

	dependency = get_object_or_404(CapVersion,
					capability__slug=d_slug, name=d_version, counter=d_counter)

	item_version.capabilities.remove(dependency)
	item_version.save()
	return render_to_response('json/dependency_removed.json', locals(),
				context_instance=RequestContext(r),
				mimetype='application/json')
<|endoftext|>


<commit_before>from datetime import timedelta

from django.utils import timezone

from celery.task import periodic_task
from celery.task.schedules import crontab

from core.models.group import Group
from core.models.user import AtmosphereUser
from core.models.provider import Provider

from service.allocation import check_over_allocation
from service.driver import get_admin_driver

from threepio import logger

@periodic_task(run_every=crontab(hour='*', minute='*/15', day_of_week='*'),
               # 5min before task expires, 5min to run task
               expires=5*60, time_limit=5*60, retry=0)
def monitor_instances():
    """
    Update instances for each active provider.
    """
    for p in Provider.objects.filter(active=True, end_date=None):
        monitor_instances_for(p)

def monitor_instances_for(provider):
    """
    Update instances for provider.
    """
    #For now, lets just ignore everything that isn't openstack.
    if 'openstack' not in provider.type.name.lower():
        return

    admin_driver = get_admin_driver(provider)
    meta = admin_driver.meta(admin_driver=admin_driver)
    logger.info("Retrieving all tenants..")
    all_tenants = admin_driver._connection._keystone_list_tenants()
    logger.info("Retrieved %s tenants. Retrieving all instances.."
                % len(all_tenants))
    all_instances = meta.all_instances()
    logger.info("Retrieved %s instances." % len(all_instances))
    #Convert tenant-id to tenant-name all at once
    all_instances = _convert_tenant_id_to_names(all_instances, all_tenants)
    logger.info("Owner information added.")
    #Make a mapping of owner-to-instance
    instance_map = _make_instance_owner_map(all_instances)
    logger.info("Instance owner map created")

    for username in instance_map.keys():
        try:
            user = AtmosphereUser.objects.get(username=username)
            group = Group.objects.get(name=user.username)
            id = user.identity_set.get(provider=provider)
            im = id.identitymembership_set.get(member=group)
            if not im.allocation:
                continue
            instances = instance_map[username]
            over_allocation = over_allocation_test(im.identity, instances)
            if over_allocation:
                continue
            core_instances = im.identity.instance_set.filter(end_date=None)
            update_instances(im.identity, instances, core_instances)
        except:
            logger.exception("Unable to monitor instance: %s" % i)
            raise
    logger.info("Monitoring completed")

def _make_instance_owner_map(instances):
    owner_map = {}
    for i in instances:
        key = i.owner
        instance_list = owner_map.get(key, [])
        instance_list.append(i)
        owner_map[key] = instance_list
    return owner_map

def _convert_tenant_id_to_names(instances, tenants):
    for i in instances:
        for tenant in tenants:
            if tenant['id'] == i.owner:
                i.owner = tenant['name']
    return instances

def over_allocation_test(identity, esh_instances):
    from api import get_esh_driver
    from core.models.instance import convert_esh_instance
    from atmosphere import settings
    over_allocated, time_diff = check_over_allocation(
        identity.created_by.username, identity.id)
    if not over_allocated:
        # Nothing changed, bail.
        return False
    if settings.DEBUG:
        logger.info('Do not enforce allocations in DEBUG mode')
        return False
    driver = get_esh_driver(identity)
    for instance in esh_instances:
        #Suspend active instances, update the task in the DB
        try:
            if driver._is_active_instance(instance):
                driver.suspend_instance(instance)
        except Exception, e:
            if 'in vm_state suspended' not in e.message:
                raise
        updated_esh = driver.get_instance(instance.id)
        updated_core = convert_esh_instance(driver, updated_esh,
                                            identity.provider.id,
                                            identity.id,
                                            identity.created_by)
        updated_core.update_history(updated_esh.extra['status'],
                                    updated_esh.extra.get('task'))
    #All instances are dealt with, move along.
    return True # User was over_allocation

def update_instances(identity, esh_list, core_list):
    """
    End-date core instances that don't show up in esh_list
    && Update the values of instances that do
    """
    esh_ids = [instance.id for instance in esh_list]
    #logger.info('%s Instances for Identity %s: %s' % (len(esh_ids), identity, esh_ids))
    for core_instance in core_list:
        try:
            index = esh_ids.index(core_instance.provider_alias)
        except ValueError:
            logger.info("Did not find instance %s in ID List: %s" %
                        (core_instance.provider_alias, esh_ids))
            core_instance.end_date_all()
            continue
        esh_instance = esh_list[index]
        core_instance.update_history(
            esh_instance.extra['status'],
            esh_instance.extra.get('task') or
            esh_instance.extra.get('metadata', {}).get('tmp_status'))
    return
<commit_msg>Changes Steve made for production. 	modified:   service/tasks/allocation.py<commit_after>from datetime import timedelta

from django.utils import timezone

from celery.task import periodic_task
from celery.task.schedules import crontab

from core.models.group import Group
from core.models.user import AtmosphereUser
from core.models.provider import Provider

from service.allocation import check_over_allocation
from service.driver import get_admin_driver

from threepio import logger

@periodic_task(run_every=crontab(hour='*', minute='*/15', day_of_week='*'),
               # 5min before task expires, 5min to run task
               expires=5*60, time_limit=5*60, retry=0)
def monitor_instances():
    """
    Update instances for each active provider.
    """
    for p in Provider.objects.filter(active=True, end_date=None):
        monitor_instances_for(p)

def get_instance_owner_map(provider):
    admin_driver = get_admin_driver(provider)
    meta = admin_driver.meta(admin_driver=admin_driver)
    logger.info("Retrieving all tenants..")
    all_tenants = admin_driver._connection._keystone_list_tenants()
    logger.info("Retrieved %s tenants. Retrieving all instances.."
                % len(all_tenants))
    all_instances = meta.all_instances()
    logger.info("Retrieved %s instances." % len(all_instances))
    #Convert tenant-id to tenant-name all at once
    all_instances = _convert_tenant_id_to_names(all_instances, all_tenants)
    logger.info("Owner information added.")
    #Make a mapping of owner-to-instance
    instance_map = _make_instance_owner_map(all_instances)
    logger.info("Instance owner map created")
    return instance_map

def monitor_instances_for(provider):
    """
    Update instances for provider.
    """
    #For now, lets just ignore everything that isn't openstack.
    if 'openstack' not in provider.type.name.lower():
        return
    instance_map = get_instance_owner_map(provider)
    for username in instance_map.keys():
        try:
            user = AtmosphereUser.objects.get(username=username)
            group = Group.objects.get(name=user.username)
            id = user.identity_set.get(provider=provider)
            im = id.identitymembership_set.get(member=group)
            if not im.allocation:
                continue
            instances = instance_map[username]
            over_allocation = over_allocation_test(im.identity, instances)
            if over_allocation:
                continue
            core_instances = im.identity.instance_set.filter(end_date=None)
            update_instances(im.identity, instances, core_instances)
        except:
            logger.exception("Unable to monitor User:%s" % username)
            raise
    logger.info("Monitoring completed")

def _make_instance_owner_map(instances):
    owner_map = {}
    for i in instances:
        key = i.owner
        instance_list = owner_map.get(key, [])
        instance_list.append(i)
        owner_map[key] = instance_list
    return owner_map

def _convert_tenant_id_to_names(instances, tenants):
    for i in instances:
        for tenant in tenants:
            if tenant['id'] == i.owner:
                i.owner = tenant['name']
    return instances

def over_allocation_test(identity, esh_instances):
    from api import get_esh_driver
    from core.models.instance import convert_esh_instance
    from atmosphere import settings
    over_allocated, time_diff = check_over_allocation(
        identity.created_by.username, identity.id)
    if not over_allocated:
        # Nothing changed, bail.
        return False
    if settings.DEBUG:
        logger.info('Do not enforce allocations in DEBUG mode')
        return False
    driver = get_esh_driver(identity)
    for instance in esh_instances:
        #Suspend active instances, update the task in the DB
        try:
            if driver._is_active_instance(instance):
                driver.suspend_instance(instance)
        except Exception, e:
            if 'in vm_state suspended' not in e.message:
                raise
        updated_esh = driver.get_instance(instance.id)
        updated_core = convert_esh_instance(driver, updated_esh,
                                            identity.provider.id,
                                            identity.id,
                                            identity.created_by)
        updated_core.update_history(updated_esh.extra['status'],
                                    updated_esh.extra.get('task'))
    #All instances are dealt with, move along.
    return True # User was over_allocation

def update_instances(identity, esh_list, core_list):
    """
    End-date core instances that don't show up in esh_list
    && Update the values of instances that do
    """
    esh_ids = [instance.id for instance in esh_list]
    #logger.info('%s Instances for Identity %s: %s' % (len(esh_ids), identity, esh_ids))
    for core_instance in core_list:
        try:
            index = esh_ids.index(core_instance.provider_alias)
        except ValueError:
            logger.info("Did not find instance %s in ID List: %s" %
                        (core_instance.provider_alias, esh_ids))
            core_instance.end_date_all()
            continue
        esh_instance = esh_list[index]
        core_instance.update_history(
            esh_instance.extra['status'],
            esh_instance.extra.get('task') or
            esh_instance.extra.get('metadata', {}).get('tmp_status'))
    return
<|endoftext|>


<commit_before>__author__ = 'jpi'

import json

from django.views.generic import View
from django.http import HttpResponse
from SPARQLWrapper import SPARQLWrapper, JSON
from rest_framework.generics import GenericAPIView, ListCreateAPIView, RetrieveUpdateDestroyAPIView, ListAPIView

from stadtgedaechtnis_backend.utils import get_nearby_locations
from stadtgedaechtnis_backend.services.views import GZIPAPIView
from stadtgedaechtnis_backend.services.serializer.serializers import *
from stadtgedaechtnis_backend.services.serializer.generics import MultipleRequestSerializerAPIView
from stadtgedaechtnis_backend.services.authentication.permissions import IsAuthenticatedOrReadOnlyOrModerated

RETURN_TYPE_JSON = "application/json"

class GetNearbyPlacesDBPedia(View):
    """
    Returns a list of places to a given location.
    Parameters: lat - Latitude, lon - Longitude
    A query to DBpedia is made and the results are given in JSON.
    """

    def get(self, request, *args, **kwargs):
        lat, lon = float(kwargs["lat"]), float(kwargs["lon"])
        min_lat, max_lat = lat - 0.01, lat + 0.01
        min_lon, max_lon = lon - 0.01, lon + 0.01

        # build query
        sparql = SPARQLWrapper("http://dbpedia.org/sparql")
        sparql.setQuery("select distinct ?link, ?name, ?latitude, ?longitude where "
                        "{?link geo:lat ?latitude . ?link geo:long ?longitude . ?link foaf:name ?name "
                        "filter (xsd:decimal(?latitude) >= " + str(min_lat) + ") "
                        "filter (xsd:decimal(?latitude) <= " + str(max_lat) + ") "
                        "filter (xsd:decimal(?longitude) >= " + str(min_lon) + ") "
                        "filter (xsd:decimal(?longitude) <= " + str(max_lon) + ")}")
        sparql.setReturnFormat(JSON)
        # query DBpedia
        places = sparql.query().convert()

        result = {"entries": []}

        # iterate over results
        for place in places["results"]["bindings"]:
            result["entries"].append({
                "name": place["name"]["value"],
                "url": place["link"]["value"],
                "lat": place["latitude"]["value"],
                "lon": place["longitude"]["value"]
            })

        return HttpResponse(json.dumps(result),
                            content_type="application/json")

class LocationView(GZIPAPIView, GenericAPIView):
    """
    Base class for use with Locations
    """
    queryset = Location.objects.all()
    serializer_class = LocationSerializer
    permission_classes = (IsAuthenticatedOrReadOnlyOrModerated, )

class LocationListCreate(LocationView, ListCreateAPIView, MultipleRequestSerializerAPIView):
    """
    List all locations or create a new one
    """
    serializer_classes = {
        "GET": LocationSerializer,
        "POST": LocationSerializerWithUniqueID,
    }

class LocationList(LocationView, ListAPIView):
    """
    Simply lists all locations
    """

class LocationListWithStoryIDs(LocationList):
    """
    Gets all the location and the story IDs
    """
    serializer_class = LocationSerializerWithStoryIDs

class LocationListWithStoryTitle(LocationList):
    """
    Retrieves a list of locations. Also includes a list of attached story IDs.
    """
    serializer_class = LocationSerializerWithStoryTitle

class LocationListWithStories(LocationList):
    """
    Retrieves a list of locations. Also includes all the story information attached to these locations.
    """
    serializer_class = LocationSerializerWithStories

class SingleLocation(LocationView, RetrieveUpdateDestroyAPIView):
    """
    Gets a single Location.
    """

class SingleLocationWithStoryIDs(SingleLocation, LocationListWithStoryIDs):
    """
    Retrieves a single location by its ID.
    Also includes a list of attached story IDs.
    """

class SingleLocationWithStoryTitle(SingleLocation, LocationListWithStoryTitle):
    """
    Retrieves a single location by its ID.
    Also includes a list of attached story title.
    """

class SingleLocationWithStories(SingleLocation, LocationListWithStories):
    """
    Retrieves a single location by its ID.
    Also includes a list of attached stories.
    """

class LocationListNearby(LocationList):
    """
    Retrieves a list of locations to given lat and lon coordinates.
    """
    def get_queryset(self):
        return get_nearby_locations(self.kwargs["lat"], self.kwargs["lon"],
                                    self.kwargs["maxlat"] if "maxlat" in self.kwargs else 0,
                                    self.kwargs["maxlon"] if "maxlon" in self.kwargs else 0, True)

class LocationListNearbyWithStoryIDs(LocationListNearby, LocationListWithStoryIDs):
    """
    Retrieves a list of locations to given lat and lon coordinates. Also includes a list of attached story IDs.
    """

class LocationListNearbyWithImages(LocationListNearby):
    """
    Retrieves a list of locations. Also includes all the story information attached to these locations.
    """
    serializer_class = LocationSerializerWithStoryImages

class SingleLocationWithStoriesImage(SingleLocation, LocationListNearbyWithImages):
    """
    Retrieves a single location by ids ID.
    also includes al ist of attached stories and images.
    """

class LocationListNearbyWithStoryTitle(LocationListWithStoryTitle, LocationListNearby):
    """
    Retrieves a list of locations to given lat and lon coordinates. Also includes a list of attached story titles.
    """

class LocationListNearbyWithStories(LocationListWithStories, LocationListNearby):
    """
    Retrieves a list of locations to given lat and lon coordinates.
    Also includes all the story information attached to these locations.
    """
<commit_msg>Fix error for single location with image and story<commit_after>__author__ = 'jpi'

import json

from django.views.generic import View
from django.http import HttpResponse
from SPARQLWrapper import SPARQLWrapper, JSON
from rest_framework.generics import GenericAPIView, ListCreateAPIView, RetrieveUpdateDestroyAPIView, ListAPIView

from stadtgedaechtnis_backend.utils import get_nearby_locations
from stadtgedaechtnis_backend.services.views import GZIPAPIView
from stadtgedaechtnis_backend.services.serializer.serializers import *
from stadtgedaechtnis_backend.services.serializer.generics import MultipleRequestSerializerAPIView
from stadtgedaechtnis_backend.services.authentication.permissions import IsAuthenticatedOrReadOnlyOrModerated

RETURN_TYPE_JSON = "application/json"

class GetNearbyPlacesDBPedia(View):
    """
    Returns a list of places to a given location.
    Parameters: lat - Latitude, lon - Longitude
    A query to DBpedia is made and the results are given in JSON.
    """

    def get(self, request, *args, **kwargs):
        lat, lon = float(kwargs["lat"]), float(kwargs["lon"])
        min_lat, max_lat = lat - 0.01, lat + 0.01
        min_lon, max_lon = lon - 0.01, lon + 0.01

        # build query
        sparql = SPARQLWrapper("http://dbpedia.org/sparql")
        sparql.setQuery("select distinct ?link, ?name, ?latitude, ?longitude where "
                        "{?link geo:lat ?latitude . ?link geo:long ?longitude . ?link foaf:name ?name "
                        "filter (xsd:decimal(?latitude) >= " + str(min_lat) + ") "
                        "filter (xsd:decimal(?latitude) <= " + str(max_lat) + ") "
                        "filter (xsd:decimal(?longitude) >= " + str(min_lon) + ") "
                        "filter (xsd:decimal(?longitude) <= " + str(max_lon) + ")}")
        sparql.setReturnFormat(JSON)
        # query DBpedia
        places = sparql.query().convert()

        result = {"entries": []}

        # iterate over results
        for place in places["results"]["bindings"]:
            result["entries"].append({
                "name": place["name"]["value"],
                "url": place["link"]["value"],
                "lat": place["latitude"]["value"],
                "lon": place["longitude"]["value"]
            })

        return HttpResponse(json.dumps(result),
                            content_type="application/json")

class LocationView(GZIPAPIView, GenericAPIView):
    """
    Base class for use with Locations
    """
    queryset = Location.objects.all()
    serializer_class = LocationSerializer
    permission_classes = (IsAuthenticatedOrReadOnlyOrModerated, )

class LocationListCreate(LocationView, ListCreateAPIView, MultipleRequestSerializerAPIView):
    """
    List all locations or create a new one
    """
    serializer_classes = {
        "GET": LocationSerializer,
        "POST": LocationSerializerWithUniqueID,
    }

class LocationList(LocationView, ListAPIView):
    """
    Simply lists all locations
    """

class LocationListWithStoryIDs(LocationList):
    """
    Gets all the location and the story IDs
    """
    serializer_class = LocationSerializerWithStoryIDs

class LocationListWithStoryTitle(LocationList):
    """
    Retrieves a list of locations. Also includes a list of attached story IDs.
    """
    serializer_class = LocationSerializerWithStoryTitle

class LocationListWithStories(LocationList):
    """
    Retrieves a list of locations. Also includes all the story information attached to these locations.
    """
    serializer_class = LocationSerializerWithStories

class SingleLocation(LocationView, RetrieveUpdateDestroyAPIView):
    """
    Gets a single Location.
    """

class SingleLocationWithStoryIDs(SingleLocation, LocationListWithStoryIDs):
    """
    Retrieves a single location by its ID.
    Also includes a list of attached story IDs.
    """

class SingleLocationWithStoryTitle(SingleLocation, LocationListWithStoryTitle):
    """
    Retrieves a single location by its ID.
    Also includes a list of attached story title.
    """

class SingleLocationWithStories(SingleLocation, LocationListWithStories):
    """
    Retrieves a single location by its ID.
    Also includes a list of attached stories.
    """

class LocationListNearby(LocationList):
    """
    Retrieves a list of locations to given lat and lon coordinates.
    """
    def get_queryset(self):
        return get_nearby_locations(self.kwargs["lat"], self.kwargs["lon"],
                                    self.kwargs["maxlat"] if "maxlat" in self.kwargs else 0,
                                    self.kwargs["maxlon"] if "maxlon" in self.kwargs else 0, True)

class LocationListNearbyWithStoryIDs(LocationListNearby, LocationListWithStoryIDs):
    """
    Retrieves a list of locations to given lat and lon coordinates. Also includes a list of attached story IDs.
    """

class LocationListNearbyWithImages(LocationListNearby):
    """
    Retrieves a list of locations. Also includes all the story information attached to these locations.
    """
    serializer_class = LocationSerializerWithStoryImages

class SingleLocationWithStoriesImage(SingleLocation):
    """
    Retrieves a single location by its ID.
    also includes al ist of attached stories and images.
    """
    serializer_class = LocationSerializerWithStoryImages

class LocationListNearbyWithStoryTitle(LocationListWithStoryTitle, LocationListNearby):
    """
    Retrieves a list of locations to given lat and lon coordinates. Also includes a list of attached story titles.
    """

class LocationListNearbyWithStories(LocationListWithStories, LocationListNearby):
    """
    Retrieves a list of locations to given lat and lon coordinates.
    Also includes all the story information attached to these locations.
    """
<|endoftext|>


<commit_before># coding=utf-8

# Transcript Downloader
# Downloads from a specified list of URLs
# Then saves them all to disk

from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
import json
import os

class MySpider(BaseSpider):
    root_directory = os.path.dirname(os.path.abspath(os.curdir))

    # Name the spider as required by scrapy
    name = "download"

    # Set a download delay of 5 seconds.
    # This makes the downloads take longer, but it's kinder on the UCSB servers.
    download_delay = 5

    # Import the list of URLs to download from - this comes from our earlier file.
    if 'urls.json' in os.listdir(os.curdir):
        with file('urls.json', 'r') as f:
            start_urls = json.load(f)

    # Once again, the allowed domain is set.
    allowed_domains = ['ucsb.edu']

    # parse method, run every time the spider is run
    def parse(self, response):
        hxs = HtmlXPathSelector(response)

        # The only purpose of the following code is to give the files sensible names.

        # Extract the date of the debate
        date = hxs.select("//span[@class='docdate']/text()").extract()
        for date in date:
            debateDate = date
            break

        # Extract the name description of the debate
        name = hxs.select("//span[@class='paperstitle']/text()").extract()
        for name in name:
            debateName = name
            break

        fileName = debateDate + " " + debateName + ".html"

        os.mkdir(os.path.join(self.root_directory, 'html-files'))
        # The files are then saved to a directory, for access later.
        with open(os.path.join(self.root_directory, 'html-files', fileName), 'w') as f:
            f.write(response.body)
<commit_msg>compatibility for others to clone github repo<commit_after># coding=utf-8

# Transcript Downloader
# Downloads from a specified list of URLs
# Then saves them all to disk

from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
import json
import os

class MySpider(BaseSpider):
    root_directory = os.path.dirname(os.path.abspath(os.curdir))

    # Name the spider as required by scrapy
    name = "download"

    # Set a download delay of 5 seconds.
    # This makes the downloads take longer, but it's kinder on the UCSB servers.
    download_delay = 5

    # Import the list of URLs to download from - this comes from our earlier file.
    if 'urls.json' in os.listdir(os.curdir):
        with file('urls.json', 'r') as f:
            start_urls = json.load(f)

    # Once again, the allowed domain is set.
    allowed_domains = ['ucsb.edu']

    # parse method, run every time the spider is run
    def parse(self, response):
        hxs = HtmlXPathSelector(response)

        # The only purpose of the following code is to give the files sensible names.

        # Extract the date of the debate
        date = hxs.select("//span[@class='docdate']/text()").extract()
        for date in date:
            debateDate = date
            break

        # Extract the name description of the debate
        name = hxs.select("//span[@class='paperstitle']/text()").extract()
        for name in name:
            debateName = name
            break

        fileName = debateDate + " " + debateName + ".html"

        if 'html-files' not in os.listdir(self.root_directory):
            os.mkdir(os.path.join(self.root_directory, 'html-files'))
        # The files are then saved to a directory, for access later.
        with open(os.path.join(self.root_directory, 'html-files', fileName), 'w') as f:
            f.write(response.body)
<|endoftext|>


<commit_before>import numpy as np
import torch

from syft.frameworks.torch.overload_torch import overloaded
from syft.frameworks.torch.tensors.interpreters import AbstractTensor

class LargePrecisionTensor(AbstractTensor):
    """LargePrecisionTensor allows handling of numbers bigger than LongTensor

    Some systems using Syft require larger types than those supported natively. This tensor type supports arbitrarily
    large values by packing them in smaller ones.
    Typically a user will require to enlarge a float number by fixing its precision
        tensor.fix_prec()

    The large value is defined by `precision_fractional`.

    The smaller values are of type `internal_type`. The split of the large number into the smaller values
    is in the range ±2**(size - 1).

    By default operations are done with NumPy. This implies unpacking the representation and packing it again.

    Check the tests to see how to play with the different parameters.
    """

    def __init__(
        self,
        owner=None,
        id=None,
        tags=None,
        description=None,
        base: int = 10,
        precision_fractional=0,
        internal_type=torch.int32,
        verbose=False,
    ):
        """Initializes a LargePrecisionTensor.

        Args:
            owner (BaseWorker): An optional BaseWorker object to specify the worker on which
                the tensor is located.
            id (str or int): An optional string or integer id of the LargePrecisionTensor.
            tags (list): list of tags for searching.
            description (str): a description of this tensor.
            base (int): The base that will be used to to calculate the precision.
            precision_fractional (int): The precision required by the caller.
            internal_type (dtype): The large tensor will be stored using tensor of this type.
        """
        super().__init__(id=id, owner=owner, tags=tags, description=description)
        self.base = base
        self.internal_type = internal_type
        self.precision_fractional = precision_fractional
        self.verbose = verbose

    def _create_internal_representation(self):
        """Decompose a tensor into an array of numbers that represent such tensor with the required precision"""
        result = []
        self_scaled = self.child.numpy() * self.base ** self.precision_fractional
        result = LargePrecisionTensor._split_number(
            self_scaled, self.internal_precision, self.internal_type
        )
        return torch.tensor(result, dtype=self.internal_type)

    @staticmethod
    def _expand_item(a_number, max_length):
        return [0] * (max_length - len(a_number)) + a_number

    @property
    def internal_precision(self):
        """"The internal precision used to decompose the large numbers is the size of the type - 1.
        The large number is decomposed into positive smaller numbers. This could provoke overflow if any of these
        smaller parts are bigger than type_precision/2.
        """
        return type_precision[self.internal_type] - 1

    def get_class_attributes(self):
        """
        Specify all the attributes need to build a wrapper correctly when returning a response.
        """
        return {
            "base": self.base,
            "internal_type": self.internal_type,
            "precision_fractional": self.precision_fractional,
        }

    @overloaded.method
    def add(self, self_, other):
        return self_ + other

    __add__ = add

    def __iadd__(self, other):
        """Add two fixed precision tensors together.
        """
        self.child = self.add(other).child

        return self

    add_ = __iadd__

    @overloaded.method
    def sub(self, self_, other):
        return self_ - other

    __sub__ = sub

    def __isub__(self, other):
        """Add two fixed precision tensors together.
        """
        self.child = self.sub(other).child

        return self

    sub_ = __isub__

    @overloaded.method
    def mul(self, self_, other):
        # We need to divide the result of the multiplication by the precision
        return (self_ * other) / (self.base ** self.precision_fractional)

    __mul__ = mul

    def __imul__(self, other):
        self.child = self.mul(other).child
        return self

    mul_ = __imul__

    @overloaded.method
    def mod(self, self_, other):
        return self_ % other

    __mod__ = mod

    def fix_large_precision(self):
        self.child = self._create_internal_representation()
        return self

    def float_precision(self):
        """
        Restore the tensor from the internal representation.

        Returns:
            tensor: the original tensor.
        """
        result = self._internal_representation_to_large_ints()
        result /= self.base ** self.precision_fractional
        # At this point the value is an object type. Force cast to float before creating torch.tensor
        return torch.from_numpy(result.reshape(self.child.shape[:-1]).astype(np.float32))

    @staticmethod
    def create_tensor_from_numpy(ndarray, **kwargs):
        """Decompose a NumPy array into an array of numbers that represent such tensor with the required precision.

        Typically this private method is called on the result of an operation.
        """
        internal_type = kwargs["internal_type"]
        internal_precision = type_precision[internal_type] - 1

        result = LargePrecisionTensor._split_number(ndarray, internal_precision, internal_type)
        return torch.tensor(result, dtype=internal_type)

    @staticmethod
    def _split_number(number, bits, internal_type) -> np.array:
        """Splits a number in numbers of a smaller power.

        Args:
            number (int): the number to split.
            bits (int): the bits to use in the split.

        Returns:
            list: a list of numbers representing the original one.

        """
        sign_mask = np.where(number > 0, 1, -1)
        if internal_type == torch.uint8:
            assert np.all(sign_mask == 1)
        number = np.where(number > 0, number, -number)

        base = 2 ** bits
        number_parts = []
        while np.any(number):
            # number, part = np.divmod(number, base)  # Not sure why this doesn't work
            part = number % base
            number = number // base
            number_parts.append(part * sign_mask)
        res = np.array(number_parts[::-1], dtype=np.int)
        return res.transpose(*range(1, res.ndim), 0)

    def _internal_representation_to_large_ints(self) -> np.array:
        """Creates an numpy array containing the objective large numbers."""
        ndarray = self.child.numpy()
        ndarray = ndarray.reshape(-1, ndarray.shape[-1])
        result = []
        for elem in ndarray:
            result.append(LargePrecisionTensor._restore_large_number(elem, self.internal_precision))
        return np.array(result).reshape(self.child.shape[:-1])

    @staticmethod
    def _restore_large_number(number_parts, bits) -> int:
        """Rebuilds a number from a numpy array.

        Args:
            number_parts (ndarray): the numpy array of numbers representing the original one.
            bits (int): the bits used in the split.

        Returns:
            Number: the large number represented by this tensor
        """

        def _restore_recursive(parts, acc, base):
            if len(parts) == 0:
                return acc
            return _restore_recursive(parts[1:], acc * base + parts[0].item(), base)

        return _restore_recursive(number_parts, 0, 2 ** bits)

# The size of each type
type_precision = {
    torch.uint8: 8,
    torch.int8: 8,
    torch.int16: 16,
    torch.short: 16,
    torch.int32: 32,
    torch.int: 32,
    torch.int64: 64,
    torch.long: 64,
}
<commit_msg>some cleaning<commit_after>import numpy as np
import torch

from syft.frameworks.torch.overload_torch import overloaded
from syft.frameworks.torch.tensors.interpreters import AbstractTensor

class LargePrecisionTensor(AbstractTensor):
    """LargePrecisionTensor allows handling of numbers bigger than LongTensor

    Some systems using Syft require larger types than those supported natively. This tensor type supports arbitrarily
    large values by packing them in smaller ones.
    Typically a user will require to enlarge a float number by fixing its precision
        tensor.fix_prec()

    The large value is defined by `precision_fractional`.

    The smaller values are of type `internal_type`. The split of the large number into the smaller values
    is in the range ±2**(size - 1).

    By default operations are done with NumPy. This implies unpacking the representation and packing it again.

    Check the tests to see how to play with the different parameters.
    """

    def __init__(
        self,
        owner=None,
        id=None,
        tags=None,
        description=None,
        base: int = 10,
        precision_fractional=0,
        internal_type=torch.int32,
        verbose=False,
    ):
        """Initializes a LargePrecisionTensor.

        Args:
            owner (BaseWorker): An optional BaseWorker object to specify the worker on which
                the tensor is located.
            id (str or int): An optional string or integer id of the LargePrecisionTensor.
            tags (list): list of tags for searching.
            description (str): a description of this tensor.
            base (int): The base that will be used to to calculate the precision.
            precision_fractional (int): The precision required by the caller.
            internal_type (dtype): The large tensor will be stored using tensor of this type.
        """
        super().__init__(id=id, owner=owner, tags=tags, description=description)
        self.base = base
        self.internal_type = internal_type
        self.precision_fractional = precision_fractional
        self.verbose = verbose

    def _create_internal_representation(self):
        """Decompose a tensor into an array of numbers that represent such tensor with the required precision"""
        self_scaled = self.child.numpy() * self.base ** self.precision_fractional
        result = LargePrecisionTensor._split_number(
            self_scaled, self.internal_precision, self.internal_type
        )
        return torch.tensor(result, dtype=self.internal_type)

    @staticmethod
    def _expand_item(a_number, max_length):
        return [0] * (max_length - len(a_number)) + a_number

    @property
    def internal_precision(self):
        """"The internal precision used to decompose the large numbers is the size of the type - 1.
        The large number is decomposed into positive smaller numbers. This could provoke overflow if any of these
        smaller parts are bigger than type_precision/2.
        """
        return type_precision[self.internal_type] - 1

    def get_class_attributes(self):
        """
        Specify all the attributes need to build a wrapper correctly when returning a response.
        """
        return {
            "base": self.base,
            "internal_type": self.internal_type,
            "precision_fractional": self.precision_fractional,
        }

    @overloaded.method
    def add(self, self_, other):
        return self_ + other

    __add__ = add

    def __iadd__(self, other):
        """Add two fixed precision tensors together.
        """
        self.child = self.add(other).child

        return self

    add_ = __iadd__

    @overloaded.method
    def sub(self, self_, other):
        return self_ - other

    __sub__ = sub

    def __isub__(self, other):
        """Add two fixed precision tensors together.
        """
        self.child = self.sub(other).child

        return self

    sub_ = __isub__

    @overloaded.method
    def mul(self, self_, other):
        # We need to divide the result of the multiplication by the precision
        return (self_ * other) / (self.base ** self.precision_fractional)

    __mul__ = mul

    def __imul__(self, other):
        self.child = self.mul(other).child
        return self

    mul_ = __imul__

    @overloaded.method
    def mod(self, self_, other):
        return self_ % other

    __mod__ = mod

    def fix_large_precision(self):
        self.child = self._create_internal_representation()
        return self

    def float_precision(self):
        """
        Restore the tensor from the internal representation.

        Returns:
            tensor: the original tensor.
        """
        result = self._internal_representation_to_large_ints()
        result /= self.base ** self.precision_fractional
        # At this point the value is an object type. Force cast to float before creating torch.tensor
        return torch.from_numpy(result.reshape(self.child.shape[:-1]).astype(np.float32))

    @staticmethod
    def create_tensor_from_numpy(ndarray, **kwargs):
        """Decompose a NumPy array into an array of numbers that represent such tensor with the required precision.

        Typically this private method is called on the result of an operation.
        """
        internal_type = kwargs["internal_type"]
        internal_precision = type_precision[internal_type] - 1

        result = LargePrecisionTensor._split_number(ndarray, internal_precision, internal_type)
        return torch.tensor(result, dtype=internal_type)

    @staticmethod
    def _split_number(number, bits, internal_type) -> np.array:
        """Splits a number in numbers of a smaller power.

        Args:
            number (int): the number to split.
            bits (int): the bits to use in the split.

        Returns:
            list: a list of numbers representing the original one.

        """
        sign_mask = np.where(number > 0, 1, -1)
        if internal_type == torch.uint8:
            assert np.all(
                sign_mask == 1
            ), "LargePrecisionTensors with negative values cannot be represented with uint8"
        number = np.where(number > 0, number, -number)

        base = 2 ** bits
        number_parts = []
        while np.any(number):
            # number, part = np.divmod(number, base)  # Not sure why this doesn't work
            part = number % base
            number = number // base
            number_parts.append(part * sign_mask)
        res = np.array(number_parts[::-1], dtype=np.int)
        return res.transpose(*range(1, res.ndim), 0)

    def _internal_representation_to_large_ints(self) -> np.array:
        """Creates an numpy array containing the objective large numbers."""
        ndarray = self.child.numpy()
        ndarray = ndarray.reshape(-1, ndarray.shape[-1])
        result = []
        for elem in ndarray:
            result.append(LargePrecisionTensor._restore_large_number(elem, self.internal_precision))
        return np.array(result).reshape(self.child.shape[:-1])

    @staticmethod
    def _restore_large_number(number_parts, bits) -> int:
        """Rebuilds a number from a numpy array.

        Args:
            number_parts (ndarray): the numpy array of numbers representing the original one.
            bits (int): the bits used in the split.

        Returns:
            Number: the large number represented by this tensor
        """

        def _restore_recursive(parts, acc, base):
            if len(parts) == 0:
                return acc
            return _restore_recursive(parts[1:], acc * base + parts[0].item(), base)

        return _restore_recursive(number_parts, 0, 2 ** bits)

# The size of each type
type_precision = {
    torch.uint8: 8,
    torch.int8: 8,
    torch.int16: 16,
    torch.short: 16,
    torch.int32: 32,
    torch.int: 32,
    torch.int64: 64,
    torch.long: 64,
}
<|endoftext|>


<commit_before># -*- test-case-name: admin.test.test_packaging -*-
# Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
Helper utilities for Flocker packaging.
"""

import sys
from subprocess import check_call, check_output
from tempfile import mkdtemp

from twisted.python.filepath import FilePath
from twisted.python import usage

from characteristic import attributes

from .release import make_rpm_version

@attributes(['steps'])
class BuildSequence(object):
    """
    Run the supplied ``steps`` consecutively.

    :ivar tuple steps: A sequence of steps.
    """
    def run(self):
        for step in self.steps:
            step.run()

@attributes(['target_path'])
class InstallVirtualEnv(object):
    """
    Install a virtualenv in the supplied ``target_path``.

    We call ``virtualenv`` as a subprocess rather than as a library, so that we
    can turn off Python byte code compilation.

    :ivar FilePath target_path: The path to a directory in which to create the
        virtualenv.
    """
    def run(self):
        check_call(
            ['virtualenv', '--quiet', '--system-site-packages',
             self.target_path.path],
            env=dict(PYTHONDONTWRITEBYTECODE='1')
        )

@attributes(['virtualenv_path', 'package_uri'])
class InstallApplication(object):
    """
    Install the supplied ``package_uri`` using ``pip`` from the supplied
    ``virtualenv_path``.

    After installing the package and its dependencies, the virtualenv is made
    ``relocatable`` to remove and absolute paths and shebang lines in scripts.

    :ivar FilePath virtualenv_path: The path to an existing ``virtualenv``.
    :ivar bytes package_uri: A ``pip install`` compatible package URI.
    """
    def run(self):
        python_path = self.virtualenv_path.child('bin').child('python').path
        pip_path = self.virtualenv_path.child('bin').child('pip').path
        check_call(
            [python_path, pip_path, '--quiet', 'install', self.package_uri]
        )
        check_call(
            ['virtualenv', '--quiet', '--relocatable',
             self.virtualenv_path.path],
            env=dict(PYTHONDONTWRITEBYTECODE='1')
        )

@attributes(['virtualenv_path', 'package_name'])
class GetPackageVersion(object):
    """
    Record the version of ``package_name`` installed in ``virtualenv_path`` by
    parsing the output of ``pip show``.

    :ivar FilePath virtualenv_path: The path of the ``virtualenv`` containing
        the package.
    :ivar bytes package_name: The name of the package whose version will be
        recorded.
    :ivar version: The version string of the supplied package. Default is
        ``None`` until the step has been run or if the supplied
        ``package_name`` is not found.
    """
    version = None

    def run(self):
        python_path = self.virtualenv_path.child('bin').child('python').path
        pip_path = self.virtualenv_path.child('bin').child('pip').path
        # Can't just call pip directly, because in tests its shebang line
        # becomes too long.
        output = check_output(
            [python_path, pip_path, 'show', self.package_name])

        for line in output.splitlines():
            parts = [part.strip() for part in line.split(':', 1)]
            if len(parts) == 2:
                key, value = parts
                if key.lower() == 'version':
                    self.version = value
                    return

@attributes(
    ['destination_path', 'source_path', 'name', 'prefix', 'epoch',
     'rpm_version', 'license', 'url', 'vendor', 'maintainer', 'architecture',
     'description'])
class BuildRpm(object):
    """
    Use ``fpm`` to build an RPM file from the supplied ``source_path``.

    :ivar FilePath destination_path: The path in which to save the resulting
        RPM package file.
    :ivar FilePath source_path: The path to a directory whose contents will be
        packaged.
    :ivar bytes name: The name of the package.
    :ivar FilePath prefix: The path beneath which the packaged files will be
         installed.
    :ivar bytes epoch: An integer string tag used to help RPM determine version
        number ordering.
    :ivar rpm_version rpm_version: An object representing an RPM style version
        containing a release and a version attribute.
    :ivar bytes license: The name of the license under which this package is
        released.
    :ivar bytes url: The URL of the source of this package.
    :ivar unicode vendor: The name of the package vendor.
    :ivar bytes maintainer: The email address of the package maintainer.
    :ivar bytes architecture: The OS architecture for which this package is
        targeted. Default ``None`` means architecture independent.
    :ivar unicode description: A description of the package.
    """
    def run(self):
        architecture = self.architecture
        if architecture is None:
            architecture = 'all'

        check_call([
            'fpm',
            '-s', 'dir',
            '-t', 'rpm',
            '--package', self.destination_path.path,
            '--name', self.name,
            '--prefix', self.prefix.path,
            '--version', self.rpm_version.version,
            '--epoch', self.epoch,
            '--iteration', self.rpm_version.release,
            '--license', self.license,
            '--url', self.url,
            '--vendor', self.vendor,
            '--maintainer', self.maintainer,
            '--architecture', architecture,
            '--description', self.description,
            '--exclude', '*.pyc',
            '.'], cwd=self.source_path.path
        )

@attributes(['package_version_step'])
class DelayedRpmVersion(object):
    _rpm_version = None

    @property
    def rpm_version(self):
        if self._rpm_version is None:
            self._rpm_version = make_rpm_version(
                self.package_version_step.version
            )
        return self._rpm_version

    @property
    def version(self):
        return self.rpm_version.version

    @property
    def release(self):
        return self.rpm_version.release

def sumo_rpm_builder(destination_path, package_uri, target_dir=None):
    """
    Build an RPM file containing the supplied `package` and all its
    dependencies.

    Motivation:
    * We depend on libraries which are not packaged for the target OS.
    * We depend on newer versions of libraries which have not yet been included
      in the target OS.

    Disadvantages:
    * We won't be able to take advantage of library security updates shipped by
      the target OS.
      * But by shipping our own separate dependency packages we will need to be
        responsible for shipping security patches in those packages.
      * And rather than being responsible only for the security of Flocker, we
        become responsible for the security of all other packages that depend
        on that package.
    * Packages will be larger.

    Plan:
    * Create a temporary working dir.
    * Create virtualenv with `--system-site-packages`
      * Allows certain python libraries to be supplied by the operating system.
    * Install flocker from wheel file (which will include all the
      dependencies).
      * We'll need to keep track of which of our dependencies are provided on
        each platform and somehow omit those for from the build for that
        platform.
    * Generate an RPM version number.
    * Run `fpm` supplying the virtualenv path and version number.

    Followup Issues:
    * Update all pinned dependencies to instead be minimum dependencies.
      * This means that as and when sufficiently new versions of our
        dependencies are introduced upstream, we can remove them from our sumo
        build.
      * Those dependencies which are either too old or which are not packaged
        will be imported from the sumo virtualenv in preference.
      * Eventually we hope that all our dependencies will filter upstream and
        we will no longer have to bundle them; at which point the `flocker`
        package itself may be ready to be packaged by upstream distributions.

    Ticket refs:
         * https://github.com/ClusterHQ/flocker/issues/88

    Issue: CI integration (??):
    Update buildbot to build RPMs using new build scripts
    * Issue: create deb, mac, gentoo build slave
    * Issue: install from resulting package from repo and run test suite

    Issue: Client package build (??):
    Sumo packaging of flocker-deploy
    * For deb, RPM, and mac (via homebrew or ...)
    * Proper mac packages. See
      http://stackoverflow.com/questions/11487596/making-os-x-installer-packages-like-a-pro-xcode4-developer-id-mountain-lion-re

    Client package CI integration

    Misc:
    * separate stable and testing repos for deb and rpm
    * update python-flocker.spec.in requirements (remove most of them)
    * maybe even remove the spec file template and generate_spec function
      entirely (do we need it?)
    * do we still need to build an SRPM?
    * automatically build a wheel
    * automatically build an sdist
    """
    if target_dir is None:
        target_dir = FilePath(mkdtemp())

    get_package_version_step = GetPackageVersion(
        virtualenv_path=target_dir, package_name='Flocker')

    return BuildSequence(
        steps=(
            InstallVirtualEnv(target_path=target_dir),
            InstallApplication(virtualenv_path=target_dir,
                               package_uri=package_uri),
            get_package_version_step,
            BuildRpm(
                destination_path=destination_path,
                source_path=target_dir,
                name='Flocker',
                prefix=FilePath('/opt/flocker'),
                epoch=b'0',
                rpm_version=DelayedRpmVersion(
                    package_version_step=get_package_version_step),
                license='ASL 2.0',
                url='https://clusterhq.com',
                vendor='ClusterHQ',
                maintainer='noreply@build.clusterhq.com',
                architecture=None,
                description='A Docker orchestration and volume management tool',
            )
        )
    )

from textwrap import dedent
class BuildOptions(usage.Options):
    """
    Command line options for the ``build-package`` tool.
    """
    synopsis = 'build-rpm [options] <package-uri>'

    optParameters = [
        ['destination-path', 'd', '.',
         'The path to a directory in which to create package files and '
         'artifacts.'],
    ]

    longdesc = dedent("""\
    Arguments:

    <package-uri>: The Python package url or path to install using ``pip``.
    """)

    def parseArgs(self, package_uri):
        """
        The Python package to install.
        """
        self['package-uri'] = package_uri

    def postOptions(self):
        """
        Coerce to ``FilePath`` where appropriate.
        """
        self['destination-path'] = FilePath(self['destination-path'])

class BuildScript(object):
    build_command = staticmethod(sumo_rpm_builder)

    def __init__(self, sys_module=None):
        """
        """
        if sys_module is None:
            sys_module = sys
        self.sys_module = sys_module

    def main(self, top_level=None, base_path=None):
        """
        Build a package.

        :param list argv: The arguments passed to the script.
        """
        options = BuildOptions()

        try:
            options.parseOptions(self.sys_module.argv[1:])
        except usage.UsageError as e:
            self.sys_module.stderr.write("%s\n" % (options,))
            self.sys_module.stderr.write("%s\n" % (e,))
            raise SystemExit(1)

        self.build_command(
            destination_path=options['destination-path'],
            package_uri=options['package-uri']
        ).run()

main = BuildScript().main
<commit_msg>docstrings<commit_after># -*- test-case-name: admin.test.test_packaging -*-
# Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
Helper utilities for Flocker packaging.
"""

import sys
from subprocess import check_call, check_output
from tempfile import mkdtemp

from twisted.python.filepath import FilePath
from twisted.python import usage

from characteristic import attributes

from .release import make_rpm_version

@attributes(['steps'])
class BuildSequence(object):
    """
    Run the supplied ``steps`` consecutively.

    :ivar tuple steps: A sequence of steps.
    """
    def run(self):
        for step in self.steps:
            step.run()

@attributes(['target_path'])
class InstallVirtualEnv(object):
    """
    Install a virtualenv in the supplied ``target_path``.

    We call ``virtualenv`` as a subprocess rather than as a library, so that we
    can turn off Python byte code compilation.

    :ivar FilePath target_path: The path to a directory in which to create the
        virtualenv.
    """
    def run(self):
        check_call(
            ['virtualenv', '--quiet', '--system-site-packages',
             self.target_path.path],
            env=dict(PYTHONDONTWRITEBYTECODE='1')
        )

@attributes(['virtualenv_path', 'package_uri'])
class InstallApplication(object):
    """
    Install the supplied ``package_uri`` using ``pip`` from the supplied
    ``virtualenv_path``.

    After installing the package and its dependencies, the virtualenv is made
    ``relocatable`` to remove and absolute paths and shebang lines in scripts.

    :ivar FilePath virtualenv_path: The path to an existing ``virtualenv``.
    :ivar bytes package_uri: A ``pip install`` compatible package URI.
    """
    def run(self):
        python_path = self.virtualenv_path.child('bin').child('python').path
        pip_path = self.virtualenv_path.child('bin').child('pip').path
        check_call(
            [python_path, pip_path, '--quiet', 'install', self.package_uri]
        )
        check_call(
            ['virtualenv', '--quiet', '--relocatable',
             self.virtualenv_path.path],
            env=dict(PYTHONDONTWRITEBYTECODE='1')
        )

@attributes(['virtualenv_path', 'package_name'])
class GetPackageVersion(object):
    """
    Record the version of ``package_name`` installed in ``virtualenv_path`` by
    parsing the output of ``pip show``.

    :ivar FilePath virtualenv_path: The path of the ``virtualenv`` containing
        the package.
    :ivar bytes package_name: The name of the package whose version will be
        recorded.
    :ivar version: The version string of the supplied package. Default is
        ``None`` until the step has been run or if the supplied
        ``package_name`` is not found.
    """
    version = None

    def run(self):
        python_path = self.virtualenv_path.child('bin').child('python').path
        pip_path = self.virtualenv_path.child('bin').child('pip').path
        # Can't just call pip directly, because in tests its shebang line
        # becomes too long.
        output = check_output(
            [python_path, pip_path, 'show', self.package_name])

        for line in output.splitlines():
            parts = [part.strip() for part in line.split(':', 1)]
            if len(parts) == 2:
                key, value = parts
                if key.lower() == 'version':
                    self.version = value
                    return

@attributes(
    ['destination_path', 'source_path', 'name', 'prefix', 'epoch',
     'rpm_version', 'license', 'url', 'vendor', 'maintainer', 'architecture',
     'description'])
class BuildRpm(object):
    """
    Use ``fpm`` to build an RPM file from the supplied ``source_path``.

    :ivar FilePath destination_path: The path in which to save the resulting
        RPM package file.
    :ivar FilePath source_path: The path to a directory whose contents will be
        packaged.
    :ivar bytes name: The name of the package.
    :ivar FilePath prefix: The path beneath which the packaged files will be
         installed.
    :ivar bytes epoch: An integer string tag used to help RPM determine version
        number ordering.
    :ivar rpm_version rpm_version: An object representing an RPM style version
        containing a release and a version attribute.
    :ivar bytes license: The name of the license under which this package is
        released.
    :ivar bytes url: The URL of the source of this package.
    :ivar unicode vendor: The name of the package vendor.
    :ivar bytes maintainer: The email address of the package maintainer.
    :ivar bytes architecture: The OS architecture for which this package is
        targeted. Default ``None`` means architecture independent.
    :ivar unicode description: A description of the package.
    """
    def run(self):
        architecture = self.architecture
        if architecture is None:
            architecture = 'all'

        check_call([
            'fpm',
            '-s', 'dir',
            '-t', 'rpm',
            '--package', self.destination_path.path,
            '--name', self.name,
            '--prefix', self.prefix.path,
            '--version', self.rpm_version.version,
            '--epoch', self.epoch,
            '--iteration', self.rpm_version.release,
            '--license', self.license,
            '--url', self.url,
            '--vendor', self.vendor,
            '--maintainer', self.maintainer,
            '--architecture', architecture,
            '--description', self.description,
            '--exclude', '*.pyc',
            '.'], cwd=self.source_path.path
        )

@attributes(['package_version_step'])
class DelayedRpmVersion(object):
    """
    Pretend to be an ``rpm_version`` instance providing a ``version`` and
    ``release`` attribute.

    The values of these attributes will be calculated from the Python version
    string read from a previous ``GetPackageVersion`` build step.

    :ivar GetPackageVersion package_version_step: An instance of
        ``GetPackageVersion`` whose ``run`` method has been called and from
        which the version string will be read.
    """
    _rpm_version = None

    @property
    def rpm_version(self):
        """
        :return: An ``rpm_version`` and cache it.
        """
        if self._rpm_version is None:
            self._rpm_version = make_rpm_version(
                self.package_version_step.version
            )
        return self._rpm_version

    @property
    def version(self):
        """
        :return: The ``version`` string.
        """
        return self.rpm_version.version

    @property
    def release(self):
        """
        :return: The ``release`` string.
        """
        return self.rpm_version.release

def sumo_rpm_builder(destination_path, package_uri, target_dir=None):
    """
    Build an RPM file containing the supplied `package` and all its
    dependencies.

    Motivation:
    * We depend on libraries which are not packaged for the target OS.
    * We depend on newer versions of libraries which have not yet been included
      in the target OS.

    Disadvantages:
    * We won't be able to take advantage of library security updates shipped by
      the target OS.
      * But by shipping our own separate dependency packages we will need to be
        responsible for shipping security patches in those packages.
      * And rather than being responsible only for the security of Flocker, we
        become responsible for the security of all other packages that depend
        on that package.
    * Packages will be larger.

    Plan:
    * Create a temporary working dir.
    * Create virtualenv with `--system-site-packages`
      * Allows certain python libraries to be supplied by the operating system.
    * Install flocker from wheel file (which will include all the
      dependencies).
      * We'll need to keep track of which of our dependencies are provided on
        each platform and somehow omit those for from the build for that
        platform.
    * Generate an RPM version number.
    * Run `fpm` supplying the virtualenv path and version number.

    Followup Issues:
    * Update all pinned dependencies to instead be minimum dependencies.
      * This means that as and when sufficiently new versions of our
        dependencies are introduced upstream, we can remove them from our sumo
        build.
      * Those dependencies which are either too old or which are not packaged
        will be imported from the sumo virtualenv in preference.
      * Eventually we hope that all our dependencies will filter upstream and
        we will no longer have to bundle them; at which point the `flocker`
        package itself may be ready to be packaged by upstream distributions.

    Ticket refs:
         * https://github.com/ClusterHQ/flocker/issues/88

    Issue: CI integration (??):
    Update buildbot to build RPMs using new build scripts
    * Issue: create deb, mac, gentoo build slave
    * Issue: install from resulting package from repo and run test suite

    Issue: Client package build (??):
    Sumo packaging of flocker-deploy
    * For deb, RPM, and mac (via homebrew or ...)
    * Proper mac packages. See
      http://stackoverflow.com/questions/11487596/making-os-x-installer-packages-like-a-pro-xcode4-developer-id-mountain-lion-re

    Client package CI integration

    Misc:
    * separate stable and testing repos for deb and rpm
    * update python-flocker.spec.in requirements (remove most of them)
    * maybe even remove the spec file template and generate_spec function
      entirely (do we need it?)
    * do we still need to build an SRPM?
    * automatically build a wheel
    * automatically build an sdist
    """
    if target_dir is None:
        target_dir = FilePath(mkdtemp())

    get_package_version_step = GetPackageVersion(
        virtualenv_path=target_dir, package_name='Flocker')

    return BuildSequence(
        steps=(
            InstallVirtualEnv(target_path=target_dir),
            InstallApplication(virtualenv_path=target_dir,
                               package_uri=package_uri),
            get_package_version_step,
            BuildRpm(
                destination_path=destination_path,
                source_path=target_dir,
                name='Flocker',
                prefix=FilePath('/opt/flocker'),
                epoch=b'0',
                rpm_version=DelayedRpmVersion(
                    package_version_step=get_package_version_step),
                license='ASL 2.0',
                url='https://clusterhq.com',
                vendor='ClusterHQ',
                maintainer='noreply@build.clusterhq.com',
                architecture=None,
                description='A Docker orchestration and volume management tool',
            )
        )
    )

from textwrap import dedent
class BuildOptions(usage.Options):
    """
    Command line options for the ``build-package`` tool.
    """
    synopsis = 'build-rpm [options] <package-uri>'

    optParameters = [
        ['destination-path', 'd', '.',
         'The path to a directory in which to create package files and '
         'artifacts.'],
    ]

    longdesc = dedent("""\
    Arguments:

    <package-uri>: The Python package url or path to install using ``pip``.
    """)

    def parseArgs(self, package_uri):
        """
        The Python package to install.
        """
        self['package-uri'] = package_uri

    def postOptions(self):
        """
        Coerce to ``FilePath`` where appropriate.
        """
        self['destination-path'] = FilePath(self['destination-path'])

class BuildScript(object):
    build_command = staticmethod(sumo_rpm_builder)

    def __init__(self, sys_module=None):
        """
        """
        if sys_module is None:
            sys_module = sys
        self.sys_module = sys_module

    def main(self, top_level=None, base_path=None):
        """
        Build a package.

        :param list argv: The arguments passed to the script.
        """
        options = BuildOptions()

        try:
            options.parseOptions(self.sys_module.argv[1:])
        except usage.UsageError as e:
            self.sys_module.stderr.write("%s\n" % (options,))
            self.sys_module.stderr.write("%s\n" % (e,))
            raise SystemExit(1)

        self.build_command(
            destination_path=options['destination-path'],
            package_uri=options['package-uri']
        ).run()

main = BuildScript().main
<|endoftext|>


<commit_before># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for TargetColumn."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf

class RegressionTargetColumnTest(tf.test.TestCase):

  # TODO(zakaria): test multilabel regresssion.
  def testRegression(self):
    target_column = tf.contrib.layers.regression_target()
    with tf.Graph().as_default(), tf.Session() as sess:
      prediction = tf.constant([[1.], [1.], [3.]])
      labels = tf.constant([[0.], [1.], [1.]])
      self.assertAlmostEqual(
          5. / 3, sess.run(target_column.loss(prediction, labels, {})))

  def testRegressionWithWeights(self):
    target_column = tf.contrib.layers.regression_target(
        weight_column_name="label_weight")
    with tf.Graph().as_default(), tf.Session() as sess:
      features = {"label_weight": tf.constant([[2.], [5.], [0.]])}
      prediction = tf.constant([[1.], [1.], [3.]])
      labels = tf.constant([[0.], [1.], [1.]])
      self.assertAlmostEqual(
          2. / 7,
          sess.run(target_column.loss(prediction, labels, features)),
          places=3)
      self.assertAlmostEqual(
          2. / 3,
          sess.run(target_column.training_loss(prediction, labels, features)),
          places=3)

class MultiClassTargetColumnTest(tf.test.TestCase):

  def testBinaryClassification(self):
    target_column = tf.contrib.layers.multi_class_target(n_classes=2)
    with tf.Graph().as_default(), tf.Session() as sess:
      logits = tf.constant([[1.], [1.]])
      labels = tf.constant([[1.], [0.]])
      # logloss: z:label, x:logit
      # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
      self.assertAlmostEqual(0.81326175,
                             sess.run(target_column.loss(logits, labels, {})))

  def testBinaryClassificationWithWeights(self):
    target_column = tf.contrib.layers.multi_class_target(
        n_classes=2, weight_column_name="label_weight")
    with tf.Graph().as_default(), tf.Session() as sess:
      features = {"label_weight": tf.constant([[1.], [0.]])}
      logits = tf.constant([[1.], [1.]])
      labels = tf.constant([[1.], [0.]])
      # logloss: z:label, x:logit
      # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
      self.assertAlmostEqual(
          .31326166, sess.run(target_column.loss(logits, labels, features)))

  def testBinaryEvalMetrics(self):
    target_column = tf.contrib.layers.multi_class_target(n_classes=2)
    with tf.Graph().as_default(), tf.Session() as sess:
      logits = tf.constant([[1.], [1.], [-1.]])
      labels = tf.constant([[1.], [0.], [1.]])
      eval_dict = target_column.get_eval_ops({}, logits, labels)
      # TODO(zakaria): test all metrics
      accuracy_op, update_op = eval_dict["accuracy/threshold_0.500000_mean"]
      sess.run(tf.global_variables_initializer())
      sess.run(tf.local_variables_initializer())
      sess.run(update_op)
      self.assertAlmostEqual(1.0 / 3, sess.run(accuracy_op))

  def testMultiClass(self):
    target_column = tf.contrib.layers.multi_class_target(n_classes=3)
    with tf.Graph().as_default(), tf.Session() as sess:
      logits = tf.constant([[1., 0., 0.]])
      labels = tf.constant([2])
      # logloss: z:label, x:logit
      # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
      self.assertAlmostEqual(1.5514446,
                             sess.run(target_column.loss(logits, labels, {})))

  def testMultiClassWithWeight(self):
    target_column = tf.contrib.layers.multi_class_target(
        n_classes=3, weight_column_name="label_weight")
    with tf.Graph().as_default(), tf.Session() as sess:
      features = {"label_weight": tf.constant([0.1])}
      logits = tf.constant([[1., 0., 0.]])
      labels = tf.constant([2])
      # logloss: z:label, x:logit
      # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
      self.assertAlmostEqual(
          1.5514446, sess.run(target_column.loss(logits, labels, features)))

  def testMultiClassWithInvalidNClass(self):
    try:
      tf.contrib.layers.multi_class_target(n_classes=1)
      self.fail("Softmax with no n_classes did not raise error.")
    except ValueError:
      # Expected
      pass

  def testMultiClassEvalMetrics(self):
    target_column = tf.contrib.layers.multi_class_target(n_classes=3)
    with tf.Graph().as_default(), tf.Session() as sess:
      logits = tf.constant([[1., 0., 0.]])
      labels = tf.constant([2])
      eval_dict = target_column.get_eval_ops({}, logits, labels)
      loss_op, update_op = eval_dict["loss"]
      sess.run(tf.global_variables_initializer())
      sess.run(tf.local_variables_initializer())
      sess.run(update_op)
      # logloss: z:label, x:logit
      # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
      self.assertAlmostEqual(1.5514446, sess.run(loss_op))

  def testBinarySVMDefaultWeights(self):
    target_column = tf.contrib.layers.binary_svm_target()
    predictions = tf.constant([[-0.5], [1.2]])
    labels = tf.constant([0, 1])
    loss = target_column.loss(predictions, labels, {})
    # Prediction for first example is in the right side of the hyperplane (i.e.,
    # < 0) but it is within the [-1,1] margin. There is a 0.5 loss incurred by
    # this example. The 2nd prediction is outside the margin so it incurs no
    # loss at all. The overall (normalized) loss is therefore 0.5/(1+1) = 0.25.
    with tf.Session() as sess:
      self.assertAlmostEqual(0.25, sess.run(loss))

  def testBinarySVMWithWeights(self):
    target_column = tf.contrib.layers.binary_svm_target(
        weight_column_name="weights")
    predictions = tf.constant([[-0.7], [0.2]])
    labels = tf.constant([0, 1])
    features = {"weights": tf.constant([2.0, 10.0])}
    loss = target_column.loss(predictions, labels, features)
    training_loss = target_column.training_loss(predictions, labels, features)
    # Prediction for both examples are in the right side of the hyperplane but
    # within the margin. The (weighted) loss incurred is 2*0.3=0.6 and 10*0.8=8
    # respectively. The overall (normalized) loss is therefore 8.6/12.
    with tf.Session() as sess:
      self.assertAlmostEqual(8.6 / 12, sess.run(loss), places=3)
      self.assertAlmostEqual(8.6 / 2, sess.run(training_loss), places=3)

if __name__ == "__main__":
  tf.test.main()
<commit_msg>Loosen equality test. Change: 138155531<commit_after># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for TargetColumn."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf

class RegressionTargetColumnTest(tf.test.TestCase):

  # TODO(zakaria): test multilabel regresssion.
  def testRegression(self):
    target_column = tf.contrib.layers.regression_target()
    with tf.Graph().as_default(), tf.Session() as sess:
      prediction = tf.constant([[1.], [1.], [3.]])
      labels = tf.constant([[0.], [1.], [1.]])
      self.assertAlmostEqual(
          5. / 3, sess.run(target_column.loss(prediction, labels, {})))

  def testRegressionWithWeights(self):
    target_column = tf.contrib.layers.regression_target(
        weight_column_name="label_weight")
    with tf.Graph().as_default(), tf.Session() as sess:
      features = {"label_weight": tf.constant([[2.], [5.], [0.]])}
      prediction = tf.constant([[1.], [1.], [3.]])
      labels = tf.constant([[0.], [1.], [1.]])
      self.assertAlmostEqual(
          2. / 7,
          sess.run(target_column.loss(prediction, labels, features)),
          places=3)
      self.assertAlmostEqual(
          2. / 3,
          sess.run(target_column.training_loss(prediction, labels, features)),
          places=3)

class MultiClassTargetColumnTest(tf.test.TestCase):

  def testBinaryClassification(self):
    target_column = tf.contrib.layers.multi_class_target(n_classes=2)
    with tf.Graph().as_default(), tf.Session() as sess:
      logits = tf.constant([[1.], [1.]])
      labels = tf.constant([[1.], [0.]])
      # logloss: z:label, x:logit
      # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
      self.assertAlmostEqual(0.81326175,
                             sess.run(target_column.loss(logits, labels, {})))

  def testBinaryClassificationWithWeights(self):
    target_column = tf.contrib.layers.multi_class_target(
        n_classes=2, weight_column_name="label_weight")
    with tf.Graph().as_default(), tf.Session() as sess:
      features = {"label_weight": tf.constant([[1.], [0.]])}
      logits = tf.constant([[1.], [1.]])
      labels = tf.constant([[1.], [0.]])
      # logloss: z:label, x:logit
      # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
      self.assertAlmostEqual(
          .31326166,
          sess.run(target_column.loss(logits, labels, features)),
          delta=1e-6)

  def testBinaryEvalMetrics(self):
    target_column = tf.contrib.layers.multi_class_target(n_classes=2)
    with tf.Graph().as_default(), tf.Session() as sess:
      logits = tf.constant([[1.], [1.], [-1.]])
      labels = tf.constant([[1.], [0.], [1.]])
      eval_dict = target_column.get_eval_ops({}, logits, labels)
      # TODO(zakaria): test all metrics
      accuracy_op, update_op = eval_dict["accuracy/threshold_0.500000_mean"]
      sess.run(tf.global_variables_initializer())
      sess.run(tf.local_variables_initializer())
      sess.run(update_op)
      self.assertAlmostEqual(1.0 / 3, sess.run(accuracy_op))

  def testMultiClass(self):
    target_column = tf.contrib.layers.multi_class_target(n_classes=3)
    with tf.Graph().as_default(), tf.Session() as sess:
      logits = tf.constant([[1., 0., 0.]])
      labels = tf.constant([2])
      # logloss: z:label, x:logit
      # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
      self.assertAlmostEqual(1.5514446,
                             sess.run(target_column.loss(logits, labels, {})))

  def testMultiClassWithWeight(self):
    target_column = tf.contrib.layers.multi_class_target(
        n_classes=3, weight_column_name="label_weight")
    with tf.Graph().as_default(), tf.Session() as sess:
      features = {"label_weight": tf.constant([0.1])}
      logits = tf.constant([[1., 0., 0.]])
      labels = tf.constant([2])
      # logloss: z:label, x:logit
      # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
      self.assertAlmostEqual(
          1.5514446, sess.run(target_column.loss(logits, labels, features)))

  def testMultiClassWithInvalidNClass(self):
    try:
      tf.contrib.layers.multi_class_target(n_classes=1)
      self.fail("Softmax with no n_classes did not raise error.")
    except ValueError:
      # Expected
      pass

  def testMultiClassEvalMetrics(self):
    target_column = tf.contrib.layers.multi_class_target(n_classes=3)
    with tf.Graph().as_default(), tf.Session() as sess:
      logits = tf.constant([[1., 0., 0.]])
      labels = tf.constant([2])
      eval_dict = target_column.get_eval_ops({}, logits, labels)
      loss_op, update_op = eval_dict["loss"]
      sess.run(tf.global_variables_initializer())
      sess.run(tf.local_variables_initializer())
      sess.run(update_op)
      # logloss: z:label, x:logit
      # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
      self.assertAlmostEqual(1.5514446, sess.run(loss_op))

  def testBinarySVMDefaultWeights(self):
    target_column = tf.contrib.layers.binary_svm_target()
    predictions = tf.constant([[-0.5], [1.2]])
    labels = tf.constant([0, 1])
    loss = target_column.loss(predictions, labels, {})
    # Prediction for first example is in the right side of the hyperplane (i.e.,
    # < 0) but it is within the [-1,1] margin. There is a 0.5 loss incurred by
    # this example. The 2nd prediction is outside the margin so it incurs no
    # loss at all. The overall (normalized) loss is therefore 0.5/(1+1) = 0.25.
    with tf.Session() as sess:
      self.assertAlmostEqual(0.25, sess.run(loss))

  def testBinarySVMWithWeights(self):
    target_column = tf.contrib.layers.binary_svm_target(
        weight_column_name="weights")
    predictions = tf.constant([[-0.7], [0.2]])
    labels = tf.constant([0, 1])
    features = {"weights": tf.constant([2.0, 10.0])}
    loss = target_column.loss(predictions, labels, features)
    training_loss = target_column.training_loss(predictions, labels, features)
    # Prediction for both examples are in the right side of the hyperplane but
    # within the margin. The (weighted) loss incurred is 2*0.3=0.6 and 10*0.8=8
    # respectively. The overall (normalized) loss is therefore 8.6/12.
    with tf.Session() as sess:
      self.assertAlmostEqual(8.6 / 12, sess.run(loss), places=3)
      self.assertAlmostEqual(8.6 / 2, sess.run(training_loss), places=3)

if __name__ == "__main__":
  tf.test.main()
<|endoftext|>