<commit_before># -*- coding: utf-8 -*-

import os
import time
import logging

import base64
try: import cPickle as pickle
except: import pickle

import datetime
from django.utils import timezone

import json
#from django.utils import simplejson

from pprint import pprint

from django.shortcuts import render_to_response
from django.http import HttpResponseRedirect
from django.template import RequestContext

from django.http import HttpResponseForbidden
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import redirect, render

logger = logging.getLogger(__name__)

import boto.ec2
import boto.ec2.cloudwatch
#from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType

from django.contrib.auth.models import User
from userprofile.models import Profile as userprofile

from amazon import s3_funcs
from amazon import s3_funcs_shortcuts

from cloud_storage.models import Files
from cloud_storage.models import Uploaded_Files

from django.template.defaultfilters import filesizeformat, upper
from django.contrib.humanize.templatetags.humanize import naturalday

from cloudly.templatetags.cloud_extras import clear_filename, get_file_extension

from cloud_vms.models import Cache

import decimal
from django.db.models.base import ModelState

def date_handler(obj):
	return obj.isoformat() if hasattr(obj, 'isoformat') else obj

def ajax_vms_refresh(request):

	user = request.user

	try:
		profile = userprofile.objects.get(user=request.user)
	except: return HttpResponseRedirect("/")

	print 'Refreshing', user, 'VMs cache..'

	aws_access_key = profile.aws_access_key
	aws_secret_key = profile.aws_secret_key
	aws_ec2_verified = profile.aws_ec2_verified

	aws_virtual_machines = {}

	if aws_ec2_verified:

		vms_cache = Cache.objects.get_or_create(user=user)
		vms_cache = vms_cache[0]

		vms_cache.is_updating = True
		vms_cache.save()

		aws_regions = profile.aws_enabled_regions.split(',')
		print 'AWS regions', aws_regions

		for ec2_region in aws_regions:

			if(ec2_region):

				ec2conn = boto.ec2.connect_to_region(ec2_region,aws_access_key_id=aws_access_key,aws_secret_access_key=aws_secret_key)
				cloudwatch = boto.ec2.cloudwatch.connect_to_region(ec2_region,aws_access_key_id=aws_access_key,aws_secret_access_key=aws_secret_key)

				try:
					reservations = ec2conn.get_all_instances()
				except:
					vms_cache.is_updating = False
					vms_cache.vms_response = ""
					vms_cache.save()
					print vms_cache.is_updating
					print vms_cache.vms_response
					return HttpResponse("access denied")

				instances = [i for r in reservations for i in r.instances]

				for instance in instances:

					if not instance: continue

					instance_metrics = {}
					instance_metrics['groups'] = {}
					instance_metrics['block_device_mapping'] = {}

					print '** instance', instance.id, instance.private_ip_address

					volumes = []
					for volume in ec2conn.get_all_volumes(filters={'attachment.instance-id': instance.id}):
						volumes.append([volume.id, volume.iops, volume.size,])

					groups = []
					for group in instance.__dict__['groups']:
						groups.append([group.id, group.name,])

					instance_metrics['groups'] = groups
					instance_metrics['block_device_mapping'] = volumes
					instance_metrics['architecture'] = instance.architecture
					instance_metrics['client_token'] = instance.client_token
					instance_metrics['dns_name'] = instance.dns_name
					instance_metrics['private_ip_address'] = instance.private_ip_address
					instance_metrics['hypervisor'] = instance.hypervisor
					instance_metrics['id'] = instance.id
					instance_metrics['image_id'] = instance.image_id
					instance_metrics['instance_type'] = instance.instance_type
					instance_metrics['ip_address'] = instance.ip_address
					instance_metrics['key_name'] = instance.key_name
					instance_metrics['launch_time'] = instance.launch_time
					instance_metrics['monitored'] = instance.monitored
					instance_metrics['persistent'] = instance.persistent
					instance_metrics['ramdisk'] = instance.ramdisk
					instance_metrics['root_device_name'] = instance.root_device_name
					instance_metrics['root_device_type'] = instance.root_device_type
					instance_metrics['tags'] = instance.tags
					instance_metrics['virtualization_type'] = instance.virtualization_type
					instance_metrics['vpc_id'] = instance.vpc_id
					instance_metrics['region'] = {"endpoint":instance.region.endpoint,"name":instance.region.name,}
					instance_metrics['state'] = {"state":instance.state,"code":instance.state_code,"state_reason":instance.state_reason,}

					try:
						ec2conn.monitor_instance(str(instance.id))
					except:
						print instance.id, 'instance not in a monitorable state!'.upper()
						continue

					# Here is where you define start - end for the Logs...............
					end = datetime.datetime.utcnow()
					start = end - datetime.timedelta(hours=1)

					# And the list of possible values on the aws response....
					# ['Minimum', 'Maximum', 'Sum', 'Average', 'SampleCount']
					# ['Seconds', 'Percent', 'Bytes', 'Bits', 'Count', 'Bytes/Second', 'Bits/Second', 'Count/Second']
					# print ec2conn.list_metrics()

					# CPUUtilization
					try:
						metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="CPUUtilization")[0]
					except: continue

					cpu_utilization_datapoints = metric.query(start, end, 'Average', 'Percent')
					instance_metrics['cpu_utilization_datapoints'] = json.dumps(cpu_utilization_datapoints,default=date_handler)

					# DiskReadOps
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="DiskReadOps")[0]
					disk_readops_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['disk_readops_datapoints'] = disk_readops_datapoints

					# DiskWriteOps
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="DiskWriteOps")[0]
					disk_writeops_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['disk_writeops_datapoints'] = disk_writeops_datapoints

					# DiskReadBytes
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="DiskReadBytes")[0]
					disk_readbytes_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['disk_readbytes_datapoints'] = disk_readbytes_datapoints

					# DiskWriteBytes
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="DiskWriteBytes")[0]
					disk_writebytes_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['disk_writebytes_datapoints'] = disk_writebytes_datapoints

					# NetworkIn
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="NetworkIn")[0]
					networkin_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['networkin_datapoints'] = networkin_datapoints

					# NetworkOut
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="NetworkOut")[0]
					networkout_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['networkout_datapoints'] = networkout_datapoints

					aws_virtual_machines[instance.id] = instance_metrics

		print 'aws_virtual_machines'

		# XXX Note This is how this should be - but I get this fucked up error on the prod......
		# XXX Note Can't pickle <type '_hashlib.HASH'>: attribute lookup _hashlib.HASH failed
		# XXX Note Specifically, it's the Python Bug that "cannot be fixed" http://bugs.python.org/issue11771
		# XXX Note Quote: There is no way to implement generic pickling for hash objects that would work across all implementations.
		#try:
		vms_cache.vms_response = base64.b64encode(pickle.dumps(aws_virtual_machines, pickle.HIGHEST_PROTOCOL))
		#except:
		#	return HttpResponse("Fuck shit wank bugger Can't pickle <type '_hashlib.HASH'>: attribute lookup _hashlib.HASH failed problem..")

		print 'xxxxx', aws_virtual_machines

		vms_cache.last_seen = timezone.now()
		vms_cache.is_updating = False
		vms_cache.save()

		print 'VMs cache was succesfully updated.'

	return HttpResponse("ALLDONE")

def ajax_virtual_machines(request):

	print '-- ajax virtual machines'

	if not request.user.is_authenticated():
		print 'anonymous'
		return HttpResponseRedirect("/")

	print request.user

	return render_to_response('ajax_virtual_machines.html', locals(), context_instance=RequestContext(request))

<commit_msg>applying same json dumping feature on the DiskReadOps<commit_after># -*- coding: utf-8 -*-

import os
import time
import logging

import base64
try: import cPickle as pickle
except: import pickle

import datetime
from django.utils import timezone

import json
#from django.utils import simplejson

from pprint import pprint

from django.shortcuts import render_to_response
from django.http import HttpResponseRedirect
from django.template import RequestContext

from django.http import HttpResponseForbidden
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import redirect, render

logger = logging.getLogger(__name__)

import boto.ec2
import boto.ec2.cloudwatch
#from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType

from django.contrib.auth.models import User
from userprofile.models import Profile as userprofile

from amazon import s3_funcs
from amazon import s3_funcs_shortcuts

from cloud_storage.models import Files
from cloud_storage.models import Uploaded_Files

from django.template.defaultfilters import filesizeformat, upper
from django.contrib.humanize.templatetags.humanize import naturalday

from cloudly.templatetags.cloud_extras import clear_filename, get_file_extension

from cloud_vms.models import Cache

import decimal
from django.db.models.base import ModelState

def date_handler(obj):
	return obj.isoformat() if hasattr(obj, 'isoformat') else obj

def ajax_vms_refresh(request):

	user = request.user

	try:
		profile = userprofile.objects.get(user=request.user)
	except: return HttpResponseRedirect("/")

	print 'Refreshing', user, 'VMs cache..'

	aws_access_key = profile.aws_access_key
	aws_secret_key = profile.aws_secret_key
	aws_ec2_verified = profile.aws_ec2_verified

	aws_virtual_machines = {}

	if aws_ec2_verified:

		vms_cache = Cache.objects.get_or_create(user=user)
		vms_cache = vms_cache[0]

		vms_cache.is_updating = True
		vms_cache.save()

		aws_regions = profile.aws_enabled_regions.split(',')
		print 'AWS regions', aws_regions

		for ec2_region in aws_regions:

			if(ec2_region):

				ec2conn = boto.ec2.connect_to_region(ec2_region,aws_access_key_id=aws_access_key,aws_secret_access_key=aws_secret_key)
				cloudwatch = boto.ec2.cloudwatch.connect_to_region(ec2_region,aws_access_key_id=aws_access_key,aws_secret_access_key=aws_secret_key)

				try:
					reservations = ec2conn.get_all_instances()
				except:
					vms_cache.is_updating = False
					vms_cache.vms_response = ""
					vms_cache.save()
					print vms_cache.is_updating
					print vms_cache.vms_response
					return HttpResponse("access denied")

				instances = [i for r in reservations for i in r.instances]

				for instance in instances:

					if not instance: continue

					instance_metrics = {}
					instance_metrics['groups'] = {}
					instance_metrics['block_device_mapping'] = {}

					print '** instance', instance.id, instance.private_ip_address

					volumes = []
					for volume in ec2conn.get_all_volumes(filters={'attachment.instance-id': instance.id}):
						volumes.append([volume.id, volume.iops, volume.size,])

					groups = []
					for group in instance.__dict__['groups']:
						groups.append([group.id, group.name,])

					instance_metrics['groups'] = groups
					instance_metrics['block_device_mapping'] = volumes
					instance_metrics['architecture'] = instance.architecture
					instance_metrics['client_token'] = instance.client_token
					instance_metrics['dns_name'] = instance.dns_name
					instance_metrics['private_ip_address'] = instance.private_ip_address
					instance_metrics['hypervisor'] = instance.hypervisor
					instance_metrics['id'] = instance.id
					instance_metrics['image_id'] = instance.image_id
					instance_metrics['instance_type'] = instance.instance_type
					instance_metrics['ip_address'] = instance.ip_address
					instance_metrics['key_name'] = instance.key_name
					instance_metrics['launch_time'] = instance.launch_time
					instance_metrics['monitored'] = instance.monitored
					instance_metrics['persistent'] = instance.persistent
					instance_metrics['ramdisk'] = instance.ramdisk
					instance_metrics['root_device_name'] = instance.root_device_name
					instance_metrics['root_device_type'] = instance.root_device_type
					instance_metrics['tags'] = instance.tags
					instance_metrics['virtualization_type'] = instance.virtualization_type
					instance_metrics['vpc_id'] = instance.vpc_id
					instance_metrics['region'] = {"endpoint":instance.region.endpoint,"name":instance.region.name,}
					instance_metrics['state'] = {"state":instance.state,"code":instance.state_code,"state_reason":instance.state_reason,}

					try:
						ec2conn.monitor_instance(str(instance.id))
					except:
						print instance.id, 'instance not in a monitorable state!'.upper()
						continue

					# Here is where you define start - end for the Logs...............
					end = datetime.datetime.utcnow()
					start = end - datetime.timedelta(hours=1)

					# And the list of possible values on the aws response....
					# ['Minimum', 'Maximum', 'Sum', 'Average', 'SampleCount']
					# ['Seconds', 'Percent', 'Bytes', 'Bits', 'Count', 'Bytes/Second', 'Bits/Second', 'Count/Second']
					# print ec2conn.list_metrics()

					# CPUUtilization
					try:
						metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="CPUUtilization")[0]
					except: continue

					cpu_utilization_datapoints = metric.query(start, end, 'Average', 'Percent')
					instance_metrics['cpu_utilization_datapoints'] = json.dumps(cpu_utilization_datapoints,default=date_handler)

					# DiskReadOps
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="DiskReadOps")[0]
					disk_readops_datapoints = metric.query(start, end, 'Average', '')
					instance_metrics['disk_readops_datapoints'] = json.dumps(disk_readops_datapoints,default=date_handler)

					# DiskWriteOps
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="DiskWriteOps")[0]
					disk_writeops_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['disk_writeops_datapoints'] = disk_writeops_datapoints

					# DiskReadBytes
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="DiskReadBytes")[0]
					disk_readbytes_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['disk_readbytes_datapoints'] = disk_readbytes_datapoints

					# DiskWriteBytes
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="DiskWriteBytes")[0]
					disk_writebytes_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['disk_writebytes_datapoints'] = disk_writebytes_datapoints

					# NetworkIn
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="NetworkIn")[0]
					networkin_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['networkin_datapoints'] = networkin_datapoints

					# NetworkOut
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="NetworkOut")[0]
					networkout_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['networkout_datapoints'] = networkout_datapoints

					aws_virtual_machines[instance.id] = instance_metrics

		print 'aws_virtual_machines'

		# XXX Note This is how this should be - but I get this fucked up error on the prod......
		# XXX Note Can't pickle <type '_hashlib.HASH'>: attribute lookup _hashlib.HASH failed
		# XXX Note Specifically, it's the Python Bug that "cannot be fixed" http://bugs.python.org/issue11771
		# XXX Note Quote: There is no way to implement generic pickling for hash objects that would work across all implementations.
		#try:
		vms_cache.vms_response = base64.b64encode(pickle.dumps(aws_virtual_machines, pickle.HIGHEST_PROTOCOL))
		#except:
		#	return HttpResponse("Fuck shit wank bugger Can't pickle <type '_hashlib.HASH'>: attribute lookup _hashlib.HASH failed problem..")

		print 'xxxxx', aws_virtual_machines

		vms_cache.last_seen = timezone.now()
		vms_cache.is_updating = False
		vms_cache.save()

		print 'VMs cache was succesfully updated.'

	return HttpResponse("ALLDONE")

def ajax_virtual_machines(request):

	print '-- ajax virtual machines'

	if not request.user.is_authenticated():
		print 'anonymous'
		return HttpResponseRedirect("/")

	print request.user

	return render_to_response('ajax_virtual_machines.html', locals(), context_instance=RequestContext(request))

<|endoftext|>


<commit_before>#!/usr/bin/python
#
#   Copyright 2012 Red Hat, Inc.
#   Portions Copyright (C) 2012,2013  Chris Lalancette <clalancette@gmail.com>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import logging
import zope
import oz.TDL
import oz.GuestFactory
import oz.ozutil
import guestfs
# TODO: We've had to migrate to lxml here because of Oz changes
#       see if we can't move the libvirt stuff as well
# For now we import both
import libxml2
import lxml
import ConfigParser
import tempfile
import base64
from imgfac.ApplicationConfiguration import ApplicationConfiguration
from imgfac.CloudDelegate import CloudDelegate
from imgfac.PersistentImageManager import PersistentImageManager
from imgfac.ReservationManager import ReservationManager

# This makes extensive use of parameters with some sensible defaults
# Try to keep an accurate list up here

# Parameter     -  Default -
# Description

# utility_image - <base_image_id>
# Description: UUID of the image that will be launched to do the modification of the
#              the base_image referenced in this target_image build.  Note that the
#              utility image should itself be a base image and can, if constructed properly,
#              be the same as the base image that is being modified.  The plugin makes a copy
#              of the utility image before launching it, which allows safe modification during
#              the target_image creation process.

# input_image_file - /input_image.raw (but only if input_image_device is not specified)
# Description: The name of the file on the working space disk where the base_image is presented

# input_image_device - None
# Description: The name of the device where the base_image is presented to the utility VM.
#              (e.g. vdc)

# NOTE: You can specify one or the other of these options but not both.  If neither are specified
#       you will end up with the default value for input_image_file.

# utility_cpus - None
# Description: Number of CPUs in the utility VM - this can also be set in the global Oz config
#              The lmc Live CD creation process benefits greatly from extra CPUs during the squashfs
#              creation step.  The performance improvement is almost perfectly O(n) w.r.t CPU.

# utility_customizations - None
# Description: A partial TDL document to drive the actions of the utility VM - only repos, packages,
#              files and commands will be used - all other content is ignored

# results_location - /results/images/boot.iso

# Description: Location inside of the working space image from which to extract the results.

# Borrowed from Oz by Chris Lalancette
def data_from_type(name, contenttype, content):
    '''
    A function to get data out of some content, possibly decoding it depending
    on the content type.  This function understands three types of content:
    raw (where no decoding is necessary), base64 (where the data needs to be
    base64 decoded), and url (where the data needs to be downloaded).  Because
    the data might be large, all data is sent to file handle, which is returned
    from the function.
    '''

    out = tempfile.NamedTemporaryFile()
    if contenttype == 'raw':
        out.write(content)
    elif contenttype == 'base64':
        base64.decode(StringIO.StringIO(content), out)
    elif contenttype == 'url':
        url = urlparse.urlparse(content)
        if url.scheme == "file":
            with open(url.netloc + url.path) as f:
                out.write("".join(f.readlines()))
        else:
            oz.ozutil.http_download_file(content, out.fileno(), False, None)
    else:
        raise oz.OzException.OzException("Type for %s must be 'raw', 'url' or 'base64'" % (name))

    # make sure the data is flushed to disk for uses of the file through
    # the name
    out.flush()
    out.seek(0)

    return out

class IndirectionCloud(object):
    zope.interface.implements(CloudDelegate)

    def __init__(self):
        super(IndirectionCloud, self).__init__()
        self.app_config = ApplicationConfiguration().configuration
        self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
        self.pim = PersistentImageManager.default_manager()
        self.res_mgr = ReservationManager()

    def builder_should_create_target_image(self, builder, target, image_id, template, parameters):
        # This plugin wants to be the only thing operating on the input image
        # We do all our work here and then return False which stops any additional activity

        # User may specify a utility image - if they do not we assume we can use the input image
        utility_image_id = parameters.get('utility_image', image_id)

        # The utility image is what we actually re-animate with Oz
        # We borrow these variable names from code that is very similar to the Oz/TinMan OS plugin
        self.active_image = self.pim.image_with_id(utility_image_id)
        if not self.active_image:
            raise Exception("Could not find utility image with ID (%s)" % (utility_image_id) )
        self.tdlobj = oz.TDL.TDL(xmlstring=self.active_image.template)

        # Later on, we will either copy in the base_image content as a file, or expose it as a device
        # to the utility VM.  We cannot do both.  Detect invalid input here before doing any long running
        # work
        input_image_device = parameters.get('input_image_device', None)
        input_image_file = parameters.get('input_image_filename', None)

        if input_image_device and input_image_file:
            raise Exception("You can specify either an input_image_device or an input_image_file but not both")

        if (not input_image_device) and (not input_image_file):
            input_image_file="/input_image.raw"

        # We remove any packages, commands and files from the original TDL - these have already been
        # installed/executed.  We leave the repos in place, as it is possible that commands executed
        # later may depend on them
        self.tdlobj.packages = [ ]
        self.tdlobj.commands = { }
        self.tdlobj.files = { }

        # This creates a new Oz object - replaces the auto-generated disk file location with
        # the copy of the utility image made above, and prepares an initial libvirt_xml string
        self._init_oz()
        utility_image_tmp = self.app_config['imgdir'] + "/tmp-utility-image-" + str(builder.target_image.identifier)
        self.guest.diskimage = utility_image_tmp
        # Below we will create this file as a qcow2 image using the original utility image as
        # a backing store - For the follow-on XML generation to work correctly, we need to force
        # Oz to use qcow2 as the image type
        self.guest.image_type = 'qcow2'

        if 'utility_cpus' in parameters:
            self.guest.install_cpus = int(parameters['utility_cpus'])

        libvirt_xml = self.guest._generate_xml("hd", None)
        libvirt_doc = libxml2.parseDoc(libvirt_xml)

        # Now we create a second disk image as working/scratch space
        # Hardcode at 30G
        # TODO: Make configurable
        # Make it, format it, copy in the base_image
        working_space_image = self.app_config['imgdir'] + "/working-space-image-" + str(builder.target_image.identifier)
        self.create_ext2_image(working_space_image)

        # Modify the libvirt_xml used with Oz to contain a reference to a second "working space" disk image
        working_space_device = parameters.get('working_space_device', 'vdb')
        self.add_disk(libvirt_doc, working_space_image, working_space_device)

        self.log.debug("Updated domain XML with working space image:\n%s" % (libvirt_xml))

        # We expect to find a partial TDL document in this parameter - this is what drives the
        # tasks performed by the utility image
        if 'utility_customizations' in parameters:
            self.oz_refresh_customizations(parameters['utility_customizations'])
        else:
            self.log.info('No additional repos, packages, files or commands specified for utility tasks')

        # Create a qcow2 image using the original utility image file (which may be read-only) as a
        # backing store.
        self.log.debug("Creating temporary writeable qcow2 working copy of utlity image (%s) as (%s)" % (self.active_image.data, utility_image_tmp))
        self.guest._internal_generate_diskimage(image_filename=utility_image_tmp, backing_filename=self.active_image.data)

        if input_image_file:
            # Here we finally involve the actual Base Image content - it is made available for the utlity image to modify
            self.copy_content_to_image(builder.base_image.data, working_space_image, input_image_file)
        else:
            # Note that we know that one or the other of these are set because of code earlier
            self.add_disk(libvirt_doc, builder.base_image.data, input_image_device)

        # Run all commands, repo injection, etc specified
        try:
            self.log.debug("Launching utility image and running any customizations specified")
            libvirt_xml = libvirt_doc.serialize(None, 1)
            self.guest.customize(libvirt_xml)
            self.log.debug("Utility image tasks complete")
        finally:
            self.log.debug("Cleaning up install artifacts")
            self.guest.cleanup_install()

        # After shutdown, extract the results
        results_location = parameters.get('results_location', "/results/images/boot.iso")
        self.copy_content_from_image(results_location, working_space_image, builder.target_image.data)

        # TODO: Remove working_space image and utility_image_tmp
        return False

    def add_disk(self, libvirt_doc, disk_image_file, device_name):
	devices = libvirt_doc.xpathEval("/domain/devices")[0]
	new_dev = devices.newChild(None, "disk", None)
	new_dev.setProp("type", "file")
	new_dev.setProp("device", "disk")
	source = new_dev.newChild(None, "source", None)
	source.setProp("file", disk_image_file)
	target = new_dev.newChild(None, "target", None)
	target.setProp("dev", device_name)
	target.setProp("bus", self.guest.disk_bus)

    def oz_refresh_customizations(self, partial_tdl):
        # This takes our already created and well formed TDL object with already blank customizations
        # and attempts to add in any additional customizations found in partial_tdl
        # partial_tdl need not contain the <os>, <name> or <description> sections
        # if it does they will be ignored
        # TODO: Submit an Oz patch to make this shorter or a utility function within the TDL class

        doc = lxml.etree.fromstring(partial_tdl)
        self.tdlobj.doc = doc

        packageslist = doc.xpath('/template/packages/package')
        self.tdlobj._add_packages(packageslist)

        for afile in doc.xpath('/template/files/file'):
            name = afile.get('name')
            if name is None:
                raise Exception("File without a name was given")
            contenttype = afile.get('type')
            if contenttype is None:
                contenttype = 'raw'

            content = afile.text
            if content:
                content = content.strip()
            else:
                content = ''
            self.tdlobj.files[name] = data_from_type(name, contenttype, content)

        repositorieslist = doc.xpath('/template/repositories/repository')
        self.tdlobj._add_repositories(repositorieslist)

        self.tdlobj.commands = self.tdlobj._parse_commands()

    def _init_oz(self):
        # populate a config object to pass to OZ; this allows us to specify our
        # own output dir but inherit other Oz behavior
        self.oz_config = ConfigParser.SafeConfigParser()
        if self.oz_config.read("/etc/oz/oz.cfg") != []:
            self.oz_config.set('paths', 'output_dir', self.app_config["imgdir"])
            if "oz_data_dir" in self.app_config:
                self.oz_config.set('paths', 'data_dir', self.app_config["oz_data_dir"])
            if "oz_screenshot_dir" in self.app_config:
                self.oz_config.set('paths', 'screenshot_dir', self.app_config["oz_screenshot_dir"])
        else:
            raise ImageFactoryException("No Oz config file found. Can't continue.")

        # Use the factory function from Oz directly
        try:
            # Force uniqueness by overriding the name in the TDL
            self.tdlobj.name = "factory-build-" + self.active_image.identifier
            self.guest = oz.GuestFactory.guest_factory(self.tdlobj, self.oz_config, None)
            # Oz just selects a random port here - This could potentially collide if we are unlucky
            self.guest.listen_port = self.res_mgr.get_next_listen_port()
        except libvirtError, e:
            raise ImageFactoryException("Cannot connect to libvirt.  Make sure libvirt is running. [Original message: %s]" %  e.message)
        except OzException, e:
            if "Unsupported" in e.message:
                raise ImageFactoryException("TinMan plugin does not support distro (%s) update (%s) in TDL" % (self.tdlobj.distro, self.tdlobj.update) )
            else:
                raise e

    def create_ext2_image(self, image_file, image_size=(1024*1024*1024*30)):
        # Why ext2?  Why not?  There's no need for the overhead of journaling.  This disk will be mounted once and thrown away.
        self.log.debug("Creating disk image of size (%d) in file (%s) with single partition containint ext2 filesystem" % (image_size, image_file))
        raw_fs_image=open(image_file,"w")
        raw_fs_image.truncate(image_size)
        raw_fs_image.close()
        g = guestfs.GuestFS()
        g.add_drive(image_file)
        g.launch()
        g.part_disk("/dev/sda","msdos")
        g.part_set_mbr_id("/dev/sda",1,0x83)
        g.mkfs("ext2", "/dev/sda1")
        g.sync()

    def copy_content_to_image(self, filename, target_image, target_filename):
        self.log.debug("Copying file (%s) into disk image (%s)" % (filename, target_image))
        g = guestfs.GuestFS()
        g.add_drive(target_image)
        g.launch()
        g.mount_options ("", "/dev/sda1", "/")
        g.upload(filename, target_filename)
        g.sync()

    def copy_content_from_image(self, filename, target_image, destination_file):
        self.log.debug("Copying file (%s) out of disk image (%s) into (%s)" % (filename, target_image, destination_file))
        g = guestfs.GuestFS()
        g.add_drive(target_image)
        g.launch()
        g.mount_options ("", "/dev/sda1", "/")
        g.download(filename,destination_file)
        g.sync()

<commit_msg>clean up the temporary image files in Indirection<commit_after>#!/usr/bin/python
#
#   Copyright 2012 Red Hat, Inc.
#   Portions Copyright (C) 2012,2013  Chris Lalancette <clalancette@gmail.com>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import logging
import zope
import oz.TDL
import oz.GuestFactory
import oz.ozutil
import guestfs
# TODO: We've had to migrate to lxml here because of Oz changes
#       see if we can't move the libvirt stuff as well
# For now we import both
import libxml2
import lxml
import ConfigParser
import tempfile
import base64
import os
import os.path
from imgfac.ApplicationConfiguration import ApplicationConfiguration
from imgfac.CloudDelegate import CloudDelegate
from imgfac.PersistentImageManager import PersistentImageManager
from imgfac.ReservationManager import ReservationManager

# This makes extensive use of parameters with some sensible defaults
# Try to keep an accurate list up here

# Parameter     -  Default -
# Description

# utility_image - <base_image_id>
# Description: UUID of the image that will be launched to do the modification of the
#              the base_image referenced in this target_image build.  Note that the
#              utility image should itself be a base image and can, if constructed properly,
#              be the same as the base image that is being modified.  The plugin makes a copy
#              of the utility image before launching it, which allows safe modification during
#              the target_image creation process.

# input_image_file - /input_image.raw (but only if input_image_device is not specified)
# Description: The name of the file on the working space disk where the base_image is presented

# input_image_device - None
# Description: The name of the device where the base_image is presented to the utility VM.
#              (e.g. vdc)

# NOTE: You can specify one or the other of these options but not both.  If neither are specified
#       you will end up with the default value for input_image_file.

# utility_cpus - None
# Description: Number of CPUs in the utility VM - this can also be set in the global Oz config
#              The lmc Live CD creation process benefits greatly from extra CPUs during the squashfs
#              creation step.  The performance improvement is almost perfectly O(n) w.r.t CPU.

# utility_customizations - None
# Description: A partial TDL document to drive the actions of the utility VM - only repos, packages,
#              files and commands will be used - all other content is ignored

# results_location - /results/images/boot.iso

# Description: Location inside of the working space image from which to extract the results.

# Borrowed from Oz by Chris Lalancette
def data_from_type(name, contenttype, content):
    '''
    A function to get data out of some content, possibly decoding it depending
    on the content type.  This function understands three types of content:
    raw (where no decoding is necessary), base64 (where the data needs to be
    base64 decoded), and url (where the data needs to be downloaded).  Because
    the data might be large, all data is sent to file handle, which is returned
    from the function.
    '''

    out = tempfile.NamedTemporaryFile()
    if contenttype == 'raw':
        out.write(content)
    elif contenttype == 'base64':
        base64.decode(StringIO.StringIO(content), out)
    elif contenttype == 'url':
        url = urlparse.urlparse(content)
        if url.scheme == "file":
            with open(url.netloc + url.path) as f:
                out.write("".join(f.readlines()))
        else:
            oz.ozutil.http_download_file(content, out.fileno(), False, None)
    else:
        raise oz.OzException.OzException("Type for %s must be 'raw', 'url' or 'base64'" % (name))

    # make sure the data is flushed to disk for uses of the file through
    # the name
    out.flush()
    out.seek(0)

    return out

class IndirectionCloud(object):
    zope.interface.implements(CloudDelegate)

    def __init__(self):
        super(IndirectionCloud, self).__init__()
        self.app_config = ApplicationConfiguration().configuration
        self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__))
        self.pim = PersistentImageManager.default_manager()
        self.res_mgr = ReservationManager()

    def builder_should_create_target_image(self, builder, target, image_id, template, parameters):
        # This plugin wants to be the only thing operating on the input image
        # We do all our work here and then return False which stops any additional activity

        self.working_space_image = None
        self.utility_image_tmp = None
        try:
            self._builder_should_create_target_image(builder, target, image_id, template, parameters)
        finally:
            self.log.debug("Cleaning up temporary utility image and working space image")
            for fname in [ self.working_space_image, self.utility_image_tmp ]:
                if fname and os.path.isfile(fname):
                    os.unlink(fname)
        return False

    def _builder_should_create_target_image(self, builder, target, image_id, template, parameters):
        # User may specify a utility image - if they do not we assume we can use the input image
        utility_image_id = parameters.get('utility_image', image_id)

        # The utility image is what we actually re-animate with Oz
        # We borrow these variable names from code that is very similar to the Oz/TinMan OS plugin
        self.active_image = self.pim.image_with_id(utility_image_id)
        if not self.active_image:
            raise Exception("Could not find utility image with ID (%s)" % (utility_image_id) )
        self.tdlobj = oz.TDL.TDL(xmlstring=self.active_image.template)

        # Later on, we will either copy in the base_image content as a file, or expose it as a device
        # to the utility VM.  We cannot do both.  Detect invalid input here before doing any long running
        # work
        input_image_device = parameters.get('input_image_device', None)
        input_image_file = parameters.get('input_image_filename', None)

        if input_image_device and input_image_file:
            raise Exception("You can specify either an input_image_device or an input_image_file but not both")

        if (not input_image_device) and (not input_image_file):
            input_image_file="/input_image.raw"

        # We remove any packages, commands and files from the original TDL - these have already been
        # installed/executed.  We leave the repos in place, as it is possible that commands executed
        # later may depend on them
        self.tdlobj.packages = [ ]
        self.tdlobj.commands = { }
        self.tdlobj.files = { }

        # This creates a new Oz object - replaces the auto-generated disk file location with
        # a copy of the utility image we will make later, and prepares an initial libvirt_xml string
        self._init_oz()
        self.utility_image_tmp = self.app_config['imgdir'] + "/tmp-utility-image-" + str(builder.target_image.identifier)
        self.guest.diskimage = self.utility_image_tmp
        # Below we will create this file as a qcow2 image using the original utility image as
        # a backing store - For the follow-on XML generation to work correctly, we need to force
        # Oz to use qcow2 as the image type
        self.guest.image_type = 'qcow2'

        if 'utility_cpus' in parameters:
            self.guest.install_cpus = int(parameters['utility_cpus'])

        libvirt_xml = self.guest._generate_xml("hd", None)
        libvirt_doc = libxml2.parseDoc(libvirt_xml)

        # Now we create a second disk image as working/scratch space
        # Hardcode at 30G
        # TODO: Make configurable
        # Make it, format it, copy in the base_image
        self.working_space_image = self.app_config['imgdir'] + "/working-space-image-" + str(builder.target_image.identifier)
        self.create_ext2_image(self.working_space_image)

        # Modify the libvirt_xml used with Oz to contain a reference to a second "working space" disk image
        working_space_device = parameters.get('working_space_device', 'vdb')
        self.add_disk(libvirt_doc, self.working_space_image, working_space_device)

        self.log.debug("Updated domain XML with working space image:\n%s" % (libvirt_xml))

        # We expect to find a partial TDL document in this parameter - this is what drives the
        # tasks performed by the utility image
        if 'utility_customizations' in parameters:
            self.oz_refresh_customizations(parameters['utility_customizations'])
        else:
            self.log.info('No additional repos, packages, files or commands specified for utility tasks')

        # Create a qcow2 image using the original utility image file (which may be read-only) as a
        # backing store.
        self.log.debug("Creating temporary writeable qcow2 working copy of utlity image (%s) as (%s)" % (self.active_image.data, self.utility_image_tmp))
        self.guest._internal_generate_diskimage(image_filename=self.utility_image_tmp, backing_filename=self.active_image.data)

        if input_image_file:
            # Here we finally involve the actual Base Image content - it is made available for the utlity image to modify
            self.copy_content_to_image(builder.base_image.data, self.working_space_image, input_image_file)
        else:
            # Note that we know that one or the other of these are set because of code earlier
            self.add_disk(libvirt_doc, builder.base_image.data, input_image_device)

        # Run all commands, repo injection, etc specified
        try:
            self.log.debug("Launching utility image and running any customizations specified")
            libvirt_xml = libvirt_doc.serialize(None, 1)
            self.guest.customize(libvirt_xml)
            self.log.debug("Utility image tasks complete")
        finally:
            self.log.debug("Cleaning up install artifacts")
            self.guest.cleanup_install()

        # After shutdown, extract the results
        results_location = parameters.get('results_location', "/results/images/boot.iso")
        self.copy_content_from_image(results_location, self.working_space_image, builder.target_image.data)

    def add_disk(self, libvirt_doc, disk_image_file, device_name):
	devices = libvirt_doc.xpathEval("/domain/devices")[0]
	new_dev = devices.newChild(None, "disk", None)
	new_dev.setProp("type", "file")
	new_dev.setProp("device", "disk")
	source = new_dev.newChild(None, "source", None)
	source.setProp("file", disk_image_file)
	target = new_dev.newChild(None, "target", None)
	target.setProp("dev", device_name)
	target.setProp("bus", self.guest.disk_bus)

    def oz_refresh_customizations(self, partial_tdl):
        # This takes our already created and well formed TDL object with already blank customizations
        # and attempts to add in any additional customizations found in partial_tdl
        # partial_tdl need not contain the <os>, <name> or <description> sections
        # if it does they will be ignored
        # TODO: Submit an Oz patch to make this shorter or a utility function within the TDL class

        doc = lxml.etree.fromstring(partial_tdl)
        self.tdlobj.doc = doc

        packageslist = doc.xpath('/template/packages/package')
        self.tdlobj._add_packages(packageslist)

        for afile in doc.xpath('/template/files/file'):
            name = afile.get('name')
            if name is None:
                raise Exception("File without a name was given")
            contenttype = afile.get('type')
            if contenttype is None:
                contenttype = 'raw'

            content = afile.text
            if content:
                content = content.strip()
            else:
                content = ''
            self.tdlobj.files[name] = data_from_type(name, contenttype, content)

        repositorieslist = doc.xpath('/template/repositories/repository')
        self.tdlobj._add_repositories(repositorieslist)

        self.tdlobj.commands = self.tdlobj._parse_commands()

    def _init_oz(self):
        # populate a config object to pass to OZ; this allows us to specify our
        # own output dir but inherit other Oz behavior
        self.oz_config = ConfigParser.SafeConfigParser()
        if self.oz_config.read("/etc/oz/oz.cfg") != []:
            self.oz_config.set('paths', 'output_dir', self.app_config["imgdir"])
            if "oz_data_dir" in self.app_config:
                self.oz_config.set('paths', 'data_dir', self.app_config["oz_data_dir"])
            if "oz_screenshot_dir" in self.app_config:
                self.oz_config.set('paths', 'screenshot_dir', self.app_config["oz_screenshot_dir"])
        else:
            raise ImageFactoryException("No Oz config file found. Can't continue.")

        # Use the factory function from Oz directly
        try:
            # Force uniqueness by overriding the name in the TDL
            self.tdlobj.name = "factory-build-" + self.active_image.identifier
            self.guest = oz.GuestFactory.guest_factory(self.tdlobj, self.oz_config, None)
            # Oz just selects a random port here - This could potentially collide if we are unlucky
            self.guest.listen_port = self.res_mgr.get_next_listen_port()
        except libvirtError, e:
            raise ImageFactoryException("Cannot connect to libvirt.  Make sure libvirt is running. [Original message: %s]" %  e.message)
        except OzException, e:
            if "Unsupported" in e.message:
                raise ImageFactoryException("TinMan plugin does not support distro (%s) update (%s) in TDL" % (self.tdlobj.distro, self.tdlobj.update) )
            else:
                raise e

    def create_ext2_image(self, image_file, image_size=(1024*1024*1024*30)):
        # Why ext2?  Why not?  There's no need for the overhead of journaling.  This disk will be mounted once and thrown away.
        self.log.debug("Creating disk image of size (%d) in file (%s) with single partition containint ext2 filesystem" % (image_size, image_file))
        raw_fs_image=open(image_file,"w")
        raw_fs_image.truncate(image_size)
        raw_fs_image.close()
        g = guestfs.GuestFS()
        g.add_drive(image_file)
        g.launch()
        g.part_disk("/dev/sda","msdos")
        g.part_set_mbr_id("/dev/sda",1,0x83)
        g.mkfs("ext2", "/dev/sda1")
        g.sync()

    def copy_content_to_image(self, filename, target_image, target_filename):
        self.log.debug("Copying file (%s) into disk image (%s)" % (filename, target_image))
        g = guestfs.GuestFS()
        g.add_drive(target_image)
        g.launch()
        g.mount_options ("", "/dev/sda1", "/")
        g.upload(filename, target_filename)
        g.sync()

    def copy_content_from_image(self, filename, target_image, destination_file):
        self.log.debug("Copying file (%s) out of disk image (%s) into (%s)" % (filename, target_image, destination_file))
        g = guestfs.GuestFS()
        g.add_drive(target_image)
        g.launch()
        g.mount_options ("", "/dev/sda1", "/")
        g.download(filename,destination_file)
        g.sync()

<|endoftext|>


<commit_before>#######################################################
## Key Master - Hackerspace Adelaide NFC Door reader ##
#######################################################

# scard documentation:
# http://pyscard.sourceforge.net/epydoc/smartcard.scard.scard-module.html

# Post successful card scan to hackadl.org

# When running on a Raspberri Pi, you need to install these:
# $ sudo apt-get install python-pyscard
# $ sudo apt-get install pcscd
# $ sudo apt-get install pcsc-tools

# If the reader isn't showing up, solve it using this:
# http://enjoy-rfid.blogspot.com.au/2015/03/raspberry-pi-nfc.html
# Which gets you to create a blacklist file:
# /etc/modprobe.d/raspi-blacklist.conf
# That just has:
# blacklist pn533
# blacklist nfc

# For a hacky way to make this run at boot on the RaspberryPi,
# running this from /etc/rc.local

import urllib
import urllib2
import base64
import sys
import argparse

from select import select
from smartcard.scard import *

import imp
try:
  imp.find_module('wiringpi2')
  hasWiringPi = True
except ImportError:
  hasWiringPi = False

if hasWiringPi:
  # For GPIO pin control
  import wiringpi2 as wiringpi
  from time import sleep
  wiringpi.wiringPiSetupGpio()
  # Connect the pieso buzzer to GPIO 23
  # Ground to the third pin down from the top on the right column.
  # Power to the eighth pin down from the top on the right column.
  wiringpi.softToneCreate(23)

def hexarray(array):
  return ":".join(["{:02x}".format(b) for b in array])

def b64array(array):
  return base64.b64encode("".join([chr(b) for b in array]))

hresult, hcontext = SCardEstablishContext(SCARD_SCOPE_USER)

assert hresult==SCARD_S_SUCCESS

hresult, readers = SCardListReaders(hcontext, [])

assert len(readers)>0

reader = readers[0]
timeout = 10 # Timeout when there isn't any input
url = 'https://members.hackerspace-adelaide.org.au/'
devUrl = 'http://localhost:3000/'

# Parse arguments handed in when running

parser = argparse.ArgumentParser()
parser.add_argument("-d", "--development", help="Run in development environment talking to localhost:3000.", action="store_true")
parser.add_argument("-l", "--lookup", help="Run in lookup mode, only for admins.", action="store_true")
app_args = parser.parse_args()

# Welcome message

print '\n###### Hackadl.org ######'
print '######  Oh hello.  ######\n'

# Welcome song

wiringpi.softToneWrite(23, 1000)
sleep(0.05)
wiringpi.softToneWrite(23, 0)
sleep(0.05)
wiringpi.softToneWrite(23, 1000)
sleep(0.05)
wiringpi.softToneWrite(23, 0)
sleep(0.05)
wiringpi.softToneWrite(23, 1000)
sleep(0.05)
wiringpi.softToneWrite(23, 0)
sleep(0.05)
wiringpi.softToneWrite(23, 1500)
sleep(0.5)
wiringpi.softToneWrite(23, 0)

if app_args.development:
  # Development mode
  url = devUrl
else:
  print 'Development (d) or production (p)?'
  rlist, _, _ = select([sys.stdin], [], [], timeout)
  if rlist:
    answer = sys.stdin.readline()
    if answer[0] == 'd':
      url = devUrl
  else:
    print "No input. Defaulting to production."

if app_args.lookup:
  # Lookup mode
  url += 'lookup'
else:
  print 'Lookup (l) or checkin (c)?'
  rlist, _, _ = select([sys.stdin], [], [], timeout)
  if rlist:
    answer = sys.stdin.readline()
    if answer[0] == 'l':
      url += 'lookup'
    else:
      url += 'checkin'
  else:
    url += 'checkin'
    print "No input. Defaulting to checkin."

print 'URL: ' + url

## NFC reader code

readerstates = []
for i in xrange(len(readers)):
    readerstates += [ (readers[i], SCARD_STATE_UNAWARE) ]
hresult, newstates = SCardGetStatusChange(hcontext, 0, readerstates)

while True:
  hresult, newstates = SCardGetStatusChange(hcontext, 5000, newstates)
  for reader, eventstate, atr in newstates:
    if eventstate & SCARD_STATE_PRESENT:
      print 'Card found'
      hresult, hcard, dwActiveProtocol = SCardConnect(
      hcontext,
      reader,
      SCARD_SHARE_SHARED,
      SCARD_PROTOCOL_T0 | SCARD_PROTOCOL_T1)
      hresult, reader, state, protocol, atr = SCardStatus(hcard)
      print 'ATR:', hexarray(atr)
      hresult, response = SCardTransmit(hcard,dwActiveProtocol,[0xFF,0xCA,0x00,0x00,0x00])
      if response[-2:] == [0x90,0x00]:
        # Last two bytes 90 & 00 means success!
        # Remove them before printing the ID.
        id = response[:-2]
        print 'ID:', hexarray(id)
        # POST the card data
        try:
          data = urllib.urlencode({'atr' : b64array(atr), 'id' : b64array(response)})
          content = urllib2.urlopen(url, data).read()
          print content
          if hasWiringPi and content == "null":
            # Play bad sound
            wiringpi.softToneWrite(23, 2000)
            sleep(0.5)
            wiringpi.softToneWrite(23, 1000)
            sleep(0.5)
            wiringpi.softToneWrite(23, 0)
          else:
            # Play good sound
            for x in xrange(2000, 3000, 100):
              wiringpi.softToneWrite(23, x)
              sleep(0.05)
            for x in xrange(3000, 2000, -100):
              wiringpi.softToneWrite(23, x)
              sleep(0.05)
            wiringpi.softToneWrite(23, 0)
        except Exception, e:
          print e
          # Play bad sound
          wiringpi.softToneWrite(23, 2000)
          sleep(0.5)
          wiringpi.softToneWrite(23, 1000)
          sleep(0.5)
          wiringpi.softToneWrite(23, 750)
          sleep(0.5)
          wiringpi.softToneWrite(23, 0)
        else:
          pass
      else:
        # Unsuccessful read.
        print 'ID: error! Response: ', hexarray(response)
        print

    # elif eventstate & SCARD_STATE_EMPTY:
    	# Reader is empty, but commenting printing that to stop spamming on loop
    	# print 'Reader empty\n'

    # else:
      # Ignoring other event states too.
      # print 'Unknown event state', eventstate
<commit_msg>Print messages to syslog as well as the screen.<commit_after>#######################################################
## Key Master - Hackerspace Adelaide NFC Door reader ##
#######################################################

# scard documentation:
# http://pyscard.sourceforge.net/epydoc/smartcard.scard.scard-module.html

# Post successful card scan to hackadl.org

# When running on a Raspberri Pi, you need to install these:
# $ sudo apt-get install python-pyscard
# $ sudo apt-get install pcscd
# $ sudo apt-get install pcsc-tools

# If the reader isn't showing up, solve it using this:
# http://enjoy-rfid.blogspot.com.au/2015/03/raspberry-pi-nfc.html
# Which gets you to create a blacklist file:
# /etc/modprobe.d/raspi-blacklist.conf
# That just has:
# blacklist pn533
# blacklist nfc

# For a hacky way to make this run at boot on the RaspberryPi,
# running this from /etc/rc.local

import urllib
import urllib2
import base64
import sys
import argparse
import syslog

# Tag logs to syslog with keymaster
syslog.openlog('keymaster')

from select import select
from smartcard.scard import *

import imp
try:
  imp.find_module('wiringpi2')
  hasWiringPi = True
except ImportError:
  hasWiringPi = False

if hasWiringPi:
  # For GPIO pin control
  import wiringpi2 as wiringpi
  from time import sleep
  wiringpi.wiringPiSetupGpio()
  # Connect the pieso buzzer to GPIO 23
  # Ground to the third pin down from the top on the right column.
  # Power to the eighth pin down from the top on the right column.
  wiringpi.softToneCreate(23)

def hexarray(array):
  return ":".join(["{:02x}".format(b) for b in array])

def b64array(array):
  return base64.b64encode("".join([chr(b) for b in array]))

def printToScreenAndSyslog(message)
  print(message)
  syslog.syslog(message)

hresult, hcontext = SCardEstablishContext(SCARD_SCOPE_USER)

assert hresult==SCARD_S_SUCCESS

hresult, readers = SCardListReaders(hcontext, [])

assert len(readers)>0

reader = readers[0]
timeout = 10 # Timeout when there isn't any input
url = 'https://members.hackerspace-adelaide.org.au/'
devUrl = 'http://localhost:3000/'

# Parse arguments handed in when running

parser = argparse.ArgumentParser()
parser.add_argument("-d", "--development", help="Run in development environment talking to localhost:3000.", action="store_true")
parser.add_argument("-l", "--lookup", help="Run in lookup mode, only for admins.", action="store_true")
app_args = parser.parse_args()

# Welcome message

printToScreenAndSyslog('\n###### Hackadl.org ######')
printToScreenAndSyslog('######  Oh hello.  ######\n')

# Welcome song

wiringpi.softToneWrite(23, 1000)
sleep(0.05)
wiringpi.softToneWrite(23, 0)
sleep(0.05)
wiringpi.softToneWrite(23, 1000)
sleep(0.05)
wiringpi.softToneWrite(23, 0)
sleep(0.05)
wiringpi.softToneWrite(23, 1000)
sleep(0.05)
wiringpi.softToneWrite(23, 0)
sleep(0.05)
wiringpi.softToneWrite(23, 1500)
sleep(0.5)
wiringpi.softToneWrite(23, 0)

if app_args.development:
  # Development mode
  url = devUrl
else:
  printToScreenAndSyslog('Development (d) or production (p)?')
  rlist, _, _ = select([sys.stdin], [], [], timeout)
  if rlist:
    answer = sys.stdin.readline()
    if answer[0] == 'd':
      url = devUrl
  else:
    printToScreenAndSyslog("No input. Defaulting to production.")

if app_args.lookup:
  # Lookup mode
  url += 'lookup'
else:
  printToScreenAndSyslog('Lookup (l) or checkin (c)?')
  rlist, _, _ = select([sys.stdin], [], [], timeout)
  if rlist:
    answer = sys.stdin.readline()
    if answer[0] == 'l':
      url += 'lookup'
    else:
      url += 'checkin'
  else:
    url += 'checkin'
    printToScreenAndSyslog("No input. Defaulting to checkin.")

printToScreenAndSyslog('URL: ' + url)

## NFC reader code

readerstates = []
for i in xrange(len(readers)):
    readerstates += [ (readers[i], SCARD_STATE_UNAWARE) ]
hresult, newstates = SCardGetStatusChange(hcontext, 0, readerstates)

while True:
  hresult, newstates = SCardGetStatusChange(hcontext, 5000, newstates)
  for reader, eventstate, atr in newstates:
    if eventstate & SCARD_STATE_PRESENT:
      printToScreenAndSyslog('Card found')
      hresult, hcard, dwActiveProtocol = SCardConnect(
      hcontext,
      reader,
      SCARD_SHARE_SHARED,
      SCARD_PROTOCOL_T0 | SCARD_PROTOCOL_T1)
      hresult, reader, state, protocol, atr = SCardStatus(hcard)
      printToScreenAndSyslog('ATR:', hexarray(atr))
      hresult, response = SCardTransmit(hcard,dwActiveProtocol,[0xFF,0xCA,0x00,0x00,0x00])
      if response[-2:] == [0x90,0x00]:
        # Last two bytes 90 & 00 means success!
        # Remove them before printing the ID.
        id = response[:-2]
        printToScreenAndSyslog('ID:', hexarray(id))
        # POST the card data
        try:
          data = urllib.urlencode({'atr' : b64array(atr), 'id' : b64array(response)})
          content = urllib2.urlopen(url, data).read()
          printToScreenAndSyslog(content)
          if hasWiringPi and content == "null":
            # Play bad sound
            wiringpi.softToneWrite(23, 2000)
            sleep(0.5)
            wiringpi.softToneWrite(23, 1000)
            sleep(0.5)
            wiringpi.softToneWrite(23, 0)
          else:
            # Play good sound
            for x in xrange(2000, 3000, 100):
              wiringpi.softToneWrite(23, x)
              sleep(0.05)
            for x in xrange(3000, 2000, -100):
              wiringpi.softToneWrite(23, x)
              sleep(0.05)
            wiringpi.softToneWrite(23, 0)
        except Exception, e:
          printToScreenAndSyslog(e)
          # Play bad sound
          wiringpi.softToneWrite(23, 2000)
          sleep(0.5)
          wiringpi.softToneWrite(23, 1000)
          sleep(0.5)
          wiringpi.softToneWrite(23, 750)
          sleep(0.5)
          wiringpi.softToneWrite(23, 0)
        else:
          pass
      else:
        # Unsuccessful read.
        printToScreenAndSyslog('ID: error! Response: ', hexarray(response))
        printToScreenAndSyslog()
    # elif eventstate & SCARD_STATE_EMPTY:
    	# Reader is empty, but commenting printing that to stop spamming on loop
    	# print 'Reader empty\n'

    # else:
      # Ignoring other event states too.
      # print 'Unknown event state', eventstate
<|endoftext|>


<commit_before>from gevent import monkey; monkey.patch_all()

import gevent

from gevent.queue import Queue
from gevent.timeout import Timeout

from random import sample
import urllib2
import logging

class DefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
    def http_error_default(self, req, fp, code, msg, headers):
        result = urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
        result.status = code
        return result

class HttpWhoHas(object):

    def __init__(self, per_cluster=3, user_agent='HttpWhoHas.py', proxy=None, timeout=5):
        self.clusters = {}
        self.per_cluster = per_cluster
        self.user_agent = user_agent
        self.timeout = timeout

        if proxy:
            urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler({'http': proxy})))

        urllib2.install_opener(urllib2.build_opener(DefaultErrorHandler()))

        self.logger = logging.getLogger('httpwhohas')

    def set_cluster(self, name, ips, headers=None):
        self.clusters[name] = {
            'ips': ips,
            'headers': headers if headers else {},
            'per_cluster': min(self.per_cluster, len(ips)),
        }

    def _do_req(self, name, req, res):
        full_url = req.get_full_url()
        try:
            resp = urllib2.urlopen(req, timeout=self.timeout)
            status_code = resp.code
            if status_code in (200, 304) and not hasattr(req, 'redirect_dict'):
                host = req.get_header('Host')
                modified = status_code == 200
                self.logger.debug('found url=%s filer=%s host=%s modified=%s', full_url, name, host, modified)
                res.put({
                        'filer': name,
                        'url': full_url,
                        'host': host,
                        'modified': modified,
                        'headers': dict(resp.headers),
                })
            else:
                self.logger.debug('%s url=%s returned code %d', name, full_url, status_code)
        except (urllib2.HTTPError, urllib2.URLError) as exc:
            self.logger.debug('%s url=%s error: %s', name, full_url, exc)
        except Exception as exc:
            self.logger.exception('%s url=%s got an exception', name, full_url)

    def _do_reqs(self, reqs, res):
        jobs = [gevent.spawn(self._do_req, name, req, res) for name, req in reqs]
        gevent.joinall(jobs, timeout=self.timeout)
        res.put(None)
        gevent.killall(jobs)

    def resolve(self, filename, etag=None, last_modified=None):
        self.logger.debug('resolving %s', filename)
        res = Queue()
        reqs = []
        for name, info in self.clusters.items():
            headers = {'User-Agent': self.user_agent}
            headers.update(info['headers'])
            if etag:
                headers['If-None-Match'] = etag
            if last_modified:
                headers['If-modified-since'] = last_modified
            for ip in sample(info['ips'], info['per_cluster']):
                self.logger.debug('looking for %s on %s with headers %s', filename, ip, headers)
                req = urllib2.Request('http://%s%s' % (ip, filename), headers=headers)
                req.get_method = lambda: 'HEAD'
                reqs.append((name, req))

        gevent.spawn(self._do_reqs, reqs, res)
        result = res.get()
        if result:
            self.logger.debug('found %s on %s: url=%s host=%s', filename, result['filer'], result['url'], result['host'])
        else:
            self.logger.debug('%s not found', filename)
        return result
<commit_msg>use etag if present, then fallback to last-modified<commit_after>from gevent import monkey; monkey.patch_all()

import gevent

from gevent.queue import Queue
from gevent.timeout import Timeout

from random import sample
import urllib2
import logging

class DefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
    def http_error_default(self, req, fp, code, msg, headers):
        result = urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
        result.status = code
        return result

class HttpWhoHas(object):

    def __init__(self, per_cluster=3, user_agent='HttpWhoHas.py', proxy=None, timeout=5):
        self.clusters = {}
        self.per_cluster = per_cluster
        self.user_agent = user_agent
        self.timeout = timeout

        if proxy:
            urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler({'http': proxy})))

        urllib2.install_opener(urllib2.build_opener(DefaultErrorHandler()))

        self.logger = logging.getLogger('httpwhohas')

    def set_cluster(self, name, ips, headers=None):
        self.clusters[name] = {
            'ips': ips,
            'headers': headers if headers else {},
            'per_cluster': min(self.per_cluster, len(ips)),
        }

    def _do_req(self, name, req, res):
        full_url = req.get_full_url()
        try:
            resp = urllib2.urlopen(req, timeout=self.timeout)
            status_code = resp.code
            if status_code in (200, 304) and not hasattr(req, 'redirect_dict'):
                host = req.get_header('Host')
                modified = status_code == 200
                self.logger.debug('found url=%s filer=%s host=%s modified=%s', full_url, name, host, modified)
                res.put({
                        'filer': name,
                        'url': full_url,
                        'host': host,
                        'modified': modified,
                        'headers': dict(resp.headers),
                })
            else:
                self.logger.debug('%s url=%s returned code %d', name, full_url, status_code)
        except (urllib2.HTTPError, urllib2.URLError) as exc:
            self.logger.debug('%s url=%s error: %s', name, full_url, exc)
        except Exception as exc:
            self.logger.exception('%s url=%s got an exception', name, full_url)

    def _do_reqs(self, reqs, res):
        jobs = [gevent.spawn(self._do_req, name, req, res) for name, req in reqs]
        gevent.joinall(jobs, timeout=self.timeout)
        res.put(None)
        gevent.killall(jobs)

    def resolve(self, filename, etag=None, last_modified=None):
        self.logger.debug('resolving %s', filename)
        res = Queue()
        reqs = []
        for name, info in self.clusters.items():
            headers = {'User-Agent': self.user_agent}
            headers.update(info['headers'])
            if etag:
                headers['If-None-Match'] = etag
            elif last_modified:
                headers['If-Modified-Since'] = last_modified
            for ip in sample(info['ips'], info['per_cluster']):
                self.logger.debug('looking for %s on %s with headers %s', filename, ip, headers)
                req = urllib2.Request('http://%s%s' % (ip, filename), headers=headers)
                req.get_method = lambda: 'HEAD'
                reqs.append((name, req))

        gevent.spawn(self._do_reqs, reqs, res)
        result = res.get()
        if result:
            self.logger.debug('found %s on %s: url=%s host=%s', filename, result['filer'], result['url'], result['host'])
        else:
            self.logger.debug('%s not found', filename)
        return result
<|endoftext|>


<commit_before>"""
This is a monkeypatch for the django user model to allow longer usernames up to 255 characters in length.

It patches:
    django.contrib.auth.models.User.username
    django.contrib.auth.forms: AuthenticationForm, UserCreationForm, UserChangeForm
"""
from django.utils.translation import ugettext_lazy as _
from django.contrib.auth.models import User
from django.db.models.signals import pre_save
from django.core.validators import validate_email

User._meta.get_field('username').verbose_name = _('email')
User._meta.get_field('username').max_length = 255
User._meta.get_field('username').help_text = _("Required. 255 characters or fewer. Letters, numbers and @/./+/-/_ characters")
User._meta.get_field('username').validators[0].limit_value = 255

User._meta.get_field('email').max_length = 255
User._meta.get_field('email').validators[0].limit_value = 255

from django.contrib.auth.forms import AuthenticationForm

AuthenticationForm.base_fields['username'].max_length = 255  # I guess not needed
AuthenticationForm.base_fields['username'].widget.attrs['maxlength'] = 255  # html
AuthenticationForm.base_fields['username'].validators[0].limit_value = 255

from django.contrib.auth.forms import UserCreationForm

UserCreationForm.base_fields['username'].label = _('Email')
UserCreationForm.base_fields['username'].max_length = 255
UserCreationForm.base_fields['username'].widget.attrs['maxlength'] = 255  # html
UserCreationForm.base_fields['username'].validators[0].limit_value = 255
UserCreationForm.base_fields['username'].validators[1] = validate_email
UserCreationForm.base_fields['username'].error_messages['invalid'] = _(u'Enter a valid e-mail address.')
UserCreationForm.base_fields['username'].help_text = _("Required. 255 characters or fewer. Must be a valid email address")

from django.contrib.auth.forms import UserChangeForm

UserChangeForm.base_fields['username'].label = _('Email')
UserChangeForm.base_fields['username'].max_length = 255
UserChangeForm.base_fields['username'].widget.attrs['maxlength'] = 255  # html
UserChangeForm.base_fields['username'].validators[0].limit_value = 255
UserChangeForm.base_fields['username'].validators[1] = validate_email
UserChangeForm.base_fields['username'].error_messages['invalid'] = _(u'Enter a valid e-mail address.')
UserChangeForm.base_fields['username'].help_text = _("Required. 255 characters or fewer. Must be a valid email address")

old_init = UserChangeForm.__init__
def new_init(self, *args, **kwargs):
    if args:
        data = args[0]
    else:
        data = {}
    ret = old_init(self, *args, **kwargs)
    self.fields['email'].label = 'Username'
    self.fields['email'].max_length = 255
    self.fields['email'].validators[0].limit_value = 255
    self.fields['email'].help_text = _("This will be set to the email")
    self.fields['email'].widget.attrs['maxlength'] = 254  # html
    self.fields['email'].widget.attrs['readonly'] = 'readonly'
    self.fields['email'].widget.attrs['disabled'] = 'disabled'

    if data:
        data[self['email'].name] = data[self['username'].name]
    return ret

UserChangeForm.__init__ = new_init

old_clean = UserChangeForm.clean
def new_clean(self, *args, **kwargs):
    ret = old_clean(self, *args, **kwargs)
    if 'email' in self._errors:
        del self._errors['email']
    return ret

UserChangeForm.clean = new_clean

def copy_username_to_email(sender, instance, *args, **kwargs):
    """
    We remove the ``email`` field above, and this method "brings it back",
    by copying the username field to the email.
    """
    instance.email = instance.username

pre_save.connect(copy_username_to_email, User)
<commit_msg>added error correction for fix user UserCreationForm init method<commit_after>"""
This is a monkeypatch for the django user model to allow longer usernames up to 255 characters in length.

It patches:
    django.contrib.auth.models.User.username
    django.contrib.auth.forms: AuthenticationForm, UserCreationForm, UserChangeForm
"""
from django.utils.translation import ugettext_lazy as _
from django.contrib.auth.models import User
from django.db.models.signals import pre_save
from django.core.validators import validate_email

User._meta.get_field('username').verbose_name = _('email')
User._meta.get_field('username').max_length = 255
User._meta.get_field('username').help_text = _("Required. 255 characters or fewer. Letters, numbers and @/./+/-/_ characters")
User._meta.get_field('username').validators[0].limit_value = 255

User._meta.get_field('email').max_length = 255
User._meta.get_field('email').validators[0].limit_value = 255

from django.contrib.auth.forms import AuthenticationForm

AuthenticationForm.base_fields['username'].max_length = 255  # I guess not needed
AuthenticationForm.base_fields['username'].widget.attrs['maxlength'] = 255  # html
AuthenticationForm.base_fields['username'].validators[0].limit_value = 255

from django.contrib.auth.forms import UserCreationForm

UserCreationForm.base_fields['username'].label = _('Email')
UserCreationForm.base_fields['username'].max_length = 255
UserCreationForm.base_fields['username'].widget.attrs['maxlength'] = 255  # html
UserCreationForm.base_fields['username'].validators[0].limit_value = 255
UserCreationForm.base_fields['username'].validators[1] = validate_email
UserCreationForm.base_fields['username'].error_messages['invalid'] = _(u'Enter a valid e-mail address.')
UserCreationForm.base_fields['username'].help_text = _("Required. 255 characters or fewer. Must be a valid email address")

from django.contrib.auth.forms import UserChangeForm

UserChangeForm.base_fields['username'].label = _('Email')
UserChangeForm.base_fields['username'].max_length = 255
UserChangeForm.base_fields['username'].widget.attrs['maxlength'] = 255  # html
UserChangeForm.base_fields['username'].validators[0].limit_value = 255
UserChangeForm.base_fields['username'].validators[1] = validate_email
UserChangeForm.base_fields['username'].error_messages['invalid'] = _(u'Enter a valid e-mail address.')
UserChangeForm.base_fields['username'].help_text = _("Required. 255 characters or fewer. Must be a valid email address")

old_init = UserChangeForm.__init__
def new_init(self, *args, **kwargs):
    if args:
        data = args[0]
    else:
        data = {}
    ret = old_init(self, *args, **kwargs)
    try:
        self.fields['email'].label = 'Username'
        self.fields['email'].max_length = 255
        self.fields['email'].validators[0].limit_value = 255
        self.fields['email'].help_text = _("This will be set to the email")
        self.fields['email'].widget.attrs['maxlength'] = 254  # html
        self.fields['email'].widget.attrs['readonly'] = 'readonly'
        self.fields['email'].widget.attrs['disabled'] = 'disabled'

        if data:
            data[self['email'].name] = data[self['username'].name]
    except KeyError:
        pass
    return ret

UserChangeForm.__init__ = new_init

old_clean = UserChangeForm.clean
def new_clean(self, *args, **kwargs):
    ret = old_clean(self, *args, **kwargs)
    if 'email' in self._errors:
        del self._errors['email']
    return ret

UserChangeForm.clean = new_clean

def copy_username_to_email(sender, instance, *args, **kwargs):
    """
    We remove the ``email`` field above, and this method "brings it back",
    by copying the username field to the email.
    """
    instance.email = instance.username

pre_save.connect(copy_username_to_email, User)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import time
import logging

import base64
try: import cPickle as pickle
except: import pickle

import datetime
from django.utils import timezone

import json
#from django.utils import simplejson

from pprint import pprint

from django.shortcuts import render_to_response
from django.http import HttpResponseRedirect
from django.template import RequestContext

from django.http import HttpResponseForbidden
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import redirect, render

logger = logging.getLogger(__name__)

import boto.ec2
import boto.ec2.cloudwatch
#from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType

from django.contrib.auth.models import User
from userprofile.models import Profile as userprofile

from amazon import s3_funcs
from amazon import s3_funcs_shortcuts

from cloud_storage.models import Files
from cloud_storage.models import Uploaded_Files

from django.template.defaultfilters import filesizeformat, upper
from django.contrib.humanize.templatetags.humanize import naturalday

from cloudly.templatetags.cloud_extras import clear_filename, get_file_extension

from cloud_vms.models import Cache

import decimal
from django.db.models.base import ModelState

def date_handler(obj):
	return obj.isoformat() if hasattr(obj, 'isoformat') else obj

def ajax_vms_refresh(request):

	user = request.user

	try:
		profile = userprofile.objects.get(user=request.user)
	except: return HttpResponseRedirect("/")

	print 'Refreshing', user, 'VMs cache..'

	aws_access_key = profile.aws_access_key
	aws_secret_key = profile.aws_secret_key
	aws_ec2_verified = profile.aws_ec2_verified

	aws_virtual_machines = {}

	if aws_ec2_verified:

		vms_cache = Cache.objects.get_or_create(user=user)
		vms_cache = vms_cache[0]

		vms_cache.is_updating = True
		vms_cache.save()

		aws_regions = profile.aws_enabled_regions.split(',')
		print 'AWS regions', aws_regions

		for ec2_region in aws_regions:

			if(ec2_region):

				ec2conn = boto.ec2.connect_to_region(ec2_region,aws_access_key_id=aws_access_key,aws_secret_access_key=aws_secret_key)
				cloudwatch = boto.ec2.cloudwatch.connect_to_region(ec2_region,aws_access_key_id=aws_access_key,aws_secret_access_key=aws_secret_key)

				try:
					reservations = ec2conn.get_all_instances()
				except:
					vms_cache.is_updating = False
					vms_cache.vms_response = ""
					vms_cache.save()
					print vms_cache.is_updating
					print vms_cache.vms_response
					return HttpResponse("access denied")

				instances = [i for r in reservations for i in r.instances]

				for instance in instances:

					if not instance: continue

					instance_metrics = {}

					print '-'*1000
					pprint(instance.__dict__)

					instance_metrics = {}
					instance_metrics['groups'] = {}
					instance_metrics['block_device_mapping'] = {}

					print '** instance', instance.id, instance.private_ip_address

					volumes = []
					for volume in ec2conn.get_all_volumes(filters={'attachment.instance-id': instance.id}):
						volumes.append([volume.id, volume.iops, volume.size,])

					groups = []
					for group in instance.__dict__['groups']:
						groups.append([group.id, group.name,])

					instance_metrics['groups'] = groups
					instance_metrics['block_device_mapping'] = volumes
					instance_metrics['architecture'] = instance.architecture
					instance_metrics['client_token'] = instance.client_token
					instance_metrics['dns_name'] = instance.dns_name
					instance_metrics['private_ip_address'] = instance.private_ip_address
					instance_metrics['hypervisor'] = instance.hypervisor
					instance_metrics['id'] = instance.id
					instance_metrics['image_id'] = instance.image_id
					instance_metrics['instance_type'] = instance.instance_type
					instance_metrics['ip_address'] = instance.ip_address
					instance_metrics['key_name'] = instance.key_name
					instance_metrics['launch_time'] = instance.launch_time
					instance_metrics['monitored'] = instance.monitored
					instance_metrics['persistent'] = instance.persistent
					instance_metrics['ramdisk'] = instance.ramdisk

					try:
						ec2conn.monitor_instance(str(instance.id))
					except:
						print instance.id, 'instance not in a monitorable state!'.upper()
						print instance.id, 'reason:', instance.state_reason['message']
						continue

					end = datetime.datetime.utcnow()
					start = end - datetime.timedelta(hours=1)

					# ['Minimum', 'Maximum', 'Sum', 'Average', 'SampleCount']
					# ['Seconds', 'Percent', 'Bytes', 'Bits', 'Count', 'Bytes/Second', 'Bits/Second', 'Count/Second']

					# CPUUtilization
					try:
						metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="CPUUtilization")[0]
					except: continue

					cpu_utilization_datapoints = metric.query(start, end, 'Average', 'Percent')
					#instance_metrics['cpu_utilization_datapoints'] = cpu_utilization_datapoints

					# DiskReadOps
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="DiskReadOps")[0]
					disk_readops_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['disk_readops_datapoints'] = disk_readops_datapoints

					# DiskWriteOps
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="DiskWriteOps")[0]
					disk_writeops_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['disk_writeops_datapoints'] = disk_writeops_datapoints

					# DiskReadBytes
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="DiskReadBytes")[0]
					disk_readbytes_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['disk_readbytes_datapoints'] = disk_readbytes_datapoints

					# DiskWriteBytes
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="DiskWriteBytes")[0]
					disk_writebytes_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['disk_writebytes_datapoints'] = disk_writebytes_datapoints

					# NetworkIn
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="NetworkIn")[0]
					networkin_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['networkin_datapoints'] = networkin_datapoints

					# NetworkOut
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="NetworkOut")[0]
					networkout_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['networkout_datapoints'] = networkout_datapoints

					aws_virtual_machines[instance.id] = instance_metrics

		print 'aws_virtual_machines'

		# XXX Note This is how this should be - but I get this fucked up error on the prod......
		# XXX Note Can't pickle <type '_hashlib.HASH'>: attribute lookup _hashlib.HASH failed
		# XXX Note Specifically, it's the Python Bug that "cannot be fixed" http://bugs.python.org/issue11771
		# XXX Note Quote: There is no way to implement generic pickling for hash objects that would work across all implementations.
		try:
			vms_cache.vms_response = base64.b64encode(pickle.dumps(aws_virtual_machines, pickle.HIGHEST_PROTOCOL))
		except:
			return HttpResponse("Fuck shit wank bugger Can't pickle <type '_hashlib.HASH'>: attribute lookup _hashlib.HASH failed problem..")

		print 'xxxxx', aws_virtual_machines

		vms_cache.last_seen = timezone.now()
		vms_cache.is_updating = False
		vms_cache.save()

		print 'VMs cache was succesfully updated.'

	return HttpResponse("ALLDONE")

def ajax_virtual_machines(request):

	print '-- ajax virtual machines'

	if not request.user.is_authenticated():
		print 'anonymous'
		return HttpResponseRedirect("/")

	print request.user

	return render_to_response('ajax_virtual_machines.html', locals(), context_instance=RequestContext(request))

<commit_msg>added on rest of the instance metrics parameters that arent objects<commit_after># -*- coding: utf-8 -*-

import os
import time
import logging

import base64
try: import cPickle as pickle
except: import pickle

import datetime
from django.utils import timezone

import json
#from django.utils import simplejson

from pprint import pprint

from django.shortcuts import render_to_response
from django.http import HttpResponseRedirect
from django.template import RequestContext

from django.http import HttpResponseForbidden
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import redirect, render

logger = logging.getLogger(__name__)

import boto.ec2
import boto.ec2.cloudwatch
#from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType

from django.contrib.auth.models import User
from userprofile.models import Profile as userprofile

from amazon import s3_funcs
from amazon import s3_funcs_shortcuts

from cloud_storage.models import Files
from cloud_storage.models import Uploaded_Files

from django.template.defaultfilters import filesizeformat, upper
from django.contrib.humanize.templatetags.humanize import naturalday

from cloudly.templatetags.cloud_extras import clear_filename, get_file_extension

from cloud_vms.models import Cache

import decimal
from django.db.models.base import ModelState

def date_handler(obj):
	return obj.isoformat() if hasattr(obj, 'isoformat') else obj

def ajax_vms_refresh(request):

	user = request.user

	try:
		profile = userprofile.objects.get(user=request.user)
	except: return HttpResponseRedirect("/")

	print 'Refreshing', user, 'VMs cache..'

	aws_access_key = profile.aws_access_key
	aws_secret_key = profile.aws_secret_key
	aws_ec2_verified = profile.aws_ec2_verified

	aws_virtual_machines = {}

	if aws_ec2_verified:

		vms_cache = Cache.objects.get_or_create(user=user)
		vms_cache = vms_cache[0]

		vms_cache.is_updating = True
		vms_cache.save()

		aws_regions = profile.aws_enabled_regions.split(',')
		print 'AWS regions', aws_regions

		for ec2_region in aws_regions:

			if(ec2_region):

				ec2conn = boto.ec2.connect_to_region(ec2_region,aws_access_key_id=aws_access_key,aws_secret_access_key=aws_secret_key)
				cloudwatch = boto.ec2.cloudwatch.connect_to_region(ec2_region,aws_access_key_id=aws_access_key,aws_secret_access_key=aws_secret_key)

				try:
					reservations = ec2conn.get_all_instances()
				except:
					vms_cache.is_updating = False
					vms_cache.vms_response = ""
					vms_cache.save()
					print vms_cache.is_updating
					print vms_cache.vms_response
					return HttpResponse("access denied")

				instances = [i for r in reservations for i in r.instances]

				for instance in instances:

					if not instance: continue

					instance_metrics = {}

					print '-'*1000
					pprint(instance.__dict__)

					instance_metrics = {}
					instance_metrics['groups'] = {}
					instance_metrics['block_device_mapping'] = {}

					print '** instance', instance.id, instance.private_ip_address

					volumes = []
					for volume in ec2conn.get_all_volumes(filters={'attachment.instance-id': instance.id}):
						volumes.append([volume.id, volume.iops, volume.size,])

					groups = []
					for group in instance.__dict__['groups']:
						groups.append([group.id, group.name,])

					instance_metrics['groups'] = groups
					instance_metrics['block_device_mapping'] = volumes
					instance_metrics['architecture'] = instance.architecture
					instance_metrics['client_token'] = instance.client_token
					instance_metrics['dns_name'] = instance.dns_name
					instance_metrics['private_ip_address'] = instance.private_ip_address
					instance_metrics['hypervisor'] = instance.hypervisor
					instance_metrics['id'] = instance.id
					instance_metrics['image_id'] = instance.image_id
					instance_metrics['instance_type'] = instance.instance_type
					instance_metrics['ip_address'] = instance.ip_address
					instance_metrics['key_name'] = instance.key_name
					instance_metrics['launch_time'] = instance.launch_time
					instance_metrics['monitored'] = instance.monitored
					instance_metrics['persistent'] = instance.persistent
					instance_metrics['ramdisk'] = instance.ramdisk
					instance_metrics['root_device_name'] = instance.root_device_name
					instance_metrics['root_device_type'] = instance.root_device_type
					instance_metrics['tags'] = instance.tags
					instance_metrics['virtualization_type'] = instance.virtualization_type
					instance_metrics['vpc_id'] = instance.vpc_id

					try:
						ec2conn.monitor_instance(str(instance.id))
					except:
						print instance.id, 'instance not in a monitorable state!'.upper()
						print instance.id, 'reason:', instance.state_reason['message']
						continue

					end = datetime.datetime.utcnow()
					start = end - datetime.timedelta(hours=1)

					# ['Minimum', 'Maximum', 'Sum', 'Average', 'SampleCount']
					# ['Seconds', 'Percent', 'Bytes', 'Bits', 'Count', 'Bytes/Second', 'Bits/Second', 'Count/Second']

					# CPUUtilization
					try:
						metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="CPUUtilization")[0]
					except: continue

					cpu_utilization_datapoints = metric.query(start, end, 'Average', 'Percent')
					#instance_metrics['cpu_utilization_datapoints'] = cpu_utilization_datapoints

					# DiskReadOps
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="DiskReadOps")[0]
					disk_readops_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['disk_readops_datapoints'] = disk_readops_datapoints

					# DiskWriteOps
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="DiskWriteOps")[0]
					disk_writeops_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['disk_writeops_datapoints'] = disk_writeops_datapoints

					# DiskReadBytes
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="DiskReadBytes")[0]
					disk_readbytes_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['disk_readbytes_datapoints'] = disk_readbytes_datapoints

					# DiskWriteBytes
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="DiskWriteBytes")[0]
					disk_writebytes_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['disk_writebytes_datapoints'] = disk_writebytes_datapoints

					# NetworkIn
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="NetworkIn")[0]
					networkin_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['networkin_datapoints'] = networkin_datapoints

					# NetworkOut
					metric = cloudwatch.list_metrics(dimensions={'InstanceId':instance.id}, metric_name="NetworkOut")[0]
					networkout_datapoints = metric.query(start, end, 'Average', '')
					#instance_metrics['networkout_datapoints'] = networkout_datapoints

					aws_virtual_machines[instance.id] = instance_metrics

		print 'aws_virtual_machines'

		# XXX Note This is how this should be - but I get this fucked up error on the prod......
		# XXX Note Can't pickle <type '_hashlib.HASH'>: attribute lookup _hashlib.HASH failed
		# XXX Note Specifically, it's the Python Bug that "cannot be fixed" http://bugs.python.org/issue11771
		# XXX Note Quote: There is no way to implement generic pickling for hash objects that would work across all implementations.
		try:
			vms_cache.vms_response = base64.b64encode(pickle.dumps(aws_virtual_machines, pickle.HIGHEST_PROTOCOL))
		except:
			return HttpResponse("Fuck shit wank bugger Can't pickle <type '_hashlib.HASH'>: attribute lookup _hashlib.HASH failed problem..")

		print 'xxxxx', aws_virtual_machines

		vms_cache.last_seen = timezone.now()
		vms_cache.is_updating = False
		vms_cache.save()

		print 'VMs cache was succesfully updated.'

	return HttpResponse("ALLDONE")

def ajax_virtual_machines(request):

	print '-- ajax virtual machines'

	if not request.user.is_authenticated():
		print 'anonymous'
		return HttpResponseRedirect("/")

	print request.user

	return render_to_response('ajax_virtual_machines.html', locals(), context_instance=RequestContext(request))

<|endoftext|>


<commit_before>from . import controllers
from flask import session, render_template, redirect, url_for, request, flash
from flask import Blueprint

home_bp = Blueprint('home', __name__, template_folder='templates')

@home_bp.route('/')
def index():
    return render_template('home.html', session=session)

@home_bp.route('/register', methods=['GET', 'POST'])
def register():
    """Register the user."""
    if session.get('user_id'):
        return redirect(url_for('home.index'))
    error = None
    if request.method == 'POST':
        error = controllers.signup(
            request.form['ushpa'],
            request.form['email'],
            request.form['password'],
            request.form['password2'],
            request.form['firstname'],
            request.form['lastname']
        )
        if not error:
            flash('You were successfully registered and can login now')
            return redirect(url_for('home.login'))
    return render_template('register.html', error=error)

@home_bp.route('/login', methods=['GET', 'POST'])
def login():
    """User login"""
    if session.get('user_id'):
        return redirect(url_for('home.index'))
    error = None
    if request.method == 'POST':
        user = controllers.authenticate_user(request.form['ushpa'], request.form['password'])
        if user:
            flash('Welcome back, %s' % user['firstname'])
            session['user_id'] = str(user['_id'])
            session['user_ushpa'] = user['ushpa']
            session['firstname'] = user['firstname']
            session['lastname'] = user['lastname']
            return redirect(url_for('home.index'))
        else:
            error = 'Invalid username or password'
    return render_template('login.html', error=error, session=session)

@home_bp.route('/logout')
def logout(msg='You were logged out'):
    """Logs the user out."""
    flash(msg)
    session.pop('user_id', None)
    return redirect(url_for('home.index'))
<commit_msg>clear all session data<commit_after>from . import controllers
from flask import session, render_template, redirect, url_for, request, flash
from flask import Blueprint

home_bp = Blueprint('home', __name__, template_folder='templates')

@home_bp.route('/')
def index():
    return render_template('home.html', session=session)

@home_bp.route('/register', methods=['GET', 'POST'])
def register():
    """Register the user."""
    if session.get('user_id'):
        return redirect(url_for('home.index'))
    error = None
    if request.method == 'POST':
        error = controllers.signup(
            request.form['ushpa'],
            request.form['email'],
            request.form['password'],
            request.form['password2'],
            request.form['firstname'],
            request.form['lastname']
        )
        if not error:
            flash('You were successfully registered and can login now')
            return redirect(url_for('home.login'))
    return render_template('register.html', error=error)

@home_bp.route('/login', methods=['GET', 'POST'])
def login():
    """User login"""
    if session.get('user_id'):
        return redirect(url_for('home.index'))
    error = None
    if request.method == 'POST':
        user = controllers.authenticate_user(request.form['ushpa'], request.form['password'])
        if user:
            flash('Welcome back, %s' % user['firstname'])
            session['user_id'] = str(user['_id'])
            session['user_ushpa'] = user['ushpa']
            session['firstname'] = user['firstname']
            session['lastname'] = user['lastname']
            return redirect(url_for('home.index'))
        else:
            error = 'Invalid username or password'
    return render_template('login.html', error=error, session=session)

@home_bp.route('/logout')
def logout(msg='You were logged out'):
    """Logs the user out."""
    flash(msg)
    session.clear()
    return redirect(url_for('home.index'))
<|endoftext|>