<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

from django import template
from django import http
from django.conf import settings
from django.contrib import messages
from django.shortcuts import redirect
from django.shortcuts import render_to_response
from django.contrib.auth.decorators import login_required
from glance.client import ClientConnectionError as GlanceClientConnectionError
from glance.common import exception as glance_exception

from django_openstack import api
from django_openstack import forms

class DeleteImage(forms.SelfHandlingForm):
    image_id = forms.CharField(required=True)

    def handle(self, request, data):
        image_id = data['image_id']
        try:
            api.image_delete(request, image_id)
        except GlanceClientConnectionError, e:
            messages.error(request, "Error connecting to glance: %s" % e.message)
        except glance_exception.Error, e:
            messages.error(request, "Error deleting image: %s" % e.message)
        return redirect(request.build_absolute_uri())

class ToggleImage(forms.SelfHandlingForm):
    image_id = forms.CharField(required=True)

    def handle(self, request, data):
        image_id = data['image_id']
        try:
            api.image_update(request, image_id, image_meta={'is_public': False})
        except GlanceClientConnectionError, e:
            messages.error(request, "Error connecting to glance: %s" % e.message)
        except glance_exception.Error, e:
            messages.error(request, "Error updating image: %s" % e.message)
        return redirect(request.build_absolute_uri())

class UpdateImageForm(forms.Form):
    name = forms.CharField(max_length="5", label="Name")
    kernel = forms.CharField(max_length="5", label="Kernel ID", required=False)
    ramdisk = forms.CharField(max_length="5", label="Ramdisk ID", required=False)
    architecture = forms.CharField(label="Architecture", required=False)
    #project_id = forms.CharField(label="Project ID")
    container_format = forms.CharField(label="Container Format", required=False)
    disk_format = forms.CharField(label="Disk Format")
    #is_public = forms.BooleanField(label="Publicly Available", required=False)

class UploadImageForm(forms.Form):
    name = forms.CharField(max_length="5", label="Name")
    image_file = forms.FileField(required=False)
    # is_public = forms.BooleanField(label="Publicly Available", required=False, initial=True)

@login_required
def index(request):
    for f in (DeleteImage, ToggleImage):
        _, handled = f.maybe_handle(request)
        if handled:
            return handled

    # We don't have any way of showing errors for these, so don't bother
    # trying to reuse the forms from above
    delete_form = DeleteImage()
    toggle_form = ToggleImage()

    images = []
    try:
        images = api.image_list_detailed(request)
        if not images:
            messages.info(request, "There are currently no images.")
    except GlanceClientConnectionError, e:
        messages.error(request, "Error connecting to glance: %s" % e.message)
    except glance_exception.Error, e:
        messages.error(request, "Error retrieving image list: %s" % e.message)

    return render_to_response('syspanel_images.html', {
        'delete_form': delete_form,
        'toggle_form': toggle_form,
        'images': images,
    }, context_instance = template.RequestContext(request))

@login_required
def update(request, image_id):
    try:
        image = api.image_get(request, image_id)
    except GlanceClientConnectionError, e:
        messages.error(request, "Error connecting to glance: %s" % e.message)
    except glance_exception.Error, e:
        messages.error(request, "Error retrieving image %s: %s" % (image_id, e.message))

    if request.method == "POST":
        form = UpdateImageForm(request.POST)
        if form.is_valid():
            image_form = form.clean()
            metadata = {
                'is_public': True,
                'disk_format': image_form['disk_format'],
                'container_format': image_form['container_format'],
                'name': image_form['name'],
            }
            try:
                # TODO add public flag to properties
                metadata['properties'] = {}
                if image_form['kernel']:
                    metadata['properties']['kernel_id'] = image_form['kernel']
                if image_form['ramdisk']:
                    metadata['properties']['ramdisk_id'] = image_form['ramdisk']
                if image_form['architecture']:
                    metadata['properties']['architecture'] = image_form['architecture']
                api.image_update(request, image_id, metadata)
                messages.success(request, "Image was successfully updated.")
            except GlanceClientConnectionError, e:
                messages.error(request, "Error connecting to glance: %s" % e.message)
            except glance_exception.Error, e:
                messages.error(request, "Error updating image: %s" % e.message)
            return redirect("syspanel_images")
        else:
            messages.error(request, "Image could not be uploaded, please try agian.")
            form = UpdateImageForm(request.POST)
            return render_to_response('syspanel_image_update.html',{
                'image': image,
                'form': form,
            }, context_instance = template.RequestContext(request))
    else:
        form = UpdateImageForm(initial={
                'name': image.get('name', ''),
                'kernel': image['properties'].get('kernel_id', ''),
                'ramdisk': image['properties'].get('ramdisk_id', ''),
                'is_public': image.get('is_public', ''),
                'location': image.get('location', ''),
                'state': image['properties'].get('image_state', ''),
                'architecture': image['properties'].get('architecture', ''),
                'project_id': image['properties'].get('project_id', ''),
                'container_format': image.get('container_format', ''),
                'disk_format': image.get('disk_format', ''),
            })

        return render_to_response('syspanel_image_update.html',{
            'image': image,
            'form': form,
        }, context_instance = template.RequestContext(request))

@login_required
def upload(request):
    if request.method == "POST":
        form = UploadImageForm(request.POST)
        if form.is_valid():
            image = form.clean()
            metadata = {'is_public': True,
                        'disk_format': 'ami',
                        'container_format': 'ami',
                        'name': image.get('name', None)}
            try:
                api.image_create(request, metadata, image['image_file'])
            except GlanceClientConnectionError, e:
                messages.error(request, "Error connecting to glance: %s" % e.message)
            except glance_exception.Error, e:
                messages.error(request, "Error adding image: %s" % e.message)
        else:
            messages.error(request, "Image could not be uploaded, please try agian.")
            form = UploadImageForm(request.POST)
            return render_to_response('syspanel_image_upload.html',{
                'form': form,
            }, context_instance = template.RequestContext(request))

        return redirect('syspanel_images')
    else:
        form = UploadImageForm()
        return render_to_response('syspanel_image_upload.html',{
            'form': form,
        }, context_instance = template.RequestContext(request))
<commit_msg>increase number of characters for image names<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

from django import template
from django import http
from django.conf import settings
from django.contrib import messages
from django.shortcuts import redirect
from django.shortcuts import render_to_response
from django.contrib.auth.decorators import login_required
from glance.client import ClientConnectionError as GlanceClientConnectionError
from glance.common import exception as glance_exception

from django_openstack import api
from django_openstack import forms

class DeleteImage(forms.SelfHandlingForm):
    image_id = forms.CharField(required=True)

    def handle(self, request, data):
        image_id = data['image_id']
        try:
            api.image_delete(request, image_id)
        except GlanceClientConnectionError, e:
            messages.error(request, "Error connecting to glance: %s" % e.message)
        except glance_exception.Error, e:
            messages.error(request, "Error deleting image: %s" % e.message)
        return redirect(request.build_absolute_uri())

class ToggleImage(forms.SelfHandlingForm):
    image_id = forms.CharField(required=True)

    def handle(self, request, data):
        image_id = data['image_id']
        try:
            api.image_update(request, image_id, image_meta={'is_public': False})
        except GlanceClientConnectionError, e:
            messages.error(request, "Error connecting to glance: %s" % e.message)
        except glance_exception.Error, e:
            messages.error(request, "Error updating image: %s" % e.message)
        return redirect(request.build_absolute_uri())

class UpdateImageForm(forms.Form):
    name = forms.CharField(max_length="25", label="Name")
    kernel = forms.CharField(max_length="25", label="Kernel ID", required=False)
    ramdisk = forms.CharField(max_length="25", label="Ramdisk ID", required=False)
    architecture = forms.CharField(label="Architecture", required=False)
    #project_id = forms.CharField(label="Project ID")
    container_format = forms.CharField(label="Container Format", required=False)
    disk_format = forms.CharField(label="Disk Format")
    #is_public = forms.BooleanField(label="Publicly Available", required=False)

class UploadImageForm(forms.Form):
    name = forms.CharField(max_length="25", label="Name")
    image_file = forms.FileField(required=False)
    # is_public = forms.BooleanField(label="Publicly Available", required=False, initial=True)

@login_required
def index(request):
    for f in (DeleteImage, ToggleImage):
        _, handled = f.maybe_handle(request)
        if handled:
            return handled

    # We don't have any way of showing errors for these, so don't bother
    # trying to reuse the forms from above
    delete_form = DeleteImage()
    toggle_form = ToggleImage()

    images = []
    try:
        images = api.image_list_detailed(request)
        if not images:
            messages.info(request, "There are currently no images.")
    except GlanceClientConnectionError, e:
        messages.error(request, "Error connecting to glance: %s" % e.message)
    except glance_exception.Error, e:
        messages.error(request, "Error retrieving image list: %s" % e.message)

    return render_to_response('syspanel_images.html', {
        'delete_form': delete_form,
        'toggle_form': toggle_form,
        'images': images,
    }, context_instance = template.RequestContext(request))

@login_required
def update(request, image_id):
    try:
        image = api.image_get(request, image_id)
    except GlanceClientConnectionError, e:
        messages.error(request, "Error connecting to glance: %s" % e.message)
    except glance_exception.Error, e:
        messages.error(request, "Error retrieving image %s: %s" % (image_id, e.message))

    if request.method == "POST":
        form = UpdateImageForm(request.POST)
        if form.is_valid():
            image_form = form.clean()
            metadata = {
                'is_public': True,
                'disk_format': image_form['disk_format'],
                'container_format': image_form['container_format'],
                'name': image_form['name'],
            }
            try:
                # TODO add public flag to properties
                metadata['properties'] = {}
                if image_form['kernel']:
                    metadata['properties']['kernel_id'] = image_form['kernel']
                if image_form['ramdisk']:
                    metadata['properties']['ramdisk_id'] = image_form['ramdisk']
                if image_form['architecture']:
                    metadata['properties']['architecture'] = image_form['architecture']
                api.image_update(request, image_id, metadata)
                messages.success(request, "Image was successfully updated.")
            except GlanceClientConnectionError, e:
                messages.error(request, "Error connecting to glance: %s" % e.message)
            except glance_exception.Error, e:
                messages.error(request, "Error updating image: %s" % e.message)
            return redirect("syspanel_images")
        else:
            messages.error(request, "Image could not be uploaded, please try agian.")
            form = UpdateImageForm(request.POST)
            return render_to_response('syspanel_image_update.html',{
                'image': image,
                'form': form,
            }, context_instance = template.RequestContext(request))
    else:
        form = UpdateImageForm(initial={
                'name': image.get('name', ''),
                'kernel': image['properties'].get('kernel_id', ''),
                'ramdisk': image['properties'].get('ramdisk_id', ''),
                'is_public': image.get('is_public', ''),
                'location': image.get('location', ''),
                'state': image['properties'].get('image_state', ''),
                'architecture': image['properties'].get('architecture', ''),
                'project_id': image['properties'].get('project_id', ''),
                'container_format': image.get('container_format', ''),
                'disk_format': image.get('disk_format', ''),
            })

        return render_to_response('syspanel_image_update.html',{
            'image': image,
            'form': form,
        }, context_instance = template.RequestContext(request))

@login_required
def upload(request):
    if request.method == "POST":
        form = UploadImageForm(request.POST)
        if form.is_valid():
            image = form.clean()
            metadata = {'is_public': True,
                        'disk_format': 'ami',
                        'container_format': 'ami',
                        'name': image.get('name', None)}
            try:
                api.image_create(request, metadata, image['image_file'])
            except GlanceClientConnectionError, e:
                messages.error(request, "Error connecting to glance: %s" % e.message)
            except glance_exception.Error, e:
                messages.error(request, "Error adding image: %s" % e.message)
        else:
            messages.error(request, "Image could not be uploaded, please try agian.")
            form = UploadImageForm(request.POST)
            return render_to_response('syspanel_image_upload.html',{
                'form': form,
            }, context_instance = template.RequestContext(request))

        return redirect('syspanel_images')
    else:
        form = UploadImageForm()
        return render_to_response('syspanel_image_upload.html',{
            'form': form,
        }, context_instance = template.RequestContext(request))
<|endoftext|>


<commit_before>import os
import logging
import yaml

from imp import load_source
from urllib import urlretrieve
from uuid import uuid4

from eeagent.util import unmake_id
from pidantic.pyon.persistence import PyonDataObject, PyonProcDataObject
from pidantic.pidantic_exceptions import PIDanticUsageException, PIDanticExecutionException

class Pyon(object):

    cols = ['directory', 'pyon_name', 'module', 'module_uri', 'cls', 'process_name', 'config',
    ]

    def __init__(self, pyon_db, pyon_container=None, proc=None, name=None,
            data_object=None, dirpath=None, log=logging):
        if data_object is None and name is None:
            raise PIDanticUsageException("You must specify a socket")

        self._log = log
        self._pyon_db = pyon_db
        if data_object:
            self._data_object = data_object
        else:
            if not dirpath:
                msg = "You must set the dirpath keyword argument"
                raise PIDanticUsageException(msg)
            self._working_dir = os.path.join(dirpath, name)
            try:
                os.makedirs(self._working_dir)
            except Exception:
                msg = "The directory %s already exists" % (self._working_dir)
                self._log.warning(msg)

            data_object = PyonDataObject()
            data_object.name = name
            self._pyon_db.db_obj_add(data_object)
            self._pyon_db.db_commit()
            self._data_object = data_object

        self._container = pyon_container

    def get_process_status(self, name):
        # TODO: this will probably change once pyon migrates away from fail
        # fast

        pyon_id = self._get_pyon_process_id(name)
        proc = self._container.proc_manager.get(pyon_id)
        if proc:
            status = proc.running
        else:
            status = None
        return status

    def create_process_db(self, **kwargs):

        process_object = PyonProcDataObject()
        for key in kwargs:
            if key not in self.cols:
                raise PIDanticUsageException('invalid key %s' % (key))
            process_object.__setattr__(key, kwargs[key])
        if not process_object.directory:
            process_object.directory = self._working_dir
        self._pyon_db.db_obj_add(process_object)
        data_object = self._data_object
        data_object.processes.append(process_object)
        self._pyon_db.db_commit()

        return process_object

    def run_process(self, process_object):

        try:
            config = yaml.load(process_object.config)
        except AttributeError:
            config = None

        if process_object.module_uri is not None:
            module_file = self.download_module(process_object.module_uri)
            module = self.load_module(module_file, process_object.module)
            process_object.module = module
            self._pyon_db.db_commit()

        pyon_id = self._container.spawn_process(name=process_object.pyon_name,
                module=process_object.module, cls=process_object.cls,
                config=config)
        process_object.pyon_process_id = pyon_id
        self._pyon_db.db_commit()
        return pyon_id

    def download_module(self, module_uri):
        try:
            module_file, _ = urlretrieve(module_uri)
        except:
            msg = "Unable to download code module %s" % module_uri
            self._log.exception(msg)
            raise PIDanticExecutionException(msg)
        return module_file

    def load_module(self, module_file, module):
        module_name = self._unique_module_name(module)
        try:
            load_source(module_name, module_file)
        except:
            #TODO throw right exception
            msg = "Unable to load code module %s" % module
            self._log.exception(msg)
            raise PIDanticExecutionException(msg)

        return module_name

    def _unique_module_name(self, module=None):

        unique = uuid4().hex
        if module:
            unique = ".".join([unique, module])
            unique = unique.replace(".", "_")

        return unique

    def getState(self):
        return self._container._is_started

    def terminate_process(self, name):

        pyon_id = self._get_pyon_process_id(name)
        terminate_result = self._container.terminate_process(pyon_id)
        return terminate_result

    def _get_pyon_process_id(self, name):
        process_object = None
        data_object = self._data_object
        for p in data_object.processes:
            if p.pyon_name == name:
                process_object = p
                break
        if not process_object:
            msg = "%s is not a known process name" % (name)
            raise PIDanticUsageException(msg)

        pyon_id = process_object.pyon_process_id
        return pyon_id

    def remove_process(self, name):
        process_object = None
        data_object = self._data_object
        for p in data_object.processes:
            if p.pyon_name == name:
                process_object = p
                break
        if not process_object:
            msg = "%s is not a known process name" % (name)
            raise PIDanticUsageException(msg)

        data_object.processes.remove(process_object)
        self._pyon_db.db_obj_delete(process_object)
        return

    def terminate(self):
        # TODO: How should terminate work in Pyon mode
        self._log.error("terminate is disabled for now")
        return

    def ping(self):
        return self._container._is_started

    def delete(self):
        data_object = self._data_object
        for p in data_object.processes:
            self._pyon_db.db_obj_delete(p)
        self._pyon_db.db_obj_delete(data_object)
        self._pyon_db.db_commit()

    def get_data_object(self):
        return self._data_object

    def get_all_state(self):
        return dict(self._container.proc_manager.procs)

    def get_name(self):
        data_object = self._data_object
        return data_object.process_name

    def get_error_message(self):
        return ""

<commit_msg>Add locks around pyon proc_manager operations<commit_after>import os
import yaml
import logging
import threading

from imp import load_source
from urllib import urlretrieve
from uuid import uuid4

from eeagent.util import unmake_id
from pidantic.pyon.persistence import PyonDataObject, PyonProcDataObject
from pidantic.pidantic_exceptions import PIDanticUsageException, PIDanticExecutionException

def proc_manager_lock(func):
    def call(self, *args, **kwargs):

        with self._proc_manager_lock:
            return func(self, *args, **kwargs)
    return call

class Pyon(object):

    cols = ['directory', 'pyon_name', 'module', 'module_uri', 'cls', 'process_name', 'config', ]

    def __init__(self, pyon_db, pyon_container=None, proc=None, name=None,
            data_object=None, dirpath=None, log=logging):
        if data_object is None and name is None:
            raise PIDanticUsageException("You must specify a data_object or name")

        if pyon_container is None:
            raise PIDanticUsageException("You must supply a Pyon container")

        self._log = log
        self._pyon_db = pyon_db
        self._container = pyon_container
        self._proc_manager_lock = threading.RLock()
        if data_object:
            self._data_object = data_object
        else:
            if not dirpath:
                msg = "You must set the dirpath keyword argument"
                raise PIDanticUsageException(msg)
            self._working_dir = os.path.join(dirpath, name)
            try:
                os.makedirs(self._working_dir)
            except Exception:
                msg = "The directory %s already exists" % (self._working_dir)
                self._log.warning(msg)

            data_object = PyonDataObject()
            data_object.name = name
            self._pyon_db.db_obj_add(data_object)
            self._pyon_db.db_commit()
            self._data_object = data_object

    @proc_manager_lock
    def get_process_status(self, name):
        # TODO: this will probably change once pyon migrates away from fail
        # fast

        pyon_id = self._get_pyon_process_id(name)
        proc = self._container.proc_manager.get(pyon_id)
        if proc:
            status = proc.running
        else:
            status = None
        return status

    def create_process_db(self, **kwargs):

        process_object = PyonProcDataObject()
        for key in kwargs:
            if key not in self.cols:
                raise PIDanticUsageException('invalid key %s' % (key))
            process_object.__setattr__(key, kwargs[key])
        if not process_object.directory:
            process_object.directory = self._working_dir
        self._pyon_db.db_obj_add(process_object)
        data_object = self._data_object
        data_object.processes.append(process_object)
        self._pyon_db.db_commit()

        return process_object

    @proc_manager_lock
    def run_process(self, process_object):

        try:
            config = yaml.load(process_object.config)
        except AttributeError:
            config = None

        if process_object.module_uri is not None:
            module_file = self.download_module(process_object.module_uri)
            module = self.load_module(module_file, process_object.module)
            process_object.module = module
            self._pyon_db.db_commit()

        pyon_id = self._container.spawn_process(name=process_object.pyon_name,
                module=process_object.module, cls=process_object.cls,
                config=config)
        process_object.pyon_process_id = pyon_id
        self._pyon_db.db_commit()
        return pyon_id

    def download_module(self, module_uri):
        try:
            module_file, _ = urlretrieve(module_uri)
        except:
            msg = "Unable to download code module %s" % module_uri
            self._log.exception(msg)
            raise PIDanticExecutionException(msg)
        return module_file

    def load_module(self, module_file, module):
        module_name = self._unique_module_name(module)
        try:
            load_source(module_name, module_file)
        except:
            #TODO throw right exception
            msg = "Unable to load code module %s" % module
            self._log.exception(msg)
            raise PIDanticExecutionException(msg)

        return module_name

    def _unique_module_name(self, module=None):

        unique = uuid4().hex
        if module:
            unique = ".".join([unique, module])
            unique = unique.replace(".", "_")

        return unique

    def getState(self):
        return self._container._is_started

    @proc_manager_lock
    def terminate_process(self, name):

        pyon_id = self._get_pyon_process_id(name)
        terminate_result = self._container.terminate_process(pyon_id)
        return terminate_result

    def _get_pyon_process_id(self, name):
        process_object = None
        data_object = self._data_object
        for p in data_object.processes:
            if p.pyon_name == name:
                process_object = p
                break
        if not process_object:
            msg = "%s is not a known process name" % (name)
            raise PIDanticUsageException(msg)

        pyon_id = process_object.pyon_process_id
        return pyon_id

    def remove_process(self, name):
        process_object = None
        data_object = self._data_object
        for p in data_object.processes:
            if p.pyon_name == name:
                process_object = p
                break
        if not process_object:
            msg = "%s is not a known process name" % (name)
            raise PIDanticUsageException(msg)

        data_object.processes.remove(process_object)
        self._pyon_db.db_obj_delete(process_object)
        return

    def terminate(self):
        # TODO: How should terminate work in Pyon mode
        self._log.error("terminate is disabled for now")
        return

    def ping(self):
        return self._container._is_started

    def delete(self):
        data_object = self._data_object
        for p in data_object.processes:
            self._pyon_db.db_obj_delete(p)
        self._pyon_db.db_obj_delete(data_object)
        self._pyon_db.db_commit()

    def get_data_object(self):
        return self._data_object

    def get_all_state(self):
        return dict(self._container.proc_manager.procs)

    def get_name(self):
        data_object = self._data_object
        return data_object.process_name

    def get_error_message(self):
        return ""

<|endoftext|>


<commit_before>__author__ = 'pavle'

from troup.store import InMemorySyncedStore
from troup.infrastructure import AsyncIOWebSocketServer, IncomingChannelWSAdapter, ChannelManager, message_bus, bus
from troup.system import StatsTracker
from troup.messaging import message, serialize, deserialize, deserialize_dict, Message
import threading
from troup.threading import IntervalTimer
from troup.apps import App
import random
from math import ceil

class Node:

    def __init__(self, node_id, config):
        self.node_id = node_id
        self.config = config
        self.store = self._build_store_()
        self.channel_manager = None
        self.aio_server = None
        self.stats_tracker = None
        self.sync_manager = None
        self.bus = message_bus

    def _start_channel_manager_(self):
        aio_srv = AsyncIOWebSocketServer(host=self.config['server'].get('hostname'), port=self.config['server']['port'], web_socket_class=IncomingChannelWSAdapter)
        def start_aio_server():
            print('AIO Server start')
            aio_srv.start()
            print('AIO Server end')
        th = threading.Thread(target=start_aio_server)
        th.start()
        print('Notify aio server to start')
        channel_manager = ChannelManager(aio_srv)
        self.aio_server = aio_srv
        return channel_manager

    def __register_message_dispatcher__(self):
        def on_channel_data(message_str, channel):
            try:
                msg = deserialize(message_str)
                if msg.type:
                    self.bus.publish(msg.type, msg)
                else:
                    self.bus.publish('__message.genericType', msg)
            except Exception as e:
                self.log.exception('Failed to handle channel data', e)
        self.channel_manager.on('channel.data', on_channel_data)

    def _build_store_(self):
        store = InMemorySyncedStore(root_path=self.config['store']['path'])
        return store

    def _start_stats_tracker_(self):
        self.stats_tracker = StatsTracker(period=self.config['stats']['update_interval'])
        print('stats tracking ON')

    def _start_sync_manager_(self):
        self.sync_manager = SyncManager(node=self, channel_manager=self.channel_manager, event_processor=None, sync_interval=10000, sync_percent=0.3)
        self.sync_manager.start()

        neighbours = self.config.get('neighbours')
        if neighbours:
            for url in neighbours:
                name, sep, endpoint = url.partition(':')
                node = NodeInfo(name=name, stats=None, apps=None, endpoint=endpoint)
                self.sync_manager.register_node(node)
                print('Added neighbour %s [%s]' % (name, endpoint))

    def get_available_apps(self):
        return [app.name for app in self.store.apps]

    def get_stats(self):
        pass

    def get_apps(self):
        pass

    def query(self, q):
        pass

    def run_app(self, app_name):
        pass

    def start(self):
        self.channel_manager = self._start_channel_manager_()
        print('Node %s started' % self.node_id)
        self._start_stats_tracker_()
        self._start_sync_manager_()
        self.__register_message_dispatcher__()

    def stop(self):
        print('node stop')
        if self.stats_tracker:
            self.stats_tracker.stop_tracking()
            print('Statistics tracking has stopped')
        if self.aio_server:
            self.aio_server.stop()
            print('Async I/O Server notified to stop')
        if self.sync_manager:
            self.sync_manager.stop()

    def get_node_info(self):
        return NodeInfo(name=self.node_id, stats=self.stats_tracker.get_stats(), apps=self.get_apps(), endpoint=self.aio_server.get_server_endpoint())

    @bus.subscribe('task')
    def __on_task__(self, task):
        print('Received task: %s' % task)

    @bus.subscribe('command')
    def __on_command__(self, command):
        print('Received command: %s' % command)

class NodeInfo:
    def __init__(self, name=None, stats=None, apps=None, endpoint=None):
        self.name = name
        self.stats = stats
        self.apps = apps
        self.endpoint = endpoint

class EventProcessor:
    def __init__(self, node):
        self.node = node
        self.handlers = {}

    def process(self, event, message):
        handlers = self.handlers.get(event)
        if handlers:
            for handler in handlers:
                handler(message, event, self)

    def register_handler(self, event_name, handler):
        handlers = self.handlers.get(event_name)
        if not handlers:
            handlers = self.handlers[event_name] = []
        handlers.append(handler)

def node_info_from_dict(node_dict):
    apps = []
    dapps = node_dict.get('apps') or []

    for dapp in dapps:
        app = deserialize_dict(dapp, App)
        apps.append(app)
    node = deserialize_dict(node_dict, NodeInfo)
    node.apps = apps
    return node

class RandomBuffer:

    def __init__(self, nodes_dict):
        self.nodes_dict = nodes_dict
        self.buffer = []
        self._shuffle_()

    def next(self, n):
        n = int(ceil(n))
        while len(self.buffer) < n:
            self._shuffle_()
        shuffled = self.buffer[0:n]
        self.buffer = self.buffer[n:]
        return shuffled

    def _shuffle_(self):
        nodes = [name for name, node in self.nodes_dict.items()]
        random.shuffle(nodes)
        self.buffer = self.buffer + nodes

class SyncManager:

    def __init__(self, node, channel_manager, event_processor, sync_interval=60000, sync_percent=0.3):
        self.node = node
        self.channel_manager = channel_manager
        self.event_processor= event_processor
        self.sync_percent = sync_percent
        self.known_nodes = {}
        self.random_buffer = RandomBuffer(self.known_nodes)
        self.sync_timer = IntervalTimer(offset=sync_interval, interval=sync_interval, target=self.sync_random_nodes)

    def _on_message_(self, msg_str, channel):
        msg = deserialize(msg_str, as_type=Message)
        #print('Got message [%s]' % msg.__dict__)
        if msg.data.get('type') == 'sync-message':
            self._on_sync_message_(msg)

    def _on_sync_message_(self, msg):
        #print('Got sync message -> %s' % msg)
        #print(' : From node %s(%s)' % (msg.data['node']['name'], msg.data['node']['endpoint']))
        nodes = [node_info_from_dict(msg.data['node'])]

        known_nodes = [node_info_from_dict(node) for node in msg.data['known_nodes']]
        nodes = nodes + known_nodes
        self._merge_nodes_list_(nodes)

    def _merge_nodes_list_(self, nodes):
        for node in nodes:
            if node.name == self.node.node_id:
                continue
            if not self.known_nodes.get(node.name):
                self.known_nodes[node.name] = node
                print('Node %s has joined' % node.name)
                self._print_known_nodes_()
            self._merge_node_(node)

    def _print_known_nodes_(self):
        print(' Members: ')
        for name, node in self.known_nodes.items():
            print('   %s[%s]' %(name, node.endpoint) )

    def _merge_node_(self, node):
        existing = self.known_nodes[node.name]
        self.known_nodes[node.name] = node
        if existing.endpoint != node.endpoint:
            self.channel_manager.close_channel(existing.endpoint)

    def _on_closed_channel_(self, channel):
        to_remove = []
        for name, node in self.known_nodes.items():
            if node.endpoint == channel.to_url:
                to_remove.append(name)

        for name in to_remove:
            del self.known_nodes[name]

    def start(self):
        self.sync_timer.start()

        self.channel_manager.on('channel.closed', self._on_closed_channel_)
        self.channel_manager.on('channel.data', self._on_message_)

    def stop(self):
        self.sync_timer.cancel()
        self.channel_manager.remove_listener('channel.closed', self._on_closed_channel_)

    def register_node(self, node):
        if self.known_nodes.get(node.name):
            self._merge_node_(node)
        else:
            self.known_nodes[node.name] = node

    def unregister_node(self, node):
        pass

    def sync_random_nodes(self):
        nodes = self.random_buffer.next(len(self.known_nodes) * self.sync_percent)

        for name in nodes:
            if self.known_nodes.get(name):
                node = self.known_nodes[name]
                print('Sync with %s [%s]' % (name, node.endpoint))
                self.channel_manager.send(to_url=node.endpoint, data=serialize(self.get_sync_message(), indent=2))

    def sync_one_node(self, node, this_node_info):
        pass

    def get_sync_message(self):
        return message().value('node', self.node.get_node_info()).\
            value('known_nodes', [n for k,n in self.known_nodes.items()]).\
            value('type', 'sync-message').build()

<commit_msg>Fixed message data.<commit_after>__author__ = 'pavle'

from troup.store import InMemorySyncedStore
from troup.infrastructure import AsyncIOWebSocketServer, IncomingChannelWSAdapter, ChannelManager, message_bus, bus
from troup.system import StatsTracker
from troup.messaging import message, serialize, deserialize, deserialize_dict, Message
import threading
from troup.threading import IntervalTimer
from troup.apps import App
import random
from math import ceil

class Node:

    def __init__(self, node_id, config):
        self.node_id = node_id
        self.config = config
        self.store = self._build_store_()
        self.channel_manager = None
        self.aio_server = None
        self.stats_tracker = None
        self.sync_manager = None
        self.bus = message_bus

    def _start_channel_manager_(self):
        aio_srv = AsyncIOWebSocketServer(host=self.config['server'].get('hostname'), port=self.config['server']['port'], web_socket_class=IncomingChannelWSAdapter)
        def start_aio_server():
            print('AIO Server start')
            aio_srv.start()
            print('AIO Server end')
        th = threading.Thread(target=start_aio_server)
        th.start()
        print('Notify aio server to start')
        channel_manager = ChannelManager(aio_srv)
        self.aio_server = aio_srv
        return channel_manager

    def __register_message_dispatcher__(self):
        def on_channel_data(message_str, channel):
            try:
                msg = deserialize(message_str)
                if msg.data.get('type'):
                    self.bus.publish(msg.data['type'], msg)
                else:
                    self.bus.publish('__message.genericType', msg)
            except Exception as e:
                self.log.exception('Failed to handle channel data', e)
        self.channel_manager.on('channel.data', on_channel_data)

    def _build_store_(self):
        store = InMemorySyncedStore(root_path=self.config['store']['path'])
        return store

    def _start_stats_tracker_(self):
        self.stats_tracker = StatsTracker(period=self.config['stats']['update_interval'])
        print('stats tracking ON')

    def _start_sync_manager_(self):
        self.sync_manager = SyncManager(node=self, channel_manager=self.channel_manager, event_processor=None, sync_interval=10000, sync_percent=0.3)
        self.sync_manager.start()

        neighbours = self.config.get('neighbours')
        if neighbours:
            for url in neighbours:
                name, sep, endpoint = url.partition(':')
                node = NodeInfo(name=name, stats=None, apps=None, endpoint=endpoint)
                self.sync_manager.register_node(node)
                print('Added neighbour %s [%s]' % (name, endpoint))

    def get_available_apps(self):
        return [app.name for app in self.store.apps]

    def get_stats(self):
        pass

    def get_apps(self):
        pass

    def query(self, q):
        pass

    def run_app(self, app_name):
        pass

    def start(self):
        self.channel_manager = self._start_channel_manager_()
        print('Node %s started' % self.node_id)
        self._start_stats_tracker_()
        self._start_sync_manager_()
        self.__register_message_dispatcher__()

    def stop(self):
        print('node stop')
        if self.stats_tracker:
            self.stats_tracker.stop_tracking()
            print('Statistics tracking has stopped')
        if self.aio_server:
            self.aio_server.stop()
            print('Async I/O Server notified to stop')
        if self.sync_manager:
            self.sync_manager.stop()

    def get_node_info(self):
        return NodeInfo(name=self.node_id, stats=self.stats_tracker.get_stats(), apps=self.get_apps(), endpoint=self.aio_server.get_server_endpoint())

    @bus.subscribe('task')
    def __on_task__(self, task):
        print('Received task: %s' % task)

    @bus.subscribe('command')
    def __on_command__(self, command):
        print('Received command: %s' % command)

class NodeInfo:
    def __init__(self, name=None, stats=None, apps=None, endpoint=None):
        self.name = name
        self.stats = stats
        self.apps = apps
        self.endpoint = endpoint

class EventProcessor:
    def __init__(self, node):
        self.node = node
        self.handlers = {}

    def process(self, event, message):
        handlers = self.handlers.get(event)
        if handlers:
            for handler in handlers:
                handler(message, event, self)

    def register_handler(self, event_name, handler):
        handlers = self.handlers.get(event_name)
        if not handlers:
            handlers = self.handlers[event_name] = []
        handlers.append(handler)

def node_info_from_dict(node_dict):
    apps = []
    dapps = node_dict.get('apps') or []

    for dapp in dapps:
        app = deserialize_dict(dapp, App)
        apps.append(app)
    node = deserialize_dict(node_dict, NodeInfo)
    node.apps = apps
    return node

class RandomBuffer:

    def __init__(self, nodes_dict):
        self.nodes_dict = nodes_dict
        self.buffer = []
        self._shuffle_()

    def next(self, n):
        n = int(ceil(n))
        while len(self.buffer) < n:
            self._shuffle_()
        shuffled = self.buffer[0:n]
        self.buffer = self.buffer[n:]
        return shuffled

    def _shuffle_(self):
        nodes = [name for name, node in self.nodes_dict.items()]
        random.shuffle(nodes)
        self.buffer = self.buffer + nodes

class SyncManager:

    def __init__(self, node, channel_manager, event_processor, sync_interval=60000, sync_percent=0.3):
        self.node = node
        self.channel_manager = channel_manager
        self.event_processor= event_processor
        self.sync_percent = sync_percent
        self.known_nodes = {}
        self.random_buffer = RandomBuffer(self.known_nodes)
        self.sync_timer = IntervalTimer(offset=sync_interval, interval=sync_interval, target=self.sync_random_nodes)

    def _on_message_(self, msg_str, channel):
        msg = deserialize(msg_str, as_type=Message)
        #print('Got message [%s]' % msg.__dict__)
        if msg.data.get('type') == 'sync-message':
            self._on_sync_message_(msg)

    def _on_sync_message_(self, msg):
        #print('Got sync message -> %s' % msg)
        #print(' : From node %s(%s)' % (msg.data['node']['name'], msg.data['node']['endpoint']))
        nodes = [node_info_from_dict(msg.data['node'])]

        known_nodes = [node_info_from_dict(node) for node in msg.data['known_nodes']]
        nodes = nodes + known_nodes
        self._merge_nodes_list_(nodes)

    def _merge_nodes_list_(self, nodes):
        for node in nodes:
            if node.name == self.node.node_id:
                continue
            if not self.known_nodes.get(node.name):
                self.known_nodes[node.name] = node
                print('Node %s has joined' % node.name)
                self._print_known_nodes_()
            self._merge_node_(node)

    def _print_known_nodes_(self):
        print(' Members: ')
        for name, node in self.known_nodes.items():
            print('   %s[%s]' %(name, node.endpoint) )

    def _merge_node_(self, node):
        existing = self.known_nodes[node.name]
        self.known_nodes[node.name] = node
        if existing.endpoint != node.endpoint:
            self.channel_manager.close_channel(existing.endpoint)

    def _on_closed_channel_(self, channel):
        to_remove = []
        for name, node in self.known_nodes.items():
            if node.endpoint == channel.to_url:
                to_remove.append(name)

        for name in to_remove:
            del self.known_nodes[name]

    def start(self):
        self.sync_timer.start()

        self.channel_manager.on('channel.closed', self._on_closed_channel_)
        self.channel_manager.on('channel.data', self._on_message_)

    def stop(self):
        self.sync_timer.cancel()
        self.channel_manager.remove_listener('channel.closed', self._on_closed_channel_)

    def register_node(self, node):
        if self.known_nodes.get(node.name):
            self._merge_node_(node)
        else:
            self.known_nodes[node.name] = node

    def unregister_node(self, node):
        pass

    def sync_random_nodes(self):
        nodes = self.random_buffer.next(len(self.known_nodes) * self.sync_percent)

        for name in nodes:
            if self.known_nodes.get(name):
                node = self.known_nodes[name]
                print('Sync with %s [%s]' % (name, node.endpoint))
                self.channel_manager.send(to_url=node.endpoint, data=serialize(self.get_sync_message(), indent=2))

    def sync_one_node(self, node, this_node_info):
        pass

    def get_sync_message(self):
        return message().value('node', self.node.get_node_info()).\
            value('known_nodes', [n for k,n in self.known_nodes.items()]).\
            value('type', 'sync-message').build()

<|endoftext|>


<commit_before>from django.conf import settings
import django_filters
from django_filters import CharFilter
import graphene
from graphene_django.filter import DjangoFilterConnectionField
from graphene_django.types import DjangoObjectType
from graphql_jwt.decorators import login_required

from .models import Library, Camera, Lens, Photo, Tag, PhotoTag

class LibraryType(DjangoObjectType):
    class Meta:
        model = Library

class CameraType(DjangoObjectType):
    class Meta:
        model = Camera

class LensType(DjangoObjectType):
    class Meta:
        model = Lens

class PhotoTagType(DjangoObjectType):
    class Meta:
        model = PhotoTag

class CustomNode(graphene.Node):

    class Meta:
        name = 'Node'

    @staticmethod
    def to_global_id(type, id):
        return id

class PhotoInterface(graphene.Interface):
    photo_tags__tag__id = graphene.String()
    multi_filter = graphene.String()

class PhotoNode(DjangoObjectType):
    url = graphene.String()
    location = graphene.String()
    location_tags = graphene.List(PhotoTagType)
    object_tags = graphene.List(PhotoTagType)
    color_tags = graphene.List(PhotoTagType)
    style_tags = graphene.List(PhotoTagType)
    width = graphene.Int()
    height = graphene.Int()

    class Meta:
        model = Photo
        interfaces = (CustomNode, PhotoInterface)

    def resolve_location(self, info):
        if self.latitude and self.longitude:
            return '{},{}'.format(self.latitude, self.longitude)
        return None

    def resolve_url(self, info):
        size = settings.THUMBNAIL_SIZES[-1]
        return self.thumbnail_url(size)

    def resolve_location_tags(self, info):
        return self.photo_tags.filter(tag__type='L')

    def resolve_object_tags(self, info):
        return self.photo_tags.filter(tag__type='O')

    def resolve_color_tags(self, info):
        return self.photo_tags.filter(tag__type='C')

    def resolve_style_tags(self, info):
        return self.photo_tags.filter(tag__type='S')

    def resolve_width(self, info):
        return self.dimensions[0]

    def resolve_height(self, info):
        return self.dimensions[1]

class PhotoFilter(django_filters.FilterSet):
    multi_filter = CharFilter(method='multi_filter_filter')

    class Meta:
        model = Photo
        fields = {
            'aperture': ['exact'],
            'camera__id': ['exact'],
            'camera__make': ['exact', 'icontains'],
            'lens__id': ['exact'],
            'photo_tags__tag__id': ['exact', 'in', 'icontains'],
            'photo_tags__tag__name': ['exact', 'icontains', 'in'],
            'library__id': ['exact'],
        }

    def sanitize(self, value_list):
        return [v for v in value_list if v != '']  # Remove empty items

    def customize(self, value):
        return value

    def multi_filter_filter(self, queryset, name, value):
        filters = value.split(',')
        filters = self.sanitize(filters)
        filters = map(self.customize, filters)

        has_tags = False
        for filter_val in filters:
            if ':' in filter_val:
                key, val = filter_val.split(':')
                if key == 'tag':
                    queryset = queryset.filter(photo_tags__tag__id=val)
                    has_tags = True
                elif key == 'camera':
                    queryset = queryset.filter(camera__id=val)
                elif key == 'lens':
                    queryset = queryset.filter(lens__id=val)
                elif key == 'aperture':
                    queryset = queryset.filter(aperture=val)
                elif key == 'exposure':
                    queryset = queryset.filter(exposure=val)
                elif key == 'isoSpeed':
                    queryset = queryset.filter(iso_speed=val)
                elif key == 'focalLength':
                    queryset = queryset.filter(focal_length=val)
                elif key == 'flash':
                    queryset = queryset.filter(
                        flash=val == 'on' and True or False)
                elif key == 'meeteringMode':
                    queryset = queryset.filter(metering_mode=val)
                elif key == 'driveMode':
                    queryset = queryset.filter(drive_mode=val)
                elif key == 'shootingMode':
                    queryset = queryset.filter(shooting_mode=val)
        if has_tags:
            queryset.order_by('-photo_tags__significance')
        return queryset.distinct()

class LocationTagType(DjangoObjectType):
    class Meta:
        model = Tag

class ObjectTagType(DjangoObjectType):
    class Meta:
        model = Tag

class PersonTagType(DjangoObjectType):
    class Meta:
        model = Tag

class ColorTagType(DjangoObjectType):
    class Meta:
        model = Tag

class StyleTagType(DjangoObjectType):
    class Meta:
        model = Tag

class Query(graphene.ObjectType):
    all_libraries = graphene.List(LibraryType)

    camera = graphene.Field(CameraType, id=graphene.UUID(), make=graphene.String(), model=graphene.String())
    all_cameras = graphene.List(CameraType)

    lens = graphene.Field(LensType, id=graphene.UUID(), name=graphene.String())
    all_lenses = graphene.List(LensType)

    all_apertures = graphene.List(graphene.Float)
    all_exposures = graphene.List(graphene.String)
    all_iso_speeds = graphene.List(graphene.Int)
    all_focal_lengths = graphene.List(graphene.Float)
    all_metering_modes = graphene.List(graphene.String)
    all_drive_modes = graphene.List(graphene.String)
    all_shooting_modes = graphene.List(graphene.String)

    photo = graphene.Field(PhotoNode, id=graphene.UUID())
    all_photos = DjangoFilterConnectionField(PhotoNode, filterset_class=PhotoFilter)

    all_location_tags = graphene.List(LocationTagType)
    all_object_tags = graphene.List(ObjectTagType)
    all_person_tags = graphene.List(PersonTagType)
    all_color_tags = graphene.List(ColorTagType)
    all_style_tags = graphene.List(StyleTagType)

    def resolve_all_libraries(self, info, **kwargs):
        user = info.context.user
        return Library.objects.filter(users__user=user)

    def resolve_camera(self, info, **kwargs):
        id = kwargs.get('id')
        make = kwargs.get('make')
        model = kwargs.get('model')

        if id is not None:
            return Camera.objects.get(pk=id)

        if make is not None and model is not None:
            return Camera.objects.get(make=make, model=model)

        return None

    @login_required
    def resolve_all_cameras(self, info, **kwargs):
        user = info.context.user
        return Camera.objects.filter(library__users__user=user)

    def resolve_lens(self, info, **kwargs):
        id = kwargs.get('id')
        name = kwargs.get('name')

        if id is not None:
            return Lens.objects.get(pk=id)

        if name is not None:
            return Lens.objects.get(name=name)

        return None

    def resolve_all_lenses(self, info, **kwargs):
        user = info.context.user
        return Lens.objects.filter(library__users__user=user)

    def resolve_all_apertures(self, info, **kwargs):
        user = info.context.user
        return Photo.objects.filter(library__users__user=user).exclude(aperture__isnull=True).values_list('aperture', flat=True).distinct().order_by('aperture')

    def resolve_all_exposures(self, info, **kwargs):
        user = info.context.user
        return Photo.objects.filter(library__users__user=user).exclude(exposure__isnull=True).values_list('exposure', flat=True).distinct().order_by('exposure')

    def resolve_all_iso_speeds(self, info, **kwargs):
        user = info.context.user
        return Photo.objects.filter(library__users__user=user).exclude(iso_speed__isnull=True).values_list('iso_speed', flat=True).distinct().order_by('iso_speed')

    def resolve_all_focal_lengths(self, info, **kwargs):
        user = info.context.user
        return Photo.objects.filter(library__users__user=user).exclude(focal_length__isnull=True).values_list('focal_length', flat=True).distinct().order_by('focal_length')

    def resolve_all_metering_modes(self, info, **kwargs):
        user = info.context.user
        return Photo.objects.filter(library__users__user=user).exclude(metering_mode__isnull=True).values_list('metering_mode', flat=True).distinct().order_by('metering_mode')

    def resolve_all_drive_modes(self, info, **kwargs):
        user = info.context.user
        return Photo.objects.filter(library__users__user=user).exclude(drive_mode__isnull=True).values_list('drive_mode', flat=True).distinct().order_by('drive_mode')

    def resolve_all_shooting_modes(self, info, **kwargs):
        user = info.context.user
        return Photo.objects.filter(library__users__user=user).exclude(shooting_mode__isnull=True).values_list('shooting_mode', flat=True).distinct().order_by('shooting_mode')

    def resolve_photo(self, info, **kwargs):
        id = kwargs.get('id')
        if id is not None:
            return Photo.objects.get(pk=id)
        return None

    @login_required
    def resolve_all_photos(self, info, **kwargs):
        user = info.context.user
        return Photo.objects.filter(library__users__user=user)

    def resolve_all_location_tags(self, info, **kwargs):
        user = info.context.user
        return Tag.objects.filter(library__users__user=user, type='L')

    def resolve_all_object_tags(self, info, **kwargs):
        user = info.context.user
        return Tag.objects.filter(library__users__user=user, type='O')

    def resolve_all_person_tags(self, info, **kwargs):
        user = info.context.user
        return Tag.objects.filter(library__users__user=user, type='P')

    def resolve_all_color_tags(self, info, **kwargs):
        user = info.context.user
        return Tag.objects.filter(library__users__user=user, type='C')

    def resolve_all_style_tags(self, info, **kwargs):
        user = info.context.user
        return Tag.objects.filter(library__users__user=user, type='S')
<commit_msg>Update schema.py<commit_after>from django.conf import settings
import django_filters
from django_filters import CharFilter
import graphene
from graphene_django.filter import DjangoFilterConnectionField
from graphene_django.types import DjangoObjectType
from graphql_jwt.decorators import login_required

from .models import Library, Camera, Lens, Photo, Tag, PhotoTag

class LibraryType(DjangoObjectType):
    class Meta:
        model = Library

class CameraType(DjangoObjectType):
    class Meta:
        model = Camera

class LensType(DjangoObjectType):
    class Meta:
        model = Lens

class PhotoTagType(DjangoObjectType):
    class Meta:
        model = PhotoTag

class CustomNode(graphene.Node):

    class Meta:
        name = 'Node'

    @staticmethod
    def to_global_id(type, id):
        return id

class PhotoInterface(graphene.Interface):
    photo_tags__tag__id = graphene.String()
    multi_filter = graphene.String()

class PhotoNode(DjangoObjectType):
    url = graphene.String()
    location = graphene.String()
    location_tags = graphene.List(PhotoTagType)
    object_tags = graphene.List(PhotoTagType)
    color_tags = graphene.List(PhotoTagType)
    style_tags = graphene.List(PhotoTagType)
    width = graphene.Int()
    height = graphene.Int()

    class Meta:
        model = Photo
        interfaces = (CustomNode, PhotoInterface)

    def resolve_location(self, info):
        if self.latitude and self.longitude:
            return '{},{}'.format(self.latitude, self.longitude)
        return None

    def resolve_url(self, info):
        size = settings.THUMBNAIL_SIZES[-1]
        return self.thumbnail_url(size)

    def resolve_location_tags(self, info):
        return self.photo_tags.filter(tag__type='L')

    def resolve_object_tags(self, info):
        return self.photo_tags.filter(tag__type='O')

    def resolve_color_tags(self, info):
        return self.photo_tags.filter(tag__type='C')

    def resolve_style_tags(self, info):
        return self.photo_tags.filter(tag__type='S')

    def resolve_width(self, info):
        return self.dimensions[0]

    def resolve_height(self, info):
        return self.dimensions[1]

class PhotoFilter(django_filters.FilterSet):
    multi_filter = CharFilter(method='multi_filter_filter')

    class Meta:
        model = Photo
        fields = {
            'aperture': ['exact'],
            'camera__id': ['exact'],
            'camera__make': ['exact', 'icontains'],
            'lens__id': ['exact'],
            'photo_tags__tag__id': ['exact', 'in', 'icontains'],
            'photo_tags__tag__name': ['exact', 'icontains', 'in'],
            'library__id': ['exact'],
        }

    def sanitize(self, value_list):
        return [v for v in value_list if v != '']  # Remove empty items

    def customize(self, value):
        return value

    def multi_filter_filter(self, queryset, name, value):
        filters = value.split(',')
        filters = self.sanitize(filters)
        filters = map(self.customize, filters)

        has_tags = False
        for filter_val in filters:
            if ':' in filter_val:
                key, val = filter_val.split(':')
                if key == 'tag':
                    queryset = queryset.filter(photo_tags__tag__id=val)
                    has_tags = True
                elif key == 'camera':
                    queryset = queryset.filter(camera__id=val)
                elif key == 'lens':
                    queryset = queryset.filter(lens__id=val)
                elif key == 'aperture':
                    queryset = queryset.filter(aperture=val)
                elif key == 'exposure':
                    queryset = queryset.filter(exposure=val)
                elif key == 'isoSpeed':
                    queryset = queryset.filter(iso_speed=val)
                elif key == 'focalLength':
                    queryset = queryset.filter(focal_length=val)
                elif key == 'flash':
                    queryset = queryset.filter(
                        flash=val == 'on' and True or False)
                elif key == 'meeteringMode':
                    queryset = queryset.filter(metering_mode=val)
                elif key == 'driveMode':
                    queryset = queryset.filter(drive_mode=val)
                elif key == 'shootingMode':
                    queryset = queryset.filter(shooting_mode=val)
        if has_tags:
            queryset.order_by('-photo_tags__significance')
        return queryset.distinct()

class LocationTagType(DjangoObjectType):
    class Meta:
        model = Tag

class ObjectTagType(DjangoObjectType):
    class Meta:
        model = Tag

class PersonTagType(DjangoObjectType):
    class Meta:
        model = Tag

class ColorTagType(DjangoObjectType):
    class Meta:
        model = Tag

class StyleTagType(DjangoObjectType):
    class Meta:
        model = Tag

class Query(graphene.ObjectType):
    all_libraries = graphene.List(LibraryType)

    camera = graphene.Field(CameraType, id=graphene.UUID(), make=graphene.String(), model=graphene.String())
    all_cameras = graphene.List(CameraType)

    lens = graphene.Field(LensType, id=graphene.UUID(), name=graphene.String())
    all_lenses = graphene.List(LensType)

    all_apertures = graphene.List(graphene.Float)
    all_exposures = graphene.List(graphene.String)
    all_iso_speeds = graphene.List(graphene.Int)
    all_focal_lengths = graphene.List(graphene.Float)
    all_metering_modes = graphene.List(graphene.String)
    all_drive_modes = graphene.List(graphene.String)
    all_shooting_modes = graphene.List(graphene.String)

    photo = graphene.Field(PhotoNode, id=graphene.UUID())
    all_photos = DjangoFilterConnectionField(PhotoNode, filterset_class=PhotoFilter)

    all_location_tags = graphene.List(LocationTagType)
    all_object_tags = graphene.List(ObjectTagType)
    all_person_tags = graphene.List(PersonTagType)
    all_color_tags = graphene.List(ColorTagType)
    all_style_tags = graphene.List(StyleTagType)

    def resolve_all_libraries(self, info, **kwargs):
        user = info.context.user
        return Library.objects.filter(users__user=user)

    def resolve_camera(self, info, **kwargs):
        id = kwargs.get('id')
        make = kwargs.get('make')
        model = kwargs.get('model')

        if id is not None:
            return Camera.objects.get(pk=id)

        if make is not None and model is not None:
            return Camera.objects.get(make=make, model=model)

        return None

    @login_required
    def resolve_all_cameras(self, info, **kwargs):
        user = info.context.user
        return Camera.objects.filter(library__users__user=user)

    def resolve_lens(self, info, **kwargs):
        id = kwargs.get('id')
        name = kwargs.get('name')

        if id is not None:
            return Lens.objects.get(pk=id)

        if name is not None:
            return Lens.objects.get(name=name)

        return None

    def resolve_all_lenses(self, info, **kwargs):
        user = info.context.user
        return Lens.objects.filter(library__users__user=user)

    def resolve_all_apertures(self, info, **kwargs):
        user = info.context.user
        return Photo.objects.filter(library__users__user=user).exclude(aperture__isnull=True).values_list('aperture', flat=True).distinct().order_by('aperture')

    def resolve_all_exposures(self, info, **kwargs):
        user = info.context.user
        return Photo.objects.filter(library__users__user=user).exclude(exposure__isnull=True).values_list('exposure', flat=True).distinct().order_by('exposure')

    def resolve_all_iso_speeds(self, info, **kwargs):
        user = info.context.user
        return Photo.objects.filter(library__users__user=user).exclude(iso_speed__isnull=True).values_list('iso_speed', flat=True).distinct().order_by('iso_speed')

    def resolve_all_focal_lengths(self, info, **kwargs):
        user = info.context.user
        return Photo.objects.filter(library__users__user=user).exclude(focal_length__isnull=True).values_list('focal_length', flat=True).distinct().order_by('focal_length')

    def resolve_all_metering_modes(self, info, **kwargs):
        user = info.context.user
        return Photo.objects.filter(library__users__user=user).exclude(metering_mode__isnull=True).values_list('metering_mode', flat=True).distinct().order_by('metering_mode')

    def resolve_all_drive_modes(self, info, **kwargs):
        user = info.context.user
        return Photo.objects.filter(library__users__user=user).exclude(drive_mode__isnull=True).values_list('drive_mode', flat=True).distinct().order_by('drive_mode')

    def resolve_all_shooting_modes(self, info, **kwargs):
        user = info.context.user
        return Photo.objects.filter(library__users__user=user).exclude(shooting_mode__isnull=True).values_list('shooting_mode', flat=True).distinct().order_by('shooting_mode')

    def resolve_photo(self, info, **kwargs):
        id = kwargs.get('id')
        if id is not None:
            return Photo.objects.get(pk=id)
        return None

    @login_required
    def resolve_all_photos(self, info, **kwargs):
        user = info.context.user
        return Photo.objects.filter(library__users__user=user)

    def resolve_all_location_tags(self, info, **kwargs):
        user = info.context.user
        return Tag.objects.filter(library__users__user=user, type='L')

    def resolve_all_object_tags(self, info, **kwargs):
        user = info.context.user
        return Tag.objects.filter(library__users__user=user, type='O')

    def resolve_all_person_tags(self, info, **kwargs):
        user = info.context.user
        return Tag.objects.filter(library__users__user=user, type='P')

    def resolve_all_color_tags(self, info, **kwargs):
        user = info.context.user
        return Tag.objects.filter(library__users__user=user, type='C')

    def resolve_all_style_tags(self, info, **kwargs):
        user = info.context.user
        return Tag.objects.filter(library__users__user=user, type='S')

    def save(self, new_image=False, *args, **kwargs):
    if new_image:
        image=(self.image,model.photo)
        self.image_small=image(name,image)
    super(Model, self).save(*args, **kwargs)
<|endoftext|>


<commit_before>'''
Linearly scale the expression range of one gene to be between 0 and 1.
If a reference dataset is provided, then the scaling of one gene in the
target dataset in done using the minimun and range of that gene in the
reference dataset.
'''

import sys
import argparse
sys.path.insert(0,'Data_collection_processing/')
from pcl import PCLfile

#parse the command input
parser = argparse.ArgumentParser(description="Linearly scale the expression range of one gene to be between 0 and 1. If a reference dataset is provided, then the scaling of one gene in the target dataset in done using the minimun and range of that gene in the reference dataset.")
parser.add_argument('tar',help='the target file for zero one normalization')
parser.add_argument('out',help='the output file after zero one normalization')
parser.add_argument('ref',help='the reference file. If reference file is \'None\', then zero one normalization will be done based on target file itself.')
args=parser.parse_args()

def zero_one_normal(tar=None, out=None, ref=None):
    '''
    tar: the target file for zero one normalization
    out: the output file after zero one normalization
    ref: the reference file. If reference file is 'None',
         then zero one normalization will be done based on
         target file itself.
    '''

    if ref=='None':
        tar_data=PCLfile(tar, skip_col = 0)
        tar_data.zero_one_normalization()
        tar_data.write_pcl(out)
    else:
        ref_data = PCLfile(ref, skip_col=0)
        tar_data = PCLfile(tar, skip_col=0)
        for i in xrange(ref_data.data_matrix.shape[0]):
            row_minimum = ref_data.data_matrix[i,:].min()
            row_maximum = ref_data.data_matrix[i,:].max()
            row_range = row_maximum - row_minimum
            tar_data.data_matrix[i,:] = (tar_data.data_matrix[i,:] - row_minimum)/row_range
            #bound the values to be between 0 and 1
            tar_data.data_matrix[i,:] = [0 if x< 0 else x for x in tar_data.data_matrix[i,:]]
            tar_data.data_matrix[i,:] = [1 if x> 1 else x for x in tar_data.data_matrix[i,:]]
        tar_data.write_pcl(out)

zero_one_normal(tar=args.tar, out=args.out, ref=args.ref)<commit_msg>Update zero_one_normalization.py<commit_after>'''
Linearly scale the expression range of one gene to be between 0 and 1.
If a reference dataset is provided, then the scaling of one gene in the
target dataset in done using the minimun and range of that gene in the
reference dataset.
'''

import sys
import argparse
sys.path.insert(0,'Data_collection_processing/')
from pcl import PCLfile

#parse the command input
parser = argparse.ArgumentParser(description="Linearly scale the expression range of one gene to be between 0 and 1. If a reference dataset is provided, then the scaling of one gene in the target dataset in done using the minimun and range of that gene in the reference dataset.")
parser.add_argument('tar',help='the target file for zero one normalization')
parser.add_argument('out',help='the output file after zero one normalization')
parser.add_argument('ref',help='the reference file. If reference file is \'None\', then zero one normalization will be done based on target file itself.')
args=parser.parse_args()

def zero_one_normal(tar=None, out=None, ref=None):
    '''
    tar: the target file for zero one normalization
    out: the output file after zero one normalization
    ref: the reference file. If reference file is 'None',
         then zero one normalization will be done based on
         target file itself.
    '''

    if ref=='None':
        tar_data=PCLfile(tar, skip_col = 0)
        tar_data.zero_one_normalization()
        tar_data.write_pcl(out)
    else:
        ref_data = PCLfile(ref, skip_col=0)
        tar_data = PCLfile(tar, skip_col=0)
        for i in xrange(ref_data.data_matrix.shape[0]):
            row_minimum = ref_data.data_matrix[i,:].min()
            row_maximum = ref_data.data_matrix[i,:].max()
            row_range = row_maximum - row_minimum
            tar_data.data_matrix[i,:] = (tar_data.data_matrix[i,:] - row_minimum)/row_range
            #bound the values to be between 0 and 1
            tar_data.data_matrix[i,:] = [0 if x< 0 else x for x in tar_data.data_matrix[i,:]]
            tar_data.data_matrix[i,:] = [1 if x> 1 else x for x in tar_data.data_matrix[i,:]]
        tar_data.write_pcl(out)

zero_one_normal(tar=args.tar, out=args.out, ref=args.ref)
<|endoftext|>


<commit_before>import os
import logging

try:
    from cmreslogging.handlers import CMRESHandler
except ImportError:
    _es_logging_enabled = False
else:
    _es_logging_enabled = True

class OptionalModuleMissing(Exception):
    ''' Error raised a required module is missing for a optional/extra provider
    '''

    def __init__(self, module_names, reason):
        self.module_names = module_names
        self.reason = reason

    def __repr__(self):
        return "Unable to initialize logger.Missing:{0},  Reason:{1}".format(
            self.module_names, self.reason
        )

class NullHandler(logging.Handler):
    """Setup default logging to /dev/null since this is library."""

    def emit(self, record):
        pass

def get_db_logger(host='search-parsl-logging-test-2yjkk2wuoxukk2wdpiicl7mcrm.us-east-1.es.amazonaws.com',
                  port=443,
                  enable_es_logging=False,
                  index_name="my_python_index",
                  version='1.0.0',
                  **kwargs):
    """
    Parameters
    ----------
    host : str, optional
        URL to the elasticsearch cluster. Skip the http(s)://
    port : int, optional
        Port to use to access the elasticsearch cluster
    enable_es_logging : Bool, optional
        Set to True to enable logging to elasticsearch
    index_name : str, optional
        Index name to use for elasticsearch

    Returns
    -------
    logging.logger object

    Raises
    ------
    OptionalModuleMissing

    """
    logger = logging.getLogger(__file__)
    if enable_es_logging:
        if not _es_logging_enabled:
            raise OptionalModuleMissing(
                ['cmreslogging'], "Logging to ElasticSearch requires the cmreslogging module")

        handler = CMRESHandler(hosts=[{'host': host,
                                       'port': port}],
                               use_ssl=True,
                               auth_type=CMRESHandler.AuthType.NO_AUTH,
                               es_index_name=index_name,
                               es_additional_fields={
                                   'Campaign': "test",
                                   # use the name of the user's home directory as their username since there
                                   # does not seem to be a portable way to do this
                                   'Version': version,
                                   'Username': os.path.split(os.path.expanduser('~'))[-1]})
        logger = logging.getLogger("ParslElasticsearch")
        logger.setLevel(logging.INFO)
        logger.addHandler(handler)
    else:
        logger.addHandler(NullHandler())

    return logger
<commit_msg>Logging library is cmreslogging but it is on pip as CMRESHandler.<commit_after>import os
import logging

try:
    from cmreslogging.handlers import CMRESHandler
except ImportError:
    _es_logging_enabled = False
else:
    _es_logging_enabled = True

class OptionalModuleMissing(Exception):
    ''' Error raised a required module is missing for a optional/extra provider
    '''

    def __init__(self, module_names, reason):
        self.module_names = module_names
        self.reason = reason

    def __repr__(self):
        return "Unable to initialize logger.Missing:{0},  Reason:{1}".format(
            self.module_names, self.reason
        )

class NullHandler(logging.Handler):
    """Setup default logging to /dev/null since this is library."""

    def emit(self, record):
        pass

def get_db_logger(host='search-parsl-logging-test-2yjkk2wuoxukk2wdpiicl7mcrm.us-east-1.es.amazonaws.com',
                  port=443,
                  enable_es_logging=False,
                  index_name="my_python_index",
                  version='1.0.0',
                  **kwargs):
    """
    Parameters
    ----------
    host : str, optional
        URL to the elasticsearch cluster. Skip the http(s)://
    port : int, optional
        Port to use to access the elasticsearch cluster
    enable_es_logging : Bool, optional
        Set to True to enable logging to elasticsearch
    index_name : str, optional
        Index name to use for elasticsearch

    Returns
    -------
    logging.logger object

    Raises
    ------
    OptionalModuleMissing

    """
    logger = logging.getLogger(__file__)
    if enable_es_logging:
        if not _es_logging_enabled:
            raise OptionalModuleMissing(
                ['CMRESHandler'], "Logging to ElasticSearch requires the cmreslogging module")

        handler = CMRESHandler(hosts=[{'host': host,
                                       'port': port}],
                               use_ssl=True,
                               auth_type=CMRESHandler.AuthType.NO_AUTH,
                               es_index_name=index_name,
                               es_additional_fields={
                                   'Campaign': "test",
                                   # use the name of the user's home directory as their username since there
                                   # does not seem to be a portable way to do this
                                   'Version': version,
                                   'Username': os.path.split(os.path.expanduser('~'))[-1]})
        logger = logging.getLogger("ParslElasticsearch")
        logger.setLevel(logging.INFO)
        logger.addHandler(handler)
    else:
        logger.addHandler(NullHandler())

    return logger
<|endoftext|>


<commit_before>#
# profile_fitting_reciprocal_space.py
#
#  Copyright (C) 2013 Diamond Light Source
#
#  Author: James Parkhurst
#
#  This code is distributed under the BSD license, a copy of which is
#  included in the root directory of this package.

from __future__ import division

class ProfileFittingReciprocalSpace(object):
  ''' Class to do reciprocal space profile fitting. '''

  reference_counter = 0

  def __init__(self, **kwargs):
    ''' Initialise the algorithm. '''
    from dials.algorithms.peak_finding.spot_matcher import SpotMatcher
    from math import pi

    # Set the parameters
    self.grid_size = kwargs['grid_size']
    self.threshold = kwargs['threshold']
    self.frame_interval = kwargs['frame_interval']
    self.bbox_nsigma = kwargs['n_sigma']
    self.sigma_b = kwargs['sigma_b'] * pi / 180
    self.sigma_m = kwargs['sigma_m'] * pi / 180

  def __call__(self, experiment, reflections):
    ''' Do the integration.

    Transform the profiles to reciprocal space. Select the reflections
    to use in the reference profiles. Learn the reference profiles and
    integrate the intensities via profile fitting.

    '''
    from dials.model.serialize import dump
    assert("flags" in reflections)
    self._transform_profiles(experiment, reflections)
    self.learner = self._learn_references(experiment, reflections)
    counter = ProfileFittingReciprocalSpace.reference_counter
    dump.reference(self.learner.locate(), "reference_%d.pickle" % counter)
    ProfileFittingReciprocalSpace.reference_counter += 1
    return self._integrate_intensities(self.learner, reflections)

  def _transform_profiles(self, experiment, reflections):
    ''' Transform the reflection profiles to reciprocal space. '''
    from dials.util.command_line import Command
    from dials.algorithms.reflection_basis import transform as rbt
    from dials.array_family import flex

    # Initialise the reciprocal space transform
    Command.start('Initialising reciprocal space transform')
    spec = rbt.TransformSpec(experiment, self.sigma_b, self.sigma_m,
                             5, self.grid_size)
    Command.end('Initialised reciprocal space transform')

    # Transform the reflections to reciprocal space
    Command.start('Transforming reflections to reciprocal space')
    s1 = reflections['s1']
    phi = reflections['xyzcal.mm'].parts()[2]
    shoebox = reflections['shoebox']
    rs_shoebox = flex.transformed_shoebox(spec, s1, phi, shoebox)
    reflections['rs_shoebox'] = rs_shoebox
    Command.end('Transformed {0} reflections'.format(len(reflections)))

  def _learn_references(self, experiment, reflections):
    ''' Learn the reference profiles. '''
    from dials.algorithms.integration.profile import GridSampler
    from dials.algorithms.integration.profile import GridReferenceLearner
    from dials.algorithms.integration.profile import XdsCircleSampler
    from dials.algorithms.integration.profile import XdsCircleReferenceLearner
    from dials.array_family import flex

    # Match the predictions with the strong spots
    #sind, pind = self.match(strong, reflections)
    pind = reflections.get_flags(reflections.flags.reference_spot)

    # Create the reference profile sampler
    assert(len(experiment.detector) == 1)
    image_size = experiment.detector[0].get_image_size()
    num_frames = experiment.scan.get_num_images()
    volume_size = image_size + (num_frames,)
    sampler = GridSampler(volume_size, (3, 3, 1))
    #sampler = XdsCircleSampler(volume_size, 1)

    # Configure the reference learner
    grid_size = (self.grid_size * 2 + 1,) * 3
    learner = GridReferenceLearner(sampler, grid_size, self.threshold)
    #learner = XdsCircleReferenceLearner(sampler, grid_size, self.threshold)
    profiles = reflections['rs_shoebox'].select(pind)
    coords = reflections['xyzcal.px'].select(pind)
    learner.learn(profiles, coords)

    # Return the learner
    return learner

  def _integrate_intensities(self, learner, reflections):
    ''' Integrate the intensities. '''
    from dials.util.command_line import Command
    from dials.algorithms.integration import ProfileFittingReciprocalSpaceAlgorithm

    # Configure the integration algorithm with the locator class
    integrate = ProfileFittingReciprocalSpaceAlgorithm(learner.locate())

    # Perform the integration
    Command.start('Integrating reflections in reciprocal space')
    profiles = reflections['rs_shoebox']
    coords = reflections['xyzcal.px']
    intensity = integrate(profiles, coords)
    mask = intensity.parts()[1] > 0
    reflections['intensity.raw.value'] = intensity.parts()[0]
    reflections['intensity.raw.variance'] = intensity.parts()[1]
    reflections['profile.correlation'] = intensity.parts()[2]
    reflections.set_flags(mask, reflections.flags.integrated)
    Command.end('Integrated {0} reflections'.format(len(reflections)))

    return reflections
<commit_msg>Fixed negative variance<commit_after>#
# profile_fitting_reciprocal_space.py
#
#  Copyright (C) 2013 Diamond Light Source
#
#  Author: James Parkhurst
#
#  This code is distributed under the BSD license, a copy of which is
#  included in the root directory of this package.

from __future__ import division

class ProfileFittingReciprocalSpace(object):
  ''' Class to do reciprocal space profile fitting. '''

  reference_counter = 0

  def __init__(self, **kwargs):
    ''' Initialise the algorithm. '''
    from dials.algorithms.peak_finding.spot_matcher import SpotMatcher
    from math import pi

    # Set the parameters
    self.grid_size = kwargs['grid_size']
    self.threshold = kwargs['threshold']
    self.frame_interval = kwargs['frame_interval']
    self.bbox_nsigma = kwargs['n_sigma']
    self.sigma_b = kwargs['sigma_b'] * pi / 180
    self.sigma_m = kwargs['sigma_m'] * pi / 180

  def __call__(self, experiment, reflections):
    ''' Do the integration.

    Transform the profiles to reciprocal space. Select the reflections
    to use in the reference profiles. Learn the reference profiles and
    integrate the intensities via profile fitting.

    '''
    from dials.model.serialize import dump
    assert("flags" in reflections)
    self._transform_profiles(experiment, reflections)
    self.learner = self._learn_references(experiment, reflections)
    counter = ProfileFittingReciprocalSpace.reference_counter
    dump.reference(self.learner.locate(), "reference_%d.pickle" % counter)
    ProfileFittingReciprocalSpace.reference_counter += 1
    return self._integrate_intensities(self.learner, reflections)

  def _transform_profiles(self, experiment, reflections):
    ''' Transform the reflection profiles to reciprocal space. '''
    from dials.util.command_line import Command
    from dials.algorithms.reflection_basis import transform as rbt
    from dials.array_family import flex

    # Initialise the reciprocal space transform
    Command.start('Initialising reciprocal space transform')
    spec = rbt.TransformSpec(experiment, self.sigma_b, self.sigma_m,
                             5, self.grid_size)
    Command.end('Initialised reciprocal space transform')

    # Transform the reflections to reciprocal space
    Command.start('Transforming reflections to reciprocal space')
    s1 = reflections['s1']
    phi = reflections['xyzcal.mm'].parts()[2]
    shoebox = reflections['shoebox']
    rs_shoebox = flex.transformed_shoebox(spec, s1, phi, shoebox)
    reflections['rs_shoebox'] = rs_shoebox
    Command.end('Transformed {0} reflections'.format(len(reflections)))

  def _learn_references(self, experiment, reflections):
    ''' Learn the reference profiles. '''
    from dials.algorithms.integration.profile import GridSampler
    from dials.algorithms.integration.profile import GridReferenceLearner
    from dials.algorithms.integration.profile import XdsCircleSampler
    from dials.algorithms.integration.profile import XdsCircleReferenceLearner
    from dials.array_family import flex

    # Match the predictions with the strong spots
    #sind, pind = self.match(strong, reflections)
    pind = reflections.get_flags(reflections.flags.reference_spot)

    # Create the reference profile sampler
    assert(len(experiment.detector) == 1)
    image_size = experiment.detector[0].get_image_size()
    num_frames = experiment.scan.get_num_images()
    volume_size = image_size + (num_frames,)
    sampler = GridSampler(volume_size, (3, 3, 1))
    #sampler = XdsCircleSampler(volume_size, 1)

    # Configure the reference learner
    grid_size = (self.grid_size * 2 + 1,) * 3
    learner = GridReferenceLearner(sampler, grid_size, self.threshold)
    #learner = XdsCircleReferenceLearner(sampler, grid_size, self.threshold)
    profiles = reflections['rs_shoebox'].select(pind)
    coords = reflections['xyzcal.px'].select(pind)
    learner.learn(profiles, coords)

    # Return the learner
    return learner

  def _integrate_intensities(self, learner, reflections):
    ''' Integrate the intensities. '''
    from dials.util.command_line import Command
    from dials.algorithms.integration import ProfileFittingReciprocalSpaceAlgorithm

    # Configure the integration algorithm with the locator class
    integrate = ProfileFittingReciprocalSpaceAlgorithm(learner.locate())

    # Perform the integration
    Command.start('Integrating reflections in reciprocal space')
    profiles = reflections['rs_shoebox']
    coords = reflections['xyzcal.px']
    intensity = integrate(profiles, coords)
    I, I_var, P_cor = intensity.parts()
    mask = I_var > 0
    I_var.set_selected(mask != True, 0.0)
    reflections['intensity.raw.value'] = I
    reflections['intensity.raw.variance'] = I_var
    reflections['profile.correlation'] = P_cor
    reflections.set_flags(mask, reflections.flags.integrated)
    Command.end('Integrated {0} reflections'.format(len(reflections)))

    return reflections
<|endoftext|>


<commit_before>import webapp2
import logging
import traceback
import sys

#Format modules
import json
import yaml

import youtube_dl

class NoneFile(object):
    '''
    A file-like object that does nothing
    '''
    def write(self,msg):
        pass
    def flush(self,*args,**kaargs):
        pass
    def isatty(self):
        return False

class ScreenFile(NoneFile):
    def write(self,msg):
        logging.debug(msg)

if not hasattr(sys.stderr,'isatty'):
    #In GAE it's not defined and we must monkeypatch
    sys.stderr.isatty = lambda : False

class SimpleFileDownloader(youtube_dl.FileDownloader):
    def __init__(self,*args,**kargs):
        super(SimpleFileDownloader,self).__init__(*args,**kargs)
        self._screen_file=ScreenFile()
        self._ies = youtube_dl.gen_extractors()
        for ie in self._ies:
            ie.set_downloader(self)

    def extract(self,url):
        videos_info,ie = self._extract(url)
        if videos_info:
            self._addVideos(videos_info)
            self._addIE(ie)
        return self.videos_info,self.ie

def videos(url):
    '''
    Get a list with a dict for every video founded
    '''
    fd = SimpleFileDownloader({'outtmpl':'%(title)s'})
    res = fd.extract_info(url, download = False)#(url)
    r_type = res[0].get('_type', 'video')
    #Do not return yet playlists
    if r_type == 'video':
        videos = res
    elif r_type == 'playlist':
        videos = res[0]['entries']
    return videos

class Api(webapp2.RequestHandler):
    @property
    def out_format(self):
        """The format the response must use"""
        return self.request.get("format","json")
    @property
    def content_type(self):
        """Content type for the response"""
        if self.out_format == "json":
            return "application/json"
        if self.out_format == "yaml":
            return "application/yaml"
    def dumps(self,dic):
        """Dump a dic to a string using the format specified in the reques"""
        if self.out_format == "json":
            return json.dumps(dic)
        elif self.out_format == "yaml":
            return yaml.safe_dump(dic)
    def get(self):
        #Allow javascript get requests from other domains
        self.response.headers["Access-Control-Allow-Origin"] = "*"
        self.response.headers["Content-Type"] = self.content_type

        errors = (youtube_dl.DownloadError, youtube_dl.ExtractorError)
        try:
            url = self.request.get("url")
            vids = videos(url)
            dic = {'youtube-dl.version':youtube_dl.__version__,
                   'url':url,
                   'videos':vids}
        except errors as err:
            dic = {'error': str(err)}
            logging.error(traceback.format_exc())
        response = self.dumps(dic)
        self.response.out.write(response)
<commit_msg>Remove old method<commit_after>import webapp2
import logging
import traceback
import sys

#Format modules
import json
import yaml

import youtube_dl

class NoneFile(object):
    '''
    A file-like object that does nothing
    '''
    def write(self,msg):
        pass
    def flush(self,*args,**kaargs):
        pass
    def isatty(self):
        return False

class ScreenFile(NoneFile):
    def write(self,msg):
        logging.debug(msg)

if not hasattr(sys.stderr,'isatty'):
    #In GAE it's not defined and we must monkeypatch
    sys.stderr.isatty = lambda : False

class SimpleFileDownloader(youtube_dl.FileDownloader):
    def __init__(self,*args,**kargs):
        super(SimpleFileDownloader,self).__init__(*args,**kargs)
        self._screen_file=ScreenFile()
        self._ies = youtube_dl.gen_extractors()
        for ie in self._ies:
            ie.set_downloader(self)

def videos(url):
    '''
    Get a list with a dict for every video founded
    '''
    fd = SimpleFileDownloader({'outtmpl':'%(title)s'})
    res = fd.extract_info(url, download = False)#(url)
    r_type = res[0].get('_type', 'video')
    #Do not return yet playlists
    if r_type == 'video':
        videos = res
    elif r_type == 'playlist':
        videos = res[0]['entries']
    return videos

class Api(webapp2.RequestHandler):
    @property
    def out_format(self):
        """The format the response must use"""
        return self.request.get("format","json")
    @property
    def content_type(self):
        """Content type for the response"""
        if self.out_format == "json":
            return "application/json"
        if self.out_format == "yaml":
            return "application/yaml"
    def dumps(self,dic):
        """Dump a dic to a string using the format specified in the reques"""
        if self.out_format == "json":
            return json.dumps(dic)
        elif self.out_format == "yaml":
            return yaml.safe_dump(dic)
    def get(self):
        #Allow javascript get requests from other domains
        self.response.headers["Access-Control-Allow-Origin"] = "*"
        self.response.headers["Content-Type"] = self.content_type

        errors = (youtube_dl.DownloadError, youtube_dl.ExtractorError)
        try:
            url = self.request.get("url")
            vids = videos(url)
            dic = {'youtube-dl.version':youtube_dl.__version__,
                   'url':url,
                   'videos':vids}
        except errors as err:
            dic = {'error': str(err)}
            logging.error(traceback.format_exc())
        response = self.dumps(dic)
        self.response.out.write(response)
<|endoftext|>