<commit_before>"""Base models."""

import os
import subprocess
import shutil

from django.db import models
from django.utils.text import slugify
from django.core.urlresolvers import reverse
from django.core.validators import RegexValidator
from django.core.urlresolvers import reverse
from django.db.models.signals import post_save

from jinja2 import Template
from jinja2.runtime import Context

from django.conf import settings

from task_manager.file_tasks import unzip
from task_manager.tika_tasks import create_index

def alphanumeric_validator():
    return RegexValidator(r'^[a-zA-Z0-9-_ ]+$',
        'Only numbers, letters, underscores, dashes and spaces are allowed.')

def zipped_file_validator():
    return RegexValidator(r'.*\.(ZIP|zip)$',
        'Only compressed archive (.zip) files are allowed.')

def delete_folder_contents(folder):
    for file in os.listdir(folder):
        file_path = os.path.join(folder, file)
        try:
            if os.path.isfile(file_path):
                os.unlink(file_path)
            elif os.path.isdir(file_path):
                shitul.rmtree(file_path)
        except Exception, e:
            print(e)

class Project(models.Model):
    """Project model.

    Every application that plugs into Memex Explorer should have a
    foreign key relationship to a Project.

    Model Fields
    ------------

    name : str, 64 characters max
    slug : str, 64 characters max
        The `slug` field is derived from `name` on save, and is restricted
        to URL-safe characters.
    description : textfield

    """

    name = models.CharField(max_length=64, unique=True,
        validators=[alphanumeric_validator()])
    slug = models.SlugField(max_length=64, unique=True)
    description = models.TextField(blank=True)

    def save(self, *args, **kwargs):
        self.slug = slugify(unicode(self.name))
        super(Project, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return reverse('base:project',
            kwargs=dict(project_slug=self.slug))

    def kibana_url(self):
        return '/{}/kibana/'.format(self.name)

    def __unicode__(self):
        return self.name

class App(models.Model):
    """
    Represents information about starting an application in a container.
    """
    name = models.CharField(max_length=64, unique=True,
        validators=[alphanumeric_validator()])
    index_url = models.URLField()

    #Only one of the following can be non-blank. This is not enforced yet.
    image = models.TextField(max_length=256, blank=True, null=True)
    build = models.TextField(max_length=265, blank=True, null=True)
    command = models.TextField(max_length=256)

    def create_container_entry(self, project):
        container = Container.objects.create(
            app = self,
            project = project,
            high_port = None,
            running = True
        )
        return container

    def __unicode__(self):
        return "{} running {}".format(self.name, self.image or self.build)

class AppLink(models.Model):
    from_app = models.ForeignKey(App, related_name='links')
    to_app = models.ForeignKey(App)
    alias = models.TextField(max_length=64, null=True, blank=True)
    external = models.BooleanField(default=False)

class AppPort(models.Model):
    expose_publicly = models.BooleanField(default=False)
    app = models.ForeignKey(App, related_name='ports')
    internal_port = models.IntegerField(null=False, blank=False)
    service_name = models.TextField(max_length=64, null=True, blank=True)

    def clean(self, *args, **kwargs):
        if AppPort.objects.filter(app = self.app).filter(expose_publicly = True).exists():
            raise ValidationError("An application can only expose one port publicly. {} already exposes port {}".filter(
              self.app.name, AppPort.objects.filter(app = self.app).filter(expose_publicly = True).get().internal_port
              ))

    def __unicode__(self):
        return "{} is running on port {}".format(self.app.name, self.internal_port)

    class Meta:
        unique_together = ('app', 'internal_port')

class VolumeMount(models.Model):
    """
    When creating this app, where to mount it in the container.

    TODO: More thinking required
    """
    app = models.ForeignKey(App)
    mounted_at = models.TextField(max_length=254)
    """Where within the container is it mounted?"""
    located_at = models.TextField(max_length=254)
    """Where on the host is the directory?"""
    read_only = models.BooleanField(default=False)

class EnvVar(models.Model):
    app = models.ForeignKey(App, related_name='environment_variables')
    name = models.TextField(max_length=64)
    value = models.TextField(max_length=256, default='')

class Container(models.Model):
    """

    """
    NGINX_CONFIG_TEMPLATE_PATH = os.path.join(settings.BASE_DIR, 'base/deploy_templates/nginx-reverse-proxy.conf.jinja2')
    DOCKER_COMPOSE_TEMPLATE_PATH = os.path.join(settings.BASE_DIR, 'base/deploy_templates/docker-compose.yml.jinja2')
    NGINX_CONFIG_DESTINATION_PATH =  os.path.join(settings.BASE_DIR, 'base/nginx-reverse-proxy.conf')
    NGINX_CONFIG_COPY_PATH = '/etc/nginx/sites-enabled/default'
    DOCKER_COMPOSE_DESTINATION_PATH = os.path.join(settings.BASE_DIR, 'base/docker-compose.yml')

    app = models.ForeignKey(App)
    project = models.ForeignKey(Project)
    "What type of app should the container be running?"
    high_port = models.IntegerField(null=True, blank=True)
    "If the app exposes a port, what high port does it end up exposing it on?"
    public_path_base = models.TextField(null=True, blank=True)
    "If the app is supposed to be served to the outside world and has a base url different than /project.name/app.name, what is it?"
    running = models.BooleanField(default=False)
    "Should the container be running?"

    def slug(self):
        return Container.__slug(self.project, self.app)

    def public_urlbase(self):
        if self.public_path_base:
            return self.public_path_base
        return "/{}/{}".format(self.project.name, self.app.name)

    def docker_name(self):
        composefile_dir_name = os.path.basename(os.path.dirname(Container.DOCKER_COMPOSE_DESTINATION_PATH))
        return "{}_{}_1".format(composefile_dir_name, self.slug())

    def context_dict(self):
        #TODO: This can be dramatically sped up by actually thinking about db queries and a judicious prefectch_related
        result = {
            'slug':self.slug(),
            'command': self.app.command or '',
            'volumes' : list(VolumeMount.objects.filter(app = self.app).values('located_at', 'mounted_at')),
            'ports': [port[0] for port in AppPort.objects.filter(app=self.app).values_list('internal_port')],
            'links': [{'name': Container.__slug(self.project, link.to_app), 'alias': link.alias or ''} for link in
                        AppLink.objects.filter(from_app = self.app)],
            'environment_variables': list(EnvVar.objects.filter(app=self.app).values('name', 'value')),
        }
        if self.app.image:
            result['image'] = self.app.image
        elif self.app.build:
            result['build'] = self.app.build
        else:
            raise ValueError("container {} has neither an image not a build.".format(self.slug()))
        return result

    @staticmethod
    def __slug(project, app):
        return "{}{}".format(project.name, app.name)

    @staticmethod
    def fill_template(source, destination, context_dict):
        template = Template(open(source, 'r').read(), trim_blocks = True, lstrip_blocks = True)
        result = template.render(context_dict)
        with open(destination, 'w') as f:
            f.write(result)
            f.flush()

    @staticmethod
    def generate_container_context():
        containers = Container.objects.filter(running = True).select_related('app', 'project').all()
        return {'containers': [container.context_dict() for container in containers]} #this is going to make about 50 queries when it could make 2 or 5.

    @staticmethod
    def create_containers(cls):
        """
        Create a new docker compose file with an entry for every container that is supposed to be running.
        """
        Container.fill_template(Container.DOCKER_COMPOSE_TEMPLATE_PATH, Container.DOCKER_COMPOSE_DESTINATION_PATH, Container.generate_container_context())
        if os.path.exists(os.path.expanduser('~/miniconda/')):
            DOCKER_COMPOSE_PATH=os.path.expanduser('~/miniconda/bin/docker-compose')
        elif os.path.exists(os.path.expanduser('~/anaconda/')):
            DOCKER_COMPOSE_PATH=os.path.expanduser('~/anaconda/bin/docker-compose')
        elif os.path.exists('/miniconda/'):
            DOCKER_COMPOSE_PATH='/miniconda/bin/docker-compose'
        elif os.path.exists('/anaconda/'):
            DOCKER_COMPOSE_PATH='/anaconda/bin/docker-compose'
        else:
            DOCKER_COMPOSE_PATH='docker-compose'
        command = ["sudo",DOCKER_COMPOSE_PATH,"-f",cls.DOCKER_COMPOSE_DESTINATION_PATH,"up","-d","--no-recreate"]
        print(command)
        out = subprocess.check_output(command)

        app_ids = AppPort.objects.filter(expose_publicly = True).values_list('app_id', flat=True)
        return out

    @staticmethod
    def get_port_mappings():
        app_ports = dict(AppPort.objects.filter(expose_publicly = True).values_list('app_id', 'internal_port'))
        port_mappings = []
        for container in Container.objects.filter(app_id__in = app_ports.values()).filter(running = True).all():
            docker_port_output = subprocess.check_output(['sudo', 'docker', 'port', self.docker_name()])
            for raw_mapping in docker_port_output.split('\n'):
                print(raw_mapping)
                if '/tcp -> 0.0.0.0:' in raw_mapping:
                    if internal == app_ports[container.app_id]:
                        internal, external = raw_mapping.split('/tcp -> 0.0.0.0:')
                        container.high_port = int(external)
                        container.save()
                        port_mappings.append((container.public_urlbase(), self.high_port))
        return port_mappings

    @staticmethod
    def generate_nginx_context(port_mappings=[]):
        if port_mappings is None:
          port_mappings = Container.get_port_mappings()
        root_port = os.environ.get('ROOT_PORT', '8000')
        hostname = os.environ.get('HOST_NAME', settings.HOSTNAME)
        ip_addr = os.environ.get('IP_ADDR', settings.IP_ADDR)
        return {
            'static_root': settings.STATIC_ROOT,
            'portmaps': [{'port': mapping[1], 'urlbase': mapping[0]}
                                    for mapping in port_mappings]
                , 'root_port': root_port, 'hostname': hostname, 'ip_addr': ip_addr}

    @staticmethod
    def map_public_ports(port_mappings=None):
        """
        Create a new nginx config with an entry for every container that is supposed to be running and has a public path base.
        Then, restart nginx.
        """
        Container.fill_template(Container.NGINX_CONFIG_TEMPLATE_PATH,
            Container.NGINX_CONFIG_DESTINATION_PATH, Container.generate_nginx_context(port_mappings))
        command = ["sudo","cp",Container.NGINX_CONFIG_DESTINATION_PATH, Container.NGINX_CONFIG_COPY_PATH]
        print(command)
        subprocess.check_output(command)
        return subprocess.check_output(["sudo","service","nginx","restart"])

from task_manager.docker_tasks import start_containers

def start_container_celery(sender, instance, **kwargs):
    start_containers.delay(instance)

if settings.DEPLOYMENT:
    post_save.connect(start_container_celery, sender = Project)

def get_zipped_data_path(instance, filename):
    """
    This method must stay outside of the class definition because django
    cannot serialize unbound methods in Python 2:

    https://docs.djangoproject.com/en/dev/topics/migrations/#migration-serializing
    """
    return os.path.join(settings.MEDIA_ROOT, "indices", instance.slug, "zipped_data", filename)

class Index(models.Model):
    """Index model.

    The index model keeps track of indices that are made and what files are
    contained within them.

    Model Fields
    ------------

    name : str, 64 characters max
    description : textfield
    uploaded_data : Django FileField
    data_dolder : textfield
    project : fk to base.Project

    """
    def get_dumped_data_path(instance):
        return os.path.join(
            settings.MEDIA_ROOT,
            "indices",
            instance.slug,
            "data"
        )

    name = models.CharField(max_length=64, unique=True,
        validators=[alphanumeric_validator()])
    slug = models.SlugField(max_length=64, unique=True)
    uploaded_data = models.FileField(upload_to=get_zipped_data_path,
        validators=[zipped_file_validator()])
    data_folder = models.TextField(blank=True)
    project = models.ForeignKey(Project)

    def save(self, *args, **kwargs):
        self.slug = slugify(unicode(self.name))
        if self.uploaded_data:
            zipped_data_path = os.path.dirname(get_zipped_data_path(self, self.uploaded_data.name))
            if os.path.isdir(zipped_data_path):
                delete_folder_contents(zipped_data_path)
            super(Index, self).save(*args, **kwargs)
            self.data_folder = self.get_dumped_data_path()
            if os.path.isdir(self.data_folder):
                delete_folder_contents(self.data_folder)
            unzip.delay(self.uploaded_data.name, self.data_folder)
            if settings.DEPLOYMENT:
                create_index.delay(self)
        super(Index, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return reverse('base:index_settings',
            kwargs=dict(index_slug=self.slug, project_slug=self.project.slug))

<commit_msg>fix bug preventing nginx conf from being generted.<commit_after>"""Base models."""

import os
import subprocess
import shutil

from django.db import models
from django.utils.text import slugify
from django.core.urlresolvers import reverse
from django.core.validators import RegexValidator
from django.core.urlresolvers import reverse
from django.db.models.signals import post_save

from jinja2 import Template
from jinja2.runtime import Context

from django.conf import settings

from task_manager.file_tasks import unzip
from task_manager.tika_tasks import create_index

def alphanumeric_validator():
    return RegexValidator(r'^[a-zA-Z0-9-_ ]+$',
        'Only numbers, letters, underscores, dashes and spaces are allowed.')

def zipped_file_validator():
    return RegexValidator(r'.*\.(ZIP|zip)$',
        'Only compressed archive (.zip) files are allowed.')

def delete_folder_contents(folder):
    for file in os.listdir(folder):
        file_path = os.path.join(folder, file)
        try:
            if os.path.isfile(file_path):
                os.unlink(file_path)
            elif os.path.isdir(file_path):
                shitul.rmtree(file_path)
        except Exception, e:
            print(e)

class Project(models.Model):
    """Project model.

    Every application that plugs into Memex Explorer should have a
    foreign key relationship to a Project.

    Model Fields
    ------------

    name : str, 64 characters max
    slug : str, 64 characters max
        The `slug` field is derived from `name` on save, and is restricted
        to URL-safe characters.
    description : textfield

    """

    name = models.CharField(max_length=64, unique=True,
        validators=[alphanumeric_validator()])
    slug = models.SlugField(max_length=64, unique=True)
    description = models.TextField(blank=True)

    def save(self, *args, **kwargs):
        self.slug = slugify(unicode(self.name))
        super(Project, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return reverse('base:project',
            kwargs=dict(project_slug=self.slug))

    def kibana_url(self):
        return '/{}/kibana/'.format(self.name)

    def __unicode__(self):
        return self.name

class App(models.Model):
    """
    Represents information about starting an application in a container.
    """
    name = models.CharField(max_length=64, unique=True,
        validators=[alphanumeric_validator()])
    index_url = models.URLField()

    #Only one of the following can be non-blank. This is not enforced yet.
    image = models.TextField(max_length=256, blank=True, null=True)
    build = models.TextField(max_length=265, blank=True, null=True)
    command = models.TextField(max_length=256)

    def create_container_entry(self, project):
        container = Container.objects.create(
            app = self,
            project = project,
            high_port = None,
            running = True
        )
        return container

    def __unicode__(self):
        return "{} running {}".format(self.name, self.image or self.build)

class AppLink(models.Model):
    from_app = models.ForeignKey(App, related_name='links')
    to_app = models.ForeignKey(App)
    alias = models.TextField(max_length=64, null=True, blank=True)
    external = models.BooleanField(default=False)

class AppPort(models.Model):
    expose_publicly = models.BooleanField(default=False)
    app = models.ForeignKey(App, related_name='ports')
    internal_port = models.IntegerField(null=False, blank=False)
    service_name = models.TextField(max_length=64, null=True, blank=True)

    def clean(self, *args, **kwargs):
        if AppPort.objects.filter(app = self.app).filter(expose_publicly = True).exists():
            raise ValidationError("An application can only expose one port publicly. {} already exposes port {}".filter(
              self.app.name, AppPort.objects.filter(app = self.app).filter(expose_publicly = True).get().internal_port
              ))

    def __unicode__(self):
        return "{} is running on port {}".format(self.app.name, self.internal_port)

    class Meta:
        unique_together = ('app', 'internal_port')

class VolumeMount(models.Model):
    """
    When creating this app, where to mount it in the container.

    TODO: More thinking required
    """
    app = models.ForeignKey(App)
    mounted_at = models.TextField(max_length=254)
    """Where within the container is it mounted?"""
    located_at = models.TextField(max_length=254)
    """Where on the host is the directory?"""
    read_only = models.BooleanField(default=False)

class EnvVar(models.Model):
    app = models.ForeignKey(App, related_name='environment_variables')
    name = models.TextField(max_length=64)
    value = models.TextField(max_length=256, default='')

class Container(models.Model):
    """

    """
    NGINX_CONFIG_TEMPLATE_PATH = os.path.join(settings.BASE_DIR, 'base/deploy_templates/nginx-reverse-proxy.conf.jinja2')
    DOCKER_COMPOSE_TEMPLATE_PATH = os.path.join(settings.BASE_DIR, 'base/deploy_templates/docker-compose.yml.jinja2')
    NGINX_CONFIG_DESTINATION_PATH =  os.path.join(settings.BASE_DIR, 'base/nginx-reverse-proxy.conf')
    NGINX_CONFIG_COPY_PATH = '/etc/nginx/sites-enabled/default'
    DOCKER_COMPOSE_DESTINATION_PATH = os.path.join(settings.BASE_DIR, 'base/docker-compose.yml')

    app = models.ForeignKey(App)
    project = models.ForeignKey(Project)
    "What type of app should the container be running?"
    high_port = models.IntegerField(null=True, blank=True)
    "If the app exposes a port, what high port does it end up exposing it on?"
    public_path_base = models.TextField(null=True, blank=True)
    "If the app is supposed to be served to the outside world and has a base url different than /project.name/app.name, what is it?"
    running = models.BooleanField(default=False)
    "Should the container be running?"

    def slug(self):
        return Container.__slug(self.project, self.app)

    def public_urlbase(self):
        if self.public_path_base:
            return self.public_path_base
        return "/{}/{}".format(self.project.name, self.app.name)

    def docker_name(self):
        composefile_dir_name = os.path.basename(os.path.dirname(Container.DOCKER_COMPOSE_DESTINATION_PATH))
        return "{}_{}_1".format(composefile_dir_name, self.slug())

    def context_dict(self):
        #TODO: This can be dramatically sped up by actually thinking about db queries and a judicious prefectch_related
        result = {
            'slug':self.slug(),
            'command': self.app.command or '',
            'volumes' : list(VolumeMount.objects.filter(app = self.app).values('located_at', 'mounted_at')),
            'ports': [port[0] for port in AppPort.objects.filter(app=self.app).values_list('internal_port')],
            'links': [{'name': Container.__slug(self.project, link.to_app), 'alias': link.alias or ''} for link in
                        AppLink.objects.filter(from_app = self.app)],
            'environment_variables': list(EnvVar.objects.filter(app=self.app).values('name', 'value')),
        }
        if self.app.image:
            result['image'] = self.app.image
        elif self.app.build:
            result['build'] = self.app.build
        else:
            raise ValueError("container {} has neither an image not a build.".format(self.slug()))
        return result

    @staticmethod
    def __slug(project, app):
        return "{}{}".format(project.name, app.name)

    @staticmethod
    def fill_template(source, destination, context_dict):
        template = Template(open(source, 'r').read(), trim_blocks = True, lstrip_blocks = True)
        result = template.render(context_dict)
        with open(destination, 'w') as f:
            f.write(result)
            f.flush()

    @staticmethod
    def generate_container_context():
        containers = Container.objects.filter(running = True).select_related('app', 'project').all()
        return {'containers': [container.context_dict() for container in containers]} #this is going to make about 50 queries when it could make 2 or 5.
    @staticmethod
    def docker_compose_path():
        if os.path.exists(os.path.expanduser('~/miniconda/')):
            DOCKER_COMPOSE_PATH=os.path.expanduser('~/miniconda/bin/docker-compose')
        elif os.path.exists(os.path.expanduser('~/anaconda/')):
            DOCKER_COMPOSE_PATH=os.path.expanduser('~/anaconda/bin/docker-compose')
        elif os.path.exists('/miniconda/'):
            DOCKER_COMPOSE_PATH='/miniconda/bin/docker-compose'
        elif os.path.exists('/anaconda/'):
            DOCKER_COMPOSE_PATH='/anaconda/bin/docker-compose'
        else:
            DOCKER_COMPOSE_PATH='docker-compose'
        return DOCKER_COMPOSE_PATH

    @staticmethod
    def create_containers():
        """
        Create a new docker compose file with an entry for every container that is supposed to be running.
        """
        Container.fill_template(Container.DOCKER_COMPOSE_TEMPLATE_PATH, Container.DOCKER_COMPOSE_DESTINATION_PATH, Container.generate_container_context())
        command = ["sudo",Container.docker_compose_path(),"-f",Container.DOCKER_COMPOSE_DESTINATION_PATH,"up","-d","--no-recreate"]
        print(command)
        out = subprocess.check_output(command)

        app_ids = AppPort.objects.filter(expose_publicly = True).values_list('app_id', flat=True)
        return out

    @staticmethod
    def get_port_mappings():
        app_ports = dict(AppPort.objects.filter(expose_publicly = True).values_list('app_id', 'internal_port'))
        port_mappings = []
        for container in Container.objects.filter(app_id__in = app_ports.keys()).filter(running = True).all():
            docker_port_output = subprocess.check_output(['sudo', 'docker', 'port', container.docker_name()])
            for raw_mapping in docker_port_output.split('\n'):
                print(raw_mapping)
                if '/tcp -> 0.0.0.0:' in raw_mapping:
                    if app_ports[container.app_id] in app_ports.values():
                        internal, external = raw_mapping.split('/tcp -> 0.0.0.0:')
                        container.high_port = int(external)
                        container.save()
                        port_mappings.append((container.public_urlbase(), container.high_port))
        return port_mappings

    @staticmethod
    def generate_nginx_context(port_mappings=[]):
        if port_mappings is None:
          port_mappings = Container.get_port_mappings()
        root_port = os.environ.get('ROOT_PORT', '8000')
        hostname = os.environ.get('HOST_NAME', settings.HOSTNAME)
        ip_addr = os.environ.get('IP_ADDR', settings.IP_ADDR)
        return {
            'static_root': settings.STATIC_ROOT,
            'portmaps': [{'port': mapping[1], 'urlbase': mapping[0]}
                                    for mapping in port_mappings]
                , 'root_port': root_port, 'hostname': hostname, 'ip_addr': ip_addr}

    @staticmethod
    def map_public_ports(port_mappings=None):
        """
        Create a new nginx config with an entry for every container that is supposed to be running and has a public path base.
        Then, restart nginx.
        """
        Container.fill_template(Container.NGINX_CONFIG_TEMPLATE_PATH,
            Container.NGINX_CONFIG_DESTINATION_PATH, Container.generate_nginx_context(port_mappings))
        command = ["sudo","cp",Container.NGINX_CONFIG_DESTINATION_PATH, Container.NGINX_CONFIG_COPY_PATH]
        print(command)
        subprocess.check_output(command)
        return subprocess.check_output(["sudo","service","nginx","restart"])

from task_manager.docker_tasks import start_containers

def start_container_celery(sender, instance, **kwargs):
    start_containers.delay(instance)

if settings.DEPLOYMENT:
    post_save.connect(start_container_celery, sender = Project)

def get_zipped_data_path(instance, filename):
    """
    This method must stay outside of the class definition because django
    cannot serialize unbound methods in Python 2:

    https://docs.djangoproject.com/en/dev/topics/migrations/#migration-serializing
    """
    return os.path.join(settings.MEDIA_ROOT, "indices", instance.slug, "zipped_data", filename)

class Index(models.Model):
    """Index model.

    The index model keeps track of indices that are made and what files are
    contained within them.

    Model Fields
    ------------

    name : str, 64 characters max
    description : textfield
    uploaded_data : Django FileField
    data_dolder : textfield
    project : fk to base.Project

    """
    def get_dumped_data_path(instance):
        return os.path.join(
            settings.MEDIA_ROOT,
            "indices",
            instance.slug,
            "data"
        )

    name = models.CharField(max_length=64, unique=True,
        validators=[alphanumeric_validator()])
    slug = models.SlugField(max_length=64, unique=True)
    uploaded_data = models.FileField(upload_to=get_zipped_data_path,
        validators=[zipped_file_validator()])
    data_folder = models.TextField(blank=True)
    project = models.ForeignKey(Project)

    def save(self, *args, **kwargs):
        self.slug = slugify(unicode(self.name))
        if self.uploaded_data:
            zipped_data_path = os.path.dirname(get_zipped_data_path(self, self.uploaded_data.name))
            if os.path.isdir(zipped_data_path):
                delete_folder_contents(zipped_data_path)
            super(Index, self).save(*args, **kwargs)
            self.data_folder = self.get_dumped_data_path()
            if os.path.isdir(self.data_folder):
                delete_folder_contents(self.data_folder)
            unzip.delay(self.uploaded_data.name, self.data_folder)
            if settings.DEPLOYMENT:
                create_index.delay(self)
        super(Index, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return reverse('base:index_settings',
            kwargs=dict(index_slug=self.slug, project_slug=self.project.slug))

<|endoftext|>


<commit_before>#!/usr/bin/python
# utils/rusage.py - Utility to measure resource usage -*- python -*-
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2017 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
#
# Simple informative/supervisory wrapper around getrusage() and, optionally,
# setrlimit()
#
# By default this script lets its subprocess run to completion whatever its
# consumed limits, and only prints a limit-violation if it occurs after the
# fact. If you want to be a bit more abrupt you can run with --enforce and
# it will also run subprocesses under setrlimit().
#
# Note: setrlimit(RLIMIT_RSS) does nothing on macOS. It's unenforced.
#
# If there's a limit-violation observed (by getrusage()), it's printed to
# stderr and the wrapper exits with an error. Whether or not the violation
# was caused by enforcement or mere observation.
#
# If there's no limit-violation, the subprocess' own exit code is
# propagated, either silently or, if --verbose is passed, after printing
# the actual getrusage() memory and time values (such that they can be used
# as a limit in future runs).
#

import argparse
import resource
import subprocess
import sys

class MemAction(argparse.Action):
    def __init__(self, *args, **kwargs):
        super(MemAction, self).__init__(*args, **kwargs)

    def __call__(self, parser, namespace, v, option_string=None):
        r = None
        if v.endswith('K'):
            r = int(v[:-1]) * 1024
        elif v.endswith('M'):
            r = int(v[:-1]) * 1024 * 1024
        elif v.endswith('G'):
            r = int(v[:-1]) * 1024 * 1024 * 1024
        else:
            r = int(v)
        setattr(namespace, self.dest, r)

class TimeAction(argparse.Action):
    def __init__(self, *args, **kwargs):
        super(TimeAction, self).__init__(*args, **kwargs)

    def __call__(self, parser, namespace, v, option_string=None):
        r = None
        if v.endswith('ms'):
            r = float(v[:-2]) / 1000.0
        elif v.endswith('us'):
            r = float(v[:-2]) / 1000000.0
        else:
            r = float(v)
        setattr(namespace, self.dest, r)

parser = argparse.ArgumentParser()
parser.add_argument("--mem",
                    metavar="M",
                    help="memory (in bytes, or ..'K', 'M', 'G')",
                    action=MemAction)
parser.add_argument("--time",
                    metavar="T",
                    help="time (in secs, or ..'ms', 'us')",
                    action=TimeAction)
parser.add_argument("--enforce",
                    action='store_true',
                    default=False,
                    help="call setrlimit() before running subprocess")
parser.add_argument("--verbose",
                    action='store_true',
                    default=False,
                    help="always report status and usage")
parser.add_argument('remainder', nargs=argparse.REMAINDER,
                    help="subcommand to run under supervision")

args = parser.parse_args()
if len(args.remainder) == 0:
    parser.print_help()
    sys.exit(1)

if args.enforce:
    if args.time is not None:
        secs = max(1, int(args.time))
        (soft, hard) = resource.getrlimit(resource.RLIMIT_CPU)
        secs = min(soft, hard, secs)
        if args.verbose:
            sys.stderr.write("rusage: setrlimit(RLIMIT_CPU, %d)\n"
                             % secs)
        resource.setrlimit(resource.RLIMIT_CPU, (secs, secs))
    if args.mem is not None:
        (soft, hard) = resource.getrlimit(resource.RLIMIT_RSS)
        mem = min(soft, hard, args.mem)
        if args.verbose:
            sys.stderr.write("rusage: setrlimit(RLIMIT_RSS, %d)\n"
                             % mem)
        resource.setrlimit(resource.RLIMIT_RSS, (mem, mem))
ret = subprocess.call(args.remainder)
used = resource.getrusage(resource.RUSAGE_CHILDREN)

if args.verbose:
    sys.stderr.write("rusage: subprocess exited %d\n" % ret)

over_mem = args.mem is not None and used.ru_maxrss > args.mem
over_time = args.time is not None and used.ru_utime > args.time

if args.verbose or over_mem:
    sys.stderr.write("rusage: subprocess mem: %d bytes\n"
                     % used.ru_maxrss)
if over_mem:
    sys.stderr.write("rusage: exceeded limit: %d bytes\n"
                     % args.mem)

if args.verbose or over_time:
    sys.stderr.write("rusage: subprocess time: %.6f secs\n"
                     % used.ru_utime)
if over_time:
    sys.stderr.write("rusage:  exceeded limit: %.6f secs\n"
                     % args.time)

if over_mem or over_time:
    sys.exit(-1)
sys.exit(ret)
<commit_msg>[utils] Add a CSV-output mode to rusage script.<commit_after>#!/usr/bin/python
# utils/rusage.py - Utility to measure resource usage -*- python -*-
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2017 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
#
# Simple informative/supervisory wrapper around getrusage() and, optionally,
# setrlimit()
#
# By default this script lets its subprocess run to completion whatever its
# consumed limits, and only prints a limit-violation if it occurs after the
# fact. If you want to be a bit more abrupt you can run with --enforce and
# it will also run subprocesses under setrlimit().
#
# Note: setrlimit(RLIMIT_RSS) does nothing on macOS. It's unenforced.
#
# If there's a limit-violation observed (by getrusage()), it's printed to
# stderr and the wrapper exits with an error. Whether or not the violation
# was caused by enforcement or mere observation.
#
# If there's no limit-violation, the subprocess' own exit code is
# propagated, either silently or, if --verbose is passed, after printing
# the actual getrusage() memory and time values (such that they can be used
# as a limit in future runs).
#

import argparse
import csv
import datetime
import resource
import subprocess
import sys

class MemAction(argparse.Action):
    def __init__(self, *args, **kwargs):
        super(MemAction, self).__init__(*args, **kwargs)

    def __call__(self, parser, namespace, v, option_string=None):
        r = None
        if v.endswith('K'):
            r = int(v[:-1]) * 1024
        elif v.endswith('M'):
            r = int(v[:-1]) * 1024 * 1024
        elif v.endswith('G'):
            r = int(v[:-1]) * 1024 * 1024 * 1024
        else:
            r = int(v)
        setattr(namespace, self.dest, r)

class TimeAction(argparse.Action):
    def __init__(self, *args, **kwargs):
        super(TimeAction, self).__init__(*args, **kwargs)

    def __call__(self, parser, namespace, v, option_string=None):
        r = None
        if v.endswith('ms'):
            r = float(v[:-2]) / 1000.0
        elif v.endswith('us'):
            r = float(v[:-2]) / 1000000.0
        else:
            r = float(v)
        setattr(namespace, self.dest, r)

parser = argparse.ArgumentParser()
parser.add_argument("--mem",
                    metavar="M",
                    help="memory (in bytes, or ..'K', 'M', 'G')",
                    action=MemAction)
parser.add_argument("--time",
                    metavar="T",
                    help="time (in secs, or ..'ms', 'us')",
                    action=TimeAction)
parser.add_argument("--enforce",
                    action='store_true',
                    default=False,
                    help="call setrlimit() before running subprocess")
parser.add_argument("--verbose",
                    action='store_true',
                    default=False,
                    help="always report status and usage")
parser.add_argument("--csv",
                    action='store_true',
                    default=False,
                    help="write results as CSV")
parser.add_argument("--csv-output", default="-",
                    type=argparse.FileType('ab', 0),
                    help="Append CSV output to file")
parser.add_argument("--csv-name", type=str,
                    default=str(datetime.datetime.now()),
                    help="Label row in CSV with name")
parser.add_argument('remainder', nargs=argparse.REMAINDER,
                    help="subcommand to run under supervision")

args = parser.parse_args()
if len(args.remainder) == 0:
    parser.print_help()
    sys.exit(1)

if args.enforce:
    if args.time is not None:
        secs = max(1, int(args.time))
        (soft, hard) = resource.getrlimit(resource.RLIMIT_CPU)
        secs = min(soft, hard, secs)
        if args.verbose:
            sys.stderr.write("rusage: setrlimit(RLIMIT_CPU, %d)\n"
                             % secs)
        resource.setrlimit(resource.RLIMIT_CPU, (secs, secs))
    if args.mem is not None:
        (soft, hard) = resource.getrlimit(resource.RLIMIT_RSS)
        mem = min(soft, hard, args.mem)
        if args.verbose:
            sys.stderr.write("rusage: setrlimit(RLIMIT_RSS, %d)\n"
                             % mem)
        resource.setrlimit(resource.RLIMIT_RSS, (mem, mem))
ret = subprocess.call(args.remainder)
used = resource.getrusage(resource.RUSAGE_CHILDREN)

if args.verbose:
    sys.stderr.write("rusage: subprocess exited %d\n" % ret)

over_mem = args.mem is not None and used.ru_maxrss > args.mem
over_time = args.time is not None and used.ru_utime > args.time

if args.verbose or over_mem:
    sys.stderr.write("rusage: subprocess mem: %d bytes\n"
                     % used.ru_maxrss)
if over_mem:
    sys.stderr.write("rusage: exceeded limit: %d bytes\n"
                     % args.mem)

if args.verbose or over_time:
    sys.stderr.write("rusage: subprocess time: %.6f secs\n"
                     % used.ru_utime)
if over_time:
    sys.stderr.write("rusage:  exceeded limit: %.6f secs\n"
                     % args.time)

if args.csv:
    fieldnames = ["time", "mem", "run"]
    out = csv.DictWriter(args.csv_output, fieldnames, dialect='excel-tab')
    if args.csv_output.tell() == 0:
        out.writeheader()
    out.writerow(dict(time=used.ru_utime,
                      mem=used.ru_maxrss,
                      run=args.csv_name))

if over_mem or over_time:
    sys.exit(-1)
sys.exit(ret)
<|endoftext|>


<commit_before># Copyright 2020 The StackStorm Authors.
# Copyright 2019 Extreme Networks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
import abc
import six

from kombu.mixins import ConsumerMixin
from oslo_config import cfg

from st2common import log as logging
from st2common.util.greenpooldispatch import BufferedDispatcher
from st2common.util import concurrency

__all__ = [
    "QueueConsumer",
    "StagedQueueConsumer",
    "ActionsQueueConsumer",
    "MessageHandler",
    "StagedMessageHandler",
]

LOG = logging.getLogger(__name__)

class QueueConsumer(ConsumerMixin):
    def __init__(self, connection, queues, handler):
        self.connection = connection
        self._dispatcher = BufferedDispatcher()
        self._queues = queues
        self._handler = handler

    def shutdown(self):
        self._dispatcher.shutdown()

    def get_consumers(self, Consumer, channel):
        consumer = Consumer(
            queues=self._queues, accept=["pickle"], callbacks=[self.process]
        )

        # use prefetch_count=1 for fair dispatch. This way workers that finish an item get the next
        # task and the work does not get queued behind any single large item.
        consumer.qos(prefetch_count=1)

        return [consumer]

    def process(self, body, message):
        try:
            if not isinstance(body, self._handler.message_type):
                raise TypeError(
                    'Received an unexpected type "%s" for payload.' % type(body)
                )

            self._dispatcher.dispatch(self._process_message, body)
        except:
            LOG.exception(
                "%s failed to process message: %s", self.__class__.__name__, body
            )
        finally:
            # At this point we will always ack a message.
            message.ack()

    def _process_message(self, body):
        try:
            self._handler.process(body)
        except:
            LOG.exception(
                "%s failed to process message: %s", self.__class__.__name__, body
            )

class StagedQueueConsumer(QueueConsumer):
    """
    Used by ``StagedMessageHandler`` to effectively manage it 2 step message handling.
    """

    def process(self, body, message):
        try:
            if not isinstance(body, self._handler.message_type):
                raise TypeError(
                    'Received an unexpected type "%s" for payload.' % type(body)
                )
            response = self._handler.pre_ack_process(body)
            self._dispatcher.dispatch(self._process_message, response)
        except:
            LOG.exception(
                "%s failed to process message: %s", self.__class__.__name__, body
            )
        finally:
            # At this point we will always ack a message.
            message.ack()

class ActionsQueueConsumer(QueueConsumer):
    """
    Special Queue Consumer for action runner which uses multiple BufferedDispatcher pools:

    1. For regular (non-workflow) actions
    2. One for workflow actions

    This way we can ensure workflow actions never block non-workflow actions.
    """

    def __init__(self, connection, queues, handler):
        self.connection = connection

        self._queues = queues
        self._handler = handler

        workflows_pool_size = cfg.CONF.actionrunner.workflows_pool_size
        actions_pool_size = cfg.CONF.actionrunner.actions_pool_size
        self._workflows_dispatcher = BufferedDispatcher(
            dispatch_pool_size=workflows_pool_size, name="workflows-dispatcher"
        )
        self._actions_dispatcher = BufferedDispatcher(
            dispatch_pool_size=actions_pool_size, name="actions-dispatcher"
        )

    def process(self, body, message):
        try:
            if not isinstance(body, self._handler.message_type):
                raise TypeError(
                    'Received an unexpected type "%s" for payload.' % type(body)
                )

            action_is_workflow = getattr(body, "action_is_workflow", False)
            if action_is_workflow:
                # Use workflow dispatcher queue
                dispatcher = self._workflows_dispatcher
            else:
                # Use queue for regular or workflow actions
                dispatcher = self._actions_dispatcher

            LOG.debug('Using BufferedDispatcher pool: "%s"', str(dispatcher))
            dispatcher.dispatch(self._process_message, body)
        except:
            LOG.exception(
                "%s failed to process message: %s", self.__class__.__name__, body
            )
        finally:
            # At this point we will always ack a message.
            message.ack()

    def shutdown(self):
        self._workflows_dispatcher.shutdown()
        self._actions_dispatcher.shutdown()

class VariableMessageQueueConsumer(QueueConsumer):
    """
    Used by ``VariableMessageHandler`` to processes multiple message types.
    """

    def process(self, body, message):
        try:
            if not self._handler.message_types.get(type(body)):
                raise TypeError(
                    'Received an unexpected type "%s" for payload.' % type(body)
                )

            self._dispatcher.dispatch(self._process_message, body)
        except:
            LOG.exception(
                "%s failed to process message: %s", self.__class__.__name__, body
            )
        finally:
            # At this point we will always ack a message.
            message.ack()

@six.add_metaclass(abc.ABCMeta)
class MessageHandler(object):
    message_type = None

    def __init__(self, connection, queues):
        self._queue_consumer = self.get_queue_consumer(
            connection=connection, queues=queues
        )
        self._consumer_thread = None

    def start(self, wait=False):
        LOG.info("Starting %s...", self.__class__.__name__)
        self._consumer_thread = concurrency.spawn(self._queue_consumer.run)

        if wait:
            self.wait()

    def wait(self):
        self._consumer_thread.wait()

    def shutdown(self):
        LOG.info("Shutting down %s...", self.__class__.__name__)
        self._queue_consumer.shutdown()

    @abc.abstractmethod
    def process(self, message):
        pass

    def get_queue_consumer(self, connection, queues):
        return QueueConsumer(connection=connection, queues=queues, handler=self)

@six.add_metaclass(abc.ABCMeta)
class StagedMessageHandler(MessageHandler):
    """
    MessageHandler to deal with messages in 2 steps.
        1. pre_ack_process : This is called on the handler before ack-ing the message.
        2. process: Called after ack-in the messages
    This 2 step approach provides a way for the handler to do some hadling like saving to DB etc
    before acknowleding and then performing future processing async. This way even if the handler
    or owning process is taken down system will still maintain track of the message.
    """

    @abc.abstractmethod
    def pre_ack_process(self, message):
        """
        Called before acknowleding a message. Good place to track the message via a DB entry or some
        other applicable mechnism.

        The reponse of this method is passed into the ``process`` method. This was whatever is the
        processed version of the message can be moved forward. It is always possible to simply
        return ``message`` and have ``process`` handle the original message.
        """
        pass

    def get_queue_consumer(self, connection, queues):
        return StagedQueueConsumer(connection=connection, queues=queues, handler=self)

@six.add_metaclass(abc.ABCMeta)
class VariableMessageHandler(MessageHandler):
    """
    VariableMessageHandler processes multiple message types.
    """

    def get_queue_consumer(self, connection, queues):
        return VariableMessageQueueConsumer(
            connection=connection, queues=queues, handler=self
        )
<commit_msg>Consumer shutdown.<commit_after># Copyright 2020 The StackStorm Authors.
# Copyright 2019 Extreme Networks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
import abc
import six

from kombu.mixins import ConsumerMixin
from oslo_config import cfg

from st2common import log as logging
from st2common.util.greenpooldispatch import BufferedDispatcher
from st2common.util import concurrency

__all__ = [
    "QueueConsumer",
    "StagedQueueConsumer",
    "ActionsQueueConsumer",
    "MessageHandler",
    "StagedMessageHandler",
]

LOG = logging.getLogger(__name__)

class QueueConsumer(ConsumerMixin):
    def __init__(self, connection, queues, handler):
        self.connection = connection
        self._dispatcher = BufferedDispatcher()
        self._queues = queues
        self._handler = handler

    def shutdown(self):
        self._dispatcher.shutdown()

    def get_consumers(self, Consumer, channel):
        consumer = Consumer(
            queues=self._queues, accept=["pickle"], callbacks=[self.process]
        )

        # use prefetch_count=1 for fair dispatch. This way workers that finish an item get the next
        # task and the work does not get queued behind any single large item.
        consumer.qos(prefetch_count=1)

        return [consumer]

    def process(self, body, message):
        try:
            if not isinstance(body, self._handler.message_type):
                raise TypeError(
                    'Received an unexpected type "%s" for payload.' % type(body)
                )

            self._dispatcher.dispatch(self._process_message, body)
        except:
            LOG.exception(
                "%s failed to process message: %s", self.__class__.__name__, body
            )
        finally:
            # At this point we will always ack a message.
            message.ack()

    def _process_message(self, body):
        try:
            self._handler.process(body)
        except:
            LOG.exception(
                "%s failed to process message: %s", self.__class__.__name__, body
            )

class StagedQueueConsumer(QueueConsumer):
    """
    Used by ``StagedMessageHandler`` to effectively manage it 2 step message handling.
    """

    def process(self, body, message):
        try:
            if not isinstance(body, self._handler.message_type):
                raise TypeError(
                    'Received an unexpected type "%s" for payload.' % type(body)
                )
            response = self._handler.pre_ack_process(body)
            self._dispatcher.dispatch(self._process_message, response)
        except:
            LOG.exception(
                "%s failed to process message: %s", self.__class__.__name__, body
            )
        finally:
            # At this point we will always ack a message.
            message.ack()

class ActionsQueueConsumer(QueueConsumer):
    """
    Special Queue Consumer for action runner which uses multiple BufferedDispatcher pools:

    1. For regular (non-workflow) actions
    2. One for workflow actions

    This way we can ensure workflow actions never block non-workflow actions.
    """

    def __init__(self, connection, queues, handler):
        self.connection = connection

        self._queues = queues
        self._handler = handler

        workflows_pool_size = cfg.CONF.actionrunner.workflows_pool_size
        actions_pool_size = cfg.CONF.actionrunner.actions_pool_size
        self._workflows_dispatcher = BufferedDispatcher(
            dispatch_pool_size=workflows_pool_size, name="workflows-dispatcher"
        )
        self._actions_dispatcher = BufferedDispatcher(
            dispatch_pool_size=actions_pool_size, name="actions-dispatcher"
        )

    def process(self, body, message):
        try:
            if not isinstance(body, self._handler.message_type):
                raise TypeError(
                    'Received an unexpected type "%s" for payload.' % type(body)
                )

            action_is_workflow = getattr(body, "action_is_workflow", False)
            if action_is_workflow:
                # Use workflow dispatcher queue
                dispatcher = self._workflows_dispatcher
            else:
                # Use queue for regular or workflow actions
                dispatcher = self._actions_dispatcher

            LOG.debug('Using BufferedDispatcher pool: "%s"', str(dispatcher))
            dispatcher.dispatch(self._process_message, body)
        except:
            LOG.exception(
                "%s failed to process message: %s", self.__class__.__name__, body
            )
        finally:
            # At this point we will always ack a message.
            message.ack()

    def shutdown(self):
        self._workflows_dispatcher.shutdown()
        self._actions_dispatcher.shutdown()
        self.should_stop = True

class VariableMessageQueueConsumer(QueueConsumer):
    """
    Used by ``VariableMessageHandler`` to processes multiple message types.
    """

    def process(self, body, message):
        try:
            if not self._handler.message_types.get(type(body)):
                raise TypeError(
                    'Received an unexpected type "%s" for payload.' % type(body)
                )

            self._dispatcher.dispatch(self._process_message, body)
        except:
            LOG.exception(
                "%s failed to process message: %s", self.__class__.__name__, body
            )
        finally:
            # At this point we will always ack a message.
            message.ack()

@six.add_metaclass(abc.ABCMeta)
class MessageHandler(object):
    message_type = None

    def __init__(self, connection, queues):
        self._queue_consumer = self.get_queue_consumer(
            connection=connection, queues=queues
        )
        self._consumer_thread = None

    def start(self, wait=False):
        LOG.info("Starting %s...", self.__class__.__name__)
        self._consumer_thread = concurrency.spawn(self._queue_consumer.run)

        if wait:
            self.wait()

    def wait(self):
        self._consumer_thread.wait()

    def shutdown(self):
        LOG.info("Shutting down %s...", self.__class__.__name__)
        self._queue_consumer.shutdown()

    @abc.abstractmethod
    def process(self, message):
        pass

    def get_queue_consumer(self, connection, queues):
        return QueueConsumer(connection=connection, queues=queues, handler=self)

@six.add_metaclass(abc.ABCMeta)
class StagedMessageHandler(MessageHandler):
    """
    MessageHandler to deal with messages in 2 steps.
        1. pre_ack_process : This is called on the handler before ack-ing the message.
        2. process: Called after ack-in the messages
    This 2 step approach provides a way for the handler to do some hadling like saving to DB etc
    before acknowleding and then performing future processing async. This way even if the handler
    or owning process is taken down system will still maintain track of the message.
    """

    @abc.abstractmethod
    def pre_ack_process(self, message):
        """
        Called before acknowleding a message. Good place to track the message via a DB entry or some
        other applicable mechnism.

        The reponse of this method is passed into the ``process`` method. This was whatever is the
        processed version of the message can be moved forward. It is always possible to simply
        return ``message`` and have ``process`` handle the original message.
        """
        pass

    def get_queue_consumer(self, connection, queues):
        return StagedQueueConsumer(connection=connection, queues=queues, handler=self)

@six.add_metaclass(abc.ABCMeta)
class VariableMessageHandler(MessageHandler):
    """
    VariableMessageHandler processes multiple message types.
    """

    def get_queue_consumer(self, connection, queues):
        return VariableMessageQueueConsumer(
            connection=connection, queues=queues, handler=self
        )
<|endoftext|>


<commit_before>"""Setup for the BAF tests.

Copyright (c) 2019 Red Hat Inc.

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

import os
import sys
from fastlog import log

from auth import retrieve_access_token

# The following endpoint is used to get the access token from OSIO AUTH service
_AUTH_ENDPOINT = "/api/token/refresh"

DEFAULT_INPUT_FILE = "scenarios.csv"

def add_slash(url):
    """Add a slash at the end of URL if the slash is not already presented."""
    if url and not url.endswith('/'):
        url += '/'
    return url

def read_url_from_env_var(env_var_name):
    """Read URL from selected environment variable."""
    return add_slash(os.environ.get(env_var_name, None))

def missing_api_token_warning(env_var_name):
    """Print warning or basic info about missing API token."""
    if os.environ.get(env_var_name):
        log.success("OK: {name} environment is set and will be used as "
                    "authorization token".format(name=env_var_name))
    else:
        log.warning("Warning: the {name} environment variable is not"
                    " set.\n"
                    "Most tests that require authorization will probably fail".format(
                        name=env_var_name))

def check_api_tokens_presence():
    """Check if API token is present in environment variable(s)."""
    # we need RECOMMENDER_API_TOKEN or RECOMMENDER_REFRESH_TOKEN to be set
    if not os.environ.get("RECOMMENDER_REFRESH_TOKEN"):
        missing_api_token_warning("RECOMMENDER_API_TOKEN")
    else:
        missing_api_token_warning("RECOMMENDER_REFRESH_TOKEN")

def yes_no(string):
    """Parse 'yes' and 'Yes' strings to True, all other values to False."""
    return string is not None and string in {"yes", "Yes"}

def enabled_disabled(b):
    """Convert boolean value to 'enabled' or 'disabled'."""
    return "enabled" if b else "disabled"

def get_access_token(dry_run, refresh_token, license_service_url):
    """Get the access token if possible."""
    if not dry_run:
        log.info("Auth. token generation")
        with log.indent():
            # we can retrieve access token by using refresh/offline token
            access_token = retrieve_access_token(refresh_token, license_service_url)
            if access_token is None:
                sys.exit(-1)
            log.success("Success")
    else:
        access_token = None
    return access_token

def parse_tags(tags):
    """Parse string containing list of tags."""
    if tags is not None:
        tags = set(tags.split(","))
    return tags

def tags_as_str(tags):
    """Convert list of tags to string."""
    return " ".join(tags) if tags else "all tests"

def refresh_token_as_str(refresh_token):
    """Convert refresh token settings (set/not set) into string."""
    return "set" if refresh_token else "not set"

def setup(cli_arguments):
    """Perform BAF setup."""
    log.info("Setup")
    with log.indent():
        input_file = cli_arguments.input or DEFAULT_INPUT_FILE
        dry_run = cli_arguments.dry
        tags = parse_tags(cli_arguments.tags)

        if not dry_run:
            check_api_tokens_presence()
            license_service_url = read_url_from_env_var("OSIO_AUTH_SERVICE")
            refresh_token = os.environ.get("RECOMMENDER_REFRESH_TOKEN")

            if not license_service_url:
                log.error("OSIO_AUTH_SERVICE is not set")
                sys.exit(-1)
        else:
            license_service_url = "N/A"
            refresh_token = None

        log.info("Dry run:          " + enabled_disabled(dry_run))
        log.info("Input file:       " + input_file)
        log.info("Auth service URL: " + license_service_url)
        log.info("Run tests:        " + tags_as_str(tags))
        log.info("Refresh token:    " + refresh_token_as_str(refresh_token))
        log.success("Success")

    if api_token is not None:
        access_token = api_token
    else:
        access_token = get_access_token(dry_run, refresh_token, license_service_url)

    return {"access_token": access_token,
            "tags": tags,
            "dry_run": dry_run,
            "input_file": input_file}
<commit_msg>Fixed linter issues<commit_after>"""Setup for the BAF tests.

Copyright (c) 2019 Red Hat Inc.

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

import os
import sys
from fastlog import log

from auth import retrieve_access_token

# The following endpoint is used to get the access token from OSIO AUTH service
_AUTH_ENDPOINT = "/api/token/refresh"

DEFAULT_INPUT_FILE = "scenarios.csv"

def add_slash(url):
    """Add a slash at the end of URL if the slash is not already presented."""
    if url and not url.endswith('/'):
        url += '/'
    return url

def read_url_from_env_var(env_var_name):
    """Read URL from selected environment variable."""
    return add_slash(os.environ.get(env_var_name, None))

def missing_api_token_warning(env_var_name):
    """Print warning or basic info about missing API token."""
    if os.environ.get(env_var_name):
        log.success("OK: {name} environment is set and will be used as "
                    "authorization token".format(name=env_var_name))
    else:
        log.warning("Warning: the {name} environment variable is not"
                    " set.\n"
                    "Most tests that require authorization will probably fail".format(
                        name=env_var_name))

def check_api_tokens_presence():
    """Check if API token is present in environment variable(s)."""
    # we need RECOMMENDER_API_TOKEN or RECOMMENDER_REFRESH_TOKEN to be set
    if not os.environ.get("RECOMMENDER_REFRESH_TOKEN"):
        missing_api_token_warning("RECOMMENDER_API_TOKEN")
    else:
        missing_api_token_warning("RECOMMENDER_REFRESH_TOKEN")

def yes_no(string):
    """Parse 'yes' and 'Yes' strings to True, all other values to False."""
    return string is not None and string in {"yes", "Yes"}

def enabled_disabled(b):
    """Convert boolean value to 'enabled' or 'disabled'."""
    return "enabled" if b else "disabled"

def get_access_token(dry_run, refresh_token, license_service_url):
    """Get the access token if possible."""
    if not dry_run:
        log.info("Auth. token generation")
        with log.indent():
            # we can retrieve access token by using refresh/offline token
            access_token = retrieve_access_token(refresh_token, license_service_url)
            if access_token is None:
                sys.exit(-1)
            log.success("Success")
    else:
        access_token = None
    return access_token

def parse_tags(tags):
    """Parse string containing list of tags."""
    if tags is not None:
        tags = set(tags.split(","))
    return tags

def tags_as_str(tags):
    """Convert list of tags to string."""
    return " ".join(tags) if tags else "all tests"

def refresh_token_as_str(refresh_token):
    """Convert refresh token settings (set/not set) into string."""
    return "set" if refresh_token else "not set"

def setup(cli_arguments):
    """Perform BAF setup."""
    log.info("Setup")

    refresh_token = None
    api_token = None

    with log.indent():
        input_file = cli_arguments.input or DEFAULT_INPUT_FILE
        dry_run = cli_arguments.dry
        tags = parse_tags(cli_arguments.tags)

        if not dry_run:
            check_api_tokens_presence()
            license_service_url = read_url_from_env_var("OSIO_AUTH_SERVICE")
            refresh_token = os.environ.get("RECOMMENDER_REFRESH_TOKEN")
            api_token = os.environ.get("RECOMMENDER_API_TOKEN")

            if not license_service_url:
                log.error("OSIO_AUTH_SERVICE is not set")
                sys.exit(-1)
        else:
            license_service_url = "N/A"

        log.info("Dry run:          " + enabled_disabled(dry_run))
        log.info("Input file:       " + input_file)
        log.info("Auth service URL: " + license_service_url)
        log.info("Run tests:        " + tags_as_str(tags))
        log.info("Refresh token:    " + refresh_token_as_str(refresh_token))
        log.success("Success")

    if api_token is not None:
        access_token = api_token
    else:
        access_token = get_access_token(dry_run, refresh_token, license_service_url)

    return {"access_token": access_token,
            "tags": tags,
            "dry_run": dry_run,
            "input_file": input_file}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
import numpy
import pytest

import quadpy
from helpers import check_degree
from quadpy.ncube._helpers import integrate_monomial_over_ncube

@pytest.mark.parametrize(
    "scheme",
    [quadpy.ncube.dobrodeev_1970(n) for n in range(5, 8)]
    + [quadpy.ncube.dobrodeev_1978(n) for n in range(2, 8)]
    + [quadpy.ncube.hammer_stroud_1n(n) for n in range(3, 7)]
    + [quadpy.ncube.hammer_stroud_2n(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_1_1(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_1_2(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_2_1(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_2_2(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_3_1(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_3_2(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_3_3(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_3_4(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_3_5(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_3_6(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_5_2(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_5_3(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_5_4(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_5_5(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_5_6(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_5_7(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_5_8(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_5_9(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_7_1(n) for n in range(3, 7)],
)
def test_scheme(scheme, tol=1.0e-14):
    assert scheme.points.dtype in [numpy.float64, numpy.int64], scheme.name
    assert scheme.weights.dtype in [numpy.float64, numpy.int64], scheme.name

    n = scheme.dim
    ncube_limits = [[0.0, 1.0]] * n
    ncube = quadpy.ncube.ncube_points(*ncube_limits)
    degree = check_degree(
        lambda poly: scheme.integrate(poly, ncube),
        lambda exp: integrate_monomial_over_ncube(ncube_limits, exp),
        n,
        scheme.degree + 1,
        tol=tol,
    )
    assert degree >= scheme.degree, "observed: {}, expected: {}".format(
        degree, scheme.degree
    )
    return

if __name__ == "__main__":
    n_ = 4
    scheme_ = quadpy.ncube.Stroud(n_, "Cn 7-1")
    test_scheme(scheme_, 1.0e-14)
<commit_msg>ncube tests -> ortho<commit_after># -*- coding: utf-8 -*-
#
import math

import numpy
import pytest

import quadpy
import orthopy
from helpers import check_degree_ortho

@pytest.mark.parametrize(
    "scheme",
    [quadpy.ncube.dobrodeev_1970(n) for n in range(5, 8)]
    + [quadpy.ncube.dobrodeev_1978(n) for n in range(2, 8)]
    + [quadpy.ncube.hammer_stroud_1n(n) for n in range(3, 7)]
    + [quadpy.ncube.hammer_stroud_2n(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_1_1(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_1_2(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_2_1(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_2_2(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_3_1(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_3_2(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_3_3(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_3_4(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_3_5(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_3_6(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_5_2(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_5_3(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_5_4(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_5_5(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_5_6(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_5_7(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_5_8(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_5_9(n) for n in range(3, 7)]
    + [quadpy.ncube.stroud_cn_7_1(n) for n in range(3, 7)],
)
def test_scheme(scheme, tol=1.0e-11):
    assert scheme.points.dtype in [numpy.float64, numpy.int64], scheme.name
    assert scheme.weights.dtype in [numpy.float64, numpy.int64], scheme.name

    n = scheme.dim
    ncube_limits = [[-1.0, 1.0]] * n
    ncube = quadpy.ncube.ncube_points(*ncube_limits)

    # degree = check_degree(
    #     lambda poly: scheme.integrate(poly, ncube),
    #     lambda exp: integrate_monomial_over_ncube(ncube_limits, exp),
    #     n,
    #     scheme.degree + 1,
    #     tol=tol,
    # )
    # assert degree >= scheme.degree, "observed: {}, expected: {}".format(
    #     degree, scheme.degree
    # )

    def eval_orthopolys(x):
        return numpy.concatenate(
            orthopy.ncube.tree(x, scheme.degree + 1, symbolic=False)
        )

    vals = scheme.integrate(eval_orthopolys, ncube)

    # Put vals back into the tree structure:
    # len(approximate[k]) == k+1
    approximate = [
        vals[
            numpy.prod(range(k, k + n))
            // math.factorial(n) : numpy.prod(range(k + 1, k + 1 + n))
            // math.factorial(n)
        ]
        for k in range(scheme.degree + 2)
    ]

    exact = [numpy.zeros(len(s)) for s in approximate]
    exact[0][0] = numpy.sqrt(2.0) ** n

    degree = check_degree_ortho(approximate, exact, abs_tol=tol)

    assert degree >= scheme.degree, "{} (dim: {}) -- Observed: {}, expected: {}".format(
        scheme.name, scheme.dim, degree, scheme.degree
    )
    return

if __name__ == "__main__":
    n_ = 4
    scheme_ = quadpy.ncube.Stroud(n_, "Cn 7-1")
    test_scheme(scheme_, 1.0e-14)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from rubikscubennnsolver.RubiksCube444 import RubiksCube444, solved_444, rotate_444
from myModule import bitfield_rotate_face_90 as C_bitfield_rotate_face_90
from myModule import rotate_444 as C_rotate_444
from pprint import pformat
import datetime as dt
import logging

def pretty_time(delta):
    delta = str(delta)

    if delta.startswith('0:00:00.'):
        delta_us = int(delta.split('.')[1])
        delta_ms = int(delta_us/1000)

        if delta_ms >= 500:
            return "\033[91m%sms\033[0m" % delta_ms
        else:
            return "%sms" % delta_ms

    elif delta.startswith('0:00:01.'):
        delta_us = int(delta.split('.')[1])
        delta_ms = 1000 + int(delta_us/1000)
        return "\033[91m%sms\033[0m" % delta_ms

    else:
        return "\033[91m%s\033[0m" % delta

def python_bitfield_rotate_face_90(bitfield):
    return (
        (((bitfield >> 12) & 0xF) << 60) | # c30 -> c00
        (((bitfield >> 28) & 0xF) << 56) | # c20 -> c01
        (((bitfield >> 44) & 0xF) << 52) | # c10 -> c02
        (((bitfield >> 60) & 0xF) << 48) | # c00 -> c03

        (((bitfield >>  8) & 0xF) << 44) | # c31 -> c10
        (((bitfield >> 24) & 0xF) << 40) | # c21 -> c11
        (((bitfield >> 40) & 0xF) << 36) | # c11 -> c12
        (((bitfield >> 56) & 0xF) << 32) | # c01 -> c13

        (((bitfield >>  4) & 0xF) << 28) | # c32 -> c20
        (((bitfield >> 20) & 0xF) << 24) | # c22 -> c21
        (((bitfield >> 36) & 0xF) << 20) | # c12 -> c22
        (((bitfield >> 52) & 0xF) << 16) | # c02 -> c23

        (((bitfield >>  0) & 0xF) << 12) | # c33 -> c30
        (((bitfield >> 16) & 0xF) <<  8) | # c23 -> c31
        (((bitfield >> 32) & 0xF) <<  4) | # c13 -> c32
        (((bitfield >> 48) & 0xF) <<  0)   # c03 -> c33
    )

logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(filename)20s %(levelname)8s: %(message)s')
log = logging.getLogger(__name__)

# Color the errors and warnings in red
logging.addLevelName(logging.ERROR, "\033[91m   %s\033[0m" % logging.getLevelName(logging.ERROR))
logging.addLevelName(logging.WARNING, "\033[91m %s\033[0m" % logging.getLevelName(logging.WARNING))

COUNT = 1000000
#COUNT = 10000
cube = RubiksCube444(solved_444, 'URFDLB')
state = cube.state[:]

# For 1,000,000 this took 0:00:05.157367 at 193,897 nodes-per-sec
start_time0 = dt.datetime.now()
for x in range(COUNT):
    state = rotate_444(state[:], "U")
end_time0= dt.datetime.now()
delta = end_time0 - start_time0
print ("{:,} rotates using python rotate_444() took {} at {:,} nodes-per-sec".format(COUNT, pretty_time(delta), int(COUNT/delta.total_seconds())))

# For 1,000,000 this took 0:00:18.350457 at 54,494 nodes-per-sec
'''
cube.re_init()
start_time0 = dt.datetime.now()
for x in range(COUNT):
    cube.rotate("U")
end_time0= dt.datetime.now()
delta = end_time0 - start_time0
print ("{:,} rotates using rotate() took {} at {:,} nodes-per-sec".format(COUNT, pretty_time(delta), int(COUNT/delta.total_seconds())))
'''

cube.re_init()
#cube.print_cube()
state = cube.state[:]
#state = C_rotate_444(state[:], "U")
#cube.state = state
#cube.print_cube()

#cube.re_init()
#cube.print_cube()
state = cube.state[:]
#print("0: %s" % pformat(state))
start_time0 = dt.datetime.now()
for x in range(COUNT):
    #state = C_rotate_444(state[:], "U")
    C_rotate_444(state, "U")
    #print("%d: %s" % (x, pformat(state, width=500)))
    #cube.state = state
    #cube.print_cube()

end_time0= dt.datetime.now()
delta = end_time0 - start_time0
print ("{:,} rotates using C      rotate_444() took {} at {:,} nodes-per-sec".format(COUNT, pretty_time(delta), int(COUNT/delta.total_seconds())))

# For 1,000,000 this took 0:00:02.641851 at 378,522 nodes-per-sec
state_U = 1234567890
start_time0 = dt.datetime.now()
for x in range(COUNT):
    state_U = python_bitfield_rotate_face_90(state_U)
end_time0= dt.datetime.now()
delta = end_time0 - start_time0
print ("{:,} rotates using python bitfield_rotate_face_90() took {} at {:,} nodes-per-sec".format(COUNT, pretty_time(delta), int(COUNT/delta.total_seconds())))

# For 1,000,000 this took 196ms at 5,090,483 nodes-per-sec
state_U = 1234567890
state_F = 9876543219
state_R = 7843789758
state_B = 8747497897
state_L = 1334443112
new_state_F = 0
new_state_R = 0
new_state_B = 0
new_state_L = 0
ROW_01     = 0xFFFF000000000000
NOT_ROW_01 = 0x0000FFFFFFFFFFFF
start_time0 = dt.datetime.now()
for x in range(COUNT):
    #new_state_F = (state_F & NOT_ROW_01) | (state_R & ROW_01)
    #new_state_R = (state_R & NOT_ROW_01) | (state_B & ROW_01)
    #new_state_B = (state_B & NOT_ROW_01) | (state_L & ROW_01)
    #new_state_L = (state_L & NOT_ROW_01) | (state_F & ROW_01)
    state_U = C_bitfield_rotate_face_90(state_U)
    #state_F = new_state_F
    #state_R = new_state_R
    #state_B = new_state_B
    #state_L = new_state_L

end_time0= dt.datetime.now()
delta = end_time0 - start_time0
print ("{:,} rotates using C      bitfield_rotate_face_90() took {} at {:,} nodes-per-sec".format(COUNT, pretty_time(delta), int(COUNT/delta.total_seconds())))
<commit_msg>Removed utils/timeit.py<commit_after><|endoftext|>


<commit_before>'''Wake On LAN and computer monitoring node'''

### Libraries this Node requires
import struct, socket, re

### Parameters used by this Node
param_macAddress = Parameter('{"title":"MAC Address","schema":{"type":"string"}}')

### Functions used by this Node
def sendMagicPacket(dst_mac_addr):
    if not re.match("[0-9a-f]{2}([:])[0-9a-f]{2}(\\1[0-9a-f]{2}){4}$", dst_mac_addr.lower()):
      raise ValueError('Incorrect MAC address format')
    addr_byte = dst_mac_addr.upper().split(':')
    hw_addr = struct.pack('BBBBBB', int(addr_byte[0], 16), int(addr_byte[1], 16), int(addr_byte[2], 16), int(addr_byte[3], 16), int(addr_byte[4], 16), int(addr_byte[5], 16))
    macpck = '\xff' * 6 + hw_addr * 16
    scks = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    scks.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
    scks.sendto(macpck, ('<broadcast>', 9))

### Local actions this Node provides
def local_action_PowerOn(arg = None):
  """{"group": "Power"}"""
  lookup_local_action('Power').call('On')

def local_action_SendWOL(arg = None):
  """{"group": "Power"}"""
  print 'Sending WOL magic packet'
  sendMagicPacket(param_macAddress)

remote_action_PowerOff = RemoteAction()

### Main
def main(arg = None):
  # Start your script here.
  print 'Nodel script started.'

local_event_Status = LocalEvent({'group': 'Status', 'order': next_seq(), 'schema': {'type': 'object', 'properties': {
        'level': {'type': 'integer', 'order': 1},
        'message': {'type': 'string', 'order': 2}}}})

local_event_ComputerStatus = LocalEvent({'group': '(advanced)', 'order': 9999+next_seq(), 'schema': {'type': 'object', 'properties': {
        'level': {'type': 'integer', 'order': 1},
        'message': {'type': 'string', 'order': 2}}}})

def remote_event_ComputerStatus(arg=None):
  local_event_ComputerStatus.emit(arg)

  # this alway implies the computer is on, so set the on flag
  local_event_Power.emit('On')

  updateStatus()

# power on and power off
POWER_SCHEMA = {'type': 'string', 'enum': ['On', 'Off']}

local_event_DesiredPower = LocalEvent({'group': 'Power', 'order': next_seq(), 'schema': {'type': 'string', 'enum': ['On', 'Off']}})
local_event_Power = LocalEvent({'group': 'Power', 'order': next_seq(), 'schema': {'type': 'string', 'enum': ['Partially On', 'Partially Off', 'On', 'Off']}})

def local_action_Power(arg=None):
  '''{"group": "Power", "schema": {"type": "string", "enum": ["On", "Off"]}}'''
  local_event_DesiredPower.emit(arg)

  if arg == 'On':
    lookup_local_action('SendWOL').call()

  elif arg == 'Off':
    lookup_remote_action('PowerOff').call()

  updateStatus()

local_event_WiringStatus = LocalEvent({'group': 'Status', 'order': next_seq(), 'schema': {'type': 'string'}})

def checkWiringStatus():
  bindingState = lookup_remote_action('PowerOff').getBindingState()
  local_event_WiringStatus.emit(str(bindingState))

  updateStatus()

Timer(checkWiringStatus, 60, 7)

# performs the status aggregation
def updateStatus():
  now = date_now().getMillis()

  desiredPower = lookup_local_event('DesiredPower').getArg()
  if desiredPower is None:
    local_event_Status.emit({'level': 1, 'message': 'No desired status has been set yet.'})
    return

  elif desiredPower == 'On':
    # computer is supposed to be on so check wiring status
    wiringStatus = local_event_WiringStatus.getArg()

    if wiringStatus != 'Wired':
      last = local_event_ComputerStatus.getTimestamp()
      when = last.toString('E dd-MMM h:mm a') if last != None else "<never>"

      diff = now - (last.getMillis() if last != None else 0)
      if diff > 150000: # give it two and half minutes to boot up
        local_event_Status.emit({'level': 2, 'message': 'No connection from computer (last heard from %s)' % (when)})

      local_event_Power.emit('Partially On')

      return

    else:
      # wiring status is 'Wired', so pass through last computer status and power state
      lastStatus = local_event_ComputerStatus.getArg()
      local_event_Power.emit('On')

      if lastStatus == None:
        local_event_Status.emit({'level': 1, 'message': 'No status information from computer yet.'})

      else:
        # pass on last status
        local_event_Status.emit(lastStatus)

  else:
    # (desired power is Off, ensure off)
    timestamp = local_event_ComputerStatus.getTimestamp()
    last = 0 if timestamp == None else timestamp.getMillis()
    diff = now - last

    if diff < 180000: # give it at least 3 minutes before determining it's off
      local_event_Power.emit('Partially Off')

    else:
      local_event_Power.emit('Off')
      local_event_Status.emit({'level': 0, 'message': 'OK (*)'})

# Remote events to activate actions
def remote_event_Power(arg):
  lookup_local_action('Power').call(arg)
<commit_msg>WOL: *update* expand MAC address formats (#57)<commit_after>'''Wake On LAN and computer monitoring node'''

### Libraries this Node requires
import struct, socket, re

### Parameters used by this Node
param_macAddress = Parameter('{"title":"MAC Address","schema":{"type":"string"}}')

### Functions used by this Node
def sendMagicPacket(dst_mac_addr):
    # addr is delimited by ":" or "-"
    if re.match("^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$", dst_mac_addr.lower()):
      addr_byte = re.split(':|-', dst_mac_addr.upper())
    # addr is undelimited
    elif re.match("^([0-9A-Fa-f]{12})$", dst_mac_addr.lower()):
      addr_byte = re.findall('..', dst_mac_addr.upper())
    else:
      raise ValueError('Incorrect MAC address format')
    hw_addr = struct.pack('BBBBBB', int(addr_byte[0], 16), int(addr_byte[1], 16), int(addr_byte[2], 16), int(addr_byte[3], 16), int(addr_byte[4], 16), int(addr_byte[5], 16))
    macpck = '\xff' * 6 + hw_addr * 16
    scks = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    scks.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
    scks.sendto(macpck, ('<broadcast>', 9))

### Local actions this Node provides
def local_action_PowerOn(arg = None):
  """{"group": "Power"}"""
  lookup_local_action('Power').call('On')

def local_action_SendWOL(arg = None):
  """{"group": "Power"}"""
  print 'Sending WOL magic packet'
  sendMagicPacket(param_macAddress)

remote_action_PowerOff = RemoteAction()

### Main
def main(arg = None):
  # Start your script here.
  print 'Nodel script started.'

local_event_Status = LocalEvent({'group': 'Status', 'order': next_seq(), 'schema': {'type': 'object', 'properties': {
        'level': {'type': 'integer', 'order': 1},
        'message': {'type': 'string', 'order': 2}}}})

local_event_ComputerStatus = LocalEvent({'group': '(advanced)', 'order': 9999+next_seq(), 'schema': {'type': 'object', 'properties': {
        'level': {'type': 'integer', 'order': 1},
        'message': {'type': 'string', 'order': 2}}}})

def remote_event_ComputerStatus(arg=None):
  local_event_ComputerStatus.emit(arg)

  # this alway implies the computer is on, so set the on flag
  local_event_Power.emit('On')

  updateStatus()

# power on and power off
POWER_SCHEMA = {'type': 'string', 'enum': ['On', 'Off']}

local_event_DesiredPower = LocalEvent({'group': 'Power', 'order': next_seq(), 'schema': {'type': 'string', 'enum': ['On', 'Off']}})
local_event_Power = LocalEvent({'group': 'Power', 'order': next_seq(), 'schema': {'type': 'string', 'enum': ['Partially On', 'Partially Off', 'On', 'Off']}})

def local_action_Power(arg=None):
  '''{"group": "Power", "schema": {"type": "string", "enum": ["On", "Off"]}}'''
  local_event_DesiredPower.emit(arg)

  if arg == 'On':
    lookup_local_action('SendWOL').call()

  elif arg == 'Off':
    lookup_remote_action('PowerOff').call()

  updateStatus()

local_event_WiringStatus = LocalEvent({'group': 'Status', 'order': next_seq(), 'schema': {'type': 'string'}})

def checkWiringStatus():
  bindingState = lookup_remote_action('PowerOff').getBindingState()
  local_event_WiringStatus.emit(str(bindingState))

  updateStatus()

Timer(checkWiringStatus, 60, 7)

# performs the status aggregation
def updateStatus():
  now = date_now().getMillis()

  desiredPower = lookup_local_event('DesiredPower').getArg()
  if desiredPower is None:
    local_event_Status.emit({'level': 1, 'message': 'No desired status has been set yet.'})
    return

  elif desiredPower == 'On':
    # computer is supposed to be on so check wiring status
    wiringStatus = local_event_WiringStatus.getArg()

    if wiringStatus != 'Wired':
      last = local_event_ComputerStatus.getTimestamp()
      when = last.toString('E dd-MMM h:mm a') if last != None else "<never>"

      diff = now - (last.getMillis() if last != None else 0)
      if diff > 150000: # give it two and half minutes to boot up
        local_event_Status.emit({'level': 2, 'message': 'No connection from computer (last heard from %s)' % (when)})

      local_event_Power.emit('Partially On')

      return

    else:
      # wiring status is 'Wired', so pass through last computer status and power state
      lastStatus = local_event_ComputerStatus.getArg()
      local_event_Power.emit('On')

      if lastStatus == None:
        local_event_Status.emit({'level': 1, 'message': 'No status information from computer yet.'})

      else:
        # pass on last status
        local_event_Status.emit(lastStatus)

  else:
    # (desired power is Off, ensure off)
    timestamp = local_event_ComputerStatus.getTimestamp()
    last = 0 if timestamp == None else timestamp.getMillis()
    diff = now - last

    if diff < 180000: # give it at least 3 minutes before determining it's off
      local_event_Power.emit('Partially Off')

    else:
      local_event_Power.emit('Off')
      local_event_Status.emit({'level': 0, 'message': 'OK (*)'})

# Remote events to activate actions
def remote_event_Power(arg):
  lookup_local_action('Power').call(arg)
<|endoftext|>


<commit_before># Copyright (c) 2013 by California Institute of Technology
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of the California Institute of Technology nor
#    the names of its contributors may be used to endorse or promote
#    products derived from this software without specific prior
#    written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CALTECH
# OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
"""
TuLiP toolbox

The Temporal Logic Planning (TuLiP) Toolbox provides functions
for verifying and constructing control protocols.
"""

__all__ = ["abstract", "hybrid", "transys"]

__version__ = "1.0b"
<commit_msg>Belatedly bump version number<commit_after># Copyright (c) 2013 by California Institute of Technology
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of the California Institute of Technology nor
#    the names of its contributors may be used to endorse or promote
#    products derived from this software without specific prior
#    written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CALTECH
# OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
"""
TuLiP toolbox

The Temporal Logic Planning (TuLiP) Toolbox provides functions
for verifying and constructing control protocols.
"""

__all__ = ["abstract", "hybrid", "transys"]

__version__ = "1.1a"
<|endoftext|>