<commit_before>import redis

from django.conf.urls import url
from django.conf import settings

from tastypie.resources import Resource
from tastypie import fields
from tastypie.authorization import Authorization
from tastypie.authentication import Authentication
from tastypie.exceptions import NotFound, ImmediateHttpResponse
from tastypie.http import HttpConflict

r = redis.StrictRedis.from_url(settings.REDIS_URL)

"""
"hydra:v1:projects" = Set of projects
"hydra:v1:projects:<project>" = Metadata
GET /api/v1/projects/
    List of Projects
GET /api/v1/projects/<project>/
    Metadata on Project
    resource_uri -> /slugs/

"hydra:v1:projects:<project>:slugs" = Set of slugs
GET /api/v1/projects/<project>/slugs/
    Slugs for a project
"hydra:v1:projects:<project>:slugs:<slug>" = Actual Data
GET /api/v1/projects/<project>/slugs/<slug>/
    Detail view for slug
"""

class RedisProject(object):
    index_slug = "hydra:v1:projects"

    def __init__(self, name=None, whitelist=None, *args, **kwargs):
        super(RedisProject, self).__init__(*args, **kwargs)
        self.name = name
        if whitelist:
            self.whitelist = whitelist
        else:
            self.whitelist = self.get_whitelist()

    @classmethod
    def all_projects(cls):
        return r.smembers(cls.index_slug)

    @classmethod
    def make_key(cls, key):
        return "hydra:v1:projects:%s" % (key)

    @property
    def redis_slug(self):
        return "hydra:v1:projects:%s" % (self.name)

    def save_whitelist(self):
        for url in self.whitelist:
            r.sadd("%s:whitelist" % self.redis_slug, url)

    def save_project(self):
        r.hset(self.redis_slug, "exists", "true")
        r.sadd(self.index_slug, self.name)

    def save(self):
        self.save_whitelist()
        self.save_project()

    def delete(self):
        r.delete(self.redis_slug)

    def exists(self):
        return r.hget(self.redis_slug, "exists") == "true"

    def get_whitelist(self):
        return r.smembers("%s:whitelist" % self.redis_slug)

class ProjectResource(Resource):
    name = fields.CharField(attribute='name')
    whitelist = fields.ListField(attribute="whitelist")

    class Meta:
        resource_name = "project"
        object_class = RedisProject
        authorization = Authorization()
        authentication = Authentication()

    def get_resource_uri(self, bundle_or_obj):
        try:
            if getattr(bundle_or_obj, 'obj'):
                return "/_api/v1/project/%s/" % bundle_or_obj.obj.name
            return "/_api/v1/project/%s/" % bundle_or_obj.name
        except:
            return ""

    def obj_create(self, bundle, request=None, **kwargs):
        bundle.obj = RedisProject(bundle.data)
        bundle = self.full_hydrate(bundle)
        if kwargs.get('delete', None):
            bundle.obj.delete()
        bundle.obj.save()
        return bundle

    def obj_update(self, bundle, request=None, **kwargs):
        if bundle.data.get('resource_uri'):
            del bundle.data['resource_uri']
        return self.obj_create(bundle, request, delete=True, **kwargs)

    def obj_get(self, request=None, pk=None, **kwargs):
        proj = RedisProject(pk)
        if not proj.exists():
            raise NotFound("No object matching this pk")
        return proj

    def obj_get_list(self, request=None, **kwargs):
        ret_val = []
        filter_slug = request.GET.get('slug', None)
        if not filter_slug:
            filter_slug = request.GET.get('q', None)
        keys = RedisProject.all_projects()
        for key in keys:
            #filter
            if not filter_slug or filter_slug in key:
                obj = RedisProject(key)
                ret_val.append(obj)
        return ret_val

    def obj_delete(self, request=None, **kwargs):
        obj = RedisProject(kwargs['pk'])
        obj.delete()

    def obj_delete_list(self, request=None, **kwargs):
        for name in RedisProject.all_projects():
            obj = RedisProject(name)
            obj.delete()

    def override_urls(self):
        raw_url = r"^(?P<resource_name>%s)/(?P<pk>.+)/$"
        return [
            url(r"^(?P<resource_name>%s)/schema/$" % self._meta.resource_name, self.wrap_view('get_schema'), name="api_get_schema"),
            url(raw_url % self._meta.resource_name,
                self.wrap_view('dispatch_detail'),
                name="api_dispatch_detail"),
        ]

class RedisRedirect(object):

    def __init__(self, slug=None, project=None, urls=None, *args, **kwargs):
        super(RedisRedirect, self).__init__(*args, **kwargs)
        self.project = project
        self.slug = slug
        if urls:
            self.urls = urls
        else:
            self.get_urls()

    @property
    def index_slug(self):
        return "hydra:v1:projects:%s:slugs" % (self.project)

    @property
    def redis_slug(self):
        return "%s:%s" % (self.index_slug, self.slug)

    def all_slugs(self):
        return r.smembers(self.index_slug)

    def make_key(cls, key):
        return "%s:%s" % (self.index_slug, key)

    def save_redirect(self):
        r.sadd(self.index_slug, self.slug)

    def save_urls(self):
        for obj in self.urls:
            if hasattr(obj, 'get'):
                if obj.get('score', None):
                    r.zadd(self.redis_slug, int(obj['score']), obj['url'])
            else:
                print "WTF?!"
                r.zincrby(self.redis_slug, self.urls[0], 1)

    def save(self):
        self.save_redirect()
        self.save_urls()
        return True

    def delete(self):
        r.delete(self.redis_slug)

    def incr(self, url):
        self.save_redirect()
        r.zincrby(self.redis_slug, url, 1)

    def url_exists(self, url):
        for url_obj in self.get_urls():
            if url == url_obj['url']:
                return True
        return False

    def exists(self):
        return r.zcard(self.redis_slug)

    def get_urls(self):
        self._urls = r.zrevrange(self.redis_slug, 0, -1, withscores=True)
        self.urls = []
        for obj in self._urls:
            redirect_url, score = obj
            self.urls.append({
                'score': score,
                'url': redirect_url
            })
        return self.urls

class RedirectResource(Resource):
    slug = fields.CharField(attribute='slug')
    project = fields.CharField(attribute='project')
    urls = fields.ListField(attribute='urls', null=True)

    class Meta:
        object_class = RedisRedirect
        authorization = Authorization()
        authentication = Authentication()

    def get_resource_uri(self, bundle_or_obj):
        try:
            if getattr(bundle_or_obj, 'obj'):
                return "/_api/v1/redirect/%s/%s/" % (bundle_or_obj.obj.project,
                                                    bundle_or_obj.obj.slug)
            return "/_api/v1/redirect/%s/%s/" % (bundle_or_obj.project,
                                                bundle_or_obj.slug)
        except:
            return ""

    def obj_create(self, bundle, request=None, **kwargs):
        bundle.obj = RedisRedirect(**bundle.data)
        bundle = self.full_hydrate(bundle)
        if kwargs.get('delete', None):
            bundle.obj.delete()
        if not bundle.obj.save():
            raise ImmediateHttpResponse(
                HttpConflict("Object already exists")
            )

        return bundle

    def obj_update(self, bundle, request=None, **kwargs):
        if bundle.data.get('resource_uri'):
            del bundle.data['resource_uri']
        return self.obj_create(bundle, request, delete=True, **kwargs)

    def obj_get(self, request=None, pk=None, project=None, **kwargs):
        proj = RedisRedirect(slug=pk, project=project)
        if not proj.exists():
            raise NotFound("No object matching this pk")
        return proj

    def obj_get_list(self, request=None, project=None, **kwargs):
        ret_val = []
        filter_project = request.GET.get('project', None)
        filter_slug = request.GET.get('slug', None)
        if project:
            proj_obj = RedisRedirect(project=project)
            keys = proj_obj.all_slugs()
            for key in keys:
                if not filter_slug or filter_slug in key:
                    if not filter_project or filter_project in project:
                        obj = RedisRedirect(slug=key, project=project)
                        ret_val.append(obj)
        else:
            #Return all things for all projects
            indexes = r.keys("hydra:v1:projects:*:slugs")
            for index in indexes:
                index_project = index.split(":")[3]
                proj_obj = RedisRedirect(project=index_project)
                keys = proj_obj.all_slugs()
                for key in keys:
                    if not filter_slug or filter_slug in key:
                        if not filter_project or filter_project in index_project:
                            obj = RedisRedirect(slug=key, project=index_project)
                            ret_val.append(obj)
        return ret_val

    def obj_delete(self, request=None, **kwargs):
        obj = RedisRedirect(kwargs['pk'])
        obj.delete()

    def obj_delete_list(self, request=None, **kwargs):
        for name in RedisRedirect.all_slugs():
            obj = RedisRedirect(name)
            obj.delete()

    def override_urls(self):
        list_url = r"^(?P<resource_name>%s)/(?P<project>[^/]+)/$"
        detail_url = r"^(?P<resource_name>%s)/(?P<project>[^/]+)/(?P<pk>.+)/$"
        return [
            url(r"^(?P<resource_name>%s)/schema/$" % self._meta.resource_name, self.wrap_view('get_schema'), name="api_get_schema"),
            url(detail_url % self._meta.resource_name,
                self.wrap_view('dispatch_detail'),
                name="api_dispatch_detail"),
            url(list_url % self._meta.resource_name,
                self.wrap_view('dispatch_list'),
                name="api_dispatch_list"),
        ]
<commit_msg>Also delete the index.<commit_after>import redis

from django.conf.urls import url
from django.conf import settings

from tastypie.resources import Resource
from tastypie import fields
from tastypie.authorization import Authorization
from tastypie.authentication import Authentication
from tastypie.exceptions import NotFound, ImmediateHttpResponse
from tastypie.http import HttpConflict

r = redis.StrictRedis.from_url(settings.REDIS_URL)

"""
"hydra:v1:projects" = Set of projects
"hydra:v1:projects:<project>" = Metadata
GET /api/v1/projects/
    List of Projects
GET /api/v1/projects/<project>/
    Metadata on Project
    resource_uri -> /slugs/

"hydra:v1:projects:<project>:slugs" = Set of slugs
GET /api/v1/projects/<project>/slugs/
    Slugs for a project
"hydra:v1:projects:<project>:slugs:<slug>" = Actual Data
GET /api/v1/projects/<project>/slugs/<slug>/
    Detail view for slug
"""

class RedisProject(object):
    index_slug = "hydra:v1:projects"

    def __init__(self, name=None, whitelist=None, *args, **kwargs):
        super(RedisProject, self).__init__(*args, **kwargs)
        self.name = name
        if whitelist:
            self.whitelist = whitelist
        else:
            self.whitelist = self.get_whitelist()

    @classmethod
    def all_projects(cls):
        return r.smembers(cls.index_slug)

    @classmethod
    def make_key(cls, key):
        return "hydra:v1:projects:%s" % (key)

    @property
    def redis_slug(self):
        return "hydra:v1:projects:%s" % (self.name)

    def save_whitelist(self):
        for url in self.whitelist:
            r.sadd("%s:whitelist" % self.redis_slug, url)

    def save_project(self):
        r.hset(self.redis_slug, "exists", "true")
        r.sadd(self.index_slug, self.name)

    def save(self):
        self.save_whitelist()
        self.save_project()

    def delete(self):
        r.delete(self.redis_slug)
        r.delete(self.index_slug)

    def exists(self):
        return r.hget(self.redis_slug, "exists") == "true"

    def get_whitelist(self):
        return r.smembers("%s:whitelist" % self.redis_slug)

class ProjectResource(Resource):
    name = fields.CharField(attribute='name')
    whitelist = fields.ListField(attribute="whitelist")

    class Meta:
        resource_name = "project"
        object_class = RedisProject
        authorization = Authorization()
        authentication = Authentication()

    def get_resource_uri(self, bundle_or_obj):
        try:
            if getattr(bundle_or_obj, 'obj'):
                return "/_api/v1/project/%s/" % bundle_or_obj.obj.name
            return "/_api/v1/project/%s/" % bundle_or_obj.name
        except:
            return ""

    def obj_create(self, bundle, request=None, **kwargs):
        bundle.obj = RedisProject(bundle.data)
        bundle = self.full_hydrate(bundle)
        if kwargs.get('delete', None):
            bundle.obj.delete()
        bundle.obj.save()
        return bundle

    def obj_update(self, bundle, request=None, **kwargs):
        if bundle.data.get('resource_uri'):
            del bundle.data['resource_uri']
        return self.obj_create(bundle, request, delete=True, **kwargs)

    def obj_get(self, request=None, pk=None, **kwargs):
        proj = RedisProject(pk)
        if not proj.exists():
            raise NotFound("No object matching this pk")
        return proj

    def obj_get_list(self, request=None, **kwargs):
        ret_val = []
        filter_slug = request.GET.get('slug', None)
        if not filter_slug:
            filter_slug = request.GET.get('q', None)
        keys = RedisProject.all_projects()
        for key in keys:
            #filter
            if not filter_slug or filter_slug in key:
                obj = RedisProject(key)
                ret_val.append(obj)
        return ret_val

    def obj_delete(self, request=None, **kwargs):
        obj = RedisProject(kwargs['pk'])
        obj.delete()

    def obj_delete_list(self, request=None, **kwargs):
        for name in RedisProject.all_projects():
            obj = RedisProject(name)
            obj.delete()

    def override_urls(self):
        raw_url = r"^(?P<resource_name>%s)/(?P<pk>.+)/$"
        return [
            url(r"^(?P<resource_name>%s)/schema/$" % self._meta.resource_name, self.wrap_view('get_schema'), name="api_get_schema"),
            url(raw_url % self._meta.resource_name,
                self.wrap_view('dispatch_detail'),
                name="api_dispatch_detail"),
        ]

class RedisRedirect(object):

    def __init__(self, slug=None, project=None, urls=None, *args, **kwargs):
        super(RedisRedirect, self).__init__(*args, **kwargs)
        self.project = project
        self.slug = slug
        if urls:
            self.urls = urls
        else:
            self.get_urls()

    @property
    def index_slug(self):
        return "hydra:v1:projects:%s:slugs" % (self.project)

    @property
    def redis_slug(self):
        return "%s:%s" % (self.index_slug, self.slug)

    def all_slugs(self):
        return r.smembers(self.index_slug)

    def make_key(cls, key):
        return "%s:%s" % (self.index_slug, key)

    def save_redirect(self):
        r.sadd(self.index_slug, self.slug)

    def save_urls(self):
        for obj in self.urls:
            if hasattr(obj, 'get'):
                if obj.get('score', None):
                    r.zadd(self.redis_slug, int(obj['score']), obj['url'])
            else:
                print "WTF?!"
                r.zincrby(self.redis_slug, self.urls[0], 1)

    def save(self):
        self.save_redirect()
        self.save_urls()
        return True

    def delete(self):
        r.delete(self.redis_slug)
        r.delete(self.index_slug)

    def incr(self, url):
        self.save_redirect()
        r.zincrby(self.redis_slug, url, 1)

    def url_exists(self, url):
        for url_obj in self.get_urls():
            if url == url_obj['url']:
                return True
        return False

    def exists(self):
        return r.zcard(self.redis_slug)

    def get_urls(self):
        self._urls = r.zrevrange(self.redis_slug, 0, -1, withscores=True)
        self.urls = []
        for obj in self._urls:
            redirect_url, score = obj
            self.urls.append({
                'score': score,
                'url': redirect_url
            })
        return self.urls

class RedirectResource(Resource):
    slug = fields.CharField(attribute='slug')
    project = fields.CharField(attribute='project')
    urls = fields.ListField(attribute='urls', null=True)

    class Meta:
        object_class = RedisRedirect
        authorization = Authorization()
        authentication = Authentication()

    def get_resource_uri(self, bundle_or_obj):
        try:
            if getattr(bundle_or_obj, 'obj'):
                return "/_api/v1/redirect/%s/%s/" % (bundle_or_obj.obj.project,
                                                    bundle_or_obj.obj.slug)
            return "/_api/v1/redirect/%s/%s/" % (bundle_or_obj.project,
                                                bundle_or_obj.slug)
        except:
            return ""

    def obj_create(self, bundle, request=None, **kwargs):
        bundle.obj = RedisRedirect(**bundle.data)
        bundle = self.full_hydrate(bundle)
        if kwargs.get('delete', None):
            bundle.obj.delete()
        if not bundle.obj.save():
            raise ImmediateHttpResponse(
                HttpConflict("Object already exists")
            )

        return bundle

    def obj_update(self, bundle, request=None, **kwargs):
        if bundle.data.get('resource_uri'):
            del bundle.data['resource_uri']
        return self.obj_create(bundle, request, delete=True, **kwargs)

    def obj_get(self, request=None, pk=None, project=None, **kwargs):
        proj = RedisRedirect(slug=pk, project=project)
        if not proj.exists():
            raise NotFound("No object matching this pk")
        return proj

    def obj_get_list(self, request=None, project=None, **kwargs):
        ret_val = []
        filter_project = request.GET.get('project', None)
        filter_slug = request.GET.get('slug', None)
        if project:
            proj_obj = RedisRedirect(project=project)
            keys = proj_obj.all_slugs()
            for key in keys:
                if not filter_slug or filter_slug in key:
                    if not filter_project or filter_project in project:
                        obj = RedisRedirect(slug=key, project=project)
                        ret_val.append(obj)
        else:
            #Return all things for all projects
            indexes = r.keys("hydra:v1:projects:*:slugs")
            for index in indexes:
                index_project = index.split(":")[3]
                proj_obj = RedisRedirect(project=index_project)
                keys = proj_obj.all_slugs()
                for key in keys:
                    if not filter_slug or filter_slug in key:
                        if not filter_project or filter_project in index_project:
                            obj = RedisRedirect(slug=key, project=index_project)
                            ret_val.append(obj)
        return ret_val

    def obj_delete(self, request=None, **kwargs):
        obj = RedisRedirect(kwargs['pk'])
        obj.delete()

    def obj_delete_list(self, request=None, **kwargs):
        for name in RedisRedirect.all_slugs():
            obj = RedisRedirect(name)
            obj.delete()

    def override_urls(self):
        list_url = r"^(?P<resource_name>%s)/(?P<project>[^/]+)/$"
        detail_url = r"^(?P<resource_name>%s)/(?P<project>[^/]+)/(?P<pk>.+)/$"
        return [
            url(r"^(?P<resource_name>%s)/schema/$" % self._meta.resource_name, self.wrap_view('get_schema'), name="api_get_schema"),
            url(detail_url % self._meta.resource_name,
                self.wrap_view('dispatch_detail'),
                name="api_dispatch_detail"),
            url(list_url % self._meta.resource_name,
                self.wrap_view('dispatch_list'),
                name="api_dispatch_list"),
        ]
<|endoftext|>


<commit_before>import re

class invalidConfig(Exception):
    def __init__(self, key, line):
        self.line = line+1
        self.key = key
    def __str__(self):
        return "Invalid data on line {0}: {1}".format(self.line, self.key)

class Configuration():
    def __init__(self):
        self.config = {}
        self.template = ["server", "port", "usernick", "pass", "ident", "channels", "prefix", "admins", "loglevel"]
        self.configname = "config.txt"

        self.found = []

    def doesExist(self):
        try:
            temp = open(self.configname, "r")
        except:
            return False
        else:
            temp.close()
            return True

    def createNewConfig(self, data = {}):
        config = open(self.configname, "w")
        config.write("")
        config.close()

        config = open(self.configname, "a")

        for item in self.template:
            val = item in data and data[item] or ""
            config.write("{0} = {1}\n".format(item, val))

        config.close()

    def loadConfig(self):
        config = open(self.configname, "r")

        for i, line in enumerate(config.readlines()):

            #print i, line
            match = re.match("\s*(?P<key>\w+)\s*=\s*(?P<val>.+)?\s*",line)

            if match != None:
                key, val = match.group("key"), match.group("val")

                if val == None:
                    raise invalidConfig("key \'{0}\' has no value".format(key), i)
                elif match == None or key not in self.template:
                    raise invalidConfig("incorrect key: \'{0}\'".format(line.strip()), i)
                key, val = key.strip(), val.strip()

                self.config[key] = val
                if key not in self.found:
                    self.found.append(key)

        notFound = []
        for key in self.template:
            if key not in self.found:
                notFound.append(key)

        if len(notFound) > 0:
            raise RuntimeError("The following keys were not found in config.txt, please add them: "+", ".join(notFound))

    def getChannels(self):
        chans = self.config["channels"].split(",")
        newchans = []
        for chan in chans:
            chan = chan.strip()
            if chans[0].isalnum():
                newchans.append("#"+chan)
            else:
                newchans.append(chan)

        return newchans

    def getAdmins(self):
        admins = self.config["admins"].split(",")
        newadmins = []
        for admin in admins:
            admin = admin.strip()
            newadmins.append(admin)

        return newadmins<commit_msg>Added force_ipv6 option<commit_after>import re

class invalidConfig(Exception):
    def __init__(self, key, line):
        self.line = line+1
        self.key = key
    def __str__(self):
        return "Invalid data on line {0}: {1}".format(self.line, self.key)

class Configuration():
    def __init__(self):
        self.config = {}
        self.template = ["server", "port", "usernick", "pass", "ident", "channels", "prefix", "admins", "loglevel", "force_ipv6"]
        self.configname = "config.txt"

        self.found = []

    def doesExist(self):
        try:
            temp = open(self.configname, "r")
        except:
            return False
        else:
            temp.close()
            return True

    def createNewConfig(self, data = {}):
        config = open(self.configname, "w")
        config.write("")
        config.close()

        config = open(self.configname, "a")

        for item in self.template:
            val = item in data and data[item] or ""
            config.write("{0} = {1}\n".format(item, val))

        config.close()

    def loadConfig(self):
        config = open(self.configname, "r")

        for i, line in enumerate(config.readlines()):

            #print i, line
            match = re.match("\s*(?P<key>\w+)\s*=\s*(?P<val>.+)?\s*",line)

            if match != None:
                key, val = match.group("key"), match.group("val")

                if val == None:
                    raise invalidConfig("key \'{0}\' has no value".format(key), i)
                elif match == None or key not in self.template:
                    raise invalidConfig("incorrect key: \'{0}\'".format(line.strip()), i)
                key, val = key.strip(), val.strip()

                self.config[key] = val
                if key not in self.found:
                    self.found.append(key)

        notFound = []
        for key in self.template:
            if key not in self.found:
                notFound.append(key)

        if len(notFound) > 0:
            raise RuntimeError("The following keys were not found in config.txt, please add them: "+", ".join(notFound))

    def getChannels(self):
        chans = self.config["channels"].split(",")
        newchans = []
        for chan in chans:
            chan = chan.strip()
            if chans[0].isalnum():
                newchans.append("#"+chan)
            else:
                newchans.append(chan)

        return newchans

    def getAdmins(self):
        admins = self.config["admins"].split(",")
        newadmins = []
        for admin in admins:
            admin = admin.strip()
            newadmins.append(admin)

        return newadmins<|endoftext|>


<commit_before># -*- coding: utf-8 -*- {{{
# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et:

# Copyright (c) 2015, Battelle Memorial Institute
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and documentation
# are those of the authors and should not be interpreted as representing
# official policies, either expressed or implied, of the FreeBSD
# Project.
#
# This material was prepared as an account of work sponsored by an
# agency of the United States Government.  Neither the United States
# Government nor the United States Department of Energy, nor Battelle,
# nor any of their employees, nor any jurisdiction or organization that
# has cooperated in the development of these materials, makes any
# warranty, express or implied, or assumes any legal liability or
# responsibility for the accuracy, completeness, or usefulness or any
# information, apparatus, product, software, or process disclosed, or
# represents that its use would not infringe privately owned rights.
#
# Reference herein to any specific commercial product, process, or
# service by trade name, trademark, manufacturer, or otherwise does not
# necessarily constitute or imply its endorsement, recommendation, or
# favoring by the United States Government or any agency thereof, or
# Battelle Memorial Institute. The views and opinions of authors
# expressed herein do not necessarily state or reflect those of the
# United States Government or any agency thereof.
#
# PACIFIC NORTHWEST NATIONAL LABORATORY
# operated by BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY
# under Contract DE-AC05-76RL01830
#}}}

'''VOLTTRON platform™ agent helper classes/functions.'''

import argparse
from dateutil.parser import parse
from datetime import datetime, timedelta
import errno
import logging
import os
import pytz
import re
import stat
import sys
import syslog
import traceback

import gevent

from zmq.utils import jsonapi

from ..lib.inotify.green import inotify, IN_MODIFY

__all__ = ['load_config', 'run_agent', 'start_agent_thread']

__author__ = 'Brandon Carpenter <brandon.carpenter@pnnl.gov>'
__copyright__ = 'Copyright (c) 2015, Battelle Memorial Institute'
__license__ = 'FreeBSD'

_comment_re = re.compile(
        r'((["\'])(?:\\?.)*?\2)|(/\*.*?\*/)|((?:#|//).*?(?=\n|$))',
        re.MULTILINE | re.DOTALL)

_log = logging.getLogger(__name__)

def _repl(match):
    '''Replace the matched group with an appropriate string.'''
    # If the first group matched, a quoted string was matched and should
    # be returned unchanged.  Otherwise a comment was matched and the
    # empty string should be returned.
    return match.group(1) or ''

def strip_comments(string):
    '''Return string with all comments stripped.

    Both JavaScript-style comments (//... and /*...*/) and hash (#...)
    comments are removed.
    '''
    return _comment_re.sub(_repl, string)

def load_config(config_path):
    '''Load a JSON-encoded configuration file.'''
    return jsonapi.loads(strip_comments(open(config_path).read()))

def run_agent(cls, subscribe_address=None, publish_address=None,
              config_path=None, **kwargs):
    '''Instantiate an agent and run it in the current thread.

    Attempts to get keyword parameters from the environment if they
    are not set.
    '''
    if not subscribe_address:
        subscribe_address = os.environ.get('AGENT_SUB_ADDR')
    if subscribe_address:
        kwargs['subscribe_address'] = subscribe_address
    if not publish_address:
        publish_address = os.environ.get('AGENT_PUB_ADDR')
    if publish_address:
        kwargs['publish_address'] = publish_address
    if not config_path:
        config_path = os.environ.get('AGENT_CONFIG')
    if config_path:
        kwargs['config_path'] = config_path
    agent = cls(**kwargs)
    agent.run()

def start_agent_thread(cls, **kwargs):
    '''Instantiate an agent class and run it in a new daemon thread.

    Returns the thread object.
    '''
    import threading
    agent = cls(**kwargs)
    thread = threading.Thread(target=agent.run)
    thread.daemon = True
    thread.start()
    return thread

def isapipe(fd):
    fd = getattr(fd, 'fileno', lambda: fd)()
    return stat.S_ISFIFO(os.fstat(fd).st_mode)

def default_main(agent_class, description=None, argv=sys.argv,
                 parser_class=argparse.ArgumentParser, **kwargs):
    '''Default main entry point implementation for legacy agents.

    description and parser_class are depricated. Please avoid using them.
    '''
    try:
        # If stdout is a pipe, re-open it line buffered
        if isapipe(sys.stdout):
            # Hold a reference to the previous file object so it doesn't
            # get garbage collected and close the underlying descriptor.
            stdout = sys.stdout
            sys.stdout = os.fdopen(stdout.fileno(), 'w', 1)

        try:
            sub_addr = os.environ['AGENT_SUB_ADDR']
            pub_addr = os.environ['AGENT_PUB_ADDR']
        except KeyError as exc:
            sys.stderr.write(
                'missing environment variable: {}\n'.format(exc.args[0]))
            sys.exit(1)
        if sub_addr.startswith('ipc://') and sub_addr[6:7] != '@':
            if not os.path.exists(sub_addr[6:]):
                sys.stderr.write('warning: subscription socket does not '
                                 'exist: {}\n'.format(sub_addr[6:]))
        if pub_addr.startswith('ipc://') and pub_addr[6:7] != '@':
            if not os.path.exists(pub_addr[6:]):
                sys.stderr.write('warning: publish socket does not '
                                 'exist: {}\n'.format(pub_addr[6:]))
        config = os.environ.get('AGENT_CONFIG')
        agent = agent_class(subscribe_address=sub_addr,
                            publish_address=pub_addr,
                            config_path=config, **kwargs)
        agent.run()
    except KeyboardInterrupt:
        pass

def vip_main(agent_class, **kwargs):
    '''Default main entry point implementation for VIP agents.'''
    try:
        # If stdout is a pipe, re-open it line buffered
        if isapipe(sys.stdout):
            # Hold a reference to the previous file object so it doesn't
            # get garbage collected and close the underlying descriptor.
            stdout = sys.stdout
            sys.stdout = os.fdopen(stdout.fileno(), 'w', 1)

        # Quiet printing of KeyboardInterrupt by greenlets
        Hub = gevent.hub.Hub
        Hub.NOT_ERROR = Hub.NOT_ERROR + (KeyboardInterrupt,)

        agent_uuid = os.environ.get('AGENT_UUID')
        config = os.environ.get('AGENT_CONFIG')
        agent = agent_class(config_path=config, identity=agent_uuid, **kwargs)
        try:
            run = agent.run
        except AttributeError:
            run = agent.core.run
        task = gevent.spawn(run)
        try:
            task.join()
        finally:
            task.kill()
    except KeyboardInterrupt:
        pass

class SyslogFormatter(logging.Formatter):
    _level_map = {logging.DEBUG: syslog.LOG_DEBUG,
                  logging.INFO: syslog.LOG_INFO,
                  logging.WARNING: syslog.LOG_WARNING,
                  logging.ERROR: syslog.LOG_ERR,
                  logging.CRITICAL: syslog.LOG_CRIT,}

    def format(self, record):
        level = self._level_map.get(record.levelno, syslog.LOG_INFO)
        return '<{}>'.format(level) + super(SyslogFormatter, self).format(record)

class JsonFormatter(logging.Formatter):
    def format(self, record):
        dct = record.__dict__.copy()
        exc_info = dct.pop('exc_info', None)
        if exc_info:
            dct['exc_text'] = ''.join(traceback.format_exception(*exc_info))
        return jsonapi.dumps(dct)

class AgentFormatter(logging.Formatter):
    def __init__(self, fmt=None, datefmt=None):
        if fmt is None:
            fmt = '%(asctime)s %(composite_name)s %(levelname)s: %(message)s'
        super(AgentFormatter, self).__init__(fmt=fmt, datefmt=datefmt)

    def composite_name(self, record):
        if record.name == 'agents.log':
            cname = '(%(processName)s %(process)d) %(remote_name)s'
        elif record.name.startswith('agents.std'):
            cname = '(%(processName)s %(process)d) <{}>'.format(
                    record.name.split('.', 2)[1])
        else:
            cname = '() %(name)s'
        return cname % record.__dict__

    def format(self, record):
        if 'composite_name' not in record.__dict__:
            record.__dict__['composite_name'] = self.composite_name(record)
        if len(record.args) > 0 \
            and 'tornado.access' in record.__dict__['composite_name']:
            record.__dict__['msg'] = ','.join([str(b) for b in record.args])
            record.__dict__['args'] = []
        return super(AgentFormatter, self).format(record)

def setup_logging(level=logging.DEBUG):
    root = logging.getLogger()
    if not root.handlers:
        handler = logging.StreamHandler()
        if isapipe(sys.stderr) and '_LAUNCHED_BY_PLATFORM' in os.environ:
            handler.setFormatter(JsonFormatter())
        else:
            handler.setFormatter(logging.Formatter(
                    '%(asctime)s %(name)s %(levelname)s: %(message)s'))
        root.addHandler(handler)
    root.setLevel(level)

def format_timestamp(time_stamp):
    """Create a consistent datetime string representation based on ISO 8601 format.

    YYYY-MM-DDTHH:MM:SS.mmmmmm for unaware datetime objects.
    YYYY-MM-DDTHH:MM:SS.mmmmmm+HH:MM for aware datetime objects

    :param time_stamp: value to convert
    :type time_stamp: datetime
    :returns: datetime in string format
    :rtype: str
    """
    return time_stamp.strftime("%Y-%m-%dT%H:%M:%S.%f%z")

def parse_timestamp_string(time_stamp_str):
    """Create a datetime object from the supplied date/time string.
    Uses dateutil.parse with no extra parameters.

    :param time_stamp: value to convert
    :type time_stamp: str
    :returns: datetime object
    :rtype: datetime
    """
    return parse(time_stamp_str)

def get_aware_utc_now():
    """Create a timezone aware UTC datetime object from the system time.

    :returns: an aware UTC datetime object
    :rtype: datetime
    """
    utcnow = datetime.utcnow()
    utcnow = pytz.UTC.localize(utcnow)
    return utcnow

def process_timestamp(timestamp_string, topic=''):
    if timestamp_string is None:
        _log.error("message for {topic} missing timetamp".format(topic=topic))
        return

    try:
        timestamp = parse(timestamp_string)
    except (ValueError, TypeError) as e:
        _log.error("message for {topic} bad timetamp string: {ts_string}".format(topic=topic,
                                                                                ts_string=timestamp_string))
        return

    if timestamp.tzinfo is None:
        timestamp.replace(tzinfo=pytz.UTC)
        original_tz = None
    else:
        original_tz = timestamp.tzinfo
        timestamp = timestamp.astimezone(pytz.UTC)
    return timestamp, original_tz

def watch_file(fullpath, callback):
    '''Run callback method whenever the file changes'''
    dirname, filename = os.path.split(fullpath)
    with inotify() as inot:
        inot.add_watch(dirname, IN_MODIFY)
        for event in inot:
            if event.name == filename and event.mask & IN_MODIFY:
                callback()

def create_file_if_missing(path, permission=0o660, contents=None):
    try:
        fil = open(path)
    except IOError as exc:
        if exc.errno != errno.ENOENT:
            raise
        _log.debug('missing file %s', path)
        _log.info('creating file %s', path)
        fd = os.open(path, os.O_CREAT|os.O_WRONLY, permission)
        try:
            if contents:
                os.write(fd, contents)
        finally:
            os.close(fd)

def fix_sqlite3_datetime(sql=None):
    """Primarily for fixing the base historian cache on certain versions of python.

    Registers a new datetime converter to that uses dateutil parse. This should
    better resolve #216, #174, and #91 without the goofy workarounds that change data.

    Optional sql argument is for testing only.
    """
    if sql is None:
        import sqlite3 as sql
    sql.register_converter("timestamp", parse)
<commit_msg>Remove is_dst.<commit_after># -*- coding: utf-8 -*- {{{
# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et:

# Copyright (c) 2015, Battelle Memorial Institute
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and documentation
# are those of the authors and should not be interpreted as representing
# official policies, either expressed or implied, of the FreeBSD
# Project.
#
# This material was prepared as an account of work sponsored by an
# agency of the United States Government.  Neither the United States
# Government nor the United States Department of Energy, nor Battelle,
# nor any of their employees, nor any jurisdiction or organization that
# has cooperated in the development of these materials, makes any
# warranty, express or implied, or assumes any legal liability or
# responsibility for the accuracy, completeness, or usefulness or any
# information, apparatus, product, software, or process disclosed, or
# represents that its use would not infringe privately owned rights.
#
# Reference herein to any specific commercial product, process, or
# service by trade name, trademark, manufacturer, or otherwise does not
# necessarily constitute or imply its endorsement, recommendation, or
# favoring by the United States Government or any agency thereof, or
# Battelle Memorial Institute. The views and opinions of authors
# expressed herein do not necessarily state or reflect those of the
# United States Government or any agency thereof.
#
# PACIFIC NORTHWEST NATIONAL LABORATORY
# operated by BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY
# under Contract DE-AC05-76RL01830
#}}}

'''VOLTTRON platform™ agent helper classes/functions.'''

import argparse
from dateutil.parser import parse
from datetime import datetime, timedelta
import errno
import logging
import os
import pytz
import re
import stat
import sys
import syslog
import traceback

import gevent

from zmq.utils import jsonapi

from ..lib.inotify.green import inotify, IN_MODIFY

__all__ = ['load_config', 'run_agent', 'start_agent_thread']

__author__ = 'Brandon Carpenter <brandon.carpenter@pnnl.gov>'
__copyright__ = 'Copyright (c) 2015, Battelle Memorial Institute'
__license__ = 'FreeBSD'

_comment_re = re.compile(
        r'((["\'])(?:\\?.)*?\2)|(/\*.*?\*/)|((?:#|//).*?(?=\n|$))',
        re.MULTILINE | re.DOTALL)

_log = logging.getLogger(__name__)

def _repl(match):
    '''Replace the matched group with an appropriate string.'''
    # If the first group matched, a quoted string was matched and should
    # be returned unchanged.  Otherwise a comment was matched and the
    # empty string should be returned.
    return match.group(1) or ''

def strip_comments(string):
    '''Return string with all comments stripped.

    Both JavaScript-style comments (//... and /*...*/) and hash (#...)
    comments are removed.
    '''
    return _comment_re.sub(_repl, string)

def load_config(config_path):
    '''Load a JSON-encoded configuration file.'''
    return jsonapi.loads(strip_comments(open(config_path).read()))

def run_agent(cls, subscribe_address=None, publish_address=None,
              config_path=None, **kwargs):
    '''Instantiate an agent and run it in the current thread.

    Attempts to get keyword parameters from the environment if they
    are not set.
    '''
    if not subscribe_address:
        subscribe_address = os.environ.get('AGENT_SUB_ADDR')
    if subscribe_address:
        kwargs['subscribe_address'] = subscribe_address
    if not publish_address:
        publish_address = os.environ.get('AGENT_PUB_ADDR')
    if publish_address:
        kwargs['publish_address'] = publish_address
    if not config_path:
        config_path = os.environ.get('AGENT_CONFIG')
    if config_path:
        kwargs['config_path'] = config_path
    agent = cls(**kwargs)
    agent.run()

def start_agent_thread(cls, **kwargs):
    '''Instantiate an agent class and run it in a new daemon thread.

    Returns the thread object.
    '''
    import threading
    agent = cls(**kwargs)
    thread = threading.Thread(target=agent.run)
    thread.daemon = True
    thread.start()
    return thread

def isapipe(fd):
    fd = getattr(fd, 'fileno', lambda: fd)()
    return stat.S_ISFIFO(os.fstat(fd).st_mode)

def default_main(agent_class, description=None, argv=sys.argv,
                 parser_class=argparse.ArgumentParser, **kwargs):
    '''Default main entry point implementation for legacy agents.

    description and parser_class are depricated. Please avoid using them.
    '''
    try:
        # If stdout is a pipe, re-open it line buffered
        if isapipe(sys.stdout):
            # Hold a reference to the previous file object so it doesn't
            # get garbage collected and close the underlying descriptor.
            stdout = sys.stdout
            sys.stdout = os.fdopen(stdout.fileno(), 'w', 1)

        try:
            sub_addr = os.environ['AGENT_SUB_ADDR']
            pub_addr = os.environ['AGENT_PUB_ADDR']
        except KeyError as exc:
            sys.stderr.write(
                'missing environment variable: {}\n'.format(exc.args[0]))
            sys.exit(1)
        if sub_addr.startswith('ipc://') and sub_addr[6:7] != '@':
            if not os.path.exists(sub_addr[6:]):
                sys.stderr.write('warning: subscription socket does not '
                                 'exist: {}\n'.format(sub_addr[6:]))
        if pub_addr.startswith('ipc://') and pub_addr[6:7] != '@':
            if not os.path.exists(pub_addr[6:]):
                sys.stderr.write('warning: publish socket does not '
                                 'exist: {}\n'.format(pub_addr[6:]))
        config = os.environ.get('AGENT_CONFIG')
        agent = agent_class(subscribe_address=sub_addr,
                            publish_address=pub_addr,
                            config_path=config, **kwargs)
        agent.run()
    except KeyboardInterrupt:
        pass

def vip_main(agent_class, **kwargs):
    '''Default main entry point implementation for VIP agents.'''
    try:
        # If stdout is a pipe, re-open it line buffered
        if isapipe(sys.stdout):
            # Hold a reference to the previous file object so it doesn't
            # get garbage collected and close the underlying descriptor.
            stdout = sys.stdout
            sys.stdout = os.fdopen(stdout.fileno(), 'w', 1)

        # Quiet printing of KeyboardInterrupt by greenlets
        Hub = gevent.hub.Hub
        Hub.NOT_ERROR = Hub.NOT_ERROR + (KeyboardInterrupt,)

        agent_uuid = os.environ.get('AGENT_UUID')
        config = os.environ.get('AGENT_CONFIG')
        agent = agent_class(config_path=config, identity=agent_uuid, **kwargs)
        try:
            run = agent.run
        except AttributeError:
            run = agent.core.run
        task = gevent.spawn(run)
        try:
            task.join()
        finally:
            task.kill()
    except KeyboardInterrupt:
        pass

class SyslogFormatter(logging.Formatter):
    _level_map = {logging.DEBUG: syslog.LOG_DEBUG,
                  logging.INFO: syslog.LOG_INFO,
                  logging.WARNING: syslog.LOG_WARNING,
                  logging.ERROR: syslog.LOG_ERR,
                  logging.CRITICAL: syslog.LOG_CRIT,}

    def format(self, record):
        level = self._level_map.get(record.levelno, syslog.LOG_INFO)
        return '<{}>'.format(level) + super(SyslogFormatter, self).format(record)

class JsonFormatter(logging.Formatter):
    def format(self, record):
        dct = record.__dict__.copy()
        exc_info = dct.pop('exc_info', None)
        if exc_info:
            dct['exc_text'] = ''.join(traceback.format_exception(*exc_info))
        return jsonapi.dumps(dct)

class AgentFormatter(logging.Formatter):
    def __init__(self, fmt=None, datefmt=None):
        if fmt is None:
            fmt = '%(asctime)s %(composite_name)s %(levelname)s: %(message)s'
        super(AgentFormatter, self).__init__(fmt=fmt, datefmt=datefmt)

    def composite_name(self, record):
        if record.name == 'agents.log':
            cname = '(%(processName)s %(process)d) %(remote_name)s'
        elif record.name.startswith('agents.std'):
            cname = '(%(processName)s %(process)d) <{}>'.format(
                    record.name.split('.', 2)[1])
        else:
            cname = '() %(name)s'
        return cname % record.__dict__

    def format(self, record):
        if 'composite_name' not in record.__dict__:
            record.__dict__['composite_name'] = self.composite_name(record)
        if len(record.args) > 0 \
            and 'tornado.access' in record.__dict__['composite_name']:
            record.__dict__['msg'] = ','.join([str(b) for b in record.args])
            record.__dict__['args'] = []
        return super(AgentFormatter, self).format(record)

def setup_logging(level=logging.DEBUG):
    root = logging.getLogger()
    if not root.handlers:
        handler = logging.StreamHandler()
        if isapipe(sys.stderr) and '_LAUNCHED_BY_PLATFORM' in os.environ:
            handler.setFormatter(JsonFormatter())
        else:
            handler.setFormatter(logging.Formatter(
                    '%(asctime)s %(name)s %(levelname)s: %(message)s'))
        root.addHandler(handler)
    root.setLevel(level)

def format_timestamp(time_stamp):
    """Create a consistent datetime string representation based on ISO 8601 format.

    YYYY-MM-DDTHH:MM:SS.mmmmmm for unaware datetime objects.
    YYYY-MM-DDTHH:MM:SS.mmmmmm+HH:MM for aware datetime objects

    :param time_stamp: value to convert
    :type time_stamp: datetime
    :returns: datetime in string format
    :rtype: str
    """

    time_str = time_stamp.strftime("%Y-%m-%dT%H:%M:%S.%f")

    if time_stamp.tzinfo is not None:
        sign = '+'
        td = time_stamp.tzinfo.utcoffset(time_stamp)
        if td.days < 0:
            sign = '-'
            td = -td

        seconds = td.seconds
        minutes, seconds = divmod(seconds, 60)
        hours, minutes = divmod(minutes, 60)
        time_str += "{sign}{HH:02}:{MM:02}".format(sign=sign, HH=hours, MM=minutes)

    return time_str

def parse_timestamp_string(time_stamp_str):
    """Create a datetime object from the supplied date/time string.
    Uses dateutil.parse with no extra parameters.

    :param time_stamp: value to convert
    :type time_stamp: str
    :returns: datetime object
    :rtype: datetime
    """
    return parse(time_stamp_str)

def get_aware_utc_now():
    """Create a timezone aware UTC datetime object from the system time.

    :returns: an aware UTC datetime object
    :rtype: datetime
    """
    utcnow = datetime.utcnow()
    utcnow = pytz.UTC.localize(utcnow)
    return utcnow

def process_timestamp(timestamp_string, topic=''):
    if timestamp_string is None:
        _log.error("message for {topic} missing timetamp".format(topic=topic))
        return

    try:
        timestamp = parse(timestamp_string)
    except (ValueError, TypeError) as e:
        _log.error("message for {topic} bad timetamp string: {ts_string}".format(topic=topic,
                                                                                ts_string=timestamp_string))
        return

    if timestamp.tzinfo is None:
        timestamp.replace(tzinfo=pytz.UTC)
        original_tz = None
    else:
        original_tz = timestamp.tzinfo
        timestamp = timestamp.astimezone(pytz.UTC)
    return timestamp, original_tz

def watch_file(fullpath, callback):
    '''Run callback method whenever the file changes'''
    dirname, filename = os.path.split(fullpath)
    with inotify() as inot:
        inot.add_watch(dirname, IN_MODIFY)
        for event in inot:
            if event.name == filename and event.mask & IN_MODIFY:
                callback()

def create_file_if_missing(path, permission=0o660, contents=None):
    try:
        fil = open(path)
    except IOError as exc:
        if exc.errno != errno.ENOENT:
            raise
        _log.debug('missing file %s', path)
        _log.info('creating file %s', path)
        fd = os.open(path, os.O_CREAT|os.O_WRONLY, permission)
        try:
            if contents:
                os.write(fd, contents)
        finally:
            os.close(fd)

def fix_sqlite3_datetime(sql=None):
    """Primarily for fixing the base historian cache on certain versions of python.

    Registers a new datetime converter to that uses dateutil parse. This should
    better resolve #216, #174, and #91 without the goofy workarounds that change data.

    Optional sql argument is for testing only.
    """
    if sql is None:
        import sqlite3 as sql
    sql.register_converter("timestamp", parse)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import re
import glob
import shutil
import datetime
import sys
import getopt
import pipes
import subprocess

SOURCE_DIR = os.path.realpath(os.path.dirname(os.path.realpath(__file__)) + '/..')
BUILD_SCRIPT = SOURCE_DIR + '/build-script.py'
GYB_GEN_SOURCE_DIR = SOURCE_DIR + '/Sources/SwiftSyntax/gyb_generated/'

def escapeCmdArg(arg):
    if '"' in arg or ' ' in arg:
        return '"%s"' % arg.replace('"', '\\"')
    else:
        return arg

# Git Utils
def git(git_command, cwd):
    '''Run a git command.'''
    command = ['git'] + git_command
    check_call(command, cwd=cwd, verbose=True)

def git_tag(tag,src_path):
  '''Tag git repo'''
  git(['tag','-a',tag,'-m','Tag build ' + tag],src_path)

def git_push_tag(tag,src_path):
  '''Push git tag'''
  git(['push', 'origin', tag], src_path)

def git_clone(repo_url,src_path):
  '''Push git tag'''
  git(['clone', repo_url], src_path)

def check_call(cmd, cwd=None, env=os.environ, verbose=False):
    if verbose:
        print(' '.join([escapeCmdArg(arg) for arg in cmd]))
    return subprocess.check_call(cmd, cwd=cwd, env=env, stderr=subprocess.STDOUT)

def degyb():
    check_call([BUILD_SCRIPT, '--degyb-only'])

def main(argv):
	# Generate gyb files
	degyb()
	# Force commit gyb files
	git(git_command=['add', '-f', GYB_GEN_SOURCE_DIR + '/*.swift'], cwd=SOURCE_DIR)
	git(git_command=['commit', '-m', 'Auto-commit: add degybed files'], cwd=SOURCE_DIR)

if __name__ == "__main__":
  main(sys.argv[1:])
<commit_msg>utils: remove script for committing gyb generated files<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

#from numba import jit
import numpy as np
from time import time
from math import exp, log, pow, pi
from abel.tools import calculate_speeds, get_image_quadrants,\
                       put_image_quadrants

###########################################################################
# hasenlaw - a recursive method forwrd/inverse Abel transform algorithm
#
# Stephen Gibson - Australian National University, Australia
# Jason Gascooke - Flinders University, Australia
#
# This algorithm is adapted by Jason Gascooke from the article
#   E. W. Hansen and P-L. Law
#  "Recursive methods for computing the Abel transform and its inverse"
#   J. Opt. Soc. Am A2, 510-520 (1985) doi: 10.1364/JOSAA.2.000510
#
#  J. R. Gascooke PhD Thesis:
#   "Energy Transfer in Polyatomic-Rare Gas Collisions and Van Der Waals
#    Molecule Dissociation", Flinders University, 2000.
#
# Implemented in Python, with image quadrant co-adding, by Steve Gibson
# 2015-12-03: Vectorization and code improvements Dan Hickstein and Roman Yurchak
#             Previously the algorithm iterated over the rows of the image
#             now all of the rows are calculated simultaneously, which provides
#             the same result, but speeds up processing considerably.
###########################################################################

def fabel_hansenlaw_transform(img):
    return abel_hansenlaw_transform (img,forward=True)[::-1]  # needs flip

def iabel_hansenlaw_transform(img):
    return abel_hansenlaw_transform (img,forward=False)*np.pi  # scaling

def abel_hansenlaw_transform(img,forward=True):
    """ Forward/Inverse Abel transformation using the algorithm of:
        Hansen and Law J. Opt. Soc. Am. A 2, 510-520 (1985).

                       ∞
                       ⌠
                   -1  ⎮   g'(R)
           f(r) =  ─── ⎮ ──────────── dR      Eq. (2a)
                    π  ⎮    _________
                       ⎮   ╱  2    2
                       ⎮ ╲╱  R  - r
                       ⌡
                       r

        where f(r) = reconstructed image (source) function
              g'(R) = derivative of the projection (measured) function

        Evaluation via Eq. (17), using (16a), (16b), and (18)

        Recursion method proceeds from the outer edge of the image
        toward the image centre (origin). i.e. when n=0, R=Rmax, and
        when n=N-1, R=0. This fits well with processing the image one
        quadrant (or half) at a time.

        Parameters:
        ----------
         - img: a rows x cols numpy array = one quadrant (or half) of the image
           |                                oriented top/left
           |     +--------+      --------+
           \=>   |      * |       *      |
                 |   *    |          *   |
                 |  *     |           *  |
                 +--------+      --------+
                 |  *     |           *  |
                 |   *    |          *   |
                 |     *  |       *      |
                 +--------+      --------+
                     |
                     \=>            [ ]...[ ][ ]
                                     :     :  :
                                    [ ]...[ ][ ]
         Image centre is mid-pixel  [ ]...[ ][+]
                                    [ ]...[ ][ ]
                                     :     :  :
                                    [ ]...[ ][ ]
        Returns:
        --------
          - Aimg: a rows x cols numpy array, forward/inverse Abel transform image
    """
    img  = np.atleast_2d(img)
    N    = np.shape(img)  # shape of quadrant (half)
    Aimg = np.zeros(N)    # inverse Abel transform image

    rows,cols = N

    # constants listed in Table 1.
    h   = [0.318,0.19,0.35,0.82,1.8,3.9,8.3,19.6,48.3]
    lam = [0.0,-2.1,-6.2,-22.4,-92.5,-414.5,-1889.4,-8990.9,-47391.1]

    # Eq. (16c)
    def fGamma(Nm, lam, N, n):
        Nn1 = N - n - 1
        return 2*Nn1*(1-pow(Nm,(lam+1)))/(lam+1)
    # Eq. (18)
    def iGamma(Nm, lam, N, n):
        if lam < -1:
            return (1.0-pow(Nm,lam))/(pi*lam)
        else:
            return -np.log(Nm)/pi

    K = np.size(h)
    X = np.zeros((rows,K))

    if not forward:
        # inverse transform
        # g' - derivative of the intensity profile
        Gamma = iGamma
        if rows>1:
            gp = np.gradient(img)[1]  # second element is gradient along the columns
        else: # If there is only one row
            gp = np.atleast_2d(np.gradient(img[0]))
    else:
      # forward transform
      Gamma = fGamma
      gp = img

    # iterate along columns, starting outer edge (left side) toward image center
    for col in range(cols-1):
        Nm = (cols-col)/(cols-col-1.0)    # R0/R

        for k in range(K): # Iterate over k, the eigenvectors?
            X[:,k] = pow(Nm,lam[k])*X[:,k] +\
                     h[k]*Gamma(Nm,lam[k],cols,col)*gp[:,col] # Eq. (15 or 17)

        Aimg[:,col] = X.sum(axis=1)

    Aimg[:,cols-1] = Aimg[:,cols-2]  # special case for the center pixel

    return -Aimg

# ---- end abel_hansenlaw_transform ----

def iabel_hansenlaw (img, use_quadrants=(True,True,True,True),
                     vertical_symmetry=False, horizontal_symmetry=False,
                     calc_speeds=False, verbose=False):
    """
    Helper function for Hansen Law inverse Abel transform.

    Exploit image symmetry - select quadrants, combine to improve
                             signal

    Parameters:
    ----------
     - img: a rowsXcols numpy array
     - use_quadrants: boolean tuple, (Q0,Q1,Q2,Q3)
             +--------+--------+
             | Q1   * | *   Q0 |
             |   *    |    *   |
             |  *     |     *  |                               AQ1 | AQ0
             +--------+--------+ --(inverse Abel transform)--> ---------
             |  *     |     *  |                               AQ2 | AQ3
             |   *    |    *   |
             | Q2  *  | *   Q3 |          AQi == inverse Abel transform
             +--------+--------+                 of quadrant Qi

       (1) vertical_symmetry = True

           Combine:  Q01 = Q1 + Q2, Q23 = Q2 + Q3
           inverse image   AQ01 | AQ01
                           -----------
                           AQ23 | AQ23

       (2) horizontal_symmetry = True

           Combine: Q12 = Q1 + Q2, Q03 = Q0 + Q3
           inverse image   AQ12 | AQ03
                           -----------
                           AQ12 | AQ03

       (3) vertical_symmetry = True, horizontal = True

           Combine: Q = Q0 + Q1 + Q2 + Q3
           inverse image   AQ | AQ
                           -------  all quadrants equivalent
                           AQ | AQ

      - calc_speeds: boolean, evaluate speed profile
      - verbose: boolean, more output, timings etc.
    """
    verboseprint = print if verbose else lambda *a, **k: None

    if img.ndim == 1 or np.shape(img)[0] <= 2:
            raise ValueError('Data must be 2-dimensional.'
                             'To transform a single row, use'
                             'iabel_hansenlaw_transform().')

    rows,cols = np.shape(img)

    if not np.any(use_quadrants):
        verboseprint ("HL: Error: no image quadrants selected to use")
        return np.zeros((rows,cols))

    verboseprint ("HL: Calculating inverse Abel transform:",
                      " image size {:d}x{:d}".format(rows,cols))

    t0=time()

    # split image into quadrants
    Q0,Q1,Q2,Q3 = get_image_quadrants(img, reorient=True)

    verboseprint ("HL: Calculating inverse Abel transform ... ")

    if not vertical_symmetry and not horizontal_symmetry\
                             and np.all(use_quadrants):
        # individual quadrant inverse Abel transform
        AQ0 = iabel_hansenlaw_transform(Q0)
        AQ1 = iabel_hansenlaw_transform(Q1)
        AQ2 = iabel_hansenlaw_transform(Q2)
        AQ3 = iabel_hansenlaw_transform(Q3)

    else:  # combine selected quadrants according to assumed symmetry
        if vertical_symmetry:   # co-add quadrants
            Q0 = Q1 = Q0*use_quadrants[0]+Q1*use_quadrants[1]
            Q2 = Q3 = Q2*use_quadrants[2]+Q3*use_quadrants[3]

        if horizontal_symmetry:
            Q1 = Q2 = Q1*use_quadrants[1]+Q2*use_quadrants[2]
            Q0 = Q3 = Q0*use_quadrants[0]+Q3*use_quadrants[3]

        # HL inverse Abel transform for quadrant 1
        AQ1 = iabel_hansenlaw_transform(Q1)  # all possibilities include Q1

        if vertical_symmetry:
            AQ0 = AQ1
            AQ3 = AQ2 = iabel_hansenlaw_transform(Q2)

        if horizontal_symmetry:
            AQ2 = AQ1
            AQ3 = AQ0 = iabel_hansenlaw_transform(Q0)

    # reassemble image
    recon = put_image_quadrants ((AQ0,AQ1,AQ2,AQ3),odd_size=True)

    verboseprint ("{:.2f} seconds".format(time()-t0))

    if calc_speeds:
        verboseprint('Generating speed distribution ...')
        t1 = time()

        speeds = calculate_speeds(recon)

        verboseprint('{:.2f} seconds'.format(time() - t1))
        return recon, speeds
    else:
        return recon
<commit_msg>hansenlaw.py: forward/inverse for both f/iabel_hansen_law() and f/iabel_hansen_law_transform().<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

#from numba import jit
import numpy as np
from time import time
from math import exp, log, pow, pi
from abel.tools import calculate_speeds, get_image_quadrants,\
                       put_image_quadrants

################################################################################
# hasenlaw - a recursive method forwrd/inverse Abel transform algorithm
#
# Stephen Gibson - Australian National University, Australia
# Jason Gascooke - Flinders University, Australia
#
# This algorithm is adapted by Jason Gascooke from the article
#   E. W. Hansen and P-L. Law
#  "Recursive methods for computing the Abel transform and its inverse"
#   J. Opt. Soc. Am A2, 510-520 (1985) doi: 10.1364/JOSAA.2.000510
#
#  J. R. Gascooke PhD Thesis:
#   "Energy Transfer in Polyatomic-Rare Gas Collisions and Van Der Waals
#    Molecule Dissociation", Flinders University, 2000.
#
# Implemented in Python, with image quadrant co-adding, by Steve Gibson
# 2015-12-16: Modified to calculate the forward Abel transform
# 2015-12-03: Vectorization and code improvements Dan Hickstein and Roman Yurchak
#             Previously the algorithm iterated over the rows of the image
#             now all of the rows are calculated simultaneously, which provides
#             the same result, but speeds up processing considerably.
################################################################################

# see fabel, iabel functions at bottom

def abel_hansenlaw_transform(img, forward=True):
    """ Forward/Inverse Abel transformation using the algorithm of:
        Hansen and Law J. Opt. Soc. Am. A 2, 510-520 (1985).

                       ∞
                       ⌠
                   -1  ⎮   g'(R)
           f(r) =  ─── ⎮ ──────────── dR      Eq. (2a)
                    π  ⎮    _________
                       ⎮   ╱  2    2
                       ⎮ ╲╱  R  - r
                       ⌡
                       r

        where f(r) = reconstructed image (source) function
              g'(R) = derivative of the projection (measured) function

        Evaluation via Eq. (15 or 17), using (16a), (16b), and (16c or 18)

        Recursion method proceeds from the outer edge of the image
        toward the image centre (origin). i.e. when n=0, R=Rmax, and
        when n=N-1, R=0. This fits well with processing the image one
        quadrant (or half) at a time.

        Parameters:
        ----------
         - img: a rows x cols numpy array = one quadrant (or half) of the image
           |                                oriented top/left
           |     +--------+      --------+
           \=>   |      * |       *      |
                 |   *    |          *   |
                 |  *     |           *  |
                 +--------+      --------+
                 |  *     |           *  |
                 |   *    |          *   |
                 |     *  |       *      |
                 +--------+      --------+
                     |
                     \=>            [ ]...[ ][ ]
                                     :     :  :
                                    [ ]...[ ][ ]
         Image centre is mid-pixel  [ ]...[ ][+]
                                    [ ]...[ ][ ]
                                     :     :  :
                                    [ ]...[ ][ ]
          - forward: boolean: True = forward Abel transform
                              False = inverse Abel transform
        Returns:
        --------
          - Aimg: a rows x cols numpy array, forward/inverse Abel transform image
    """
    img  = np.atleast_2d(img)
    N    = np.shape(img)       # shape of input quadrant (half)
    Aimg = np.zeros(N)         # forward/inverse Abel transform image

    rows,cols = N

    # constants listed in Table 1.
    h   = [0.318,0.19,0.35,0.82,1.8,3.9,8.3,19.6,48.3]
    lam = [0.0,-2.1,-6.2,-22.4,-92.5,-414.5,-1889.4,-8990.9,-47391.1]

    K = np.size(h)
    X = np.zeros((rows,K))

    # Two alternative Gamma functions for forward/inverse transform
    # Eq. (16c) used for the forward transform
    def fGamma(Nm, lam, N, n):
        Nn1 = N - n - 1
        return 2*Nn1*(1-pow(Nm,(lam+1)))/(lam+1)

    # Eq. (18) used for the inverse transform
    def iGamma(Nm, lam, N, n):
        if lam < -1:
            return (1.0-pow(Nm,lam))/(pi*lam)
        else:
            return -np.log(Nm)/pi

    if forward:
        # forward transform
        Gamma = fGamma
        gp = img
    else:
        # inverse transform
        # g' - derivative of the intensity profile
        Gamma = iGamma
        if rows>1:
            gp = np.gradient(img)[1]  # second element is gradient along the columns
        else: # If there is only one row
            gp = np.atleast_2d(np.gradient(img[0]))

    # ------ The Hansen and Law algorithm ------------
    # iterate along columns, starting outer edge (left side) toward image center

    for col in range(cols-1):
        Nm = (cols-col)/(cols-col-1.0)    # R0/R

        for k in range(K): # Iterate over k, the eigenvectors?
            X[:,k] = pow(Nm,lam[k])*X[:,k] +\
                     h[k]*Gamma(Nm,lam[k],cols,col)*gp[:,col] # Eq. (15 or 17)
        Aimg[:,col] = X.sum(axis=1)

    # special case for the center pixel
    Aimg[:,cols-1] = Aimg[:,cols-2]

    return -Aimg[::-1] if forward else -Aimg*np.pi  # flip(?) or scaling

    # ---- end abel_hansenlaw_transform ----

def abel_hansenlaw (img, use_quadrants=(True,True,True,True),
                    vertical_symmetry=False, horizontal_symmetry=False,
                    calc_speeds=False, verbose=False, forward=False):
    """
    Helper function for Hansen Law inverse Abel transform.

    Exploit image symmetry - select quadrants, combine to improve
                             signal

    Parameters:
    ----------
     - img: a rowsXcols numpy array
     - use_quadrants: boolean tuple, (Q0,Q1,Q2,Q3)
             +--------+--------+
             | Q1   * | *   Q0 |
             |   *    |    *   |
             |  *     |     *  |                               AQ1 | AQ0
             +--------+--------+ --(inverse Abel transform)--> ---------
             |  *     |     *  |                               AQ2 | AQ3
             |   *    |    *   |
             | Q2  *  | *   Q3 |          AQi == inverse Abel transform
             +--------+--------+                 of quadrant Qi

       (1) vertical_symmetry = True

           Combine:  Q01 = Q1 + Q2, Q23 = Q2 + Q3
           inverse image   AQ01 | AQ01
                           -----------
                           AQ23 | AQ23

       (2) horizontal_symmetry = True

           Combine: Q12 = Q1 + Q2, Q03 = Q0 + Q3
           inverse image   AQ12 | AQ03
                           -----------
                           AQ12 | AQ03

       (3) vertical_symmetry = True, horizontal = True

           Combine: Q = Q0 + Q1 + Q2 + Q3
           inverse image   AQ | AQ
                           -------  all quadrants equivalent
                           AQ | AQ

      - calc_speeds: boolean, evaluate speed profile
      - verbose: boolean, more output, timings etc.
      - forward: boolean: True = forward Abel transform
                          False = inverse Abel transform
    """
    verboseprint = print if verbose else lambda *a, **k: None

    if img.ndim == 1 or np.shape(img)[0] <= 2:
            raise ValueError('Data must be 2-dimensional.'
                             'To transform a single row, use'
                             'iabel_hansenlaw_transform().')

    rows,cols = np.shape(img)

    if not np.any(use_quadrants):
        verboseprint ("HL: Error: no image quadrants selected to use")
        return np.zeros((rows,cols))

    verboseprint ("HL: Calculating inverse Abel transform:",
                      " image size {:d}x{:d}".format(rows,cols))

    t0=time()

    # split image into quadrants
    Q0,Q1,Q2,Q3 = get_image_quadrants(img, reorient=True)

    verboseprint ("HL: Calculating inverse Abel transform ... ")

    if not vertical_symmetry and not horizontal_symmetry\
                             and np.all(use_quadrants):
        # individual quadrant inverse Abel transform
        AQ0 = iabel_hansenlaw_transform(Q0)
        AQ1 = iabel_hansenlaw_transform(Q1)
        AQ2 = iabel_hansenlaw_transform(Q2)
        AQ3 = iabel_hansenlaw_transform(Q3)

    else:  # combine selected quadrants according to assumed symmetry
        if vertical_symmetry:   # co-add quadrants
            Q0 = Q1 = Q0*use_quadrants[0]+Q1*use_quadrants[1]
            Q2 = Q3 = Q2*use_quadrants[2]+Q3*use_quadrants[3]

        if horizontal_symmetry:
            Q1 = Q2 = Q1*use_quadrants[1]+Q2*use_quadrants[2]
            Q0 = Q3 = Q0*use_quadrants[0]+Q3*use_quadrants[3]

        # HL inverse Abel transform for quadrant 1
        AQ1 = iabel_hansenlaw_transform(Q1)  # all possibilities include Q1

        if vertical_symmetry:
            AQ0 = AQ1
            AQ3 = AQ2 = iabel_hansenlaw_transform(Q2)

        if horizontal_symmetry:
            AQ2 = AQ1
            AQ3 = AQ0 = iabel_hansenlaw_transform(Q0)

    # reassemble image
    recon = put_image_quadrants ((AQ0,AQ1,AQ2,AQ3),odd_size=True)

    verboseprint ("{:.2f} seconds".format(time()-t0))

    if calc_speeds:
        verboseprint('Generating speed distribution ...')
        t1 = time()

        speeds = calculate_speeds(recon)

        verboseprint('{:.2f} seconds'.format(time() - t1))
        return recon, speeds
    else:
        return recon

# functions to conform to naming conventions" contributing.md
def fabel_hansenlaw_transform(img):
    return abel_hansenlaw_transform(img, forward=True)

def iabel_hansenlaw_transform(img):
    return abel_hansenlaw_transform(img, forward=False)

def fabel_hansenlaw(img, use_quadrants=(True,True,True,True),
                    vertical_symmetry=False, horizontal_symmetry=False,
                    calc_speeds=False, verbose=False):

    return abel_hansenlaw(img, use_quadrants, vertical_symmetry,
                          horizontal_symmetry, calc_speeds, verbose,
                          forward=True)

def iabel_hansenlaw(img, use_quadrants=(True,True,True,True),
                    vertical_symmetry=False, horizontal_symmetry=False,
                    calc_speeds=False, verbose=False):

    return abel_hansenlaw(img, use_quadrants, vertical_symmetry,
                          horizontal_symmetry, calc_speeds, verbose,
                          forward=False)
<|endoftext|>