<commit_before>"""
Copyright (c) 2017 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.

Takes the filesystem image created by the Dockerfile generated by
pre_flatpak_create_dockerfile, extracts the tree at /var/tmp/flatpak-build
and turns it into a Flatpak application or runtime.
"""

import os
from six.moves import configparser
import re
import shutil
import subprocess
import tarfile
from textwrap import dedent

from atomic_reactor.constants import IMAGE_TYPE_OCI, IMAGE_TYPE_OCI_TAR
from atomic_reactor.plugin import PrePublishPlugin
from atomic_reactor.plugins.pre_flatpak_create_dockerfile import get_flatpak_source_info
from atomic_reactor.util import get_exported_image_metadata

# Returns flatpak's name for the current arch
def get_arch():
    return subprocess.check_output(['flatpak', '--default-arch'],
                                   universal_newlines=True).strip()

# flatpak build-init requires the sdk and runtime to be installed on the
# build system (so that subsequent build steps can execute things with
# the SDK). While it isn't impossible to download the runtime image and
# install the flatpak, that would be a lot of unnecessary complexity
# since our build step is just unpacking the filesystem we've already
# created. This is a stub implementation of 'flatpak build-init' that
# doesn't check for the SDK or use it to set up the build filesystem.
def build_init(directory, appname, sdk, runtime, runtime_branch):
    if not os.path.isdir(directory):
        os.mkdir(directory)
    with open(os.path.join(directory, "metadata"), "w") as f:
        f.write(dedent("""\
                       [Application]
                       name={appname}
                       runtime={runtime}/{arch}/{runtime_branch}
                       sdk={sdk}/{arch}/{runtime_branch}
                       """.format(appname=appname,
                                  sdk=sdk,
                                  runtime=runtime,
                                  runtime_branch=runtime_branch,
                                  arch=get_arch())))
    os.mkdir(os.path.join(directory, "files"))

# add_app_prefix('org.gimp', 'gimp, 'gimp.desktop') => org.gimp.desktop
# add_app_prefix('org.gnome', 'eog, 'eog.desktop') => org.gnome.eog.desktop
def add_app_prefix(app_id, root, full):
    prefix = app_id
    if prefix.endswith('.' + root):
        prefix = prefix[:-(1 + len(root))]
    return prefix + '.' + full

def find_desktop_files(builddir):
    desktopdir = os.path.join(builddir, 'files/share/applications')
    for (dirpath, dirnames, filenames) in os.walk(desktopdir):
        for filename in filenames:
            if filename.endswith('.desktop'):
                yield os.path.join(dirpath, filename)

def find_icons(builddir, name):
    icondir = os.path.join(builddir, 'files/share/icons/hicolor')
    for (dirpath, dirnames, filenames) in os.walk(icondir):
        for filename in filenames:
            if filename.startswith(name + '.'):
                yield os.path.join(dirpath, filename)

def update_desktop_files(app_id, builddir):
    for full_path in find_desktop_files(builddir):
        cp = configparser.RawConfigParser()
        cp.read([full_path])
        try:
            icon = cp.get('Desktop Entry', 'Icon')
        except configparser.NoOptionError:
            icon = None

        # Does it have an icon?
        if icon and not icon.startswith(app_id):
            found_icon = False

            # Rename any matching icons
            for icon_file in find_icons(builddir, icon):
                shutil.copy(icon_file,
                            os.path.join(os.path.dirname(icon_file),
                                         add_app_prefix(app_id, icon, os.path.basename(icon_file))))
                found_icon = True

            # If we renamed the icon, change the desktop file
            if found_icon:
                subprocess.check_call(['desktop-file-edit',
                                       '--set-icon',
                                       add_app_prefix(app_id, icon, icon), full_path])

        # Is the desktop file not prefixed with the app id, then prefix it
        basename = os.path.basename(full_path)
        if not basename.startswith(app_id):
            shutil.move(full_path,
                        os.path.join(os.path.dirname(full_path),
                                     add_app_prefix(app_id,
                                                    basename[:-len('.desktop')],
                                                    basename)))

class FlatpakCreateOciPlugin(PrePublishPlugin):
    key = 'flatpak_create_oci'
    is_allowed_to_fail = False

    def __init__(self, tasker, workflow):
        """
        :param tasker: DockerTasker instance
        :param workflow: DockerBuildWorkflow instance
        """
        super(FlatpakCreateOciPlugin, self).__init__(tasker, workflow)

    # Compiles a list of path mapping rules to a simple function that matches
    # against a list of fixed patterns, see below for rule syntax
    def _compile_target_rules(rules):
        ROOT = "var/tmp/flatpak-build"

        patterns = []
        for source, target in rules:
            source = re.sub("^ROOT", ROOT, source)
            if source.endswith("/"):
                patterns.append((re.compile(source + "(.*)"), target))
                patterns.append((source[:-1], target))
            else:
                patterns.append((source, target))

        def get_target_func(self, path):
            for source, target in patterns:
                if isinstance(source, str):
                    if source == path:
                        return target
                else:
                    m = source.match(path)
                    if m:
                        return os.path.join(target, m.group(1))

            return None

        return get_target_func

    # Rules for mapping paths within the exported filesystem image to their
    # location in the final flatpak filesystem
    #
    # ROOT = /var/tmp/flatpak-build
    # No trailing slash - map a directory itself exactly
    # trailing slash - map a directory and everything inside of it

    _get_target_path_runtime = _compile_target_rules([
        # We need to make sure that 'files' is created before 'files/etc',
        # which wouldn't happen if just relied on ROOT/usr/ => files.
        # Instead map ROOT => files and omit ROOT/usr
        ("ROOT", "files"),
        ("ROOT/usr", None),

        # We map ROOT/usr => files and ROOT/etc => files/etc. This creates
        # A conflict between ROOT/usr/etc and /ROOT/etc. Just assume there
        # is nothing useful in /ROOT/usr/etc.
        ("ROOT/usr/etc/", None),

        ("ROOT/usr/", "files"),
        ("ROOT/etc/", "files/etc")
    ])

    _get_target_path_app = _compile_target_rules([
        ("ROOT/app/", "files")
    ])

    def _get_target_path(self, export_path):
        if self.source.runtime:
            return self._get_target_path_runtime(export_path)
        else:
            return self._get_target_path_app(export_path)

    def _export_container(self, container_id):
        outfile = os.path.join(self.workflow.source.workdir, 'filesystem.tar.gz')

        export_stream = self.tasker.d.export(container_id)
        out_fileobj = open(outfile, "wb")
        compress_process = subprocess.Popen(['gzip', '-c'],
                                            stdin=subprocess.PIPE,
                                            stdout=out_fileobj)
        in_tf = tarfile.open(fileobj=export_stream, mode='r|')
        out_tf = tarfile.open(fileobj=compress_process.stdin, mode='w|')

        for member in in_tf:
            target_name = self._get_target_path(member.name)
            if target_name is None:
                continue

            # flatpak tries to create files/.ref and behaves badly if the root directory
            # isn't writable. It also has trouble upgrading if any directory is not
            # user-writable.
            member.mode |= 0o0200

            member.name = target_name
            if member.islnk():
                # Hard links have full paths within the archive (no leading /)
                link_target = self._get_target_path(member.linkname)
                if link_target is None:
                    self.log.debug("Skipping %s, hard link to %s", target_name, link_target)
                    continue
                member.linkname = link_target
                out_tf.addfile(member)
            elif member.issym():
                # Symlinks have the literal link target, which will be
                # relative to the chroot and doesn't need rewriting
                out_tf.addfile(member)
            else:
                f = in_tf.extractfile(member)
                out_tf.addfile(member, fileobj=f)

        in_tf.close()
        out_tf.close()
        export_stream.close()
        compress_process.stdin.close()
        if compress_process.wait() != 0:
            raise RuntimeError("gzip failed")
        out_fileobj.close()

        return outfile

    def _export_filesystem(self):
        image = self.workflow.image
        self.log.info("Creating temporary docker container")
        container_dict = self.tasker.d.create_container(image)
        container_id = container_dict['Id']

        try:
            return self._export_container(container_id)
        finally:
            self.log.info("Cleaning up docker container")
            self.tasker.d.remove_container(container_id)

    def _create_runtime_oci(self, tarred_filesystem, outfile):
        info = self.source.flatpak_json

        builddir = os.path.join(self.workflow.source.workdir, "build")
        os.mkdir(builddir)

        repo = os.path.join(self.workflow.source.workdir, "repo")
        subprocess.check_call(['ostree', 'init', '--mode=archive-z2', '--repo', repo])

        runtime_id = info['runtime']
        runtime_version = info['runtime-version']

        args = {
            'runtime_id': runtime_id,
            'arch': get_arch(),
            'runtime_version': runtime_version
        }

        METADATA_TEMPLATE = dedent("""\
            [Runtime]
            name={runtime_id}
            runtime={runtime_id}/{arch}/{runtime_version}
            sdk={runtime_id}/{arch}/{runtime_version}

            [Environment]
            LD_LIBRARY_PATH=/app/lib64:/app/lib
            GI_TYPELIB_PATH=/app/lib64/girepository-1.0
            """)

        with open(os.path.join(builddir, 'metadata'), 'w') as f:
            f.write(METADATA_TEMPLATE.format(**args))

        runtime_ref = 'runtime/{runtime_id}/{arch}/{runtime_version}'.format(**args)

        subprocess.check_call(['ostree', 'commit',
                               '--repo', repo, '--owner-uid=0',
                               '--owner-gid=0', '--no-xattrs',
                               '--branch', runtime_ref,
                               '-s', 'build of ' + runtime_ref,
                               '--tree=tar=' + tarred_filesystem,
                               '--tree=dir=' + builddir])
        subprocess.check_call(['ostree', 'summary', '-u', '--repo', repo])

        subprocess.check_call(['flatpak', 'build-bundle', repo,
                               '--oci', '--runtime',
                               outfile, runtime_id, runtime_version])

        return runtime_ref

    def _create_app_oci(self, tarred_filesystem, outfile):
        info = self.source.flatpak_json
        app_id = info['id']

        runtime_id = info['runtime']
        runtime_version = info['runtime-version']

        builddir = os.path.join(self.workflow.source.workdir, "build")
        os.mkdir(builddir)

        repo = os.path.join(self.workflow.source.workdir, "repo")

        # See comment for build_init() for why we can't use 'flatpak build-init'
        # subprocess.check_call(['flatpak', 'build-init',
        #                        builddir, app_id, runtime_id, runtime_id, runtime_version])
        build_init(builddir, app_id, runtime_id, runtime_id, runtime_version)

        # with gzip'ed tarball, tar is several seconds faster than tarfile.extractall
        subprocess.check_call(['tar', 'xCfz', builddir, tarred_filesystem])

        update_desktop_files(app_id, builddir)

        subprocess.check_call(['flatpak', 'build-finish'] + info['finish-args'] + [builddir])
        subprocess.check_call(['flatpak', 'build-export', repo, builddir])

        subprocess.check_call(['flatpak', 'build-bundle', repo, '--oci', outfile, app_id])

        app_ref = 'app/{app_id}/{arch}/master'.format(app_id=app_id, arch=get_arch())

        return app_ref

    def run(self):
        self.source = get_flatpak_source_info(self.workflow)
        if self.source is None:
            raise RuntimeError("flatpak_create_dockerfile must be run before flatpak_create_oci")

        tarred_filesystem = self._export_filesystem()
        self.log.info('filesystem tarfile written to %s', tarred_filesystem)

        outfile = os.path.join(self.workflow.source.workdir, 'flatpak-oci-image')

        if self.source.runtime:
            ref_name = self._create_runtime_oci(tarred_filesystem, outfile)
        else:
            ref_name = self._create_app_oci(tarred_filesystem, outfile)

        metadata = get_exported_image_metadata(outfile, IMAGE_TYPE_OCI)
        metadata['ref_name'] = ref_name
        self.workflow.exported_image_sequence.append(metadata)

        self.log.info('OCI image is available as %s', outfile)

        tarred_outfile = outfile + '.tar'
        with tarfile.TarFile(tarred_outfile, "w") as tf:
            tf.add(outfile, os.path.basename(outfile))

        metadata = get_exported_image_metadata(tarred_outfile, IMAGE_TYPE_OCI_TAR)
        metadata['ref_name'] = ref_name
        self.workflow.exported_image_sequence.append(metadata)

        self.log.info('OCI tarfile is available as %s', tarred_outfile)
<commit_msg>flatpak_create_oci: remove an extra directory level in the tarball<commit_after>"""
Copyright (c) 2017 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.

Takes the filesystem image created by the Dockerfile generated by
pre_flatpak_create_dockerfile, extracts the tree at /var/tmp/flatpak-build
and turns it into a Flatpak application or runtime.
"""

import os
from six.moves import configparser
import re
import shutil
import subprocess
import tarfile
from textwrap import dedent

from atomic_reactor.constants import IMAGE_TYPE_OCI, IMAGE_TYPE_OCI_TAR
from atomic_reactor.plugin import PrePublishPlugin
from atomic_reactor.plugins.pre_flatpak_create_dockerfile import get_flatpak_source_info
from atomic_reactor.util import get_exported_image_metadata

# Returns flatpak's name for the current arch
def get_arch():
    return subprocess.check_output(['flatpak', '--default-arch'],
                                   universal_newlines=True).strip()

# flatpak build-init requires the sdk and runtime to be installed on the
# build system (so that subsequent build steps can execute things with
# the SDK). While it isn't impossible to download the runtime image and
# install the flatpak, that would be a lot of unnecessary complexity
# since our build step is just unpacking the filesystem we've already
# created. This is a stub implementation of 'flatpak build-init' that
# doesn't check for the SDK or use it to set up the build filesystem.
def build_init(directory, appname, sdk, runtime, runtime_branch):
    if not os.path.isdir(directory):
        os.mkdir(directory)
    with open(os.path.join(directory, "metadata"), "w") as f:
        f.write(dedent("""\
                       [Application]
                       name={appname}
                       runtime={runtime}/{arch}/{runtime_branch}
                       sdk={sdk}/{arch}/{runtime_branch}
                       """.format(appname=appname,
                                  sdk=sdk,
                                  runtime=runtime,
                                  runtime_branch=runtime_branch,
                                  arch=get_arch())))
    os.mkdir(os.path.join(directory, "files"))

# add_app_prefix('org.gimp', 'gimp, 'gimp.desktop') => org.gimp.desktop
# add_app_prefix('org.gnome', 'eog, 'eog.desktop') => org.gnome.eog.desktop
def add_app_prefix(app_id, root, full):
    prefix = app_id
    if prefix.endswith('.' + root):
        prefix = prefix[:-(1 + len(root))]
    return prefix + '.' + full

def find_desktop_files(builddir):
    desktopdir = os.path.join(builddir, 'files/share/applications')
    for (dirpath, dirnames, filenames) in os.walk(desktopdir):
        for filename in filenames:
            if filename.endswith('.desktop'):
                yield os.path.join(dirpath, filename)

def find_icons(builddir, name):
    icondir = os.path.join(builddir, 'files/share/icons/hicolor')
    for (dirpath, dirnames, filenames) in os.walk(icondir):
        for filename in filenames:
            if filename.startswith(name + '.'):
                yield os.path.join(dirpath, filename)

def update_desktop_files(app_id, builddir):
    for full_path in find_desktop_files(builddir):
        cp = configparser.RawConfigParser()
        cp.read([full_path])
        try:
            icon = cp.get('Desktop Entry', 'Icon')
        except configparser.NoOptionError:
            icon = None

        # Does it have an icon?
        if icon and not icon.startswith(app_id):
            found_icon = False

            # Rename any matching icons
            for icon_file in find_icons(builddir, icon):
                shutil.copy(icon_file,
                            os.path.join(os.path.dirname(icon_file),
                                         add_app_prefix(app_id, icon, os.path.basename(icon_file))))
                found_icon = True

            # If we renamed the icon, change the desktop file
            if found_icon:
                subprocess.check_call(['desktop-file-edit',
                                       '--set-icon',
                                       add_app_prefix(app_id, icon, icon), full_path])

        # Is the desktop file not prefixed with the app id, then prefix it
        basename = os.path.basename(full_path)
        if not basename.startswith(app_id):
            shutil.move(full_path,
                        os.path.join(os.path.dirname(full_path),
                                     add_app_prefix(app_id,
                                                    basename[:-len('.desktop')],
                                                    basename)))

class FlatpakCreateOciPlugin(PrePublishPlugin):
    key = 'flatpak_create_oci'
    is_allowed_to_fail = False

    def __init__(self, tasker, workflow):
        """
        :param tasker: DockerTasker instance
        :param workflow: DockerBuildWorkflow instance
        """
        super(FlatpakCreateOciPlugin, self).__init__(tasker, workflow)

    # Compiles a list of path mapping rules to a simple function that matches
    # against a list of fixed patterns, see below for rule syntax
    def _compile_target_rules(rules):
        ROOT = "var/tmp/flatpak-build"

        patterns = []
        for source, target in rules:
            source = re.sub("^ROOT", ROOT, source)
            if source.endswith("/"):
                patterns.append((re.compile(source + "(.*)"), target))
                patterns.append((source[:-1], target))
            else:
                patterns.append((source, target))

        def get_target_func(self, path):
            for source, target in patterns:
                if isinstance(source, str):
                    if source == path:
                        return target
                else:
                    m = source.match(path)
                    if m:
                        return os.path.join(target, m.group(1))

            return None

        return get_target_func

    # Rules for mapping paths within the exported filesystem image to their
    # location in the final flatpak filesystem
    #
    # ROOT = /var/tmp/flatpak-build
    # No trailing slash - map a directory itself exactly
    # trailing slash - map a directory and everything inside of it

    _get_target_path_runtime = _compile_target_rules([
        # We need to make sure that 'files' is created before 'files/etc',
        # which wouldn't happen if just relied on ROOT/usr/ => files.
        # Instead map ROOT => files and omit ROOT/usr
        ("ROOT", "files"),
        ("ROOT/usr", None),

        # We map ROOT/usr => files and ROOT/etc => files/etc. This creates
        # A conflict between ROOT/usr/etc and /ROOT/etc. Just assume there
        # is nothing useful in /ROOT/usr/etc.
        ("ROOT/usr/etc/", None),

        ("ROOT/usr/", "files"),
        ("ROOT/etc/", "files/etc")
    ])

    _get_target_path_app = _compile_target_rules([
        ("ROOT/app/", "files")
    ])

    def _get_target_path(self, export_path):
        if self.source.runtime:
            return self._get_target_path_runtime(export_path)
        else:
            return self._get_target_path_app(export_path)

    def _export_container(self, container_id):
        outfile = os.path.join(self.workflow.source.workdir, 'filesystem.tar.gz')

        export_stream = self.tasker.d.export(container_id)
        out_fileobj = open(outfile, "wb")
        compress_process = subprocess.Popen(['gzip', '-c'],
                                            stdin=subprocess.PIPE,
                                            stdout=out_fileobj)
        in_tf = tarfile.open(fileobj=export_stream, mode='r|')
        out_tf = tarfile.open(fileobj=compress_process.stdin, mode='w|')

        for member in in_tf:
            target_name = self._get_target_path(member.name)
            if target_name is None:
                continue

            # flatpak tries to create files/.ref and behaves badly if the root directory
            # isn't writable. It also has trouble upgrading if any directory is not
            # user-writable.
            member.mode |= 0o0200

            member.name = target_name
            if member.islnk():
                # Hard links have full paths within the archive (no leading /)
                link_target = self._get_target_path(member.linkname)
                if link_target is None:
                    self.log.debug("Skipping %s, hard link to %s", target_name, link_target)
                    continue
                member.linkname = link_target
                out_tf.addfile(member)
            elif member.issym():
                # Symlinks have the literal link target, which will be
                # relative to the chroot and doesn't need rewriting
                out_tf.addfile(member)
            else:
                f = in_tf.extractfile(member)
                out_tf.addfile(member, fileobj=f)

        in_tf.close()
        out_tf.close()
        export_stream.close()
        compress_process.stdin.close()
        if compress_process.wait() != 0:
            raise RuntimeError("gzip failed")
        out_fileobj.close()

        return outfile

    def _export_filesystem(self):
        image = self.workflow.image
        self.log.info("Creating temporary docker container")
        container_dict = self.tasker.d.create_container(image)
        container_id = container_dict['Id']

        try:
            return self._export_container(container_id)
        finally:
            self.log.info("Cleaning up docker container")
            self.tasker.d.remove_container(container_id)

    def _create_runtime_oci(self, tarred_filesystem, outfile):
        info = self.source.flatpak_json

        builddir = os.path.join(self.workflow.source.workdir, "build")
        os.mkdir(builddir)

        repo = os.path.join(self.workflow.source.workdir, "repo")
        subprocess.check_call(['ostree', 'init', '--mode=archive-z2', '--repo', repo])

        runtime_id = info['runtime']
        runtime_version = info['runtime-version']

        args = {
            'runtime_id': runtime_id,
            'arch': get_arch(),
            'runtime_version': runtime_version
        }

        METADATA_TEMPLATE = dedent("""\
            [Runtime]
            name={runtime_id}
            runtime={runtime_id}/{arch}/{runtime_version}
            sdk={runtime_id}/{arch}/{runtime_version}

            [Environment]
            LD_LIBRARY_PATH=/app/lib64:/app/lib
            GI_TYPELIB_PATH=/app/lib64/girepository-1.0
            """)

        with open(os.path.join(builddir, 'metadata'), 'w') as f:
            f.write(METADATA_TEMPLATE.format(**args))

        runtime_ref = 'runtime/{runtime_id}/{arch}/{runtime_version}'.format(**args)

        subprocess.check_call(['ostree', 'commit',
                               '--repo', repo, '--owner-uid=0',
                               '--owner-gid=0', '--no-xattrs',
                               '--branch', runtime_ref,
                               '-s', 'build of ' + runtime_ref,
                               '--tree=tar=' + tarred_filesystem,
                               '--tree=dir=' + builddir])
        subprocess.check_call(['ostree', 'summary', '-u', '--repo', repo])

        subprocess.check_call(['flatpak', 'build-bundle', repo,
                               '--oci', '--runtime',
                               outfile, runtime_id, runtime_version])

        return runtime_ref

    def _create_app_oci(self, tarred_filesystem, outfile):
        info = self.source.flatpak_json
        app_id = info['id']

        runtime_id = info['runtime']
        runtime_version = info['runtime-version']

        builddir = os.path.join(self.workflow.source.workdir, "build")
        os.mkdir(builddir)

        repo = os.path.join(self.workflow.source.workdir, "repo")

        # See comment for build_init() for why we can't use 'flatpak build-init'
        # subprocess.check_call(['flatpak', 'build-init',
        #                        builddir, app_id, runtime_id, runtime_id, runtime_version])
        build_init(builddir, app_id, runtime_id, runtime_id, runtime_version)

        # with gzip'ed tarball, tar is several seconds faster than tarfile.extractall
        subprocess.check_call(['tar', 'xCfz', builddir, tarred_filesystem])

        update_desktop_files(app_id, builddir)

        subprocess.check_call(['flatpak', 'build-finish'] + info['finish-args'] + [builddir])
        subprocess.check_call(['flatpak', 'build-export', repo, builddir])

        subprocess.check_call(['flatpak', 'build-bundle', repo, '--oci', outfile, app_id])

        app_ref = 'app/{app_id}/{arch}/master'.format(app_id=app_id, arch=get_arch())

        return app_ref

    def run(self):
        self.source = get_flatpak_source_info(self.workflow)
        if self.source is None:
            raise RuntimeError("flatpak_create_dockerfile must be run before flatpak_create_oci")

        tarred_filesystem = self._export_filesystem()
        self.log.info('filesystem tarfile written to %s', tarred_filesystem)

        outfile = os.path.join(self.workflow.source.workdir, 'flatpak-oci-image')

        if self.source.runtime:
            ref_name = self._create_runtime_oci(tarred_filesystem, outfile)
        else:
            ref_name = self._create_app_oci(tarred_filesystem, outfile)

        metadata = get_exported_image_metadata(outfile, IMAGE_TYPE_OCI)
        metadata['ref_name'] = ref_name
        self.workflow.exported_image_sequence.append(metadata)

        self.log.info('OCI image is available as %s', outfile)

        tarred_outfile = outfile + '.tar'
        with tarfile.TarFile(tarred_outfile, "w") as tf:
            for f in os.listdir(outfile):
                tf.add(os.path.join(outfile, f), f)

        metadata = get_exported_image_metadata(tarred_outfile, IMAGE_TYPE_OCI_TAR)
        metadata['ref_name'] = ref_name
        self.workflow.exported_image_sequence.append(metadata)

        self.log.info('OCI tarfile is available as %s', tarred_outfile)
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Driver for file-based ExampleGen components in Kubeflow V2 runner."""

import argparse
import os
from typing import Any, Dict, List, Optional

from absl import app
from absl import logging
from absl.flags import argparse_flags
from tfx.components.example_gen import driver
from tfx.components.example_gen import input_processor
from tfx.components.example_gen import utils
from tfx.dsl.io import fileio
from tfx.orchestration.kubeflow.v2.container import kubeflow_v2_entrypoint_utils
from tfx.orchestration.kubeflow.v2.proto import pipeline_pb2
from tfx.proto import example_gen_pb2
from tfx.proto import range_config_pb2
from tfx.types import artifact
from tfx.types import artifact_utils
from tfx.types import standard_component_specs
from tfx.utils import proto_utils

from google.protobuf import json_format

def _run_driver(exec_properties: Dict[str, Any],
                outputs_dict: Dict[str, List[artifact.Artifact]],
                output_metadata_uri: str,
                name_from_id: Optional[Dict[int, str]] = None) -> None:
  """Runs the driver, writing its output as a ExecutorOutput proto.

  The main goal of this driver is to calculate the span and fingerprint of input
  data, allowing for the executor invocation to be skipped if the ExampleGen
  component has been previously run on the same data with the same
  configuration. This span and fingerprint are added as new custom execution
  properties to an ExecutorOutput proto and written to a GCS path. The CAIP
  pipelines system reads this file and updates MLMD with the new execution
  properties.

  Args:
    exec_properties:
      These are required to contain the following properties:
      'input_base_uri': A path from which files will be read and their
        span/fingerprint calculated.
      'input_config': A json-serialized tfx.proto.example_gen_pb2.InputConfig
        proto message.
        See https://www.tensorflow.org/tfx/guide/examplegen for more details.
      'output_config': A json-serialized tfx.proto.example_gen_pb2.OutputConfig
        proto message.
        See https://www.tensorflow.org/tfx/guide/examplegen for more details.
    outputs_dict: The mapping of the output artifacts.
    output_metadata_uri: A path at which an ExecutorOutput message will be
      written with updated execution properties and output artifacts. The CAIP
      Pipelines service will update the task's properties and artifacts prior to
      running the executor.
    name_from_id: Optional. Mapping from the converted int-typed id to str-typed
      runtime artifact name, which should be unique.
  """
  if name_from_id is None:
    name_from_id = {}

  logging.set_verbosity(logging.INFO)
  logging.info('exec_properties = %s\noutput_metadata_uri = %s',
               exec_properties, output_metadata_uri)

  input_base_uri = exec_properties.get(standard_component_specs.INPUT_BASE_KEY)

  input_config = example_gen_pb2.Input()
  proto_utils.json_to_proto(
      exec_properties[standard_component_specs.INPUT_CONFIG_KEY], input_config)

  range_config = None
  range_config_entry = exec_properties.get(
      standard_component_specs.RANGE_CONFIG_KEY)
  if range_config_entry:
    range_config = range_config_pb2.RangeConfig()
    proto_utils.json_to_proto(range_config_entry, range_config)

  processor = input_processor.FileBasedInputProcessor(input_base_uri,
                                                      input_config.splits,
                                                      range_config)
  span, version = processor.resolve_span_and_version()
  fingerprint = processor.get_input_fingerprint(span, version)

  logging.info('Calculated span: %s', span)
  logging.info('Calculated fingerprint: %s', fingerprint)

  exec_properties[utils.SPAN_PROPERTY_NAME] = span
  exec_properties[utils.FINGERPRINT_PROPERTY_NAME] = fingerprint
  exec_properties[utils.VERSION_PROPERTY_NAME] = version

  # Updates the input_config.splits.pattern.
  for split in input_config.splits:
    split.pattern = processor.get_pattern_for_span_version(
        split.pattern, span, version)
  exec_properties[standard_component_specs
                  .INPUT_CONFIG_KEY] = proto_utils.proto_to_json(input_config)

  if standard_component_specs.EXAMPLES_KEY not in outputs_dict:
    raise ValueError('Example artifact was missing in the ExampleGen outputs.')
  example_artifact = artifact_utils.get_single_instance(
      outputs_dict[standard_component_specs.EXAMPLES_KEY])

  driver.update_output_artifact(
      exec_properties=exec_properties,
      output_artifact=example_artifact.mlmd_artifact)

  # Log the output metadata file
  output_metadata = pipeline_pb2.ExecutorOutput()
  output_metadata.parameters[
      utils.FINGERPRINT_PROPERTY_NAME].string_value = fingerprint
  output_metadata.parameters[utils.SPAN_PROPERTY_NAME].string_value = str(span)
  output_metadata.parameters[
      standard_component_specs
      .INPUT_CONFIG_KEY].string_value = json_format.MessageToJson(input_config)
  output_metadata.artifacts[
      standard_component_specs.EXAMPLES_KEY].artifacts.add().CopyFrom(
          kubeflow_v2_entrypoint_utils.to_runtime_artifact(
              example_artifact, name_from_id))

  fileio.makedirs(os.path.dirname(output_metadata_uri))
  with fileio.open(output_metadata_uri, 'wb') as f:
    f.write(json_format.MessageToJson(output_metadata, sort_keys=True))

def _parse_flags(argv: List[str]) -> argparse.Namespace:
  """Command lines flag parsing."""
  parser = argparse_flags.ArgumentParser()
  parser.add_argument(
      '--json_serialized_invocation_args',
      type=str,
      required=True,
      help='JSON-serialized metadata for this execution.')
  return parser.parse_args(argv)

def main(args):
  executor_input = pipeline_pb2.ExecutorInput()
  json_format.Parse(
      args.json_serialized_invocation_args,
      executor_input,
      ignore_unknown_fields=True)

  name_from_id = {}

  exec_properties = kubeflow_v2_entrypoint_utils.parse_execution_properties(
      executor_input.inputs.parameters)
  outputs_dict = kubeflow_v2_entrypoint_utils.parse_raw_artifact_dict(
      executor_input.outputs.artifacts, name_from_id)

  _run_driver(exec_properties, outputs_dict, executor_input.outputs.output_file,
              name_from_id)

if __name__ == '__main__':
  app.run(main, flags_parser=_parse_flags)
<commit_msg>Allows unexpected flags in FBEG driver script.<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Driver for file-based ExampleGen components in Kubeflow V2 runner."""

import argparse
import os
from typing import Any, Dict, List, Optional

from absl import app
from absl import logging
from absl.flags import argparse_flags
from tfx.components.example_gen import driver
from tfx.components.example_gen import input_processor
from tfx.components.example_gen import utils
from tfx.dsl.io import fileio
from tfx.orchestration.kubeflow.v2.container import kubeflow_v2_entrypoint_utils
from tfx.orchestration.kubeflow.v2.proto import pipeline_pb2
from tfx.proto import example_gen_pb2
from tfx.proto import range_config_pb2
from tfx.types import artifact
from tfx.types import artifact_utils
from tfx.types import standard_component_specs
from tfx.utils import proto_utils

from google.protobuf import json_format

def _run_driver(exec_properties: Dict[str, Any],
                outputs_dict: Dict[str, List[artifact.Artifact]],
                output_metadata_uri: str,
                name_from_id: Optional[Dict[int, str]] = None) -> None:
  """Runs the driver, writing its output as a ExecutorOutput proto.

  The main goal of this driver is to calculate the span and fingerprint of input
  data, allowing for the executor invocation to be skipped if the ExampleGen
  component has been previously run on the same data with the same
  configuration. This span and fingerprint are added as new custom execution
  properties to an ExecutorOutput proto and written to a GCS path. The CAIP
  pipelines system reads this file and updates MLMD with the new execution
  properties.

  Args:
    exec_properties:
      These are required to contain the following properties:
      'input_base_uri': A path from which files will be read and their
        span/fingerprint calculated.
      'input_config': A json-serialized tfx.proto.example_gen_pb2.InputConfig
        proto message.
        See https://www.tensorflow.org/tfx/guide/examplegen for more details.
      'output_config': A json-serialized tfx.proto.example_gen_pb2.OutputConfig
        proto message.
        See https://www.tensorflow.org/tfx/guide/examplegen for more details.
    outputs_dict: The mapping of the output artifacts.
    output_metadata_uri: A path at which an ExecutorOutput message will be
      written with updated execution properties and output artifacts. The CAIP
      Pipelines service will update the task's properties and artifacts prior to
      running the executor.
    name_from_id: Optional. Mapping from the converted int-typed id to str-typed
      runtime artifact name, which should be unique.
  """
  if name_from_id is None:
    name_from_id = {}

  logging.set_verbosity(logging.INFO)
  logging.info('exec_properties = %s\noutput_metadata_uri = %s',
               exec_properties, output_metadata_uri)

  input_base_uri = exec_properties.get(standard_component_specs.INPUT_BASE_KEY)

  input_config = example_gen_pb2.Input()
  proto_utils.json_to_proto(
      exec_properties[standard_component_specs.INPUT_CONFIG_KEY], input_config)

  range_config = None
  range_config_entry = exec_properties.get(
      standard_component_specs.RANGE_CONFIG_KEY)
  if range_config_entry:
    range_config = range_config_pb2.RangeConfig()
    proto_utils.json_to_proto(range_config_entry, range_config)

  processor = input_processor.FileBasedInputProcessor(input_base_uri,
                                                      input_config.splits,
                                                      range_config)
  span, version = processor.resolve_span_and_version()
  fingerprint = processor.get_input_fingerprint(span, version)

  logging.info('Calculated span: %s', span)
  logging.info('Calculated fingerprint: %s', fingerprint)

  exec_properties[utils.SPAN_PROPERTY_NAME] = span
  exec_properties[utils.FINGERPRINT_PROPERTY_NAME] = fingerprint
  exec_properties[utils.VERSION_PROPERTY_NAME] = version

  # Updates the input_config.splits.pattern.
  for split in input_config.splits:
    split.pattern = processor.get_pattern_for_span_version(
        split.pattern, span, version)
  exec_properties[standard_component_specs
                  .INPUT_CONFIG_KEY] = proto_utils.proto_to_json(input_config)

  if standard_component_specs.EXAMPLES_KEY not in outputs_dict:
    raise ValueError('Example artifact was missing in the ExampleGen outputs.')
  example_artifact = artifact_utils.get_single_instance(
      outputs_dict[standard_component_specs.EXAMPLES_KEY])

  driver.update_output_artifact(
      exec_properties=exec_properties,
      output_artifact=example_artifact.mlmd_artifact)

  # Log the output metadata file
  output_metadata = pipeline_pb2.ExecutorOutput()
  output_metadata.parameters[
      utils.FINGERPRINT_PROPERTY_NAME].string_value = fingerprint
  output_metadata.parameters[utils.SPAN_PROPERTY_NAME].string_value = str(span)
  output_metadata.parameters[
      standard_component_specs
      .INPUT_CONFIG_KEY].string_value = json_format.MessageToJson(input_config)
  output_metadata.artifacts[
      standard_component_specs.EXAMPLES_KEY].artifacts.add().CopyFrom(
          kubeflow_v2_entrypoint_utils.to_runtime_artifact(
              example_artifact, name_from_id))

  fileio.makedirs(os.path.dirname(output_metadata_uri))
  with fileio.open(output_metadata_uri, 'wb') as f:
    f.write(json_format.MessageToJson(output_metadata, sort_keys=True))

def _parse_flags(argv: List[str]) -> argparse.Namespace:
  """Command lines flag parsing."""
  parser = argparse_flags.ArgumentParser()
  parser.add_argument(
      '--json_serialized_invocation_args',
      type=str,
      required=True,
      help='JSON-serialized metadata for this execution.')
  # Ignore unknown args which is expected. Beam related args are also supplied
  # as command line arguments.
  namespace, _ = parser.parse_known_args(argv)
  return namespace

def main(args):
  executor_input = pipeline_pb2.ExecutorInput()
  json_format.Parse(
      args.json_serialized_invocation_args,
      executor_input,
      ignore_unknown_fields=True)

  name_from_id = {}

  exec_properties = kubeflow_v2_entrypoint_utils.parse_execution_properties(
      executor_input.inputs.parameters)
  outputs_dict = kubeflow_v2_entrypoint_utils.parse_raw_artifact_dict(
      executor_input.outputs.artifacts, name_from_id)

  _run_driver(exec_properties, outputs_dict, executor_input.outputs.output_file,
              name_from_id)

if __name__ == '__main__':
  app.run(main, flags_parser=_parse_flags)
<|endoftext|>


<commit_before>"""Package constants."""
from configparser import ConfigParser
import logging
from os.path import expanduser

LOG = logging.getLogger(__name__)

def find_config():
    """Looks for config in CONFIG_LOCATIONS. If not found, gives a fatal error

    Returns:
        ConfigParser: found configuration file
    """
    config_locations = [
        './.foremast/foremast.cfg',
        expanduser('~/.foremast/foremast.cfg'),
        '/etc/foremast/foremast.cfg',
        ]
    config = ConfigParser()

    cfg_file = config.read(config_locations)

    if not cfg_file:
        LOG.error('No config found in the following locations: {0}\n'.format(CONFIG_LOCATIONS))

    return config

def get_configs(config, section=''):
    """Get information for a given config _section_.

    Example:
        >>> from utils import config_loader
        >>> config = config_loader.get_config(section='section')
        >>> config['key']
        'value'

    Args:
        section (str): Configuration to find.

    Returns:
        dict: Configuration information for *section*.

    Raises:
        KeyError: _section_ missing from configuration file.
    """
    if config[section]:
        return dict(config[section])
    else:
        raise KeyError('Section "{0}" missing from configuraton file.'.format(section))

#Load in consts from config
config = find_config()
API_URL = config['urls']['gate_api_url']
GIT_URL = config['urls']['git_url']
GITLAB_TOKEN = config['credentials']['gitlab_token']
SLACK_TOKEN = get_configs(config, section='credentials')['slack_token']

HEADERS = {
    'accept': '*/*',
    'content-type': 'application/json',
    'user-agent': 'foremast',
}
LOGGING_FORMAT = ('%(asctime)s [%(levelname)s] %(name)s:%(funcName)s:'
                  '%(lineno)d - %(message)s')

DOMAIN = 'example.com'
ENVS = set(('build', 'dev', 'stage', 'prod', 'prods', 'prodp'))
REGIONS = set(('us-east-1', 'us-west-2'))

ASG_WHITELIST = [
    'profileservicecoreapis'
]

<commit_msg>added most consts to config<commit_after>"""Package constants."""
from configparser import ConfigParser
import logging
from os.path import expanduser

LOG = logging.getLogger(__name__)

def find_config():
    """Looks for config in CONFIG_LOCATIONS. If not found, gives a fatal error

    Returns:
        ConfigParser: found configuration file
    """
    config_locations = [
        './.foremast/foremast.cfg',
        expanduser('~/.foremast/foremast.cfg'),
        '/etc/foremast/foremast.cfg',
        ]
    config = ConfigParser()

    cfg_file = config.read(config_locations)

    if not cfg_file:
        LOG.error('No config found in the following locations: {0}\n'.format(CONFIG_LOCATIONS))

    return config

#Load in consts from config
config = find_config()
API_URL = config['base']['gate_api_url']
GIT_URL = config['base']['git_url']
GITLAB_TOKEN = config['credentials']['gitlab_token']
SLACK_TOKEN = config['credentials']['slack_token']
DOMAIN = config['base']['domain']
ENVS = set(config['base']['envs'].split(','))
REGIONS = set(config['base']['regions'].split(','))

ASG_WHITELIST = set(config['whitelists']['asg_whitelist'].split(','))

HEADERS = {
    'accept': '*/*',
    'content-type': 'application/json',
    'user-agent': 'foremast',
}
LOGGING_FORMAT = ('%(asctime)s [%(levelname)s] %(name)s:%(funcName)s:'
                  '%(lineno)d - %(message)s')

<|endoftext|>


<commit_before>#   Foremast - Pipeline Tooling
#
#   Copyright 2016 Gogo, LLC
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""Load base config and export package constants.

The ``foremast`` configuration file is read from the following locations in
descending order. First found wins.

* ``./.foremast/foremast.cfg``
* ``~/.foremast/foremast.cfg``
* ``/etc/foremast/foremast.cfg``

.. literalinclude:: ../../src/foremast/templates/configs/foremast.cfg.example
   :language: ini
"""
import ast
import json
import logging
import sys
from configparser import ConfigParser, DuplicateSectionError
from os import getcwd, path
from os.path import exists, expanduser, expandvars

LOG = logging.getLogger(__name__)
LOGGING_FORMAT = '%(asctime)s [%(levelname)s] %(name)s:%(funcName)s:%(lineno)d - %(message)s'
SHORT_LOGGING_FORMAT = '[%(levelname)s] %(message)s'

logging.basicConfig(format=LOGGING_FORMAT)
logging.getLogger(__package__.split('.')[0]).setLevel(logging.INFO)

def validate_key_values(config_handle, section, key, default=None):
    """Warn when *key* is missing from configuration *section*.

    Args:
        config_handle (configparser.ConfigParser): Instance of configurations.
        section (str): Name of configuration section to retrieve.
        key (str): Configuration key to look up.
        default (object): Default object to use when *key* is not found.

    Returns:
        object: ``str`` when *key* exists, otherwise *default* object.
    """
    try:
        config_handle.add_section(section)
        LOG.info('Section missing from configurations: [%s]', section)
    except DuplicateSectionError:
        pass

    section_handle = config[section]

    try:
        value = section_handle[key]
    except KeyError:
        LOG.warning('[%s] missing key "%s", using %r.', section_handle.name, key, default)
        value = default

    return value

def extract_formats(config_handle):
    """Get application formats.

    Args:
        config_handle (configparser.ConfigParser): Instance of configurations.

    Returns:
        object: ``str`` when *key* exists, otherwise *default* object.
        dict: of formats in {$format_type: $format_pattern}.
        See (gogoutils.Formats) for available options.
    """
    formats = {}

    if config_handle.has_section('formats'):
        formats = dict(config_handle['formats'])

    return formats

def load_dynamic_config(configurations, config_dir=getcwd()):
    """Load and parse dynamic config"""
    # Create full path of config
    config_file = '{path}/config.py'.format(path=config_dir)

    # Insert config path so we can import it
    sys.path.insert(0, path.dirname(path.abspath(config_file)))
    try:
        config_module = __import__('config')

        for key, value in config_module.CONFIG.items():
            LOG.debug('Importing %s with key %s', key, value)
            # Update configparser object
            configurations.update({key: value})
    except ImportError:
        # Provide a default if config not found
        configurations = {}

def find_config():
    """Look for **foremast.cfg** in config_locations.

    Raises:
        SystemExit: No configuration file found.

    Returns:
        ConfigParser: found configuration file
    """
    config_locations = [
        '/etc/foremast/foremast.cfg',
        expanduser('~/.foremast/foremast.cfg'),
        './.foremast/foremast.cfg',
    ]
    configurations = ConfigParser()

    cfg_file = configurations.read(config_locations)
    dynamic_config_file = '{path}/config.py'.format(path=getcwd())

    if cfg_file:
        LOG.info('Loading static configuration file.')
    elif exists(dynamic_config_file):
        LOG.info('Loading dynamic configuration file.')
        load_dynamic_config(configurations)
    else:
        config_locations.append(dynamic_config_file)
        LOG.warning('No configuration found in the following locations:\n%s', '\n'.join(config_locations))
        LOG.warning('Using defaults...')

    return configurations

def _remove_empty_entries(entries):
    """Remove empty entries in a list"""
    valid_entries = []
    for entry in set(entries):
        if entry:
            valid_entries.append(entry)
    return sorted(valid_entries)

def _convert_string_to_native(value):
    """Convert a string to its native python type"""
    result = None

    try:
        result = ast.literal_eval(value)
    except (SyntaxError, ValueError):
        # Likely a string
        result = value.split(',')
    return result

def _generate_security_groups(config_key):
    """Read config file and generate security group dict by environment.

    Args:
        config_key (str): Configuration file key

    Returns:
        dict: of environments in {'env1': ['group1', 'group2']} format
    """
    raw_default_groups = validate_key_values(config, 'base', config_key, default='')
    default_groups = _convert_string_to_native(raw_default_groups)
    LOG.debug('Default security group for %s is %s', config_key, default_groups)

    entries = {}
    for env in ENVS:
        entries[env] = []

    if isinstance(default_groups, (list)):
        groups = _remove_empty_entries(default_groups)
        for env in entries:
            entries[env] = groups
    elif isinstance(default_groups, (dict)):
        entries.update(default_groups)

    LOG.debug('Generated security group: %s', entries)
    return entries

config = find_config()

API_URL = validate_key_values(config, 'base', 'gate_api_url')
GIT_URL = validate_key_values(config, 'base', 'git_url')
DOMAIN = validate_key_values(config, 'base', 'domain', default='example.com')
ENVS = set(validate_key_values(config, 'base', 'envs', default='').split(','))
REGIONS = set(validate_key_values(config, 'base', 'regions', default='').split(','))
ALLOWED_TYPES = set(validate_key_values(config, 'base', 'types', default='ec2,lambda,s3,datapipeline').split(','))
TEMPLATES_PATH = validate_key_values(config, 'base', 'templates_path')
AMI_JSON_URL = validate_key_values(config, 'base', 'ami_json_url')
DEFAULT_SECURITYGROUP_RULES = _generate_security_groups('default_securitygroup_rules')
DEFAULT_EC2_SECURITYGROUPS = _generate_security_groups('default_ec2_securitygroups')
DEFAULT_ELB_SECURITYGROUPS = _generate_security_groups('default_elb_securitygroups')
GITLAB_TOKEN = validate_key_values(config, 'credentials', 'gitlab_token')
SLACK_TOKEN = validate_key_values(config, 'credentials', 'slack_token')
DEFAULT_TASK_TIMEOUT = validate_key_values(config, 'task_timeouts', 'default', default=120)
TASK_TIMEOUTS = json.loads(validate_key_values(config, 'task_timeouts', 'envs', default="{}"))
ASG_WHITELIST = set(validate_key_values(config, 'whitelists', 'asg_whitelist', default='').split(','))
APP_FORMATS = extract_formats(config)
GATE_CLIENT_CERT = expandvars(expanduser(validate_key_values(config, 'base', 'gate_client_cert', default='')))
GATE_CA_BUNDLE = expandvars(expanduser(validate_key_values(config, 'base', 'gate_ca_bundle', default='')))

HEADERS = {
    'accept': '*/*',
    'content-type': 'application/json',
    'user-agent': 'foremast',
}
<commit_msg>fix: Use correct variable name<commit_after>#   Foremast - Pipeline Tooling
#
#   Copyright 2016 Gogo, LLC
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""Load base config and export package constants.

The ``foremast`` configuration file is read from the following locations in
descending order. First found wins.

* ``./.foremast/foremast.cfg``
* ``~/.foremast/foremast.cfg``
* ``/etc/foremast/foremast.cfg``

.. literalinclude:: ../../src/foremast/templates/configs/foremast.cfg.example
   :language: ini
"""
import ast
import json
import logging
import sys
from configparser import ConfigParser, DuplicateSectionError
from os import getcwd, path
from os.path import exists, expanduser, expandvars

LOG = logging.getLogger(__name__)
LOGGING_FORMAT = '%(asctime)s [%(levelname)s] %(name)s:%(funcName)s:%(lineno)d - %(message)s'
SHORT_LOGGING_FORMAT = '[%(levelname)s] %(message)s'

logging.basicConfig(format=LOGGING_FORMAT)
logging.getLogger(__package__.split('.')[0]).setLevel(logging.INFO)

def validate_key_values(config_handle, section, key, default=None):
    """Warn when *key* is missing from configuration *section*.

    Args:
        config_handle (configparser.ConfigParser): Instance of configurations.
        section (str): Name of configuration section to retrieve.
        key (str): Configuration key to look up.
        default (object): Default object to use when *key* is not found.

    Returns:
        object: ``str`` when *key* exists, otherwise *default* object.
    """
    try:
        config_handle.add_section(section)
        LOG.info('Section missing from configurations: [%s]', section)
    except DuplicateSectionError:
        pass

    section_handle = config_handle[section]

    try:
        value = section_handle[key]
    except KeyError:
        LOG.warning('[%s] missing key "%s", using %r.', section_handle.name, key, default)
        value = default

    return value

def extract_formats(config_handle):
    """Get application formats.

    Args:
        config_handle (configparser.ConfigParser): Instance of configurations.

    Returns:
        object: ``str`` when *key* exists, otherwise *default* object.
        dict: of formats in {$format_type: $format_pattern}.
        See (gogoutils.Formats) for available options.
    """
    formats = {}

    if config_handle.has_section('formats'):
        formats = dict(config_handle['formats'])

    return formats

def load_dynamic_config(configurations, config_dir=getcwd()):
    """Load and parse dynamic config"""
    # Create full path of config
    config_file = '{path}/config.py'.format(path=config_dir)

    # Insert config path so we can import it
    sys.path.insert(0, path.dirname(path.abspath(config_file)))
    try:
        config_module = __import__('config')

        for key, value in config_module.CONFIG.items():
            LOG.debug('Importing %s with key %s', key, value)
            # Update configparser object
            configurations.update({key: value})
    except ImportError:
        # Provide a default if config not found
        configurations = {}

def find_config():
    """Look for **foremast.cfg** in config_locations.

    Raises:
        SystemExit: No configuration file found.

    Returns:
        ConfigParser: found configuration file
    """
    config_locations = [
        '/etc/foremast/foremast.cfg',
        expanduser('~/.foremast/foremast.cfg'),
        './.foremast/foremast.cfg',
    ]
    configurations = ConfigParser()

    cfg_file = configurations.read(config_locations)
    dynamic_config_file = '{path}/config.py'.format(path=getcwd())

    if cfg_file:
        LOG.info('Loading static configuration file.')
    elif exists(dynamic_config_file):
        LOG.info('Loading dynamic configuration file.')
        load_dynamic_config(configurations)
    else:
        config_locations.append(dynamic_config_file)
        LOG.warning('No configuration found in the following locations:\n%s', '\n'.join(config_locations))
        LOG.warning('Using defaults...')

    return configurations

def _remove_empty_entries(entries):
    """Remove empty entries in a list"""
    valid_entries = []
    for entry in set(entries):
        if entry:
            valid_entries.append(entry)
    return sorted(valid_entries)

def _convert_string_to_native(value):
    """Convert a string to its native python type"""
    result = None

    try:
        result = ast.literal_eval(value)
    except (SyntaxError, ValueError):
        # Likely a string
        result = value.split(',')
    return result

def _generate_security_groups(config_key):
    """Read config file and generate security group dict by environment.

    Args:
        config_key (str): Configuration file key

    Returns:
        dict: of environments in {'env1': ['group1', 'group2']} format
    """
    raw_default_groups = validate_key_values(config, 'base', config_key, default='')
    default_groups = _convert_string_to_native(raw_default_groups)
    LOG.debug('Default security group for %s is %s', config_key, default_groups)

    entries = {}
    for env in ENVS:
        entries[env] = []

    if isinstance(default_groups, (list)):
        groups = _remove_empty_entries(default_groups)
        for env in entries:
            entries[env] = groups
    elif isinstance(default_groups, (dict)):
        entries.update(default_groups)

    LOG.debug('Generated security group: %s', entries)
    return entries

config = find_config()

API_URL = validate_key_values(config, 'base', 'gate_api_url')
GIT_URL = validate_key_values(config, 'base', 'git_url')
DOMAIN = validate_key_values(config, 'base', 'domain', default='example.com')
ENVS = set(validate_key_values(config, 'base', 'envs', default='').split(','))
REGIONS = set(validate_key_values(config, 'base', 'regions', default='').split(','))
ALLOWED_TYPES = set(validate_key_values(config, 'base', 'types', default='ec2,lambda,s3,datapipeline').split(','))
TEMPLATES_PATH = validate_key_values(config, 'base', 'templates_path')
AMI_JSON_URL = validate_key_values(config, 'base', 'ami_json_url')
DEFAULT_SECURITYGROUP_RULES = _generate_security_groups('default_securitygroup_rules')
DEFAULT_EC2_SECURITYGROUPS = _generate_security_groups('default_ec2_securitygroups')
DEFAULT_ELB_SECURITYGROUPS = _generate_security_groups('default_elb_securitygroups')
GITLAB_TOKEN = validate_key_values(config, 'credentials', 'gitlab_token')
SLACK_TOKEN = validate_key_values(config, 'credentials', 'slack_token')
DEFAULT_TASK_TIMEOUT = validate_key_values(config, 'task_timeouts', 'default', default=120)
TASK_TIMEOUTS = json.loads(validate_key_values(config, 'task_timeouts', 'envs', default="{}"))
ASG_WHITELIST = set(validate_key_values(config, 'whitelists', 'asg_whitelist', default='').split(','))
APP_FORMATS = extract_formats(config)
GATE_CLIENT_CERT = expandvars(expanduser(validate_key_values(config, 'base', 'gate_client_cert', default='')))
GATE_CA_BUNDLE = expandvars(expanduser(validate_key_values(config, 'base', 'gate_ca_bundle', default='')))

HEADERS = {
    'accept': '*/*',
    'content-type': 'application/json',
    'user-agent': 'foremast',
}
<|endoftext|>


<commit_before>__author__ = 'ayost'

import os
import re
import sys
from time import sleep
import boto.cloudformation
import boto.cloudformation.stack

SCRIPT_LOCATION = os.path.abspath(os.path.dirname(sys.argv[0]))

def send(stacks):
    regions = []
    for r in boto.cloudformation.regions():
        regions.append(r.name.upper())
    region = ""
    while len(region) < 1:
        answer = input("CF region: ")
        if answer.upper() in regions:
            region = answer.lower()
        else:
            print("Invalid region")
    names = []
    for stack in stacks:
        stackname = str(re.sub(r'.*[/\\]([^/\\\.]*?)\.stack', r'\1', stack)).strip()
        cf_conn = boto.cloudformation.connect_to_region(region)
        exists = False
        try:
            cf_stack = cf_conn.describe_stacks(stack_name_or_id=stackname)
            exists = True
        except:
            print("Stack does not exist. Creating...")
            exists = False
        if exists:
            print("Stack exists. updating...")
            cf_conn.update_stack(stack_name=stackname, template_body=get_template_text(stackname), capabilities=['CAPABILITY_IAM'])
        else:
            cf_stack = cf_conn.create_stack(stack_name=stackname, capabilities=['CAPABILITY_IAM'], template_body=get_template_text(stackname))
        names.append(stackname)
    results(names, region)

def results(stacks, region):
    done = []
    failed = []
    cf_success = ['CREATE_COMPLETE', 'DELETE_COMPLETE', 'UPDATE_COMPLETE']
    cf_failed = ['CREATE_FAILED', 'ROLLBACK_FAILED', 'ROLLBACK_COMPLETE', 'DELETE_FAILED']
    cf_conn = boto.cloudformation.connect_to_region(region)
    while len(done) + len(failed) < len(stacks):
        for stack in stacks:
            if stack not in failed and stack not in [d['Name'] for d in done]:
                stack_obj = cf_conn.describe_stacks(stack_name_or_id=stack)[0]
                status = stack_obj.stack_status
                if status in cf_success:
                    done.append({"Name": stack, "Outputs": stack_obj.outputs})
                if status in cf_failed:
                    failed.append(stack)
        sleep(1)
    print("All Stacks Complete")
    if len(failed) > 0:
        print("These stacks failed: ")
        print(failed)
        print("Outputs of successful stacks")
    for success in done:
        print(success['Name'])
        for output in success['Outputs']:
            print('\t' + output.key + ": " + output.value)
    return done

def get_template_text(stackname):
    template = os.path.join(SCRIPT_LOCATION, 'Stacks', stackname + '.stack')
    t_file = open(template)
    text = t_file.read()
    t_file.close()
    return text

if __name__ == '__main__':
    #send(sys.argv[1:])
    results(sys.argv[1:], 'us-east-1')<commit_msg>change main to perform actual function<commit_after>__author__ = 'ayost'

import os
import re
import sys
from time import sleep
import boto.cloudformation
import boto.cloudformation.stack

SCRIPT_LOCATION = os.path.abspath(os.path.dirname(sys.argv[0]))

def send(stacks):
    regions = []
    for r in boto.cloudformation.regions():
        regions.append(r.name.upper())
    region = ""
    while len(region) < 1:
        answer = input("CF region: ")
        if answer.upper() in regions:
            region = answer.lower()
        else:
            print("Invalid region")
    names = []
    for stack in stacks:
        stackname = str(re.sub(r'.*[/\\]([^/\\\.]*?)\.stack', r'\1', stack)).strip()
        cf_conn = boto.cloudformation.connect_to_region(region)
        exists = False
        try:
            cf_stack = cf_conn.describe_stacks(stack_name_or_id=stackname)
            exists = True
        except:
            print("Stack does not exist. Creating...")
            exists = False
        if exists:
            print("Stack exists. updating...")
            cf_conn.update_stack(stack_name=stackname, template_body=get_template_text(stackname), capabilities=['CAPABILITY_IAM'])
        else:
            cf_stack = cf_conn.create_stack(stack_name=stackname, capabilities=['CAPABILITY_IAM'], template_body=get_template_text(stackname))
        names.append(stackname)
    results(names, region)

def results(stacks, region):
    done = []
    failed = []
    cf_success = ['CREATE_COMPLETE', 'DELETE_COMPLETE', 'UPDATE_COMPLETE']
    cf_failed = ['CREATE_FAILED', 'ROLLBACK_FAILED', 'ROLLBACK_COMPLETE', 'DELETE_FAILED']
    cf_conn = boto.cloudformation.connect_to_region(region)
    while len(done) + len(failed) < len(stacks):
        for stack in stacks:
            if stack not in failed and stack not in [d['Name'] for d in done]:
                stack_obj = cf_conn.describe_stacks(stack_name_or_id=stack)[0]
                status = stack_obj.stack_status
                if status in cf_success:
                    done.append({"Name": stack, "Outputs": stack_obj.outputs})
                if status in cf_failed:
                    failed.append(stack)
        sleep(1)
    print("All Stacks Complete")
    if len(failed) > 0:
        print("These stacks failed: ")
        print(failed)
        print("Outputs of successful stacks")
    for success in done:
        print(success['Name'])
        for output in success['Outputs']:
            print('\t' + output.key + ": " + output.value)
    return done

def get_template_text(stackname):
    template = os.path.join(SCRIPT_LOCATION, 'Stacks', stackname + '.stack')
    t_file = open(template)
    text = t_file.read()
    t_file.close()
    return text

if __name__ == '__main__':
    send(sys.argv[1:])
    #results(sys.argv[1:], 'us-east-1')<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time
from datetime import datetime
from dateutil.relativedelta import relativedelta
import netsvc
import tools
from tools.safe_eval import safe_eval as eval
import pooler
from osv import fields, osv

def str2tuple(s):
    return eval('tuple(%s)' % (s or ''))

_intervalTypes = {
    'work_days': lambda interval: relativedelta(days=interval),
    'days': lambda interval: relativedelta(days=interval),
    'hours': lambda interval: relativedelta(hours=interval),
    'weeks': lambda interval: relativedelta(days=7*interval),
    'months': lambda interval: relativedelta(months=interval),
    'minutes': lambda interval: relativedelta(minutes=interval),
}

class ir_cron(osv.osv, netsvc.Agent):
    _name = "ir.cron"
    _columns = {
        'name': fields.char('Name', size=60, required=True),
        'user_id': fields.many2one('res.users', 'User', required=True),
        'active': fields.boolean('Active'),
        'interval_number': fields.integer('Interval Number',help="Repeat every x."),
        'interval_type': fields.selection( [('minutes', 'Minutes'),
            ('hours', 'Hours'), ('work_days','Work Days'), ('days', 'Days'),('weeks', 'Weeks'), ('months', 'Months')], 'Interval Unit'),
        'numbercall': fields.integer('Number of Calls', help='Number of time the function is called,\na negative number indicates that the function will always be called'),
        'doall' : fields.boolean('Repeat Missed',help="Select this if you want to run the scheduler for the missed occurencs."),
        'nextcall' : fields.datetime('Next Execution Date', required=True,help="Date when this schedular will call next time"),
        'model': fields.char('Object', size=64,help="Name of object whose function will be called when this scheduler will run. e.g. 'res.partener'"),
        'function': fields.char('Function', size=64,help="Name of the function to be called by this scheduler. "),
        'args': fields.text('Arguments',help="Arguments to be passed to the function. e.g. (uid,)"),
        'priority': fields.integer('Priority', help='0=Very Urgent\n10=Not urgent')
    }

    _defaults = {
        'nextcall' : lambda *a: time.strftime('%Y-%m-%d %H:%M:%S'),
        'priority' : lambda *a: 5,
        'user_id' : lambda obj,cr,uid,context: uid,
        'interval_number' : lambda *a: 1,
        'interval_type' : lambda *a: 'months',
        'numbercall' : lambda *a: 1,
        'active' : lambda *a: 1,
        'doall' : lambda *a: 1
    }

    def _check_args(self, cr, uid, ids, context=None):
        try:
            for this in self.browse(cr, uid, ids, context):
                str2tuple(this.args)
        except:
            return False
        return True

    _constraints = [
        (_check_args, 'Invalid arguments', ['args']),
    ]

    def _callback(self, cr, uid, model, func, args):
        args = str2tuple(args)
        m = self.pool.get(model)
        if m and hasattr(m, func):
            f = getattr(m, func)
            try:
                f(cr, uid, *args)
            except Exception, e:
                self._logger.notifyChannel('timers', netsvc.LOG_ERROR, "Job call of self.pool.get('%s').%s(cr, uid, *%r) failed" % (model, func, args))
                self._logger.notifyChannel('timers', netsvc.LOG_ERROR, tools.exception_to_unicode(e))

    def _poolJobs(self, db_name, check=False):
        try:
            db, pool = pooler.get_db_and_pool(db_name)
        except:
            return False
        cr = db.cursor()
        try:
            if not pool._init:
                now = datetime.now()
                cr.execute('select * from ir_cron where numbercall<>0 and active and nextcall<=now() order by priority')
                for job in cr.dictfetchall():
                    nextcall = datetime.strptime(job['nextcall'], '%Y-%m-%d %H:%M:%S')
                    numbercall = job['numbercall']

                    ok = False
                    while nextcall < now and numbercall:
                        if numbercall > 0:
                            numbercall -= 1
                        if not ok or job['doall']:
                            self._callback(cr, job['user_id'], job['model'], job['function'], job['args'])
                        if numbercall:
                            nextcall += _intervalTypes[job['interval_type']](job['interval_number'])
                        ok = True
                    addsql = ''
                    if not numbercall:
                        addsql = ', active=False'
                    cr.execute("update ir_cron set nextcall=%s, numbercall=%s"+addsql+" where id=%s", (nextcall.strftime('%Y-%m-%d %H:%M:%S'), numbercall, job['id']))
                    cr.commit()

            cr.execute('select min(nextcall) as min_next_call from ir_cron where numbercall<>0 and active and nextcall>=now()')
            next_call = cr.dictfetchone()['min_next_call']
            if next_call:
                next_call = time.mktime(time.strptime(next_call, '%Y-%m-%d %H:%M:%S'))
            else:
                next_call = int(time.time()) + 3600   # if do not find active cron job from database, it will run again after 1 day

            if not check:
                self.setAlarm(self._poolJobs, next_call, db_name, db_name)

        except Exception, ex:
            logger = netsvc.Logger()
            logger.notifyChannel('cron', netsvc.LOG_WARNING,
                'Exception in cron:'+str(ex))

        finally:
            cr.commit()
            cr.close()

    def restart(self, dbname):
        self.cancel(dbname)
        self._poolJobs(dbname)

    def create(self, cr, uid, vals, context=None):
        res = super(ir_cron, self).create(cr, uid, vals, context=context)
        cr.commit()
        self.restart(cr.dbname)
        return res

    def write(self, cr, user, ids, vals, context=None):
        res = super(ir_cron, self).write(cr, user, ids, vals, context=context)
        cr.commit()
        self.restart(cr.dbname)
        return res

    def unlink(self, cr, uid, ids, context=None):
        res = super(ir_cron, self).unlink(cr, uid, ids, context=context)
        cr.commit()
        self.restart(cr.dbname)
        return res
ir_cron()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<commit_msg>[IMP] base/ir_cron: better tooltips<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time
from datetime import datetime
from dateutil.relativedelta import relativedelta
import netsvc
import tools
from tools.safe_eval import safe_eval as eval
import pooler
from osv import fields, osv

def str2tuple(s):
    return eval('tuple(%s)' % (s or ''))

_intervalTypes = {
    'work_days': lambda interval: relativedelta(days=interval),
    'days': lambda interval: relativedelta(days=interval),
    'hours': lambda interval: relativedelta(hours=interval),
    'weeks': lambda interval: relativedelta(days=7*interval),
    'months': lambda interval: relativedelta(months=interval),
    'minutes': lambda interval: relativedelta(minutes=interval),
}

class ir_cron(osv.osv, netsvc.Agent):
    _name = "ir.cron"
    _columns = {
        'name': fields.char('Name', size=60, required=True),
        'user_id': fields.many2one('res.users', 'User', required=True),
        'active': fields.boolean('Active'),
        'interval_number': fields.integer('Interval Number',help="Repeat every x."),
        'interval_type': fields.selection( [('minutes', 'Minutes'),
            ('hours', 'Hours'), ('work_days','Work Days'), ('days', 'Days'),('weeks', 'Weeks'), ('months', 'Months')], 'Interval Unit'),
        'numbercall': fields.integer('Number of Calls', help='Number of time the function is called,\na negative number indicates no limit'),
        'doall' : fields.boolean('Repeat Missed', help="Enable this if you want to execute missed occurences as soon as the server restarts."),
        'nextcall' : fields.datetime('Next Execution Date', required=True, help="Next planned execution date for this scheduler"),
        'model': fields.char('Object', size=64, help="Name of object whose function will be called when this scheduler will run. e.g. 'res.partener'"),
        'function': fields.char('Function', size=64, help="Name of the method to be called on the object when this scheduler is executed."),
        'args': fields.text('Arguments', help="Arguments to be passed to the method. e.g. (uid,)"),
        'priority': fields.integer('Priority', help='0=Very Urgent\n10=Not urgent')
    }

    _defaults = {
        'nextcall' : lambda *a: time.strftime('%Y-%m-%d %H:%M:%S'),
        'priority' : lambda *a: 5,
        'user_id' : lambda obj,cr,uid,context: uid,
        'interval_number' : lambda *a: 1,
        'interval_type' : lambda *a: 'months',
        'numbercall' : lambda *a: 1,
        'active' : lambda *a: 1,
        'doall' : lambda *a: 1
    }

    def _check_args(self, cr, uid, ids, context=None):
        try:
            for this in self.browse(cr, uid, ids, context):
                str2tuple(this.args)
        except:
            return False
        return True

    _constraints = [
        (_check_args, 'Invalid arguments', ['args']),
    ]

    def _callback(self, cr, uid, model, func, args):
        args = str2tuple(args)
        m = self.pool.get(model)
        if m and hasattr(m, func):
            f = getattr(m, func)
            try:
                f(cr, uid, *args)
            except Exception, e:
                self._logger.notifyChannel('timers', netsvc.LOG_ERROR, "Job call of self.pool.get('%s').%s(cr, uid, *%r) failed" % (model, func, args))
                self._logger.notifyChannel('timers', netsvc.LOG_ERROR, tools.exception_to_unicode(e))

    def _poolJobs(self, db_name, check=False):
        try:
            db, pool = pooler.get_db_and_pool(db_name)
        except:
            return False
        cr = db.cursor()
        try:
            if not pool._init:
                now = datetime.now()
                cr.execute('select * from ir_cron where numbercall<>0 and active and nextcall<=now() order by priority')
                for job in cr.dictfetchall():
                    nextcall = datetime.strptime(job['nextcall'], '%Y-%m-%d %H:%M:%S')
                    numbercall = job['numbercall']

                    ok = False
                    while nextcall < now and numbercall:
                        if numbercall > 0:
                            numbercall -= 1
                        if not ok or job['doall']:
                            self._callback(cr, job['user_id'], job['model'], job['function'], job['args'])
                        if numbercall:
                            nextcall += _intervalTypes[job['interval_type']](job['interval_number'])
                        ok = True
                    addsql = ''
                    if not numbercall:
                        addsql = ', active=False'
                    cr.execute("update ir_cron set nextcall=%s, numbercall=%s"+addsql+" where id=%s", (nextcall.strftime('%Y-%m-%d %H:%M:%S'), numbercall, job['id']))
                    cr.commit()

            cr.execute('select min(nextcall) as min_next_call from ir_cron where numbercall<>0 and active and nextcall>=now()')
            next_call = cr.dictfetchone()['min_next_call']
            if next_call:
                next_call = time.mktime(time.strptime(next_call, '%Y-%m-%d %H:%M:%S'))
            else:
                next_call = int(time.time()) + 3600   # if do not find active cron job from database, it will run again after 1 day

            if not check:
                self.setAlarm(self._poolJobs, next_call, db_name, db_name)

        except Exception, ex:
            logger = netsvc.Logger()
            logger.notifyChannel('cron', netsvc.LOG_WARNING,
                'Exception in cron:'+str(ex))

        finally:
            cr.commit()
            cr.close()

    def restart(self, dbname):
        self.cancel(dbname)
        self._poolJobs(dbname)

    def create(self, cr, uid, vals, context=None):
        res = super(ir_cron, self).create(cr, uid, vals, context=context)
        cr.commit()
        self.restart(cr.dbname)
        return res

    def write(self, cr, user, ids, vals, context=None):
        res = super(ir_cron, self).write(cr, user, ids, vals, context=context)
        cr.commit()
        self.restart(cr.dbname)
        return res

    def unlink(self, cr, uid, ids, context=None):
        res = super(ir_cron, self).unlink(cr, uid, ids, context=context)
        cr.commit()
        self.restart(cr.dbname)
        return res
ir_cron()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    BUFSM
    ~~~~~~~~

    A simple web api for stream map cordinates with Flask and sqlite3.

"""
from sqlite3 import dbapi2 as sqlite3
import click, json
from flask import Flask, request, redirect, json, jsonify, _app_ctx_stack

# configuration
DATABASE = '/home/bufsm.db'
SECRET_KEY = 'TOKEN'

app = Flask(__name__)
app.config.from_object(__name__)
app.config.from_envvar('MINITWIT_SETTINGS', silent=True)

def get_db():
    """Opens a new database connection if there is none yet for the
    current application context.
    """
    top = _app_ctx_stack.top
    if not hasattr(top, 'sqlite_db'):
        top.sqlite_db = sqlite3.connect(app.config['DATABASE'])
        top.sqlite_db.row_factory = sqlite3.Row
    return top.sqlite_db

@app.teardown_appcontext
def close_database(exception):
    """Closes the database again at the end of the request."""
    top = _app_ctx_stack.top
    if hasattr(top, 'sqlite_db'):
        top.sqlite_db.close()

def init_db():
    """Initializes the database."""
    with app.app_context():
        db = get_db()
        with app.open_resource('schema.sql', mode='r') as f:
            db.cursor().executescript(f.read())
        db.commit()

#@app.cli.command('initdb')
def initdb_command():
    """Creates the database tables."""
    init_db()
    print('Initialized the database.')

def query_db(query, args=(), one=False):
    """Queries the database and returns a list of dictionaries."""
    cur = get_db().execute(query, args)
    rv = cur.fetchall()
    return (rv[0] if rv else None) if one else rv

def create_thing(thing, lat, lon):
    db = get_db()
    db.execute('''insert into things (thing_name, lat, lon) values (?,?,?)''',
        [thing, float(lat), float(lon)])
    db.commit()

def update_thing(lat, lon, thing_id):
    db = get_db()
    db.execute('''UPDATE things SET lat=?, lon=? WHERE thing_id=?''',
        [float(lat), float(lon), thing_id])
    db.commit()

@app.route('/api/things', methods=['POST'])
def cordinates():
    """Rest api for send and receive coordinates from bufsm's
    Add a and update things:
    curl -i -X POST -H "Content-Type: application/json" -d '{"thing":"thing_name",
        "key":"SECRET_KEY", "lat":"", "lon":""}' http://127.0.0.1:5000/api/things
    """
    thing = request.json.get('thing')
    key = request.json.get('key')
    if key == SECRET_KEY:
        thing_id = query_db('select thing_id from things where thing_name = ?',
                 [thing], one=True)
        if thing_id is None:
            create_thing(thing, request.json.get('lat'), request.json.get('lon'))
            return jsonify({ 'success': True })
        update_thing(request.json.get('lat'), request.json.get('lon'), thing_id[0])
    return jsonify({ 'success': True })

@app.route('/api/list')
def list_things():
    rcv = query_db('''SELECT lat, lon, thing_name FROM things''')
    try:
        dic = []
        for t in rcv:
            dic.append({'thing': t[2], 'lat': t[0], 'lon': t[1]})
        return json.dumps(dic)

        #return jsonify({ 'success': True, 'error':str(rcv[0][2])})
        #return jsonify(dic)
    except Exception as e:
        return jsonify({ 'success': False, 'error':str(e)})

if __name__ == '__main__':
    init_db()
    app.run(host="0.0.0.0")

<commit_msg>Geo json added<commit_after># -*- coding: utf-8 -*-
"""
    BUFSM
    ~~~~~~~~

    A simple web api for stream map cordinates with Flask and sqlite3.

"""
from sqlite3 import dbapi2 as sqlite3
import click, json
from flask import Flask, request, redirect, json, jsonify, _app_ctx_stack

# configuration
DATABASE = '/home/bufsm.db'
SECRET_KEY = 'TOKEN'

app = Flask(__name__)
app.config.from_object(__name__)
app.config.from_envvar('MINITWIT_SETTINGS', silent=True)

def get_db():
    """Opens a new database connection if there is none yet for the
    current application context.
    """
    top = _app_ctx_stack.top
    if not hasattr(top, 'sqlite_db'):
        top.sqlite_db = sqlite3.connect(app.config['DATABASE'])
        top.sqlite_db.row_factory = sqlite3.Row
    return top.sqlite_db

@app.teardown_appcontext
def close_database(exception):
    """Closes the database again at the end of the request."""
    top = _app_ctx_stack.top
    if hasattr(top, 'sqlite_db'):
        top.sqlite_db.close()

def init_db():
    """Initializes the database."""
    with app.app_context():
        db = get_db()
        with app.open_resource('schema.sql', mode='r') as f:
            db.cursor().executescript(f.read())
        db.commit()

#@app.cli.command('initdb')
def initdb_command():
    """Creates the database tables."""
    init_db()
    print('Initialized the database.')

def query_db(query, args=(), one=False):
    """Queries the database and returns a list of dictionaries."""
    cur = get_db().execute(query, args)
    rv = cur.fetchall()
    return (rv[0] if rv else None) if one else rv

def create_thing(thing, lat, lon):
    db = get_db()
    db.execute('''insert into things (thing_name, lat, lon) values (?,?,?)''',
        [thing, float(lat), float(lon)])
    db.commit()

def update_thing(lat, lon, thing_id):
    db = get_db()
    db.execute('''UPDATE things SET lat=?, lon=? WHERE thing_id=?''',
        [float(lat), float(lon), thing_id])
    db.commit()

@app.route('/api/things', methods=['POST'])
def cordinates():
    """Rest api for send and receive coordinates from bufsm's
    Add a and update things:
    curl -i -X POST -H "Content-Type: application/json" -d '{"thing":"thing_name",
        "key":"SECRET_KEY", "lat":"", "lon":""}' http://127.0.0.1:5000/api/things
    """
    thing = request.json.get('thing')
    key = request.json.get('key')
    if key == SECRET_KEY:
        thing_id = query_db('select thing_id from things where thing_name = ?',
                 [thing], one=True)
        if thing_id is None:
            create_thing(thing, request.json.get('lat'), request.json.get('lon'))
            return jsonify({ 'success': True })
        update_thing(request.json.get('lat'), request.json.get('lon'), thing_id[0])
    return jsonify({ 'success': True })

@app.route('/api/list')
def list_things():
    """
    Serving the coordinates in GEOjsom format.
    """
    rcv = query_db('''SELECT lat, lon, thing_name FROM things''')
    try:
        dic = []
        for t in rcv:
            dic.append({'geometry': {'type': 'Point', 'coordinates':\
             [t[0], t[1]]}, 'type': 'Feature', 'properties': {'thing': t[2]}})
        return json.dumps(dic)

        #return jsonify({ 'success': True, 'error':str(rcv[0][2])})
        #return jsonify(dic)
    except Exception as e:
        return jsonify({ 'success': False, 'error':str(e)})

if __name__ == '__main__':
    init_db()
    app.run(host="0.0.0.0", threaded=True)
<|endoftext|>