<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import fnmatch
import html
import json
import locale
import os
import os.path
import re
import shutil
import subprocess
import sys
import tempfile
from contextlib import closing
from distutils.version import LooseVersion
from urllib.request import urlopen

regex = re.compile(
    r"^(?P<filename>[^:]*):(?P<line>\d*):(?:(?P<column>\d*):)? (?P<message_id>[^:]*): (?P<message>.*?) \((?P<message_url>.*?)\)$",
    re.MULTILINE | re.DOTALL
)

BUILDIFIER_VERSION_PATTERN = re.compile(r"^buildifier version: ([\.\w]+)$", re.MULTILINE)

# https://github.com/bazelbuild/buildtools/blob/master/buildifier/buildifier.go#L333
# Buildifier error code for "needs formatting". We should fail on all other error codes > 0
# since they indicate a problem in how Buildifier is used.
BUILDIFIER_FORMAT_ERROR_CODE = 4

VERSION_ENV_VAR = "BUILDIFIER_VERSION"

WARNINGS_ENV_VAR = "BUILDIFIER_WARNINGS"

BUILDIFIER_RELEASES_URL = "https://api.github.com/repos/bazelbuild/buildtools/releases"

BUILDIFIER_DEFAULT_DISPLAY_URL = "https://github.com/bazelbuild/buildtools/tree/master/buildifier"

def eprint(*args, **kwargs):
    """
    Print to stderr and flush (just in case).
    """
    print(*args, flush=True, file=sys.stderr, **kwargs)

def upload_output(output):
    # Generate output usable by Buildkite's annotations.
    eprint("--- :hammer_and_wrench: Printing raw output for debugging")
    eprint(output)

    eprint("+++ :buildkite: Uploading output via 'buildkite annotate'")
    result = subprocess.run(
        ["buildkite-agent", "annotate", "--style", "warning", "--context", "buildifier"],
        input=output.encode(locale.getpreferredencoding(False)),
    )
    if result.returncode != 0:
        eprint(
            ":rotating_light: 'buildkite-agent annotate' failed with exit code {}".format(
                result.returncode
            )
        )

def print_error(failing_task, message):
    output = "##### :bazel: buildifier: error while {}:\n".format(failing_task)
    output += "
{}
".format(html.escape(message))
    if "BUILDKITE_JOB_ID" in os.environ:
        output += "\n\nSee [job {job}](#{job})\n".format(job=os.environ["BUILDKITE_JOB_ID"])

    upload_output(output)

def get_file_url(filename, line):
    commit = os.environ.get("BUILDKITE_COMMIT")
    repo = os.environ.get("BUILDKITE_PULL_REQUEST_REPO", os.environ.get("BUILDKITE_REPO", None))
    if not commit or not repo:
        return None

    # Example 1: https://github.com/bazelbuild/bazel.git
    # Example 2: git://github.com/philwo/bazel.git
    # Example 3: git@github.com:bazelbuild/bazel.git
    match = re.match(r"(?:(?:git|https?)://|git@)(github.com[:/].*)\.git", repo)
    if match:
        return "https://{}/blob/{}/{}#L{}".format(
            match[1].replace(":", "/"), commit, filename, line
        )

    return None

def run_buildifier(binary, flags, files=None, version=None, what=None):
    label = "+++ :bazel: Running "
    if version:
        label += "Buildifier " + version
    else:
        label += "unreleased Buildifier"
    if what:
        label += ": " + what

    eprint(label)

    args = [binary] + flags
    if files:
        args += files

    return subprocess.run(args, capture_output=True, universal_newlines=True)

def create_heading(issue_type, issue_count):
    return "##### :bazel: buildifier: found {} {} issue{} in your WORKSPACE, BUILD and *.bzl files\n".format(
        issue_count, issue_type, "s" if issue_count > 1 else ""
    )

def get_buildifier_info(version):
    all_releases = get_releases()
    if not all_releases:
        raise Exception("Could not get Buildifier releases from GitHub")

    resolved_version = version
    if version == "latest":
        resolved_version = str(max(LooseVersion(r) for r in all_releases))

    if resolved_version not in all_releases:
        raise Exception("Unknown Buildifier version '{}'".format(version))

    display_url, download_url = all_releases.get(resolved_version)
    return resolved_version, display_url, download_url

def get_releases():
    with closing(urlopen(BUILDIFIER_RELEASES_URL)) as res:
        body = res.read()
        content = body.decode(res.info().get_content_charset("iso-8859-1"))

    return {r["tag_name"]: get_release_urls(r) for r in json.loads(content) if not r["prerelease"]}

def get_release_urls(release):
    buildifier_assets = [
        a for a in release["assets"] if a["name"] in ("buildifier", "buildifier.linux")
    ]
    if not buildifier_assets:
        raise Exception("There is no Buildifier binary for release {}".format(release["tag_name"]))

    return release["html_url"], buildifier_assets[0]["browser_download_url"]

def download_buildifier(url):
    path = os.path.join(tempfile.mkdtemp(), "buildifier")
    with closing(urlopen(url)) as response:
        with open(path, "wb") as out_file:
            shutil.copyfileobj(response, out_file)
    os.chmod(path, 0o755)
    return path

def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    buildifier_binary = "buildifier"
    display_url = BUILDIFIER_DEFAULT_DISPLAY_URL
    version = os.environ.get(VERSION_ENV_VAR)
    if version:
        eprint("+++ :github: Downloading Buildifier version '{}'".format(version))
        try:
            version, display_url, download_url = get_buildifier_info(version)
            eprint("Downloading Buildifier {} from {}".format(version, download_url))
            buildifier_binary = download_buildifier(download_url)
        except Exception as ex:
            print_error("downloading Buildifier", str(ex))
            return 1

    # Gather all files to process.
    eprint("+++ :female-detective: Looking for WORKSPACE, BUILD, BUILD.bazel and *.bzl files")
    files = []
    build_bazel_found = False
    for root, _, filenames in os.walk("."):
        for filename in filenames:
            if fnmatch.fnmatch(filename, "BUILD.bazel"):
                build_bazel_found = True
            for pattern in ("WORKSPACE", "BUILD", "BUILD.bazel", "*.bzl"):
                if fnmatch.fnmatch(filename, pattern):
                    files.append(os.path.relpath(os.path.join(root, filename)))
    if build_bazel_found:
        eprint(
            "Found BUILD.bazel files in the workspace, thus ignoring BUILD files without suffix."
        )
        files = [fname for fname in files if not fnmatch.fnmatch(os.path.basename(fname), "BUILD")]
    if not files:
        eprint("No files found, exiting.")
        return 0

    files = sorted(files)

    # Determine Buildifier version if the user did not request a specific version.
    if not version:
        eprint("+++ :female-detective: Detecting Buildifier version")
        version_result = run_buildifier(buildifier_binary, ["--version"], what="Version info")
        match = BUILDIFIER_VERSION_PATTERN.search(version_result.stdout)
        version = match.group(1) if match and match.group(1) != "redacted" else None

    # Run formatter before linter since --lint=warn implies --mode=fix,
    # thus fixing any format issues.
    formatter_result = run_buildifier(
        buildifier_binary, ["--mode=check"], files=files, version=version, what="Format check"
    )
    if formatter_result.returncode and formatter_result.returncode != BUILDIFIER_FORMAT_ERROR_CODE:
        print_error("checking format", formatter_result.stderr)
        return formatter_result.returncode

    # Format: "<file name> # reformated"
    unformatted_files = [l.partition(" ")[0] for l in formatter_result.stdout.splitlines()]
    if unformatted_files:
        eprint(
            "+++ :construction: Found {} file(s) that must be formatted".format(
                len(unformatted_files)
            )
        )

    lint_flags = ["--lint=warn"]
    warnings = os.getenv(WARNINGS_ENV_VAR)
    if warnings:
        eprint("Running Buildifier with the following warnings: {}".format(warnings))
        lint_flags.append("--warnings={}".format(warnings))

    linter_result = run_buildifier(
        buildifier_binary, lint_flags, files=files, version=version, what="Lint checks"
    )
    if linter_result.returncode == 0 and not unformatted_files:
        # If buildifier was happy, there's nothing left to do for us.
        eprint("+++ :tada: Buildifier found nothing to complain about")
        return 0

    output = ""
    if unformatted_files:
        output = create_heading("format", len(unformatted_files))
        display_version = " {}".format(version) if version else ""
        output += (
            "Please download buildifier{} and run the following "
            "command in your workspace:

buildifier {}
"
            "\n".format(display_url, display_version, " ".join(unformatted_files))
        )

    # Parse output.
    if linter_result.returncode:
        eprint("+++ :gear: Parsing buildifier output")
        findings = list(regex.finditer(linter_result.stderr))
        output += create_heading("lint", len(findings))
        output += "
"
        for finding in findings:
            file_url = get_file_url(finding["filename"], finding["line"])
            if file_url:
                output += '{}:{}:'.format(
                    file_url, finding["filename"], finding["line"]
                )
            else:
                output += "{}:{}:".format(finding["filename"], finding["line"])
            if finding["column"]:
                output += "{}:".format(finding["column"])
            output += ' {}: {}\n'.format(
                finding["message_url"], finding["message_id"], finding["message"]
            )
        output = output.strip() + "
"

    upload_output(output)

    # Preserve buildifier's exit code.
    return max(linter_result.returncode, formatter_result.returncode)

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>formatting issues reported on stderr, not stdout. (#944)<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import fnmatch
import html
import json
import locale
import os
import os.path
import re
import shutil
import subprocess
import sys
import tempfile
from contextlib import closing
from distutils.version import LooseVersion
from urllib.request import urlopen

regex = re.compile(
    r"^(?P<filename>[^:]*):(?P<line>\d*):(?:(?P<column>\d*):)? (?P<message_id>[^:]*): (?P<message>.*?) \((?P<message_url>.*?)\)$",
    re.MULTILINE | re.DOTALL
)

BUILDIFIER_VERSION_PATTERN = re.compile(r"^buildifier version: ([\.\w]+)$", re.MULTILINE)

# https://github.com/bazelbuild/buildtools/blob/master/buildifier/buildifier.go#L333
# Buildifier error code for "needs formatting". We should fail on all other error codes > 0
# since they indicate a problem in how Buildifier is used.
BUILDIFIER_FORMAT_ERROR_CODE = 4

VERSION_ENV_VAR = "BUILDIFIER_VERSION"

WARNINGS_ENV_VAR = "BUILDIFIER_WARNINGS"

BUILDIFIER_RELEASES_URL = "https://api.github.com/repos/bazelbuild/buildtools/releases"

BUILDIFIER_DEFAULT_DISPLAY_URL = "https://github.com/bazelbuild/buildtools/tree/master/buildifier"

def eprint(*args, **kwargs):
    """
    Print to stderr and flush (just in case).
    """
    print(*args, flush=True, file=sys.stderr, **kwargs)

def upload_output(output):
    # Generate output usable by Buildkite's annotations.
    eprint("--- :hammer_and_wrench: Printing raw output for debugging")
    eprint(output)

    eprint("+++ :buildkite: Uploading output via 'buildkite annotate'")
    result = subprocess.run(
        ["buildkite-agent", "annotate", "--style", "warning", "--context", "buildifier"],
        input=output.encode(locale.getpreferredencoding(False)),
    )
    if result.returncode != 0:
        eprint(
            ":rotating_light: 'buildkite-agent annotate' failed with exit code {}".format(
                result.returncode
            )
        )

def print_error(failing_task, message):
    output = "##### :bazel: buildifier: error while {}:\n".format(failing_task)
    output += "
{}
".format(html.escape(message))
    if "BUILDKITE_JOB_ID" in os.environ:
        output += "\n\nSee [job {job}](#{job})\n".format(job=os.environ["BUILDKITE_JOB_ID"])

    upload_output(output)

def get_file_url(filename, line):
    commit = os.environ.get("BUILDKITE_COMMIT")
    repo = os.environ.get("BUILDKITE_PULL_REQUEST_REPO", os.environ.get("BUILDKITE_REPO", None))
    if not commit or not repo:
        return None

    # Example 1: https://github.com/bazelbuild/bazel.git
    # Example 2: git://github.com/philwo/bazel.git
    # Example 3: git@github.com:bazelbuild/bazel.git
    match = re.match(r"(?:(?:git|https?)://|git@)(github.com[:/].*)\.git", repo)
    if match:
        return "https://{}/blob/{}/{}#L{}".format(
            match[1].replace(":", "/"), commit, filename, line
        )

    return None

def run_buildifier(binary, flags, files=None, version=None, what=None):
    label = "+++ :bazel: Running "
    if version:
        label += "Buildifier " + version
    else:
        label += "unreleased Buildifier"
    if what:
        label += ": " + what

    eprint(label)

    args = [binary] + flags
    if files:
        args += files

    return subprocess.run(args, capture_output=True, universal_newlines=True)

def create_heading(issue_type, issue_count):
    return "##### :bazel: buildifier: found {} {} issue{} in your WORKSPACE, BUILD and *.bzl files\n".format(
        issue_count, issue_type, "s" if issue_count > 1 else ""
    )

def get_buildifier_info(version):
    all_releases = get_releases()
    if not all_releases:
        raise Exception("Could not get Buildifier releases from GitHub")

    resolved_version = version
    if version == "latest":
        resolved_version = str(max(LooseVersion(r) for r in all_releases))

    if resolved_version not in all_releases:
        raise Exception("Unknown Buildifier version '{}'".format(version))

    display_url, download_url = all_releases.get(resolved_version)
    return resolved_version, display_url, download_url

def get_releases():
    with closing(urlopen(BUILDIFIER_RELEASES_URL)) as res:
        body = res.read()
        content = body.decode(res.info().get_content_charset("iso-8859-1"))

    return {r["tag_name"]: get_release_urls(r) for r in json.loads(content) if not r["prerelease"]}

def get_release_urls(release):
    buildifier_assets = [
        a for a in release["assets"] if a["name"] in ("buildifier", "buildifier.linux")
    ]
    if not buildifier_assets:
        raise Exception("There is no Buildifier binary for release {}".format(release["tag_name"]))

    return release["html_url"], buildifier_assets[0]["browser_download_url"]

def download_buildifier(url):
    path = os.path.join(tempfile.mkdtemp(), "buildifier")
    with closing(urlopen(url)) as response:
        with open(path, "wb") as out_file:
            shutil.copyfileobj(response, out_file)
    os.chmod(path, 0o755)
    return path

def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    buildifier_binary = "buildifier"
    display_url = BUILDIFIER_DEFAULT_DISPLAY_URL
    version = os.environ.get(VERSION_ENV_VAR)
    if version:
        eprint("+++ :github: Downloading Buildifier version '{}'".format(version))
        try:
            version, display_url, download_url = get_buildifier_info(version)
            eprint("Downloading Buildifier {} from {}".format(version, download_url))
            buildifier_binary = download_buildifier(download_url)
        except Exception as ex:
            print_error("downloading Buildifier", str(ex))
            return 1

    # Gather all files to process.
    eprint("+++ :female-detective: Looking for WORKSPACE, BUILD, BUILD.bazel and *.bzl files")
    files = []
    build_bazel_found = False
    for root, _, filenames in os.walk("."):
        for filename in filenames:
            if fnmatch.fnmatch(filename, "BUILD.bazel"):
                build_bazel_found = True
            for pattern in ("WORKSPACE", "BUILD", "BUILD.bazel", "*.bzl"):
                if fnmatch.fnmatch(filename, pattern):
                    files.append(os.path.relpath(os.path.join(root, filename)))
    if build_bazel_found:
        eprint(
            "Found BUILD.bazel files in the workspace, thus ignoring BUILD files without suffix."
        )
        files = [fname for fname in files if not fnmatch.fnmatch(os.path.basename(fname), "BUILD")]
    if not files:
        eprint("No files found, exiting.")
        return 0

    files = sorted(files)

    # Determine Buildifier version if the user did not request a specific version.
    if not version:
        eprint("+++ :female-detective: Detecting Buildifier version")
        version_result = run_buildifier(buildifier_binary, ["--version"], what="Version info")
        match = BUILDIFIER_VERSION_PATTERN.search(version_result.stdout)
        version = match.group(1) if match and match.group(1) != "redacted" else None

    # Run formatter before linter since --lint=warn implies --mode=fix,
    # thus fixing any format issues.
    formatter_result = run_buildifier(
        buildifier_binary, ["--mode=check"], files=files, version=version, what="Format check"
    )
    if formatter_result.returncode and formatter_result.returncode != BUILDIFIER_FORMAT_ERROR_CODE:
        print_error("checking format", formatter_result.stderr)
        return formatter_result.returncode

    # Format: "<file name> # reformated"
    unformatted_files = [l.partition(" ")[0] for l in formatter_result.stderr.splitlines()]
    if unformatted_files:
        eprint(
            "+++ :construction: Found {} file(s) that must be formatted".format(
                len(unformatted_files)
            )
        )

    lint_flags = ["--lint=warn"]
    warnings = os.getenv(WARNINGS_ENV_VAR)
    if warnings:
        eprint("Running Buildifier with the following warnings: {}".format(warnings))
        lint_flags.append("--warnings={}".format(warnings))

    linter_result = run_buildifier(
        buildifier_binary, lint_flags, files=files, version=version, what="Lint checks"
    )
    if linter_result.returncode == 0 and not unformatted_files:
        # If buildifier was happy, there's nothing left to do for us.
        eprint("+++ :tada: Buildifier found nothing to complain about")
        return 0

    output = ""
    if unformatted_files:
        output = create_heading("format", len(unformatted_files))
        display_version = " {}".format(version) if version else ""
        output += (
            "Please download buildifier{} and run the following "
            "command in your workspace:

buildifier {}
"
            "\n".format(display_url, display_version, " ".join(unformatted_files))
        )

    # Parse output.
    if linter_result.returncode:
        eprint("+++ :gear: Parsing buildifier output")
        findings = list(regex.finditer(linter_result.stderr))
        output += create_heading("lint", len(findings))
        output += "
"
        for finding in findings:
            file_url = get_file_url(finding["filename"], finding["line"])
            if file_url:
                output += '{}:{}:'.format(
                    file_url, finding["filename"], finding["line"]
                )
            else:
                output += "{}:{}:".format(finding["filename"], finding["line"])
            if finding["column"]:
                output += "{}:".format(finding["column"])
            output += ' {}: {}\n'.format(
                finding["message_url"], finding["message_id"], finding["message"]
            )
        output = output.strip() + "
"

    upload_output(output)

    # Preserve buildifier's exit code.
    return max(linter_result.returncode, formatter_result.returncode)

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before>import numpy as np
import tensorflow as tf

from easydict import EasyDict
from luminoth.models.fasterrcnn.network import FasterRCNN
from luminoth.utils.bbox_transform import clip_boxes

class FasterRCNNetworkTest(tf.test.TestCase):

    def setUp(self):
        super(FasterRCNNetworkTest, self).setUp()
        # Setup
        self.config = EasyDict({
            'network': {
                'num_classes': 20,
                'with_rcnn': True
            },
            'train': {
                'debug': True
            },
            'anchors': {
                'base_size': 256,
                'scales': [0.5, 1, 2],
                'ratios': [0.5, 1, 2],
                'stride': 16
            },
            'loss': {
                'rpn_cls_loss_weight': 1.0,
                'rpn_reg_loss_weights': 2.0,
                'rcnn_cls_loss_weight': 1.0,
                'rcnn_reg_loss_weights': 2.0,
            },
            'pretrained': {
                'net': 'vgg_16',
                'trainable': True,
                'endpoint': 'vgg_16/conv5/conv5_1',
                'finetune_num_layers': 3,
                'weight_decay': 0.0005,
            },
            'rcnn': {
                'enabled': True,
                'layer_sizes': [4096, 4096],
                'dropout_keep_prop': 1.0,
                'activation_function': 'relu6',
                'l2_regulalization_scale': 0.0005,
                'initializer': {
                    'type': 'variance_scaling_initializer',
                    'factor': 1.0,
                    'uniform': 'True',
                    'mode': 'FAN_AVG',
                },
                'roi': {
                    'pooling_mode': 'crop',
                    'pooled_width': 7,
                    'pooled_height': 7,
                    'padding': 'VALID'
                },
                'proposals': {
                  'class_max_detections': 100,
                  'class_nms_threshold': 0.6,
                  'total_max_detections': 300,
                },
                'target': {
                  'foreground_fraction': 0.25,
                  'minibatch_size': 64,
                  'foreground_threshold': 0.5,
                  'background_threshold_high': 0.5,
                  'background_threshold_low': 0.1,
                }
             },
            'rpn': {
               'num_channels': 512,
               'kernel_shape': [3, 3],
               'initializer': {
                 'type': 'truncated_normal_initializer',
                 'mean': 0.0,
                 'stddev': 0.01,
               },
               'activation_function': 'relu6',
               'l2_regulalization_scale': 0.0005,
               'proposals': {
                 'pre_nms_top_n': 12000,
                 'post_nms_top_n': 2000,
                 'nms_threshold': 0.6,
                 'min_size': 0
               },
               'target': {
                 'allowed_border': 0,
                 'clobber_positives': False,
                 'foreground_threshold': 0.7,
                 'background_threshold_high': 0.3,
                 'background_threshold_low': 0.,
                 'foreground_fraction': 0.5,
                 'minibatch_size': 256,
               }
             }
        })
        self.image_size = (600, 800)
        self.image = np.random.randint(low=0, high=255, size=(1, 600, 800, 3))
        self.gt_boxes = np.array([
            [10, 10, 26, 28, 1],
            [10, 10, 20, 22, 1],
            [10, 11, 20, 21, 1],
            [19, 30, 31, 33, 1],
        ])

    def _run_network(self):
        image = tf.placeholder(
            tf.float32, shape=self.image.shape)
        gt_boxes = tf.placeholder(
            tf.float32, shape=self.gt_boxes.shape)
        model = FasterRCNN(self.config, debug=True)

        results = model(image, gt_boxes)

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            results = sess.run(results, feed_dict={
                gt_boxes: self.gt_boxes,
                image: self.image,
            })
            return results

    def _gen_anchors(self, config, feature_map):
        feature_map_tf = tf.placeholder(
            tf.float32, shape=feature_map.shape)
        model = FasterRCNN(config, debug=True)

        results = model._generate_anchors(feature_map)

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            results = sess.run(results, feed_dict={
                feature_map_tf: feature_map
            })
            return results

    def testBasic(self):
        """
        Test basic output of the FasterCnnNetwork
        """
        results = self._run_network()
        class_prediction = results['classification_prediction']
        rpn_prediction = results['rpn_prediction']

        # Check that every object is defined by 4 coordinates
        self.assertEqual(
            class_prediction['objects'].shape[1],
            4
        )

        # Check we get objects clipped to the image.
        self.assertAllEqual(
            clip_boxes(class_prediction['objects'], self.image_size),
            class_prediction['objects']
        )

        # Check that there are class probabilities and labels for each object
        self.assertEqual(
            class_prediction['cls_prob'].shape[0],
            class_prediction['objects'].shape[0]
        )

        self.assertEqual(
            class_prediction['objects_labels'].shape[0],
            class_prediction['objects'].shape[0]
        )

        # Check that every object label is less or equal than 'num_classes'
        self.assertTrue(
            np.less_equal(class_prediction['objects_labels'],
                          self.config.network.num_classes).all()
        )

        # Check that every roi_proposal has 4 coordinates + 1 batch index
        self.assertEqual(
            class_prediction['roi_proposals'].shape[1],
            5
        )

        # Check we get roi_proposals clipped to the image.
        self.assertAllEqual(
            clip_boxes(class_prediction['roi_proposals'], self.image_size),
            class_prediction['roi_proposals']
        )

        # Check that the sum of class probabilities is 1
        self.assertAllClose(
            np.sum(class_prediction['cls_prob'], axis=1),
            np.ones((class_prediction['cls_prob'].shape[0]))
        )

        # Check that the sum of rpn class probabilities is 1
        self.assertAllClose(
            np.sum(rpn_prediction['rpn_cls_prob'], axis=1),
            np.ones((rpn_prediction['rpn_cls_prob'].shape[0]))
        )

        # Check that every rpn proposal has 4 coordinates + 1 batch index
        self.assertEqual(
            rpn_prediction['proposals'].shape[1],
            5
        )

        # Check we get rpn proposals clipped to the image.
        self.assertAllEqual(
            clip_boxes(rpn_prediction['proposals'], self.image_size),
            rpn_prediction['proposals']
        )

    def testAnchors(self):
        """
        Tests about the anchors generated by the FasterRCNN
        """
        results = self._run_network()

        # Check we get anchors clipped to the image.
        self.assertAllEqual(
            clip_boxes(results['all_anchors'], self.image_size),
            results['all_anchors']
        )

        feature_map = np.random.randint(low=0, high=255, size=(1, 32, 32, 1))
        config = self.config
        config.anchors.base_size = 16
        config.anchors.scales = [0.5, 1, 2]
        config.anchors.ratios = [0.5, 1, 2]
        config.anchors.stride = 1  # image is 32 x 32

        anchors = self._gen_anchors(config, feature_map)

        # Check the amount of anchors generated is correct:
        # 9216 = 32^2 * config.anchor.scales * config.anchor.ratios = 1024 * 9
        self.assertEqual(anchors.shape, (9216, 4))

        anchor_widths = anchors[:, 2] - anchors[:, 0]
        anchor_heights = anchors[:, 3] - anchors[:, 1]

        # Since we are using equal scales and ratios, the set of unique heights
        # and widths must be the same.
        self.assertAllEqual(
            np.unique(anchor_widths), np.unique(anchor_heights)
        )

        anchor_areas = anchor_widths * anchor_heights

        # We have 9 possible anchors areas, minus 3 repeated ones. 6 unique.
        self.assertAllEqual(np.unique(anchor_areas).shape[0], 6)

        # Check the anchors cover all the image.
        # TODO: Check with values calculated from config.
        self.assertEqual(np.min(anchors[:, 0]), -14)
        self.assertEqual(np.max(anchors[:, 0]), 36)

        self.assertEqual(np.min(anchors[:, 1]), -14)
        self.assertEqual(np.max(anchors[:, 1]), 36)

        self.assertEqual(np.min(anchors[:, 2]), 9)
        self.assertEqual(np.max(anchors[:, 2]), 60)

        self.assertEqual(np.min(anchors[:, 3]), 9)
        self.assertEqual(np.max(anchors[:, 3]), 60)

        # Check values are sequential.
        self._assert_sequential_values(anchors[:, 0], config.anchors.stride)
        self._assert_sequential_values(anchors[:, 1], config.anchors.stride)
        self._assert_sequential_values(anchors[:, 2], config.anchors.stride)
        self._assert_sequential_values(anchors[:, 3], config.anchors.stride)

    def _assert_sequential_values(self, values, delta=1):
        unique_values = np.unique(values)
        paired_values = np.column_stack(
            (unique_values[:-1], unique_values[1:])
        )
        self.assertAllEqual(
            paired_values[:, 1] - paired_values[:, 0],
            np.ones((paired_values.shape[0], ), np.int)
        )

if __name__ == "__main__":
    tf.test.main()
<commit_msg>Fixes config change in FasterRCNN test<commit_after>import numpy as np
import tensorflow as tf

from easydict import EasyDict
from luminoth.models.fasterrcnn.network import FasterRCNN
from luminoth.utils.bbox_transform import clip_boxes

class FasterRCNNetworkTest(tf.test.TestCase):

    def setUp(self):
        super(FasterRCNNetworkTest, self).setUp()
        # Setup
        self.config = EasyDict({
            'network': {
                'num_classes': 20,
                'with_rcnn': True
            },
            'train': {
                'debug': True
            },
            'anchors': {
                'base_size': 256,
                'scales': [0.5, 1, 2],
                'ratios': [0.5, 1, 2],
                'stride': 16
            },
            'loss': {
                'rpn_cls_loss_weight': 1.0,
                'rpn_reg_loss_weights': 2.0,
                'rcnn_cls_loss_weight': 1.0,
                'rcnn_reg_loss_weights': 2.0,
            },
            'pretrained': {
                'net': 'vgg_16',
                'trainable': True,
                'endpoint': 'vgg_16/conv5/conv5_1',
                'finetune_num_layers': 3,
                'weight_decay': 0.0005,
            },
            'rcnn': {
                'enabled': True,
                'layer_sizes': [4096, 4096],
                'dropout_keep_prop': 1.0,
                'activation_function': 'relu6',
                'l2_regularization_scale': 0.0005,
                'initializer': {
                    'type': 'variance_scaling_initializer',
                    'factor': 1.0,
                    'uniform': 'True',
                    'mode': 'FAN_AVG',
                },
                'roi': {
                    'pooling_mode': 'crop',
                    'pooled_width': 7,
                    'pooled_height': 7,
                    'padding': 'VALID'
                },
                'proposals': {
                  'class_max_detections': 100,
                  'class_nms_threshold': 0.6,
                  'total_max_detections': 300,
                },
                'target': {
                  'foreground_fraction': 0.25,
                  'minibatch_size': 64,
                  'foreground_threshold': 0.5,
                  'background_threshold_high': 0.5,
                  'background_threshold_low': 0.1,
                }
             },
            'rpn': {
               'num_channels': 512,
               'kernel_shape': [3, 3],
               'initializer': {
                 'type': 'truncated_normal_initializer',
                 'mean': 0.0,
                 'stddev': 0.01,
               },
               'activation_function': 'relu6',
               'l2_regularization_scale': 0.0005,
               'proposals': {
                 'pre_nms_top_n': 12000,
                 'post_nms_top_n': 2000,
                 'nms_threshold': 0.6,
                 'min_size': 0
               },
               'target': {
                 'allowed_border': 0,
                 'clobber_positives': False,
                 'foreground_threshold': 0.7,
                 'background_threshold_high': 0.3,
                 'background_threshold_low': 0.,
                 'foreground_fraction': 0.5,
                 'minibatch_size': 256,
               }
             }
        })
        self.image_size = (600, 800)
        self.image = np.random.randint(low=0, high=255, size=(1, 600, 800, 3))
        self.gt_boxes = np.array([
            [10, 10, 26, 28, 1],
            [10, 10, 20, 22, 1],
            [10, 11, 20, 21, 1],
            [19, 30, 31, 33, 1],
        ])

    def _run_network(self):
        image = tf.placeholder(
            tf.float32, shape=self.image.shape)
        gt_boxes = tf.placeholder(
            tf.float32, shape=self.gt_boxes.shape)
        model = FasterRCNN(self.config, debug=True)

        results = model(image, gt_boxes)

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            results = sess.run(results, feed_dict={
                gt_boxes: self.gt_boxes,
                image: self.image,
            })
            return results

    def _gen_anchors(self, config, feature_map):
        feature_map_tf = tf.placeholder(
            tf.float32, shape=feature_map.shape)
        model = FasterRCNN(config, debug=True)

        results = model._generate_anchors(feature_map)

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            results = sess.run(results, feed_dict={
                feature_map_tf: feature_map
            })
            return results

    def testBasic(self):
        """
        Test basic output of the FasterCnnNetwork
        """
        results = self._run_network()
        class_prediction = results['classification_prediction']
        rpn_prediction = results['rpn_prediction']

        # Check that every object is defined by 4 coordinates
        self.assertEqual(
            class_prediction['objects'].shape[1],
            4
        )

        # Check we get objects clipped to the image.
        self.assertAllEqual(
            clip_boxes(class_prediction['objects'], self.image_size),
            class_prediction['objects']
        )

        # Check that there are class probabilities and labels for each object
        self.assertEqual(
            class_prediction['cls_prob'].shape[0],
            class_prediction['objects'].shape[0]
        )

        self.assertEqual(
            class_prediction['objects_labels'].shape[0],
            class_prediction['objects'].shape[0]
        )

        # Check that every object label is less or equal than 'num_classes'
        self.assertTrue(
            np.less_equal(class_prediction['objects_labels'],
                          self.config.network.num_classes).all()
        )

        # Check that every roi_proposal has 4 coordinates + 1 batch index
        self.assertEqual(
            class_prediction['roi_proposals'].shape[1],
            5
        )

        # Check we get roi_proposals clipped to the image.
        self.assertAllEqual(
            clip_boxes(class_prediction['roi_proposals'], self.image_size),
            class_prediction['roi_proposals']
        )

        # Check that the sum of class probabilities is 1
        self.assertAllClose(
            np.sum(class_prediction['cls_prob'], axis=1),
            np.ones((class_prediction['cls_prob'].shape[0]))
        )

        # Check that the sum of rpn class probabilities is 1
        self.assertAllClose(
            np.sum(rpn_prediction['rpn_cls_prob'], axis=1),
            np.ones((rpn_prediction['rpn_cls_prob'].shape[0]))
        )

        # Check that every rpn proposal has 4 coordinates + 1 batch index
        self.assertEqual(
            rpn_prediction['proposals'].shape[1],
            5
        )

        # Check we get rpn proposals clipped to the image.
        self.assertAllEqual(
            clip_boxes(rpn_prediction['proposals'], self.image_size),
            rpn_prediction['proposals']
        )

    def testAnchors(self):
        """
        Tests about the anchors generated by the FasterRCNN
        """
        results = self._run_network()

        # Check we get anchors clipped to the image.
        self.assertAllEqual(
            clip_boxes(results['all_anchors'], self.image_size),
            results['all_anchors']
        )

        feature_map = np.random.randint(low=0, high=255, size=(1, 32, 32, 1))
        config = self.config
        config.anchors.base_size = 16
        config.anchors.scales = [0.5, 1, 2]
        config.anchors.ratios = [0.5, 1, 2]
        config.anchors.stride = 1  # image is 32 x 32

        anchors = self._gen_anchors(config, feature_map)

        # Check the amount of anchors generated is correct:
        # 9216 = 32^2 * config.anchor.scales * config.anchor.ratios = 1024 * 9
        self.assertEqual(anchors.shape, (9216, 4))

        anchor_widths = anchors[:, 2] - anchors[:, 0]
        anchor_heights = anchors[:, 3] - anchors[:, 1]

        # Since we are using equal scales and ratios, the set of unique heights
        # and widths must be the same.
        self.assertAllEqual(
            np.unique(anchor_widths), np.unique(anchor_heights)
        )

        anchor_areas = anchor_widths * anchor_heights

        # We have 9 possible anchors areas, minus 3 repeated ones. 6 unique.
        self.assertAllEqual(np.unique(anchor_areas).shape[0], 6)

        # Check the anchors cover all the image.
        # TODO: Check with values calculated from config.
        self.assertEqual(np.min(anchors[:, 0]), -14)
        self.assertEqual(np.max(anchors[:, 0]), 36)

        self.assertEqual(np.min(anchors[:, 1]), -14)
        self.assertEqual(np.max(anchors[:, 1]), 36)

        self.assertEqual(np.min(anchors[:, 2]), 9)
        self.assertEqual(np.max(anchors[:, 2]), 60)

        self.assertEqual(np.min(anchors[:, 3]), 9)
        self.assertEqual(np.max(anchors[:, 3]), 60)

        # Check values are sequential.
        self._assert_sequential_values(anchors[:, 0], config.anchors.stride)
        self._assert_sequential_values(anchors[:, 1], config.anchors.stride)
        self._assert_sequential_values(anchors[:, 2], config.anchors.stride)
        self._assert_sequential_values(anchors[:, 3], config.anchors.stride)

    def _assert_sequential_values(self, values, delta=1):
        unique_values = np.unique(values)
        paired_values = np.column_stack(
            (unique_values[:-1], unique_values[1:])
        )
        self.assertAllEqual(
            paired_values[:, 1] - paired_values[:, 0],
            np.ones((paired_values.shape[0], ), np.int)
        )

if __name__ == "__main__":
    tf.test.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import requests
import psycopg2

import json
from multiprocessing.dummy import Pool

def write_to_db(postgres_cursor, record, lat, lon):
	"""
		postgres_cursor: psycopg2._psycopg.cursor
		record: string (json)
		lat: float
		lon: float
	"""
	jsonstring = json.dumps(record, ensure_ascii=False)
	postgres_cursor.execute("""INSERT INTO elemente (properties, location) VALUES (%s, ST_GeomFromText('POINT(%s %s)', 4326));""", (jsonstring, lat, lon))

def handle_json_response(response_data, postgres_conn_string):
	""" verarbeitet die Antwort vom Themenstadtplan

		response_data: dict
		postgres_conn_string: string

	"""

	lat = response_data["value"]["PointInWgs"]["Y"]
	lon = response_data["value"]["PointInWgs"]["X"]

	conn = psycopg2.connect(postgres_conn_string)
	cur = conn.cursor()

	for record in response_data["value"]["Records"]:
		write_to_db(cur, record, lat, lon)

	conn.commit()
	cur.close()

def scrape(parallelity=4, postgres_conn_string="dbname=themenstadtplan"):
	XMIN = 5400499
	XMAX = 5427620
	YMIN = 5649580
	YMAX = 5672231

	BASEURL = "http://stadtplan2.dresden.de/(S(jcksai1ssxvzjbveqyv1aizy))/ajaxpro/ASP.spdd_controls_mapcontainer_ascx,App_Web_o3sebx4k.ashx"
	HEADERS = {"X-AjaxPro-Method": "GetMapTipEx", "Origin": "http://stadtplan2.dresden.de", "Accept-Encoding": "gzip,deflate,sdch", "Accept-Language": "de-DE", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36", "Content-Type": "text/plain; charset=UTF-8", "Accept": "*/*", "Referer": "http://stadtplan2.dresden.de/(S(jcksai1ssxvzjbveqyv1aizy))/spdd.aspx", "Cookie": "cardo3SessionGuid=C3_4ef68a3a-59ca-4a0c-b3cc-722fdabb1c84", "Connection": "keep-alive"}

	with Pool(parallelity) as pool:
		jsonreq = '{"applicationContextType":null,"themeList":"$BaseMap#5#Tiles","gx":%.6f,"gy":%.6f,"srs":31469,"currentMapScale":128000}' % (XMIN, YMIN)

		res = requests.post(BASEURL, data=jsonreq, headers=HEADERS)
		handle_json_response(res.json, postgres_getconn())

if __name__ == "__main__":
	scrape()<commit_msg>Queryerzeugung<commit_after>#!/usr/bin/env python3
import requests
import psycopg2

import json
from multiprocessing.dummy import Pool

def write_to_db(postgres_cursor, record, lat, lon):
	"""
		postgres_cursor: psycopg2._psycopg.cursor
		record: string (json)
		lat: float
		lon: float
	"""
	jsonstring = json.dumps(record, ensure_ascii=False)
	postgres_cursor.execute("""INSERT INTO elemente (properties, location) VALUES (%s, ST_GeomFromText('POINT(%s %s)', 4326));""", (jsonstring, lat, lon))

def handle_json_response(response_data, postgres_conn_string):
	""" verarbeitet die Antwort vom Themenstadtplan

		response_data: dict
		postgres_conn_string: string

	"""

	lat = response_data["value"]["PointInWgs"]["Y"]
	lon = response_data["value"]["PointInWgs"]["X"]

	conn = psycopg2.connect(postgres_conn_string)
	cur = conn.cursor()

	for record in response_data["value"]["Records"]:
		write_to_db(cur, record, lat, lon)

	conn.commit()
	cur.close()

def build_queries(xmin, xmax, ymin, ymax, basereq, granularity=10.):
	g = 1/granularity
	xstart = int(xmin*g)
	xend = int((xmax+1)*g)
	ystart = int(ymin*g)
	yend = int((ymax+1)*g)

	for x_g in range(xstart, xend):
		for y_g in range(ystart, yend):
			x = x_g*granularity
			y = y_g*granularity
			yield(basereq % (x, y))

def scrape(parallelity=4, postgres_conn_string="dbname=themenstadtplan"):
	XMIN = 5400499.
	XMAX = 5427620.
	YMIN = 5649580.
	YMAX = 5672231.
	BASEURL = "http://stadtplan2.dresden.de/(S(jcksai1ssxvzjbveqyv1aizy))/ajaxpro/ASP.spdd_controls_mapcontainer_ascx,App_Web_o3sebx4k.ashx"
	HEADERS = {"X-AjaxPro-Method": "GetMapTipEx", "Origin": "http://stadtplan2.dresden.de", "Accept-Encoding": "gzip,deflate,sdch", "Accept-Language": "de-DE", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36", "Content-Type": "text/plain; charset=UTF-8", "Accept": "*/*", "Referer": "http://stadtplan2.dresden.de/(S(jcksai1ssxvzjbveqyv1aizy))/spdd.aspx", "Cookie": "cardo3SessionGuid=C3_4ef68a3a-59ca-4a0c-b3cc-722fdabb1c84", "Connection": "keep-alive"}
	BASEREQ = '{"applicationContextType":null,"themeList":"$BaseMap#5#Tiles","gx":%.6f,"gy":%.6f,"srs":31469,"currentMapScale":1500}'

	for query in build_queries(XMIN, XMAX, YMIN, YMAX, basereq=BASEREQ, granularity=10.):
		pass

	#with Pool(parallelity) as pool:

	#res = requests.post(BASEURL, data=jsonreq, headers=HEADERS)
	#handle_json_response(res.json(), postgres_conn_string)

if __name__ == "__main__":
	scrape()<|endoftext|>


<commit_before># Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import asyncio
import unittest
from unittest.mock import patch

from opentelemetry import context
from opentelemetry.sdk import trace
from opentelemetry.sdk.trace import export
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
    InMemorySpanExporter,
)

try:
    import contextvars  # pylint: disable=unused-import

    from opentelemetry.context.contextvars_context import (
        ContextVarsRuntimeContext,
    )
except ImportError:
    raise unittest.SkipTest("contextvars not available")

_SPAN_NAMES = [
    "test_span1",
    "test_span2",
    "test_span3",
    "test_span4",
    "test_span5",
]

def stop_loop_when(loop, cond_func, timeout=5.0):
    """Registers a periodic callback that stops the loop when cond_func() == True.
    Compatible with both Tornado and asyncio.
    """
    if cond_func() or timeout <= 0.0:
        loop.stop()
        return

    timeout -= 0.1
    loop.call_later(0.1, stop_loop_when, loop, cond_func, timeout)

class TestAsyncio(unittest.TestCase):
    @asyncio.coroutine
    def task(self, name):
        with self.tracer.start_as_current_span(name):
            context.set_value("say", "bar")

    def submit_another_task(self, name):
        self.loop.create_task(self.task(name))

    def setUp(self):
        self.token = context.attach(context.Context())
        self.tracer_provider = trace.TracerProvider()
        self.tracer = self.tracer_provider.get_tracer(__name__)
        self.memory_exporter = InMemorySpanExporter()
        span_processor = export.SimpleSpanProcessor(self.memory_exporter)
        self.tracer_provider.add_span_processor(span_processor)
        self.loop = asyncio.get_event_loop()

    def tearDown(self):
        context.detach(self.token)

    @patch(
        "opentelemetry.context._RUNTIME_CONTEXT", ContextVarsRuntimeContext()
    )
    def test_with_asyncio(self):
        with self.tracer.start_as_current_span("asyncio_test"):
            for name in _SPAN_NAMES:
                self.submit_another_task(name)

            stop_loop_when(
                self.loop,
                lambda: len(self.memory_exporter.get_finished_spans()) >= 5,
                timeout=5.0,
            )
            self.loop.run_forever()
        span_list = self.memory_exporter.get_finished_spans()
        span_names_list = [span.name for span in span_list]
        expected = [
            "test_span1",
            "test_span2",
            "test_span3",
            "test_span4",
            "test_span5",
            "asyncio_test",
        ]
        self.assertCountEqual(span_names_list, expected)
        span_names_list.sort()
        expected.sort()
        self.assertListEqual(span_names_list, expected)
        expected_parent = next(
            span for span in span_list if span.name == "asyncio_test"
        )
        for span in span_list:
            if span is expected_parent:
                continue
            self.assertEqual(span.parent, expected_parent.context)
<commit_msg>Remove deprecated @asyncio.coroutine (#2002)<commit_after># Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import asyncio
import unittest
from unittest.mock import patch

from opentelemetry import context
from opentelemetry.sdk import trace
from opentelemetry.sdk.trace import export
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
    InMemorySpanExporter,
)

try:
    import contextvars  # pylint: disable=unused-import

    from opentelemetry.context.contextvars_context import (
        ContextVarsRuntimeContext,
    )
except ImportError:
    raise unittest.SkipTest("contextvars not available")

_SPAN_NAMES = [
    "test_span1",
    "test_span2",
    "test_span3",
    "test_span4",
    "test_span5",
]

def stop_loop_when(loop, cond_func, timeout=5.0):
    """Registers a periodic callback that stops the loop when cond_func() == True.
    Compatible with both Tornado and asyncio.
    """
    if cond_func() or timeout <= 0.0:
        loop.stop()
        return

    timeout -= 0.1
    loop.call_later(0.1, stop_loop_when, loop, cond_func, timeout)

class TestAsyncio(unittest.TestCase):
    async def task(self, name):
        with self.tracer.start_as_current_span(name):
            context.set_value("say", "bar")

    def submit_another_task(self, name):
        self.loop.create_task(self.task(name))

    def setUp(self):
        self.token = context.attach(context.Context())
        self.tracer_provider = trace.TracerProvider()
        self.tracer = self.tracer_provider.get_tracer(__name__)
        self.memory_exporter = InMemorySpanExporter()
        span_processor = export.SimpleSpanProcessor(self.memory_exporter)
        self.tracer_provider.add_span_processor(span_processor)
        self.loop = asyncio.get_event_loop()

    def tearDown(self):
        context.detach(self.token)

    @patch(
        "opentelemetry.context._RUNTIME_CONTEXT", ContextVarsRuntimeContext()
    )
    def test_with_asyncio(self):
        with self.tracer.start_as_current_span("asyncio_test"):
            for name in _SPAN_NAMES:
                self.submit_another_task(name)

            stop_loop_when(
                self.loop,
                lambda: len(self.memory_exporter.get_finished_spans()) >= 5,
                timeout=5.0,
            )
            self.loop.run_forever()
        span_list = self.memory_exporter.get_finished_spans()
        span_names_list = [span.name for span in span_list]
        expected = [
            "test_span1",
            "test_span2",
            "test_span3",
            "test_span4",
            "test_span5",
            "asyncio_test",
        ]
        self.assertCountEqual(span_names_list, expected)
        span_names_list.sort()
        expected.sort()
        self.assertListEqual(span_names_list, expected)
        expected_parent = next(
            span for span in span_list if span.name == "asyncio_test"
        )
        for span in span_list:
            if span is expected_parent:
                continue
            self.assertEqual(span.parent, expected_parent.context)
<|endoftext|>


<commit_before>#
# Copyright (c) 2014 QuarksLab.
# This file is part of IRMA project.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License in the top-level directory
# of this distribution and at:
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# No part of the project, including this file, may be copied,
# modified, propagated, or distributed except according to the
# terms contained in the LICENSE file.

class ConnectionManagerError(Exception):
    """
    Error on establishing a connection to a
    remote virtual machine manager
    """
    pass

class DomainManagerError(Exception):
    """Error on managing a domain"""
    pass

class DomainError(Exception):
    """Error on creating a domain"""
    pass

class StoragePoolManagerError(Exception):
    """Error on managing a storage pool"""
    pass

class StoragePoolError(Exception):
    """Error on creating a storage pool"""
    pass

class StorageVolumeManagerError(Exception):
    """Error on managing a storage volume"""
    pass

class StorageVolumeError(Exception):
    """Error on creating a storage volume"""
    pass
<commit_msg>pep8 fix<commit_after>#
# Copyright (c) 2014 QuarksLab.
# This file is part of IRMA project.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License in the top-level directory
# of this distribution and at:
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# No part of the project, including this file, may be copied,
# modified, propagated, or distributed except according to the
# terms contained in the LICENSE file.

class ConnectionManagerError(Exception):
    """
    Error on establishing a connection to a
    remote virtual machine manager
    """
    pass

class DomainManagerError(Exception):
    """Error on managing a domain"""
    pass

class DomainError(Exception):
    """Error on creating a domain"""
    pass

class StoragePoolManagerError(Exception):
    """Error on managing a storage pool"""
    pass

class StoragePoolError(Exception):
    """Error on creating a storage pool"""
    pass

class StorageVolumeManagerError(Exception):
    """Error on managing a storage volume"""
    pass

class StorageVolumeError(Exception):
    """Error on creating a storage volume"""
    pass
<|endoftext|>


<commit_before>import json
import logging

from django.utils.translation import ugettext as _
from couchdbkit.exceptions import ResourceConflict
from django.http import HttpResponse, Http404
from django.shortcuts import render
from django.views.decorators.http import require_GET
from django.conf import settings
from django.contrib import messages
from corehq.apps.app_manager.views.apps import get_apps_base_context

from corehq.apps.app_manager.views.utils import back_to_main, bail
from corehq import toggles, privileges, feature_previews
from corehq.apps.accounting.utils import domain_has_privilege
from corehq.apps.app_manager.const import (
    SCHEDULE_CURRENT_VISIT_NUMBER,
    SCHEDULE_NEXT_DUE,
    SCHEDULE_UNSCHEDULED_VISIT,
    SCHEDULE_GLOBAL_NEXT_VISIT_DATE,
)
from corehq.apps.app_manager.util import (
    get_casedb_schema,
    get_session_schema,
    app_callout_templates,
)
from corehq.apps.fixtures.fixturegenerators import item_lists_by_domain
from corehq.apps.app_manager.dbaccessors import get_app
from corehq.apps.app_manager.models import (
    Form,
    ModuleNotFoundException,
)
from corehq.apps.app_manager.decorators import require_can_edit_apps
from corehq.apps.analytics.tasks import track_entered_form_builder_on_hubspot
from corehq.apps.analytics.utils import get_meta

logger = logging.getLogger(__name__)

@require_can_edit_apps
def form_designer(request, domain, app_id, module_id=None, form_id=None):
    meta = get_meta(request)
    track_entered_form_builder_on_hubspot.delay(request.couch_user, request.COOKIES, meta)

    app = get_app(domain, app_id)
    module = None

    try:
        module = app.get_module(module_id)
    except ModuleNotFoundException:
        return bail(request, domain, app_id, not_found="module")
    try:
        form = module.get_form(form_id)
    except IndexError:
        return bail(request, domain, app_id, not_found="form")

    if form.no_vellum:
        messages.warning(request, _(
            "You tried to edit this form in the Form Builder. "
            "However, your administrator has locked this form against editing "
            "in the form builder, so we have redirected you to "
            "the form's front page instead."
        ))
        return back_to_main(request, domain, app_id=app_id,
                            unique_form_id=form.unique_id)

    vellum_plugins = ["modeliteration", "itemset", "atwho"]
    if (toggles.COMMTRACK.enabled(domain)):
        vellum_plugins.append("commtrack")
    if toggles.VELLUM_SAVE_TO_CASE.enabled(domain):
        vellum_plugins.append("saveToCase")
    if ((app.vellum_case_management or toggles.VELLUM_EXPERIMENTAL_UI.enabled(domain)) and
            module and module.case_type and form.requires_case()):
        vellum_plugins.append("databrowser")

    vellum_features = toggles.toggles_dict(username=request.user.username,
                                           domain=domain)
    vellum_features.update(feature_previews.previews_dict(domain))
    vellum_features.update({
        'group_in_field_list': app.enable_group_in_field_list,
        'image_resize': app.enable_image_resize,
        'markdown_in_groups': app.enable_markdown_in_groups,
        'lookup_tables': domain_has_privilege(domain, privileges.LOOKUP_TABLES),
        'templated_intents': domain_has_privilege(domain, privileges.TEMPLATED_INTENTS),
        'custom_intents': domain_has_privilege(domain, privileges.CUSTOM_INTENTS),
        'rich_text': app.vellum_case_management or toggles.VELLUM_RICH_TEXT.enabled(domain)
    })

    has_schedule = (
        getattr(module, 'has_schedule', False) and
        getattr(form, 'schedule', False) and form.schedule.enabled
    )
    scheduler_data_nodes = []
    if has_schedule:
        scheduler_data_nodes = [
            SCHEDULE_CURRENT_VISIT_NUMBER,
            SCHEDULE_NEXT_DUE,
            SCHEDULE_UNSCHEDULED_VISIT,
            SCHEDULE_GLOBAL_NEXT_VISIT_DATE,
        ]
        scheduler_data_nodes.extend([
            u"next_{}".format(f.schedule_form_id)
            for f in form.get_phase().get_forms()
            if getattr(f, 'schedule', False) and f.schedule.enabled
        ])

    context = get_apps_base_context(request, domain, app)
    context.update(locals())
    context.update({
        'vellum_debug': settings.VELLUM_DEBUG,
        'nav_form': form,
        'formdesigner': True,
        'multimedia_object_map': app.get_object_map(),
        'sessionid': request.COOKIES.get('sessionid'),
        'features': vellum_features,
        'plugins': vellum_plugins,
        'app_callout_templates': next(app_callout_templates),
        'scheduler_data_nodes': scheduler_data_nodes,
        'no_header': True,
    })
    return render(request, 'app_manager/form_designer.html', context)

@require_GET
@require_can_edit_apps
def get_form_data_schema(request, domain, form_unique_id):
    """Get data schema

    One of `app_id` or `form_unique_id` is required. `app_id` is ignored
    if `form_unique_id` is provided.

    :returns: A list of data source schema definitions. A data source schema
    definition is a dictionary with the following format:
    ```
    {
        "id": string (default instance id)
        "uri": string (instance src)
        "path": string (path of root nodeset, not including `instance(...)`)
        "name": string (human readable name)
        "structure": {
            element: {
                "name": string (optional human readable name)
                "structure": {
                    nested-element: { ... }
                },
            },
            ref-element: {
                "reference": {
                    "source": string (optional data source id, defaults to this data source)
                    "subset": string (optional subset id)
                    "key": string (referenced property)
                }
            },
            @attribute: { },
            ...
        },
        "subsets": [
            {
                "id": string (unique identifier for this subset)
                "key": string (unique identifier property name)
                "name": string (optional human readable name)
                "structure": { ... }
                "related": {
                    string (relationship): string (related subset name),
                    ...
                }
            },
            ...
        ]
    }
    ```
    A structure may contain nested structure elements. A nested element
    may contain one of "structure" (a concrete structure definition) or
    "reference" (a link to some other structure definition). Any
    structure item may have a human readable "name".
    """
    data = []

    try:
        form, app = Form.get_form(form_unique_id, and_app=True)
    except ResourceConflict:
        raise Http404()
    data.append(get_session_schema(form))

    if app.domain != domain:
        raise Http404()
    if form and form.requires_case():
        data.append(get_casedb_schema(app))  # TODO use domain instead of app
    data.extend(
        sorted(item_lists_by_domain(domain), key=lambda x: x['name'].lower())
    )
    kw = {}
    if "pretty" in request.GET:
        kw["indent"] = 2
    return HttpResponse(json.dumps(data, **kw))

@require_GET
def ping(request):
    return HttpResponse("pong")
<commit_msg>Trap errors encountered during get_form_data_schema<commit_after>import json
import logging

from django.utils.translation import ugettext as _
from couchdbkit.exceptions import ResourceConflict
from django.http import HttpResponse, Http404, HttpResponseBadRequest
from django.shortcuts import render
from django.views.decorators.http import require_GET
from django.conf import settings
from django.contrib import messages
from corehq.apps.app_manager.views.apps import get_apps_base_context

from corehq.apps.app_manager.views.utils import back_to_main, bail
from corehq import toggles, privileges, feature_previews
from corehq.apps.accounting.utils import domain_has_privilege
from corehq.apps.app_manager.const import (
    SCHEDULE_CURRENT_VISIT_NUMBER,
    SCHEDULE_NEXT_DUE,
    SCHEDULE_UNSCHEDULED_VISIT,
    SCHEDULE_GLOBAL_NEXT_VISIT_DATE,
)
from corehq.apps.app_manager.util import (
    get_casedb_schema,
    get_session_schema,
    app_callout_templates,
)
from corehq.apps.fixtures.fixturegenerators import item_lists_by_domain
from corehq.apps.app_manager.dbaccessors import get_app
from corehq.apps.app_manager.models import (
    Form,
    ModuleNotFoundException,
)
from corehq.apps.app_manager.decorators import require_can_edit_apps
from corehq.apps.analytics.tasks import track_entered_form_builder_on_hubspot
from corehq.apps.analytics.utils import get_meta

logger = logging.getLogger(__name__)

@require_can_edit_apps
def form_designer(request, domain, app_id, module_id=None, form_id=None):
    meta = get_meta(request)
    track_entered_form_builder_on_hubspot.delay(request.couch_user, request.COOKIES, meta)

    app = get_app(domain, app_id)
    module = None

    try:
        module = app.get_module(module_id)
    except ModuleNotFoundException:
        return bail(request, domain, app_id, not_found="module")
    try:
        form = module.get_form(form_id)
    except IndexError:
        return bail(request, domain, app_id, not_found="form")

    if form.no_vellum:
        messages.warning(request, _(
            "You tried to edit this form in the Form Builder. "
            "However, your administrator has locked this form against editing "
            "in the form builder, so we have redirected you to "
            "the form's front page instead."
        ))
        return back_to_main(request, domain, app_id=app_id,
                            unique_form_id=form.unique_id)

    vellum_plugins = ["modeliteration", "itemset", "atwho"]
    if (toggles.COMMTRACK.enabled(domain)):
        vellum_plugins.append("commtrack")
    if toggles.VELLUM_SAVE_TO_CASE.enabled(domain):
        vellum_plugins.append("saveToCase")
    if ((app.vellum_case_management or toggles.VELLUM_EXPERIMENTAL_UI.enabled(domain)) and
            module and module.case_type and form.requires_case()):
        vellum_plugins.append("databrowser")

    vellum_features = toggles.toggles_dict(username=request.user.username,
                                           domain=domain)
    vellum_features.update(feature_previews.previews_dict(domain))
    vellum_features.update({
        'group_in_field_list': app.enable_group_in_field_list,
        'image_resize': app.enable_image_resize,
        'markdown_in_groups': app.enable_markdown_in_groups,
        'lookup_tables': domain_has_privilege(domain, privileges.LOOKUP_TABLES),
        'templated_intents': domain_has_privilege(domain, privileges.TEMPLATED_INTENTS),
        'custom_intents': domain_has_privilege(domain, privileges.CUSTOM_INTENTS),
        'rich_text': app.vellum_case_management or toggles.VELLUM_RICH_TEXT.enabled(domain)
    })

    has_schedule = (
        getattr(module, 'has_schedule', False) and
        getattr(form, 'schedule', False) and form.schedule.enabled
    )
    scheduler_data_nodes = []
    if has_schedule:
        scheduler_data_nodes = [
            SCHEDULE_CURRENT_VISIT_NUMBER,
            SCHEDULE_NEXT_DUE,
            SCHEDULE_UNSCHEDULED_VISIT,
            SCHEDULE_GLOBAL_NEXT_VISIT_DATE,
        ]
        scheduler_data_nodes.extend([
            u"next_{}".format(f.schedule_form_id)
            for f in form.get_phase().get_forms()
            if getattr(f, 'schedule', False) and f.schedule.enabled
        ])

    context = get_apps_base_context(request, domain, app)
    context.update(locals())
    context.update({
        'vellum_debug': settings.VELLUM_DEBUG,
        'nav_form': form,
        'formdesigner': True,
        'multimedia_object_map': app.get_object_map(),
        'sessionid': request.COOKIES.get('sessionid'),
        'features': vellum_features,
        'plugins': vellum_plugins,
        'app_callout_templates': next(app_callout_templates),
        'scheduler_data_nodes': scheduler_data_nodes,
        'no_header': True,
    })
    return render(request, 'app_manager/form_designer.html', context)

@require_GET
@require_can_edit_apps
def get_form_data_schema(request, domain, form_unique_id):
    """Get data schema

    One of `app_id` or `form_unique_id` is required. `app_id` is ignored
    if `form_unique_id` is provided.

    :returns: A list of data source schema definitions. A data source schema
    definition is a dictionary with the following format:
    ```
    {
        "id": string (default instance id)
        "uri": string (instance src)
        "path": string (path of root nodeset, not including `instance(...)`)
        "name": string (human readable name)
        "structure": {
            element: {
                "name": string (optional human readable name)
                "structure": {
                    nested-element: { ... }
                },
            },
            ref-element: {
                "reference": {
                    "source": string (optional data source id, defaults to this data source)
                    "subset": string (optional subset id)
                    "key": string (referenced property)
                }
            },
            @attribute: { },
            ...
        },
        "subsets": [
            {
                "id": string (unique identifier for this subset)
                "key": string (unique identifier property name)
                "name": string (optional human readable name)
                "structure": { ... }
                "related": {
                    string (relationship): string (related subset name),
                    ...
                }
            },
            ...
        ]
    }
    ```
    A structure may contain nested structure elements. A nested element
    may contain one of "structure" (a concrete structure definition) or
    "reference" (a link to some other structure definition). Any
    structure item may have a human readable "name".
    """
    data = []

    try:
        form, app = Form.get_form(form_unique_id, and_app=True)
    except ResourceConflict:
        raise Http404()

    if app.domain != domain:
        raise Http404()

    try:
        data.append(get_session_schema(form))
        if form and form.requires_case():
            data.append(get_casedb_schema(app))  # TODO use domain instead of app
    except Exception as e:
        return HttpResponseBadRequest(e)

    data.extend(
        sorted(item_lists_by_domain(domain), key=lambda x: x['name'].lower())
    )
    kw = {}
    if "pretty" in request.GET:
        kw["indent"] = 2
    return HttpResponse(json.dumps(data, **kw))

@require_GET
def ping(request):
    return HttpResponse("pong")
<|endoftext|>


<commit_before>from django.conf import settings
from django import forms
from django.http import HttpResponse, HttpRequest
from django.shortcuts import render_to_response
from django.template.defaultfilters import striptags
from django.template import RequestContext
from django.template.loader import render_to_string
from django.utils import simplejson

from snapboard.templatetags.snapboard_tags import render_filter
from snapboard.models import UserSettings

# USE_SNAPBOARD_LOGIN_FORM, USE_SNAPBOARD_SIGNIN should probably be removed
USE_SNAPBOARD_SIGNIN = getattr(settings, 'USE_SNAPBOARD_SIGNIN', False)
USE_SNAPBOARD_LOGIN_FORM = getattr(settings, 'USE_SNAPBOARD_LOGIN_FORM', False)

class RequestFormMixin(object):
    def __init__(self, data=None, files=None, request=None, *args, **kwargs):
        super(RequestFormMixin, self).__init__(data=data, files=files, *args, **kwargs)
        self.request = request

class RequestForm(RequestFormMixin, forms.Form):
    pass

class RequestModelForm(RequestFormMixin, forms.ModelForm):
    pass

def render(template_name, context, request):
    return render_to_response(template_name, context,
                              context_instance=RequestContext(request))

def renders(template_name, context):
    return render_to_string(template_name, context)

DEFAULT_USER_SETTINGS  = UserSettings()

def get_user_settings(request):
    if hasattr(request, "user"):
        if hasattr(request, "_sb_settings_cache"):
            return request._sb_settings_cache

    user = getattr(request, "user", request)

    if not user.is_authenticated():
        user_settings = DEFAULT_USER_SETTINGS
    try:
        user_settings = user.sb_usersettings
    except UserSettings.DoesNotExist:
        user_settings = DEFAULT_USER_SETTINGS

    if hasattr(request, "user"):
        request._sb_settings_cache = user_settings
    return user_settings

def sanitize(text):
    return render_filter(striptags(text), "safe")

def toggle_boolean_field(obj, field):
    '''
    Switches the a boolean value and returns the new value.
    object should be a Django Model
    '''
    setattr(obj, field, (not getattr(obj, field)))
    obj.save()
    return getattr(obj, field)

def JsonResponse(obj):
    return HttpResponse(simplejson.dumps(obj), mimetype='application/javascript')
<commit_msg>Fix a bug for non-authed userz<commit_after>from django.conf import settings
from django import forms
from django.http import HttpResponse, HttpRequest
from django.shortcuts import render_to_response
from django.template.defaultfilters import striptags
from django.template import RequestContext
from django.template.loader import render_to_string
from django.utils import simplejson

from snapboard.templatetags.snapboard_tags import render_filter
from snapboard.models import UserSettings

# USE_SNAPBOARD_LOGIN_FORM, USE_SNAPBOARD_SIGNIN should probably be removed
USE_SNAPBOARD_SIGNIN = getattr(settings, 'USE_SNAPBOARD_SIGNIN', False)
USE_SNAPBOARD_LOGIN_FORM = getattr(settings, 'USE_SNAPBOARD_LOGIN_FORM', False)

class RequestFormMixin(object):
    def __init__(self, data=None, files=None, request=None, *args, **kwargs):
        super(RequestFormMixin, self).__init__(data=data, files=files, *args, **kwargs)
        self.request = request

class RequestForm(RequestFormMixin, forms.Form):
    pass

class RequestModelForm(RequestFormMixin, forms.ModelForm):
    pass

def render(template_name, context, request):
    return render_to_response(template_name, context,
                              context_instance=RequestContext(request))

def renders(template_name, context):
    return render_to_string(template_name, context)

DEFAULT_USER_SETTINGS  = UserSettings()

def get_user_settings(request):
    if hasattr(request, "user"):
        if hasattr(request, "_sb_settings_cache"):
            return request._sb_settings_cache

    user = getattr(request, "user", request)

    if not user.is_authenticated():
        user_settings = DEFAULT_USER_SETTINGS
    else:
        try:
            user_settings = user.sb_usersettings
        except UserSettings.DoesNotExist:
            user_settings = DEFAULT_USER_SETTINGS

    if hasattr(request, "user"):
        request._sb_settings_cache = user_settings
    return user_settings

def sanitize(text):
    return render_filter(striptags(text), "safe")

def toggle_boolean_field(obj, field):
    '''
    Switches the a boolean value and returns the new value.
    object should be a Django Model
    '''
    setattr(obj, field, (not getattr(obj, field)))
    obj.save()
    return getattr(obj, field)

def JsonResponse(obj):
    return HttpResponse(simplejson.dumps(obj), mimetype='application/javascript')
<|endoftext|>


<commit_before># vim: set expandtab sw=4 ts=4:
'''
Interface to Keen IO.

Copyright (C) 2014 Dieter Adriaenssens <ruleant@users.sourceforge.net>

This file is part of buildtimetrend/python-lib
<https://github.com/buildtimetrend/python-lib/>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
'''

import os
from buildtimetrend.tools import get_logger
import copy
import keen
from keen import scoped_keys
from buildtimetrend.settings import Settings
from buildtimetrend.tools import check_file
from buildtimetrend.tools import check_dict
from buildtimetrend.tools import check_list

def keen_has_project_id():
    '''
    Check if Keen.io project ID is set.
    '''
    if "KEEN_PROJECT_ID" in os.environ or keen.project_id is not None:
        return True

    get_logger().warning("Keen.io Project ID is not set")
    return False

def keen_is_writable():
    '''
    Check if login keys for Keen IO API are set, to allow writing.
    '''
    if (keen_has_project_id() and
            "KEEN_WRITE_KEY" in os.environ or keen.write_key is not None):
        return True

    get_logger().warning("Keen.io Write Key is not set")
    return False

def keen_is_readable():
    '''
    Check if login keys for Keen IO API are set, to allow reading.
    '''
    if (keen_has_project_id() and
            "KEEN_READ_KEY" in os.environ or keen.read_key is not None):
        return True

    get_logger().warning("Keen.io Read Key is not set")
    return False

def keen_io_generate_read_key(repo):
    '''
    Create scoped key for reading only the build-stages related data.
    Param repo : github repository slug (fe. buildtimetrend/python-lib)
    '''
    logger = get_logger()

    if "KEEN_MASTER_KEY" in os.environ:
        master_key = os.getenv("KEEN_MASTER_KEY")
        privileges = {
            "filters": [{
                "property_name": "build.repo",
                "operator": "eq",
                "property_value": repo
            }],
            "allowed_operations": ["read"]
        }

        logger.info("Keen.io Read Key is created for %s", repo)
        return scoped_keys.encrypt(master_key, privileges)

    logger.warning("Keen.io Read Key was not created.")
    return None

def log_build_keen(build):
    '''Send build data to keen.io'''
    if keen_is_writable():
        get_logger().info("Sending data to Keen.io")
        keen_add_event("builds", {"build": build.to_dict()})
        keen_add_events("build_stages", build.stages_to_list())

def keen_add_event(event_collection, payload):
    '''
    Wrapper for keen.add_event(), adds project info
    Param event_collection : collection event data is submitted to
    Param payload : data that is submitted
    '''
    # add project info to this event
    payload = add_project_info_dict(payload)

    # submit list of events to Keen.io
    keen.add_event(event_collection, payload)

def keen_add_events(event_collection, payload):
    '''
    Wrapper for keen.add_events(), adds project info to each event
    Param event_collection : collection event data is submitted to
    Param payload : array of events that is submitted
    '''
    # add project info to each event
    payload = add_project_info_list(payload)

    # submit list of events to Keen.io
    keen.add_events({event_collection: payload})

def add_project_info_dict(payload):
    '''
    Adds project info to a dictonary
    Param payload: dictonary payload
    '''
    if not check_dict(payload, "payload"):
        return None

    payload_as_dict = copy.deepcopy(payload)

    payload_as_dict["buildtime_trend"] = Settings().get_project_info()

    # override timestamp, set to finished_at timestamp
    if "build" in payload and "finished_at" in payload["build"]:
        payload_as_dict["keen"] = {
            "timestamp": payload["build"]["finished_at"]["isotimestamp"]
        }

    return payload_as_dict

def add_project_info_list(payload):
    '''
    Adds project info to a list of dictionaries
    Param payload: list of dictionaries
    '''
    if not check_list(payload, "payload"):
        return None

    payload_as_list = []

    # loop over dicts in payload and add project info to each one
    for event_dict in payload:
        payload_as_list.append(add_project_info_dict(event_dict))

    return payload_as_list

def generate_overview_config_file(repo):
    '''
    Generates a config file for the overview HTML file that contains the
    graphs generated by Keen.io
    '''
    logger = get_logger()

    if not ("BUILD_TREND_SAMPLE_CONFIGFILE" in os.environ
            and "BUILD_TREND_CONFIGFILE" in os.environ
            and "KEEN_PROJECT_ID" in os.environ):
        logger.error("Trends overview config file was not created")
        return

    sample_filename = os.getenv("BUILD_TREND_SAMPLE_CONFIGFILE")
    config_file = os.getenv("BUILD_TREND_CONFIGFILE")
    keen_project_id = os.getenv("KEEN_PROJECT_ID")

    # check if sample config file exists
    if not check_file(sample_filename):
        return

    # generate read key
    read_key = keen_io_generate_read_key(repo)

    # list of string pairs to be replaced (search string : new string)
    replacements = {
        'keen_project_id': str(keen_project_id),
        'keen_read_key': str(read_key),
        'project_name': str(repo)
    }

    with open(sample_filename, 'rb') as infile, \
            open(config_file, 'w') as outfile:
        for line in infile:
            for src, target in replacements.iteritems():
                line = line.replace(src, target)
            outfile.write(line)

    logger.info("Created trends overview config file")
<commit_msg>get_avg_buildtime<commit_after># vim: set expandtab sw=4 ts=4:
'''
Interface to Keen IO.

Copyright (C) 2014 Dieter Adriaenssens <ruleant@users.sourceforge.net>

This file is part of buildtimetrend/python-lib
<https://github.com/buildtimetrend/python-lib/>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
'''

import os
from buildtimetrend.tools import get_logger
import copy
import keen
from keen import scoped_keys
from buildtimetrend.settings import Settings
from buildtimetrend.tools import check_file
from buildtimetrend.tools import check_dict
from buildtimetrend.tools import check_list

def keen_has_project_id():
    '''
    Check if Keen.io project ID is set.
    '''
    if "KEEN_PROJECT_ID" in os.environ or keen.project_id is not None:
        return True

    get_logger().warning("Keen.io Project ID is not set")
    return False

def keen_is_writable():
    '''
    Check if login keys for Keen IO API are set, to allow writing.
    '''
    if (keen_has_project_id() and
            "KEEN_WRITE_KEY" in os.environ or keen.write_key is not None):
        return True

    get_logger().warning("Keen.io Write Key is not set")
    return False

def keen_is_readable():
    '''
    Check if login keys for Keen IO API are set, to allow reading.
    '''
    if (keen_has_project_id() and
            "KEEN_READ_KEY" in os.environ or keen.read_key is not None):
        return True

    get_logger().warning("Keen.io Read Key is not set")
    return False

def keen_io_generate_read_key(repo):
    '''
    Create scoped key for reading only the build-stages related data.
    Param repo : github repository slug (fe. buildtimetrend/python-lib)
    '''
    logger = get_logger()

    if "KEEN_MASTER_KEY" in os.environ:
        master_key = os.getenv("KEEN_MASTER_KEY")
        privileges = {
            "filters": [{
                "property_name": "build.repo",
                "operator": "eq",
                "property_value": repo
            }],
            "allowed_operations": ["read"]
        }

        logger.info("Keen.io Read Key is created for %s", repo)
        return scoped_keys.encrypt(master_key, privileges)

    logger.warning("Keen.io Read Key was not created.")
    return None

def log_build_keen(build):
    '''Send build data to keen.io'''
    if keen_is_writable():
        get_logger().info("Sending data to Keen.io")
        keen_add_event("builds", {"build": build.to_dict()})
        keen_add_events("build_stages", build.stages_to_list())

def keen_add_event(event_collection, payload):
    '''
    Wrapper for keen.add_event(), adds project info
    Param event_collection : collection event data is submitted to
    Param payload : data that is submitted
    '''
    # add project info to this event
    payload = add_project_info_dict(payload)

    # submit list of events to Keen.io
    keen.add_event(event_collection, payload)

def keen_add_events(event_collection, payload):
    '''
    Wrapper for keen.add_events(), adds project info to each event
    Param event_collection : collection event data is submitted to
    Param payload : array of events that is submitted
    '''
    # add project info to each event
    payload = add_project_info_list(payload)

    # submit list of events to Keen.io
    keen.add_events({event_collection: payload})

def add_project_info_dict(payload):
    '''
    Adds project info to a dictonary
    Param payload: dictonary payload
    '''
    if not check_dict(payload, "payload"):
        return None

    payload_as_dict = copy.deepcopy(payload)

    payload_as_dict["buildtime_trend"] = Settings().get_project_info()

    # override timestamp, set to finished_at timestamp
    if "build" in payload and "finished_at" in payload["build"]:
        payload_as_dict["keen"] = {
            "timestamp": payload["build"]["finished_at"]["isotimestamp"]
        }

    return payload_as_dict

def add_project_info_list(payload):
    '''
    Adds project info to a list of dictionaries
    Param payload: list of dictionaries
    '''
    if not check_list(payload, "payload"):
        return None

    payload_as_list = []

    # loop over dicts in payload and add project info to each one
    for event_dict in payload:
        payload_as_list.append(add_project_info_dict(event_dict))

    return payload_as_list

def generate_overview_config_file(repo):
    '''
    Generates a config file for the overview HTML file that contains the
    graphs generated by Keen.io
    '''
    logger = get_logger()

    if not ("BUILD_TREND_SAMPLE_CONFIGFILE" in os.environ
            and "BUILD_TREND_CONFIGFILE" in os.environ
            and "KEEN_PROJECT_ID" in os.environ):
        logger.error("Trends overview config file was not created")
        return

    sample_filename = os.getenv("BUILD_TREND_SAMPLE_CONFIGFILE")
    config_file = os.getenv("BUILD_TREND_CONFIGFILE")
    keen_project_id = os.getenv("KEEN_PROJECT_ID")

    # check if sample config file exists
    if not check_file(sample_filename):
        return

    # generate read key
    read_key = keen_io_generate_read_key(repo)

    # list of string pairs to be replaced (search string : new string)
    replacements = {
        'keen_project_id': str(keen_project_id),
        'keen_read_key': str(read_key),
        'project_name': str(repo)
    }

    with open(sample_filename, 'rb') as infile, \
            open(config_file, 'w') as outfile:
        for line in infile:
            for src, target in replacements.iteritems():
                line = line.replace(src, target)
            outfile.write(line)

    logger.info("Created trends overview config file")

def get_avg_buildtime(repo, interval=None):
    '''
    Query Keen.io database and retrieve average buildtime

    Parameters :
    - repo : repo name (fe. buildtimetrend/service)
    '''
    if repo is None or not keen_is_readable():
        return None

    if interval == "week":
        timeframe = "this_7_days"
    elif interval == "month":
        timeframe = "this_30_days"
    elif interval == "year":
        timeframe = "this_52_weeks"
    else:
        timeframe = "this_7_days"

    return keen.average(
        "builds",
        target_property="build.duration",
        timeframe=timeframe,
        filters=[{
            "property_name": "build.repo",
            "operator": "eq",
            "property_value": str(repo)
        }]
    )
<|endoftext|>