<commit_before># Copyright 2017 datawire. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy, fnmatch, hashlib, jsonschema, os, pathspec, util, yaml
from collections import OrderedDict
from .jinja2 import render, renders
from .docker import image
from .tasks import sh, task, TaskError
from .github import Github

with open(os.path.join(os.path.dirname(__file__), "service.json")) as f:
    SCHEMA = yaml.load(f)

def load_service_yaml(path, **vars):
    with open(path, "read") as f:
        return load_service_yamls(path, f.read(), **vars)

def _dump_and_raise(rendered, e):
    task.echo("==unparseable service yaml==")
    for idx, line in enumerate(rendered.splitlines()):
        task.echo("%s: %s" % (idx + 1, line))
    task.echo("============================")
    raise TaskError("error parsing service yaml: %s" % e)

@task()
def load_service_yamls(name, content, **vars):
    if "env" not in vars:
        vars["env"] = os.environ
    rendered = renders(name, content, **vars)
    try:
        info = yaml.load(rendered)
    except yaml.parser.ParserError, e:
        _dump_and_raise(rendered, e)
    except yaml.scanner.ScannerError, e:
        _dump_and_raise(rendered, e)

    try:
        jsonschema.validate(info, SCHEMA)
        return info
    except jsonschema.ValidationError, e:
        best = jsonschema.exceptions.best_match(e.context)
        raise TaskError("error loading %s: %s" % (name, (best or e).message))

def get_ignores(directory):
    ignorefiles = [os.path.join(directory, ".gitignore"),
                   os.path.join(directory, ".forgeignore")]
    ignores = []
    for path in ignorefiles:
        if os.path.exists(path):
            with open(path) as fd:
                ignores.extend(fd.readlines())
    return ignores

def get_ancestors(path, stop="/"):
    path = os.path.abspath(path)
    stop = os.path.abspath(stop)
    if os.path.samefile(path, stop):
        return
    else:
        parent = os.path.dirname(path)
        for d in get_ancestors(parent, stop):
            yield d
        yield parent

class Discovery(object):

    def __init__(self, profile=None, branch=None):
        self.services = OrderedDict()
        self.profile = profile
        self.branch = branch

    @task()
    def search(self, directory):
        directory = os.path.abspath(directory)
        if not os.path.exists(directory):
            raise TaskError("no such directory: %s" % directory)
        if not os.path.isdir(directory):
            raise TaskError("not a directory: %s" % directory)

        base_ignores = [".git", ".forge"]
        gitdir = util.search_parents(".git", directory)
        if gitdir is None:
            gitroot = directory
        else:
            gitroot = os.path.dirname(gitdir)

        for d in get_ancestors(directory, gitroot):
            base_ignores.extend(get_ignores(d))

        found = []
        def descend(path, parent, ignores):
            if not os.path.exists(path): return
            ignores = ignores[:]

            ignores += get_ignores(path)
            spec = pathspec.PathSpec.from_lines('gitwildmatch', ignores)
            names = [n for n in os.listdir(path) if not spec.match_file(os.path.relpath(os.path.join(path, n),
                                                                                directory))]

            if "service.yaml" in names:
                svc = Service(os.path.join(path, "service.yaml"), profile=self.profile, branch=self.branch)
                if svc.name not in self.services:
                    self.services[svc.name] = svc
                found.append(svc)
                parent = svc

            if "Dockerfile" in names and parent:
                parent.dockerfiles.append(os.path.relpath(os.path.join(path, "Dockerfile"), parent.root))

            for n in names:
                child = os.path.join(path, n)
                if os.path.isdir(child):
                    descend(child, parent, ignores)
                elif parent:
                    parent.files.append(os.path.relpath(child, parent.root))

        descend(directory, None, base_ignores)
        return found

    def resolve(self, svc, dep):
        gh = Github(None)
        target = os.path.join(svc.root, ".forge", dep)
        if not os.path.exists(target):
            url = gh.remote(svc.root)
            if url is None: return False
            parts = url.split("/")
            prefix = "/".join(parts[:-1])
            remote = prefix + "/" + dep + ".git"
            if gh.exists(remote):
                task.echo("cloning %s->%s" % (remote, os.path.relpath(target, os.getcwd())))
                gh.clone(remote, target)
            else:
                raise TaskError("cannot resolve dependency: %s" % dep)
        found = self.search(target)
        return dep in [svc.name for svc in found]

    @task()
    def dependencies(self, targets):
        todo = [self.services[t] for t in targets]
        visited = set()
        added = []
        missing = []
        while todo:
            svc = todo.pop()
            if svc in visited:
                continue
            visited.add(svc)
            for r in svc.requires:
                if r not in self.services:
                    if not self.resolve(svc, r): missing.append(r)
                if r not in targets and r not in added:
                    added.append(r)
                if r in self.services:
                    todo.append(self.services[r])

        if missing:
            raise TaskError("required service(s) missing: %s" % ", ".join(missing))
        else:
            return added

def shafiles(root, files):
    result = hashlib.sha1()
    result.update("files %s\0" % len(files))
    for name in files:
        result.update("file %s\0" % name)
        try:
            with open(os.path.join(root, name)) as fd:
                result.update(fd.read())
        except IOError, e:
            if e.errno != errno.ENOENT:
                raise
    return result.hexdigest()

def is_git(path):
    if os.path.exists(os.path.join(path, ".git")):
        return True
    elif path not in ('', '/'):
        return is_git(os.path.dirname(path))
    else:
        return False

def get_version(path, dirty):
    if is_git(path):
        result = sh("git", "diff", "--quiet", "HEAD", ".", cwd=path, expected=(0, 1))
        if result.code == 0:
            line = sh("git", "log", "-n1", "--format=oneline", "--", ".", cwd=path).output.strip()
            if line:
                version = line.split()[0]
                return "%s.git" % version
    return dirty

class Service(object):

    def __init__(self, descriptor, profile=None, branch=None):
        self.descriptor = descriptor
        self.dockerfiles = []
        self.files = []
        self._info = None
        self._version = None
        if branch:
            self.branch = branch
        elif is_git(self.root):
            self.branch = sh("git", "symbolic-ref", "--short", "HEAD", cwd=self.root).output.strip()
        else:
            self.branch = None
        self.profile = profile

    @property
    def root(self):
        return os.path.dirname(self.descriptor)

    @property
    def name(self):
        info = self.info()
        if "name" in info:
            return info["name"]
        else:
            return os.path.basename(self.root)

    @property
    def version(self):
        if self._version is None:
            self._version = get_version(self.root, "%s.sha" % shafiles(self.root, self.files))
        return self._version

    def image(self, container):
        pfx = os.path.dirname(container)
        name = os.path.join(self.name, pfx) if pfx else self.name
        name = name.replace("/", "-")
        return name

    def metadata(self, registry, repo):
        metadata = OrderedDict()

        metadata["env"] = os.environ

        svc = self.info()
        if "name" not in svc:
            svc["name"] = self.name
        metadata["service"] = svc

        build = OrderedDict()
        metadata["build"] = build

        build["branch"] = self.branch

        if self.profile is None:
            profile = "default"
            if self.branch:
                for k, v in svc.get("branches", {}).items():
                    if fnmatch.fnmatch(self.branch, k):
                        profile = v
                        break
        else:
            profile = self.profile

        build["version"] = self.version
        prof = copy.deepcopy(svc.get("profiles", {}).get(profile, {}))
        build["profile"] = prof
        if "name" not in prof:
            prof["name"] = profile

        build["name"] = "%s-%s" % (svc["name"], prof["name"])

        build["images"] = OrderedDict()
        for container in self.dockerfiles:
            img = image(registry, repo, self.image(container), self.version)
            build["images"][container] = img

        return metadata

    @property
    def manifest_dir(self):
        return os.path.join(self.root, "k8s")

    def deployment(self, registry, repo, target):
        k8s_dir = self.manifest_dir
        metadata = self.metadata(registry, repo)
        render(k8s_dir, target, **metadata)

    def info(self):
        if self._info is None:
            self._info = load_service_yaml(self.descriptor, branch=self.branch)
        return self._info

    @property
    def requires(self):
        value = self.info().get("requires", ())
        if isinstance(value, basestring):
            return [value]
        else:
            return value

    @property
    def containers(self):
        info = self.info()
        containers = info.get("containers", self.dockerfiles)
        for idx, c in enumerate(containers):
            if isinstance(c, basestring):
                yield Container(self, c, index=idx)
            else:
                yield Container(self, c["dockerfile"], c.get("context", None), c.get("args", None),
                                c.get("rebuild", None), index=idx)

    def json(self):
        return {'name': self.name,
                'owner': self.name,
                'version': self.version,
                'descriptor': self.info(),
                'tasks': []}

    def __repr__(self):
        return "%s:%s" % (self.name, self.version)

class Container(object):

    def __init__(self, service, dockerfile, context=None, args=None, rebuild=None, index=None):
        self.service = service
        self.dockerfile = dockerfile
        self.context = context or os.path.dirname(self.dockerfile)
        self.args = args or {}
        self.rebuild_root = rebuild.get("root", "/") if rebuild else None
        self.rebuild_sources = rebuild.get("sources", ()) if rebuild else ()
        self.rebuild_command = rebuild.get("command") if rebuild else None
        self.index = index

    @property
    def version(self):
        return self.service.version

    @property
    def image(self):
        return self.service.image(self.dockerfile)

    @property
    def abs_dockerfile(self):
        return os.path.join(self.service.root, self.dockerfile)

    @property
    def abs_context(self):
        return os.path.join(self.service.root, self.context)

    @property
    def rebuild(self):
        return self.rebuild_sources or self.rebuild_command

    @task()
    def build(self, forge):
        if self.rebuild:
            builder = forge.docker.builder(self.abs_context, self.abs_dockerfile, self.image, self.version, self.args)
            builder.run("mkdir", "-p", self.rebuild_root)
            for src in self.rebuild_sources:
                abs_src = os.path.join(self.service.root, src)
                tgt_src = os.path.join(self.rebuild_root, src)
                if os.path.isdir(abs_src):
                    builder.run("rm", "-rf", tgt_src)
                builder.cp(abs_src, tgt_src)
            if self.rebuild_command:
                builder.run("/bin/sh", "-c", self.rebuild_command)
            builder.commit(self.image, self.version)
        else:
            forge.docker.build(self.abs_context, self.abs_dockerfile, self.image, self.version, self.args)
<commit_msg>fix branch detection for detached head<commit_after># Copyright 2017 datawire. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy, fnmatch, hashlib, jsonschema, os, pathspec, util, yaml
from collections import OrderedDict
from .jinja2 import render, renders
from .docker import image
from .tasks import sh, task, TaskError
from .github import Github

with open(os.path.join(os.path.dirname(__file__), "service.json")) as f:
    SCHEMA = yaml.load(f)

def load_service_yaml(path, **vars):
    with open(path, "read") as f:
        return load_service_yamls(path, f.read(), **vars)

def _dump_and_raise(rendered, e):
    task.echo("==unparseable service yaml==")
    for idx, line in enumerate(rendered.splitlines()):
        task.echo("%s: %s" % (idx + 1, line))
    task.echo("============================")
    raise TaskError("error parsing service yaml: %s" % e)

@task()
def load_service_yamls(name, content, **vars):
    if "env" not in vars:
        vars["env"] = os.environ
    rendered = renders(name, content, **vars)
    try:
        info = yaml.load(rendered)
    except yaml.parser.ParserError, e:
        _dump_and_raise(rendered, e)
    except yaml.scanner.ScannerError, e:
        _dump_and_raise(rendered, e)

    try:
        jsonschema.validate(info, SCHEMA)
        return info
    except jsonschema.ValidationError, e:
        best = jsonschema.exceptions.best_match(e.context)
        raise TaskError("error loading %s: %s" % (name, (best or e).message))

def get_ignores(directory):
    ignorefiles = [os.path.join(directory, ".gitignore"),
                   os.path.join(directory, ".forgeignore")]
    ignores = []
    for path in ignorefiles:
        if os.path.exists(path):
            with open(path) as fd:
                ignores.extend(fd.readlines())
    return ignores

def get_ancestors(path, stop="/"):
    path = os.path.abspath(path)
    stop = os.path.abspath(stop)
    if os.path.samefile(path, stop):
        return
    else:
        parent = os.path.dirname(path)
        for d in get_ancestors(parent, stop):
            yield d
        yield parent

class Discovery(object):

    def __init__(self, profile=None, branch=None):
        self.services = OrderedDict()
        self.profile = profile
        self.branch = branch

    @task()
    def search(self, directory):
        directory = os.path.abspath(directory)
        if not os.path.exists(directory):
            raise TaskError("no such directory: %s" % directory)
        if not os.path.isdir(directory):
            raise TaskError("not a directory: %s" % directory)

        base_ignores = [".git", ".forge"]
        gitdir = util.search_parents(".git", directory)
        if gitdir is None:
            gitroot = directory
        else:
            gitroot = os.path.dirname(gitdir)

        for d in get_ancestors(directory, gitroot):
            base_ignores.extend(get_ignores(d))

        found = []
        def descend(path, parent, ignores):
            if not os.path.exists(path): return
            ignores = ignores[:]

            ignores += get_ignores(path)
            spec = pathspec.PathSpec.from_lines('gitwildmatch', ignores)
            names = [n for n in os.listdir(path) if not spec.match_file(os.path.relpath(os.path.join(path, n),
                                                                                directory))]

            if "service.yaml" in names:
                svc = Service(os.path.join(path, "service.yaml"), profile=self.profile, branch=self.branch)
                if svc.name not in self.services:
                    self.services[svc.name] = svc
                found.append(svc)
                parent = svc

            if "Dockerfile" in names and parent:
                parent.dockerfiles.append(os.path.relpath(os.path.join(path, "Dockerfile"), parent.root))

            for n in names:
                child = os.path.join(path, n)
                if os.path.isdir(child):
                    descend(child, parent, ignores)
                elif parent:
                    parent.files.append(os.path.relpath(child, parent.root))

        descend(directory, None, base_ignores)
        return found

    def resolve(self, svc, dep):
        gh = Github(None)
        target = os.path.join(svc.root, ".forge", dep)
        if not os.path.exists(target):
            url = gh.remote(svc.root)
            if url is None: return False
            parts = url.split("/")
            prefix = "/".join(parts[:-1])
            remote = prefix + "/" + dep + ".git"
            if gh.exists(remote):
                task.echo("cloning %s->%s" % (remote, os.path.relpath(target, os.getcwd())))
                gh.clone(remote, target)
            else:
                raise TaskError("cannot resolve dependency: %s" % dep)
        found = self.search(target)
        return dep in [svc.name for svc in found]

    @task()
    def dependencies(self, targets):
        todo = [self.services[t] for t in targets]
        visited = set()
        added = []
        missing = []
        while todo:
            svc = todo.pop()
            if svc in visited:
                continue
            visited.add(svc)
            for r in svc.requires:
                if r not in self.services:
                    if not self.resolve(svc, r): missing.append(r)
                if r not in targets and r not in added:
                    added.append(r)
                if r in self.services:
                    todo.append(self.services[r])

        if missing:
            raise TaskError("required service(s) missing: %s" % ", ".join(missing))
        else:
            return added

def shafiles(root, files):
    result = hashlib.sha1()
    result.update("files %s\0" % len(files))
    for name in files:
        result.update("file %s\0" % name)
        try:
            with open(os.path.join(root, name)) as fd:
                result.update(fd.read())
        except IOError, e:
            if e.errno != errno.ENOENT:
                raise
    return result.hexdigest()

def is_git(path):
    if os.path.exists(os.path.join(path, ".git")):
        return True
    elif path not in ('', '/'):
        return is_git(os.path.dirname(path))
    else:
        return False

def get_version(path, dirty):
    if is_git(path):
        result = sh("git", "diff", "--quiet", "HEAD", ".", cwd=path, expected=(0, 1))
        if result.code == 0:
            line = sh("git", "log", "-n1", "--format=oneline", "--", ".", cwd=path).output.strip()
            if line:
                version = line.split()[0]
                return "%s.git" % version
    return dirty

class Service(object):

    def __init__(self, descriptor, profile=None, branch=None):
        self.descriptor = descriptor
        self.dockerfiles = []
        self.files = []
        self._info = None
        self._version = None
        if branch:
            self.branch = branch
        elif is_git(self.root):
            output = sh("git", "rev-parse", "--abbrev-ref", "HEAD", cwd=self.root).output.strip()
            self.branch = None if output == "HEAD" else output
        else:
            self.branch = None
        self.profile = profile

    @property
    def root(self):
        return os.path.dirname(self.descriptor)

    @property
    def name(self):
        info = self.info()
        if "name" in info:
            return info["name"]
        else:
            return os.path.basename(self.root)

    @property
    def version(self):
        if self._version is None:
            self._version = get_version(self.root, "%s.sha" % shafiles(self.root, self.files))
        return self._version

    def image(self, container):
        pfx = os.path.dirname(container)
        name = os.path.join(self.name, pfx) if pfx else self.name
        name = name.replace("/", "-")
        return name

    def metadata(self, registry, repo):
        metadata = OrderedDict()

        metadata["env"] = os.environ

        svc = self.info()
        if "name" not in svc:
            svc["name"] = self.name
        metadata["service"] = svc

        build = OrderedDict()
        metadata["build"] = build

        build["branch"] = self.branch

        if self.profile is None:
            profile = "default"
            if self.branch:
                for k, v in svc.get("branches", {}).items():
                    if fnmatch.fnmatch(self.branch, k):
                        profile = v
                        break
        else:
            profile = self.profile

        build["version"] = self.version
        prof = copy.deepcopy(svc.get("profiles", {}).get(profile, {}))
        build["profile"] = prof
        if "name" not in prof:
            prof["name"] = profile

        build["name"] = "%s-%s" % (svc["name"], prof["name"])

        build["images"] = OrderedDict()
        for container in self.dockerfiles:
            img = image(registry, repo, self.image(container), self.version)
            build["images"][container] = img

        return metadata

    @property
    def manifest_dir(self):
        return os.path.join(self.root, "k8s")

    def deployment(self, registry, repo, target):
        k8s_dir = self.manifest_dir
        metadata = self.metadata(registry, repo)
        render(k8s_dir, target, **metadata)

    def info(self):
        if self._info is None:
            self._info = load_service_yaml(self.descriptor, branch=self.branch)
        return self._info

    @property
    def requires(self):
        value = self.info().get("requires", ())
        if isinstance(value, basestring):
            return [value]
        else:
            return value

    @property
    def containers(self):
        info = self.info()
        containers = info.get("containers", self.dockerfiles)
        for idx, c in enumerate(containers):
            if isinstance(c, basestring):
                yield Container(self, c, index=idx)
            else:
                yield Container(self, c["dockerfile"], c.get("context", None), c.get("args", None),
                                c.get("rebuild", None), index=idx)

    def json(self):
        return {'name': self.name,
                'owner': self.name,
                'version': self.version,
                'descriptor': self.info(),
                'tasks': []}

    def __repr__(self):
        return "%s:%s" % (self.name, self.version)

class Container(object):

    def __init__(self, service, dockerfile, context=None, args=None, rebuild=None, index=None):
        self.service = service
        self.dockerfile = dockerfile
        self.context = context or os.path.dirname(self.dockerfile)
        self.args = args or {}
        self.rebuild_root = rebuild.get("root", "/") if rebuild else None
        self.rebuild_sources = rebuild.get("sources", ()) if rebuild else ()
        self.rebuild_command = rebuild.get("command") if rebuild else None
        self.index = index

    @property
    def version(self):
        return self.service.version

    @property
    def image(self):
        return self.service.image(self.dockerfile)

    @property
    def abs_dockerfile(self):
        return os.path.join(self.service.root, self.dockerfile)

    @property
    def abs_context(self):
        return os.path.join(self.service.root, self.context)

    @property
    def rebuild(self):
        return self.rebuild_sources or self.rebuild_command

    @task()
    def build(self, forge):
        if self.rebuild:
            builder = forge.docker.builder(self.abs_context, self.abs_dockerfile, self.image, self.version, self.args)
            builder.run("mkdir", "-p", self.rebuild_root)
            for src in self.rebuild_sources:
                abs_src = os.path.join(self.service.root, src)
                tgt_src = os.path.join(self.rebuild_root, src)
                if os.path.isdir(abs_src):
                    builder.run("rm", "-rf", tgt_src)
                builder.cp(abs_src, tgt_src)
            if self.rebuild_command:
                builder.run("/bin/sh", "-c", self.rebuild_command)
            builder.commit(self.image, self.version)
        else:
            forge.docker.build(self.abs_context, self.abs_dockerfile, self.image, self.version, self.args)
<|endoftext|>


<commit_before># Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""URL endpoint for adding new histograms to the dashboard."""

import json
import logging
import sys
import traceback

from google.appengine.api import taskqueue
from google.appengine.ext import ndb

from dashboard import add_point
from dashboard import add_point_queue
from dashboard.api import api_request_handler
from dashboard.common import datastore_hooks
from dashboard.common import histogram_helpers
from dashboard.common import stored_object
from dashboard.common import utils
from dashboard.models import histogram
from tracing.value import histogram_set
from tracing.value.diagnostics import diagnostic
from tracing.value.diagnostics import reserved_infos

SUITE_LEVEL_SPARSE_DIAGNOSTIC_NAMES = set([
    reserved_infos.ARCHITECTURES.name,
    reserved_infos.BENCHMARKS.name,
    reserved_infos.BOTS.name,
    reserved_infos.BUG_COMPONENTS.name,
    reserved_infos.GPUS.name,
    reserved_infos.MASTERS.name,
    reserved_infos.MEMORY_AMOUNTS.name,
    reserved_infos.OS_NAMES.name,
    reserved_infos.OS_VERSIONS.name,
    reserved_infos.OWNERS.name,
    reserved_infos.PRODUCT_VERSIONS.name,
    reserved_infos.TAG_MAP.name,
])

HISTOGRAM_LEVEL_SPARSE_DIAGNOSTIC_NAMES = set([
    reserved_infos.DEVICE_IDS.name,
    reserved_infos.RELATED_NAMES.name,
    reserved_infos.STORIES.name,
    reserved_infos.STORYSET_REPEATS.name,
    reserved_infos.STORY_TAGS.name,
])

SPARSE_DIAGNOSTIC_NAMES = SUITE_LEVEL_SPARSE_DIAGNOSTIC_NAMES.union(
    HISTOGRAM_LEVEL_SPARSE_DIAGNOSTIC_NAMES)

TASK_QUEUE_NAME = 'histograms-queue'

def _CheckRequest(condition, msg):
  if not condition:
    raise api_request_handler.BadRequestError(msg)

class AddHistogramsHandler(api_request_handler.ApiRequestHandler):

  def AuthorizedPost(self):
    datastore_hooks.SetPrivilegedRequest()

    try:
      data_str = self.request.get('data')
      if not data_str:
        raise api_request_handler.BadRequestError('Missing "data" parameter')

      logging.info('Received data: %s', data_str)

      histogram_dicts = json.loads(data_str)
      ProcessHistogramSet(histogram_dicts)
    except api_request_handler.BadRequestError:
      # TODO(simonhatch, eakuefner: Remove this later.
      # When this has all stabilized a bit, remove and let this 400 to clients,
      # but for now to preven the waterfall from re-uploading over and over
      # while we bug fix, let's just log the error.
      # https://github.com/catapult-project/catapult/issues/4019
      logging.error(traceback.format_exc())
    except Exception:  # pylint: disable=broad-except
      # TODO(simonhatch, eakuefner: Remove this later.
      # We shouldn't be catching ALL exceptions, this is just while the
      # stability of the endpoint is being worked on.
      logging.error(traceback.format_exc())

def _LogDebugInfo(histograms):
  hist = histograms.GetFirstHistogram()
  if not hist:
    logging.info('No histograms in data.')
    return

  log_urls = hist.diagnostics.get(reserved_infos.LOG_URLS.name)
  if log_urls:
    log_urls = list(log_urls)
    logging.info('Buildbot URL: %s', str(log_urls))
  else:
    logging.info('No LOG_URLS in data.')

def ProcessHistogramSet(histogram_dicts):
  if not isinstance(histogram_dicts, list):
    raise api_request_handler.BadRequestError(
        'HistogramSet JSON much be a list of dicts')

  bot_whitelist_future = stored_object.GetAsync(
      add_point_queue.BOT_WHITELIST_KEY)

  histograms = histogram_set.HistogramSet()
  histograms.ImportDicts(histogram_dicts)
  histograms.ResolveRelatedHistograms()
  histograms.DeduplicateDiagnostics()

  if len(histograms) == 0:
    raise api_request_handler.BadRequestError(
        'HistogramSet JSON must contain at least one histogram.')

  _LogDebugInfo(histograms)

  InlineDenseSharedDiagnostics(histograms)
  revision = ComputeRevision(histograms)
  master, bot, benchmark = _GetMasterBotBenchmarkFromHistogram(
      histograms.GetFirstHistogram())
  suite_key = utils.TestKey('%s/%s/%s' % (master, bot, benchmark))

  bot_whitelist = bot_whitelist_future.get_result()
  internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)

  # We'll skip the histogram-level sparse diagnostics because we need to
  # handle those with the histograms, below, so that we can properly assign
  # test paths.
  suite_level_sparse_diagnostic_entities = FindSuiteLevelSparseDiagnostics(
      histograms, suite_key, revision, internal_only)

  # TODO(eakuefner): Refactor master/bot computation to happen above this line
  # so that we can replace with a DiagnosticRef rather than a full diagnostic.
  new_guids_to_old_diagnostics = DeduplicateAndPut(
      suite_level_sparse_diagnostic_entities, suite_key, revision)
  for new_guid, old_diagnostic in new_guids_to_old_diagnostics.iteritems():
    histograms.ReplaceSharedDiagnostic(
        new_guid, diagnostic.Diagnostic.FromDict(old_diagnostic))

  tasks = _BatchHistogramsIntoTasks(suite_key.id(), histograms, revision)

  _QueueHistogramTasks(tasks)

def _MakeTask(params):
  return taskqueue.Task(
      url='/add_histograms_queue', payload=json.dumps(params),
      _size_check=False)

def _BatchHistogramsIntoTasks(suite_path, histograms, revision):
  params = []
  tasks = []

  base_size = _MakeTask([]).size
  estimated_size = 0

  for hist in histograms:
    diagnostics = FindHistogramLevelSparseDiagnostics(hist.guid, histograms)

    # TODO(eakuefner): Don't compute full diagnostics, because we need anyway to
    # call GetOrCreate here and in the queue.
    test_path = ComputeTestPath(suite_path, hist.guid, histograms)

    # TODO(eakuefner): Batch these better than one per task.
    task_dict = _MakeTaskDict(hist, test_path, revision, diagnostics)

    estimated_size += len(json.dumps(task_dict))

    # Creating the task directly and getting the size back is slow, so we just
    # keep a running total of estimated task size. A bit hand-wavy but the #
    # of histograms per task doesn't need to be perfect, just has to be under
    # the max task size.
    estimated_total_size = estimated_size * 1.05 + base_size + 1024
    if estimated_total_size > taskqueue.MAX_TASK_SIZE_BYTES:
      t = _MakeTask(params)
      tasks.append(t)
      params = []
      estimated_size = 0
    params.append(task_dict)

  if params:
    t = _MakeTask(params)
    tasks.append(t)

  return tasks

def _QueueHistogramTasks(tasks):
  queue = taskqueue.Queue(TASK_QUEUE_NAME)
  futures = []
  for i in xrange(0, len(tasks), taskqueue.MAX_TASKS_PER_ADD):
    f = queue.add_async(tasks[i:i + taskqueue.MAX_TASKS_PER_ADD])
    futures.append(f)
  for f in futures:
    f.get_result()

def _MakeTaskDict(hist, test_path, revision, diagnostics):
  # TODO(simonhatch): "revision" is common to all tasks, as is the majority of
  # the test path
  params = {
      'test_path': test_path,
      'revision': revision
  }

  # By changing the GUID just before serializing the task, we're making it
  # unique for each histogram. This avoids each histogram trying to write the
  # same diagnostic out (datastore contention), at the cost of copyin the
  # data. These are sparsely written to datastore anyway, so the extra
  # storage should be minimal.
  for d in diagnostics.itervalues():
    d.ResetGuid()

  diagnostics = {k: d.AsDict() for k, d in diagnostics.iteritems()}

  params['diagnostics'] = diagnostics
  params['data'] = hist.AsDict()

  return params

# TODO(eakuefner): Clean this up by making it accept raw diagnostics.
# TODO(eakuefner): Move this helper along with others to a common place.
@ndb.synctasklet
def DeduplicateAndPut(new_entities, test, rev):
  result = yield DeduplicateAndPutAsync(new_entities, test, rev)
  raise ndb.Return(result)

@ndb.tasklet
def DeduplicateAndPutAsync(new_entities, test, rev):
  query = histogram.SparseDiagnostic.query(
      ndb.AND(
          histogram.SparseDiagnostic.end_revision == sys.maxint,
          histogram.SparseDiagnostic.test == test))
  diagnostic_entities = yield query.fetch_async()
  entity_futures = []
  new_guids_to_existing_diagnostics = {}

  for new_entity in new_entities:
    old_entity = _GetDiagnosticEntityMatchingName(
        new_entity.name, diagnostic_entities)
    if old_entity is not None:
      # Case 1: One in datastore, different from new one.
      if _IsDifferent(old_entity.data, new_entity.data):
        old_entity.end_revision = rev - 1
        entity_futures.append(old_entity.put_async())
        new_entity.start_revision = rev
        new_entity.end_revision = sys.maxint
        entity_futures.append(new_entity.put_async())
      # Case 2: One in datastore, same as new one.
      else:
        new_guids_to_existing_diagnostics[new_entity.key.id()] = old_entity.data
      continue
    # Case 3: Nothing in datastore.
    entity_futures.append(new_entity.put_async())
  yield entity_futures
  raise ndb.Return(new_guids_to_existing_diagnostics)

def _GetDiagnosticEntityMatchingName(name, diagnostic_entities):
  for entity in diagnostic_entities:
    if entity.name == name:
      return entity
  return None

def _IsDifferent(diagnostic_a, diagnostic_b):
  return (diagnostic.Diagnostic.FromDict(diagnostic_a) !=
          diagnostic.Diagnostic.FromDict(diagnostic_b))

def FindSuiteLevelSparseDiagnostics(
    histograms, suite_key, revision, internal_only):
  diagnostics = {}
  for hist in histograms:
    for name, diag in hist.diagnostics.iteritems():
      if name in SUITE_LEVEL_SPARSE_DIAGNOSTIC_NAMES:
        existing_entity = diagnostics.get(name)
        if existing_entity is None:
          diagnostics[name] = histogram.SparseDiagnostic(
              id=diag.guid, data=diag.AsDict(), test=suite_key,
              start_revision=revision, end_revision=sys.maxint, name=name,
              internal_only=internal_only)
        elif existing_entity.key.id() != diag.guid:
          raise ValueError(
              name + ' diagnostics must be the same for all histograms')
  return diagnostics.values()

def FindHistogramLevelSparseDiagnostics(guid, histograms):
  hist = histograms.LookupHistogram(guid)
  diagnostics = {}
  for name, diag in hist.diagnostics.iteritems():
    if name in HISTOGRAM_LEVEL_SPARSE_DIAGNOSTIC_NAMES:
      diagnostics[name] = diag
  return diagnostics

def ComputeTestPath(suite_path, guid, histograms):
  hist = histograms.LookupHistogram(guid)
  path = '%s/%s' % (suite_path, hist.name)

  tir_label = histogram_helpers.GetTIRLabelFromHistogram(hist)
  if tir_label:
    path += '/' + tir_label

  is_ref = hist.diagnostics.get(reserved_infos.IS_REFERENCE_BUILD.name)
  if is_ref and len(is_ref) == 1:
    is_ref = list(is_ref)[0]

  # If a Histogram represents a summary across multiple stories, then its
  # 'stories' diagnostic will contain the names of all of the stories.
  # If a Histogram is not a summary, then its 'stories' diagnostic will contain
  # the singular name of its story.
  story_name = hist.diagnostics.get(reserved_infos.STORIES.name)
  if story_name and len(story_name) == 1:
    escaped_story_name = add_point.EscapeName(list(story_name)[0])
    path += '/' + escaped_story_name
    if is_ref:
      path += '_ref'
  elif is_ref:
    path += '/ref'

  return path

def _GetMasterBotBenchmarkFromHistogram(hist):
  _CheckRequest(
      reserved_infos.MASTERS.name in hist.diagnostics,
      'Histograms must have "%s" diagnostic' % reserved_infos.MASTERS.name)
  master = hist.diagnostics[reserved_infos.MASTERS.name]
  _CheckRequest(
      len(master) == 1,
      'Histograms must have exactly 1 "%s"' % reserved_infos.MASTERS.name)
  master = list(master)[0]

  _CheckRequest(
      reserved_infos.BOTS.name in hist.diagnostics,
      'Histograms must have "%s" diagnostic' % reserved_infos.BOTS.name)
  bot = hist.diagnostics[reserved_infos.BOTS.name]
  _CheckRequest(
      len(bot) == 1,
      'Histograms must have exactly 1 "%s"' % reserved_infos.BOTS.name)
  bot = list(bot)[0]

  _CheckRequest(
      reserved_infos.BENCHMARKS.name in hist.diagnostics,
      'Histograms must have "%s" diagnostic' % reserved_infos.BENCHMARKS.name)
  benchmark = hist.diagnostics[reserved_infos.BENCHMARKS.name]
  _CheckRequest(
      len(benchmark) == 1,
      'Histograms must have exactly 1 "%s"' % reserved_infos.BENCHMARKS.name)
  benchmark = list(benchmark)[0]

  return master, bot, benchmark

def ComputeRevision(histograms):
  _CheckRequest(len(histograms) > 0, 'Must upload at least one histogram')
  diagnostics = histograms.GetFirstHistogram().diagnostics
  _CheckRequest(reserved_infos.CHROMIUM_COMMIT_POSITIONS.name in diagnostics,
                'Histograms must have Chromium commit position attached')
  chromium_commit_position = list(diagnostics[
      reserved_infos.CHROMIUM_COMMIT_POSITIONS.name])

  _CheckRequest(len(chromium_commit_position) == 1,
                'Chromium commit position must have 1 value')

  # TODO(eakuefner): Allow users to specify other types of revisions to be used
  # for computing revisions of dashboard points. See
  # https://github.com/catapult-project/catapult/issues/3623.
  return chromium_commit_position[0]

def InlineDenseSharedDiagnostics(histograms):
  # TODO(eakuefner): Delete inlined diagnostics from the set
  for hist in histograms:
    diagnostics = hist.diagnostics
    for name, diag in diagnostics.iteritems():
      if name not in SPARSE_DIAGNOSTIC_NAMES:
        diag.Inline()
<commit_msg>Dashboard - Log warnings on duplicate histograms.<commit_after># Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""URL endpoint for adding new histograms to the dashboard."""

import json
import logging
import sys
import traceback

from google.appengine.api import taskqueue
from google.appengine.ext import ndb

from dashboard import add_point
from dashboard import add_point_queue
from dashboard.api import api_request_handler
from dashboard.common import datastore_hooks
from dashboard.common import histogram_helpers
from dashboard.common import stored_object
from dashboard.common import utils
from dashboard.models import histogram
from tracing.value import histogram_set
from tracing.value.diagnostics import diagnostic
from tracing.value.diagnostics import reserved_infos

SUITE_LEVEL_SPARSE_DIAGNOSTIC_NAMES = set([
    reserved_infos.ARCHITECTURES.name,
    reserved_infos.BENCHMARKS.name,
    reserved_infos.BOTS.name,
    reserved_infos.BUG_COMPONENTS.name,
    reserved_infos.GPUS.name,
    reserved_infos.MASTERS.name,
    reserved_infos.MEMORY_AMOUNTS.name,
    reserved_infos.OS_NAMES.name,
    reserved_infos.OS_VERSIONS.name,
    reserved_infos.OWNERS.name,
    reserved_infos.PRODUCT_VERSIONS.name,
    reserved_infos.TAG_MAP.name,
])

HISTOGRAM_LEVEL_SPARSE_DIAGNOSTIC_NAMES = set([
    reserved_infos.DEVICE_IDS.name,
    reserved_infos.RELATED_NAMES.name,
    reserved_infos.STORIES.name,
    reserved_infos.STORYSET_REPEATS.name,
    reserved_infos.STORY_TAGS.name,
])

SPARSE_DIAGNOSTIC_NAMES = SUITE_LEVEL_SPARSE_DIAGNOSTIC_NAMES.union(
    HISTOGRAM_LEVEL_SPARSE_DIAGNOSTIC_NAMES)

TASK_QUEUE_NAME = 'histograms-queue'

def _CheckRequest(condition, msg):
  if not condition:
    raise api_request_handler.BadRequestError(msg)

class AddHistogramsHandler(api_request_handler.ApiRequestHandler):

  def AuthorizedPost(self):
    datastore_hooks.SetPrivilegedRequest()

    try:
      data_str = self.request.get('data')
      if not data_str:
        raise api_request_handler.BadRequestError('Missing "data" parameter')

      logging.info('Received data: %s', data_str)

      histogram_dicts = json.loads(data_str)
      ProcessHistogramSet(histogram_dicts)
    except api_request_handler.BadRequestError:
      # TODO(simonhatch, eakuefner: Remove this later.
      # When this has all stabilized a bit, remove and let this 400 to clients,
      # but for now to preven the waterfall from re-uploading over and over
      # while we bug fix, let's just log the error.
      # https://github.com/catapult-project/catapult/issues/4019
      logging.error(traceback.format_exc())
    except Exception:  # pylint: disable=broad-except
      # TODO(simonhatch, eakuefner: Remove this later.
      # We shouldn't be catching ALL exceptions, this is just while the
      # stability of the endpoint is being worked on.
      logging.error(traceback.format_exc())

def _LogDebugInfo(histograms):
  hist = histograms.GetFirstHistogram()
  if not hist:
    logging.info('No histograms in data.')
    return

  log_urls = hist.diagnostics.get(reserved_infos.LOG_URLS.name)
  if log_urls:
    log_urls = list(log_urls)
    logging.info('Buildbot URL: %s', str(log_urls))
  else:
    logging.info('No LOG_URLS in data.')

def ProcessHistogramSet(histogram_dicts):
  if not isinstance(histogram_dicts, list):
    raise api_request_handler.BadRequestError(
        'HistogramSet JSON much be a list of dicts')

  bot_whitelist_future = stored_object.GetAsync(
      add_point_queue.BOT_WHITELIST_KEY)

  histograms = histogram_set.HistogramSet()
  histograms.ImportDicts(histogram_dicts)
  histograms.ResolveRelatedHistograms()
  histograms.DeduplicateDiagnostics()

  if len(histograms) == 0:
    raise api_request_handler.BadRequestError(
        'HistogramSet JSON must contain at least one histogram.')

  _LogDebugInfo(histograms)

  InlineDenseSharedDiagnostics(histograms)
  revision = ComputeRevision(histograms)
  master, bot, benchmark = _GetMasterBotBenchmarkFromHistogram(
      histograms.GetFirstHistogram())
  suite_key = utils.TestKey('%s/%s/%s' % (master, bot, benchmark))

  bot_whitelist = bot_whitelist_future.get_result()
  internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)

  # We'll skip the histogram-level sparse diagnostics because we need to
  # handle those with the histograms, below, so that we can properly assign
  # test paths.
  suite_level_sparse_diagnostic_entities = FindSuiteLevelSparseDiagnostics(
      histograms, suite_key, revision, internal_only)

  # TODO(eakuefner): Refactor master/bot computation to happen above this line
  # so that we can replace with a DiagnosticRef rather than a full diagnostic.
  new_guids_to_old_diagnostics = DeduplicateAndPut(
      suite_level_sparse_diagnostic_entities, suite_key, revision)
  for new_guid, old_diagnostic in new_guids_to_old_diagnostics.iteritems():
    histograms.ReplaceSharedDiagnostic(
        new_guid, diagnostic.Diagnostic.FromDict(old_diagnostic))

  tasks = _BatchHistogramsIntoTasks(suite_key.id(), histograms, revision)

  _QueueHistogramTasks(tasks)

def _MakeTask(params):
  return taskqueue.Task(
      url='/add_histograms_queue', payload=json.dumps(params),
      _size_check=False)

def _BatchHistogramsIntoTasks(suite_path, histograms, revision):
  params = []
  tasks = []

  base_size = _MakeTask([]).size
  estimated_size = 0

  duplicate_check = set()

  for hist in histograms:
    diagnostics = FindHistogramLevelSparseDiagnostics(hist.guid, histograms)

    # TODO(eakuefner): Don't compute full diagnostics, because we need anyway to
    # call GetOrCreate here and in the queue.
    test_path = ComputeTestPath(suite_path, hist.guid, histograms)

    if test_path in duplicate_check:
      logging.warning('Duplicate histogram detected: %s', test_path)
    duplicate_check.add(test_path)

    # TODO(eakuefner): Batch these better than one per task.
    task_dict = _MakeTaskDict(hist, test_path, revision, diagnostics)

    estimated_size += len(json.dumps(task_dict))

    # Creating the task directly and getting the size back is slow, so we just
    # keep a running total of estimated task size. A bit hand-wavy but the #
    # of histograms per task doesn't need to be perfect, just has to be under
    # the max task size.
    estimated_total_size = estimated_size * 1.05 + base_size + 1024
    if estimated_total_size > taskqueue.MAX_TASK_SIZE_BYTES:
      t = _MakeTask(params)
      tasks.append(t)
      params = []
      estimated_size = 0
    params.append(task_dict)

  if params:
    t = _MakeTask(params)
    tasks.append(t)

  return tasks

def _QueueHistogramTasks(tasks):
  queue = taskqueue.Queue(TASK_QUEUE_NAME)
  futures = []
  for i in xrange(0, len(tasks), taskqueue.MAX_TASKS_PER_ADD):
    f = queue.add_async(tasks[i:i + taskqueue.MAX_TASKS_PER_ADD])
    futures.append(f)
  for f in futures:
    f.get_result()

def _MakeTaskDict(hist, test_path, revision, diagnostics):
  # TODO(simonhatch): "revision" is common to all tasks, as is the majority of
  # the test path
  params = {
      'test_path': test_path,
      'revision': revision
  }

  # By changing the GUID just before serializing the task, we're making it
  # unique for each histogram. This avoids each histogram trying to write the
  # same diagnostic out (datastore contention), at the cost of copyin the
  # data. These are sparsely written to datastore anyway, so the extra
  # storage should be minimal.
  for d in diagnostics.itervalues():
    d.ResetGuid()

  diagnostics = {k: d.AsDict() for k, d in diagnostics.iteritems()}

  params['diagnostics'] = diagnostics
  params['data'] = hist.AsDict()

  return params

# TODO(eakuefner): Clean this up by making it accept raw diagnostics.
# TODO(eakuefner): Move this helper along with others to a common place.
@ndb.synctasklet
def DeduplicateAndPut(new_entities, test, rev):
  result = yield DeduplicateAndPutAsync(new_entities, test, rev)
  raise ndb.Return(result)

@ndb.tasklet
def DeduplicateAndPutAsync(new_entities, test, rev):
  query = histogram.SparseDiagnostic.query(
      ndb.AND(
          histogram.SparseDiagnostic.end_revision == sys.maxint,
          histogram.SparseDiagnostic.test == test))
  diagnostic_entities = yield query.fetch_async()
  entity_futures = []
  new_guids_to_existing_diagnostics = {}

  for new_entity in new_entities:
    old_entity = _GetDiagnosticEntityMatchingName(
        new_entity.name, diagnostic_entities)
    if old_entity is not None:
      # Case 1: One in datastore, different from new one.
      if _IsDifferent(old_entity.data, new_entity.data):
        old_entity.end_revision = rev - 1
        entity_futures.append(old_entity.put_async())
        new_entity.start_revision = rev
        new_entity.end_revision = sys.maxint
        entity_futures.append(new_entity.put_async())
      # Case 2: One in datastore, same as new one.
      else:
        new_guids_to_existing_diagnostics[new_entity.key.id()] = old_entity.data
      continue
    # Case 3: Nothing in datastore.
    entity_futures.append(new_entity.put_async())
  yield entity_futures
  raise ndb.Return(new_guids_to_existing_diagnostics)

def _GetDiagnosticEntityMatchingName(name, diagnostic_entities):
  for entity in diagnostic_entities:
    if entity.name == name:
      return entity
  return None

def _IsDifferent(diagnostic_a, diagnostic_b):
  return (diagnostic.Diagnostic.FromDict(diagnostic_a) !=
          diagnostic.Diagnostic.FromDict(diagnostic_b))

def FindSuiteLevelSparseDiagnostics(
    histograms, suite_key, revision, internal_only):
  diagnostics = {}
  for hist in histograms:
    for name, diag in hist.diagnostics.iteritems():
      if name in SUITE_LEVEL_SPARSE_DIAGNOSTIC_NAMES:
        existing_entity = diagnostics.get(name)
        if existing_entity is None:
          diagnostics[name] = histogram.SparseDiagnostic(
              id=diag.guid, data=diag.AsDict(), test=suite_key,
              start_revision=revision, end_revision=sys.maxint, name=name,
              internal_only=internal_only)
        elif existing_entity.key.id() != diag.guid:
          raise ValueError(
              name + ' diagnostics must be the same for all histograms')
  return diagnostics.values()

def FindHistogramLevelSparseDiagnostics(guid, histograms):
  hist = histograms.LookupHistogram(guid)
  diagnostics = {}
  for name, diag in hist.diagnostics.iteritems():
    if name in HISTOGRAM_LEVEL_SPARSE_DIAGNOSTIC_NAMES:
      diagnostics[name] = diag
  return diagnostics

def ComputeTestPath(suite_path, guid, histograms):
  hist = histograms.LookupHistogram(guid)
  path = '%s/%s' % (suite_path, hist.name)

  tir_label = histogram_helpers.GetTIRLabelFromHistogram(hist)
  if tir_label:
    path += '/' + tir_label

  is_ref = hist.diagnostics.get(reserved_infos.IS_REFERENCE_BUILD.name)
  if is_ref and len(is_ref) == 1:
    is_ref = list(is_ref)[0]

  # If a Histogram represents a summary across multiple stories, then its
  # 'stories' diagnostic will contain the names of all of the stories.
  # If a Histogram is not a summary, then its 'stories' diagnostic will contain
  # the singular name of its story.
  story_name = hist.diagnostics.get(reserved_infos.STORIES.name)
  if story_name and len(story_name) == 1:
    escaped_story_name = add_point.EscapeName(list(story_name)[0])
    path += '/' + escaped_story_name
    if is_ref:
      path += '_ref'
  elif is_ref:
    path += '/ref'

  return path

def _GetMasterBotBenchmarkFromHistogram(hist):
  _CheckRequest(
      reserved_infos.MASTERS.name in hist.diagnostics,
      'Histograms must have "%s" diagnostic' % reserved_infos.MASTERS.name)
  master = hist.diagnostics[reserved_infos.MASTERS.name]
  _CheckRequest(
      len(master) == 1,
      'Histograms must have exactly 1 "%s"' % reserved_infos.MASTERS.name)
  master = list(master)[0]

  _CheckRequest(
      reserved_infos.BOTS.name in hist.diagnostics,
      'Histograms must have "%s" diagnostic' % reserved_infos.BOTS.name)
  bot = hist.diagnostics[reserved_infos.BOTS.name]
  _CheckRequest(
      len(bot) == 1,
      'Histograms must have exactly 1 "%s"' % reserved_infos.BOTS.name)
  bot = list(bot)[0]

  _CheckRequest(
      reserved_infos.BENCHMARKS.name in hist.diagnostics,
      'Histograms must have "%s" diagnostic' % reserved_infos.BENCHMARKS.name)
  benchmark = hist.diagnostics[reserved_infos.BENCHMARKS.name]
  _CheckRequest(
      len(benchmark) == 1,
      'Histograms must have exactly 1 "%s"' % reserved_infos.BENCHMARKS.name)
  benchmark = list(benchmark)[0]

  return master, bot, benchmark

def ComputeRevision(histograms):
  _CheckRequest(len(histograms) > 0, 'Must upload at least one histogram')
  diagnostics = histograms.GetFirstHistogram().diagnostics
  _CheckRequest(reserved_infos.CHROMIUM_COMMIT_POSITIONS.name in diagnostics,
                'Histograms must have Chromium commit position attached')
  chromium_commit_position = list(diagnostics[
      reserved_infos.CHROMIUM_COMMIT_POSITIONS.name])

  _CheckRequest(len(chromium_commit_position) == 1,
                'Chromium commit position must have 1 value')

  # TODO(eakuefner): Allow users to specify other types of revisions to be used
  # for computing revisions of dashboard points. See
  # https://github.com/catapult-project/catapult/issues/3623.
  return chromium_commit_position[0]

def InlineDenseSharedDiagnostics(histograms):
  # TODO(eakuefner): Delete inlined diagnostics from the set
  for hist in histograms:
    diagnostics = hist.diagnostics
    for name, diag in diagnostics.iteritems():
      if name not in SPARSE_DIAGNOSTIC_NAMES:
        diag.Inline()
<|endoftext|>


<commit_before>"""Test dashboard.services.jira."""

import pytest
import requests_mock
import json

@pytest.fixture
def jira():
    """Fixture for JIRA."""
    from ..services import jira
    return jira

def test_setup(jira):
    """Verify our fixture."""
    assert jira

def test_query_url(jira):
    """Verify explicit arg is used for URL."""
    filter_id = "12345"
    jira_url = "https://jira.purnkleen.com"
    expected = "https://jira.purnkleen.com/rest/api/2/search?jql=filter=12345&maxResults=999"
    assert expected == jira.query_url(filter_id, jira_url)

def test_query_url_w_settings(jira, settings):
    """Verify settings used for URL implicitly."""
    filter_id = "12345"
    expected = "https://jira.purnkleen.com/rest/api/2/search?jql=filter=12345&maxResults=999"
    settings.JIRA_URL = "https://jira.purnkleen.com"
    assert expected == jira.query_url(filter_id)

def test_summarize_results(jira, settings):
    """Verify happy path results."""
    settings.JIRA_DONE = ['Donnager']
    issue_list = {
        'total': 3,
        'issues': [
            {'fields': {'status': {'name': 'In Progress'}}},
            {'fields': {'status': {'name': 'In Progress'}}},
            {'fields': {'status': {'name': 'Donnager'}}},
        ]
    }
    expected = {
        'incomplete': 2,
        'complete': 1,
        'pct_complete': 1 / 3.0,
        'total': 3,
        'errors': [],
    }
    assert jira.summarize_results(issue_list) == expected

def test_summarize_results_with_errors(jira):
    """Pass errors upstream with 0s for values."""
    issue_list = {
        'errorMessages': ["A value with ID '12245' does not exist for the field 'filter'."],
        'errors': {}
    }
    expected = {
        'incomplete': 0,
        'complete': 0,
        'pct_complete': 0,
        'total': 0,
        'errors': [
            "A value with ID '12245' does not exist for the field 'filter'.",
        ]
    }
    assert jira.summarize_results(issue_list) == expected

def test_summarize_query_weird_input(jira):
    """Return None if passed a non-int filter_id."""
    result = jira.summarize_query("FOOBAR")
    assert result == {}

def test_fetch_query_results(settings, jira):
    """Return the results."""
    expected = dict(foo="bar", baz="bat")
    expected_json = json.dumps(expected)
    with requests_mock.mock() as m:
        m.register_uri(requests_mock.ANY, requests_mock.ANY, text=expected_json)
        result = jira.fetch_query_results(43035)
        assert result == expected
<commit_msg>Test error condition with query fetcher<commit_after>"""Test dashboard.services.jira."""

from unittest.mock import patch

import pytest
import requests_mock
import json

@pytest.fixture
def jira():
    """Fixture for JIRA."""
    from ..services import jira
    return jira

def test_setup(jira):
    """Verify our fixture."""
    assert jira

def test_query_url(jira):
    """Verify explicit arg is used for URL."""
    filter_id = "12345"
    jira_url = "https://jira.purnkleen.com"
    expected = "https://jira.purnkleen.com/rest/api/2/search?jql=filter=12345&maxResults=999"
    assert expected == jira.query_url(filter_id, jira_url)

def test_query_url_w_settings(jira, settings):
    """Verify settings used for URL implicitly."""
    filter_id = "12345"
    expected = "https://jira.purnkleen.com/rest/api/2/search?jql=filter=12345&maxResults=999"
    settings.JIRA_URL = "https://jira.purnkleen.com"
    assert expected == jira.query_url(filter_id)

def test_summarize_results(jira, settings):
    """Verify happy path results."""
    settings.JIRA_DONE = ['Donnager']
    issue_list = {
        'total': 3,
        'issues': [
            {'fields': {'status': {'name': 'In Progress'}}},
            {'fields': {'status': {'name': 'In Progress'}}},
            {'fields': {'status': {'name': 'Donnager'}}},
        ]
    }
    expected = {
        'incomplete': 2,
        'complete': 1,
        'pct_complete': 1 / 3.0,
        'total': 3,
        'errors': [],
    }
    assert jira.summarize_results(issue_list) == expected

def test_summarize_results_with_errors(jira):
    """Pass errors upstream with 0s for values."""
    issue_list = {
        'errorMessages': ["A value with ID '12245' does not exist for the field 'filter'."],
        'errors': {}
    }
    expected = {
        'incomplete': 0,
        'complete': 0,
        'pct_complete': 0,
        'total': 0,
        'errors': [
            "A value with ID '12245' does not exist for the field 'filter'.",
        ]
    }
    assert jira.summarize_results(issue_list) == expected

def test_summarize_query_weird_input(jira):
    """Return None if passed a non-int filter_id."""
    result = jira.summarize_query("FOOBAR")
    assert result == {}

def test_fetch_query_results(settings, jira):
    """Return the results."""
    expected = dict(foo="bar", baz="bat")
    expected_json = json.dumps(expected)
    with requests_mock.mock() as m:
        m.register_uri(requests_mock.ANY, requests_mock.ANY, text=expected_json)
        result = jira.fetch_query_results(43035)
        assert result == expected

def test_fetch_query_results_with_errors(settings, jira):
    """JIRA responses with error messages should be logged and returned."""
    expected = dict(
        errors=[],
        errorMessages=["Was a boring conversation anyway... Luke we're gonna have company!", ],
    )
    expected_json = json.dumps(expected)
    with requests_mock.mock() as m:
        with patch("dashboard.services.jira.LOGGER") as mocked_logger:
            m.register_uri(requests_mock.ANY, requests_mock.ANY, text=expected_json)
            result = jira.fetch_query_results(43035, logger=mocked_logger)
            assert mocked_logger.warning.called
            assert result == expected
<|endoftext|>


<commit_before># -*- encoding: utf-8; py-indent-offset: 4 -*-
# +--------------------------------------------------------------------------+
# |   _____ __            ___          __    _       __                      |
# |  / ___// /___  ______/ (_)___     / /   (_)___  / /__                    |
# |  \__ \/ __/ / / / __  / / __ \   / /   / / __ \/ //_/                    |
# | ___/ / /_/ /_/ / /_/ / / /_/ /  / /___/ / / / / ,<                       |
# |/____/\__/\__,_/\__,_/_/\____/  /_____/_/_/ /_/_/|_|                      |
# |Copyright Sebastian Reimers 2013 - 2014 studio-link.de                    |
# |License: BSD-2-Clause (see LICENSE File)                                  |
# +--------------------------------------------------------------------------+

from flask import Blueprint, render_template, redirect, request, url_for, flash
from app.models.accounts import Accounts
from app.forms.dial import DialForm
from app.libs import baresip
import time
import json
import redis
import re

mod = Blueprint('calls', __name__, url_prefix='/calls')
store = redis.Redis('127.0.0.1')

@mod.route('/', methods=('GET', 'POST'))
def index():
    form = DialForm(request.form)
    db_accounts = Accounts.query.all()
    accounts = []
    accounts.append(('studio@lan', 'Local'))

    for account in db_accounts:
        sip = account.username + '@' + account.server
        accounts.append((sip, account.name))
    form.accounts.choices = accounts

    if form.validate_on_submit():
        baresip.set('user_agent', form.accounts.data)
        baresip.set('dial', form.number.data)
        store.set('call_account', form.accounts.data)
        return redirect('/calls/dial')
    if form.errors:
        for error in form.errors:
            flash(error+': '+form.errors[error][0], 'danger')

    call_account = store.get('call_account')
    if call_account:
        form.accounts.data = call_account

    return render_template('calls/index.html', form=form)

@mod.route('/dial')
def dial():
    return render_template('calls/dial.html',
                           call_number=store.get('call_number'),
                           call_account=store.get('call_account'),
                           baresip=baresip.get('list')
                           )

@mod.route('/answer')
def answer():
    baresip.set('answer', store.get('baresip_ua'))
    store.set('oncall', 'true')
    store.set('baresip_status', 'ACCEPTED')
    store.set('oncalltext', 'CALL: ACCEPTED')
    return json.dumps({'return': True})

@mod.route('/hangup')
def hangup():
    baresip.set('hangup', store.get('baresip_ua'))
    store.set('oncall', 'false')
    store.set('baresip_status', '')
    return redirect('/calls/')
<commit_msg>fix call hangup bug (empty user agent)<commit_after># -*- encoding: utf-8; py-indent-offset: 4 -*-
# +--------------------------------------------------------------------------+
# |   _____ __            ___          __    _       __                      |
# |  / ___// /___  ______/ (_)___     / /   (_)___  / /__                    |
# |  \__ \/ __/ / / / __  / / __ \   / /   / / __ \/ //_/                    |
# | ___/ / /_/ /_/ / /_/ / / /_/ /  / /___/ / / / / ,<                       |
# |/____/\__/\__,_/\__,_/_/\____/  /_____/_/_/ /_/_/|_|                      |
# |Copyright Sebastian Reimers 2013 - 2014 studio-link.de                    |
# |License: BSD-2-Clause (see LICENSE File)                                  |
# +--------------------------------------------------------------------------+

from flask import Blueprint, render_template, redirect, request, url_for, flash
from app.models.accounts import Accounts
from app.forms.dial import DialForm
from app.libs import baresip
import time
import json
import redis
import re

mod = Blueprint('calls', __name__, url_prefix='/calls')
store = redis.Redis('127.0.0.1')

@mod.route('/', methods=('GET', 'POST'))
def index():
    form = DialForm(request.form)
    db_accounts = Accounts.query.all()
    accounts = []
    accounts.append(('studio@lan', 'Local'))

    for account in db_accounts:
        sip = account.username + '@' + account.server
        accounts.append((sip, account.name))
    form.accounts.choices = accounts

    if form.validate_on_submit():
        baresip.set('user_agent', form.accounts.data)
        baresip.set('dial', form.number.data)
        store.set('call_account', form.accounts.data)
        return redirect('/calls/dial')
    if form.errors:
        for error in form.errors:
            flash(error+': '+form.errors[error][0], 'danger')

    call_account = store.get('call_account')
    if call_account:
        form.accounts.data = call_account

    return render_template('calls/index.html', form=form)

@mod.route('/dial')
def dial():
    return render_template('calls/dial.html',
                           call_number=store.get('call_number'),
                           call_account=store.get('call_account'),
                           baresip=baresip.get('list')
                           )

@mod.route('/answer')
def answer():
    baresip.set('answer', store.get('baresip_ua'))
    store.set('oncall', 'true')
    store.set('baresip_status', 'ACCEPTED')
    store.set('oncalltext', 'CALL: ACCEPTED')
    return json.dumps({'return': True})

@mod.route('/hangup')
def hangup():
    agent = store.get('baresip_ua')
    if not agent:
        agent = store.get('call_account')
    baresip.set('hangup', agent)
    store.set('oncall', 'false')
    store.set('baresip_status', '')
    return redirect('/calls/')
<|endoftext|>


<commit_before>import json
import urlparse
import requests

# Freshdesk uses some exotic return codes

HTTP_ALREADY_EXISTS = 422

class FreshDeskObjects(object):
    """ Abstract class to wrap a type of freshdesk resource.
    """
    def __init__(self, client):
        self.client = client

    def api_endpoint(self, id=None):
        if id:
            return '/{}s/{}.json'.format(self.api_name, id)
        else:
            return '/{}s.json'.format(self.api_name)

    # CRUD methods

    def create(self, **kwargs):
        return self.client.req(
            requests.post, self.api_endpoint(), self.wrapper_name,
            **kwargs)

    def update(self, id, **kwargs):
        return self.client.req(
            requests.put, self.api_endpoint(id), self.wrapper_name,
            **kwargs)

    def delete(self, id):
        return self.client.req(
            requests.delete, self.api_endpoint(id), self.wrapper_name)

    def get(self, id):
        return self.client.req(
            requests.get, self.api_endpoint(id), self.wrapper_name)

    def get_list(self, remove_pagination=False, **params):
        """
        :type  remove_pagination boolean
        :param remove_pagination if True, will fetch all pages and present
                                 result as a single list (use with caution)
        :rtype                   a list of dicts
        """
        full_list = []
        page = 1
        while True:
            resp = self.client.req(
                requests.get, self.api_endpoint(), self.wrapper_name,
                params=params)
            full_list += resp
            if len(resp) <= 0 or not remove_pagination:
                break
            else:
                page += 1
                params['page'] = page

        return full_list

class FreshDeskContacts(FreshDeskObjects):
    """ http://freshdesk.com/api#user
    """
    api_name = 'contact'
    wrapper_name = 'user'

    # Status names (default is state=verified)
    DELETED    = 'deleted'
    ALL        = 'all'         # Do not include deleted accounts
    VERIFIED   = 'verified'
    UNVERIFIED = 'unverified'

    def create(self, name, email, **kwargs):
        return super(FreshDeskContacts, self).create(name=name, email=email, **kwargs)

    def create_or_enable(self, name, email, **kwargs):
        """ If creation fails because user exists in deleted state, restore it.

        Freshdesk do not allow two users with same email, even if the collision
        is with a deleted user.
        """
        try:
            return self.create(name=name, email=email, **kwargs)
        except FreshDeskClient.APIError as e:
            contacts = self.get_list(
                state='deleted', query='email is {}'.format(email))
            if ((len(contacts) > 0) and
                (e.resp.status_code == HTTP_ALREADY_EXISTS)):
                contact = contacts[0]
                self.update(contact['id'], name=name, deleted=False, **kwargs)
                # update local version as well
                contact[name] = name
                contact.update(kwargs)

                return contact
            else:
                raise

    def get_or_create(self, name, email, **kwargs):
        """ Try to get the contact by email, and creates it if it do not exist.

        :returns a boolean (true if created) and the object dict itself
        """
        try:
            obj = self.create_or_enable(name, email, **kwargs)
            created = True
        except FreshDeskClient.APIError as e:
            if e.resp.status_code == HTTP_ALREADY_EXISTS:
                contacts = self.get_list(
                    state='all', query='email is {}'.format(email))
                obj = contacts[0]
                created = False
            else:
                raise
        return created, obj

class FreshDeskCustomers(FreshDeskObjects):
    """ http://freshdesk.com/api#companies
    """
    api_name = 'customer'
    wrapper_name = api_name

    def create(self, name, **kwargs):
        return super(FreshDeskCustomers, self).create(name=name, **kwargs)

class FreshDeskClient(object):
    """ Simple wrapper arround freshdesk REST API

    This wrapper is non neutral, it includes Oasiswork policy about default
    settings.

        cli = FreshDeskClient('https://mycompany.freshdesk.com', 'qwertyui')
        cli.contacts.get_list(state='all')

    """
    class APIError(Exception):
        def __init__(self, resp):
            self.resp = resp
        def __str__(self):
            return 'HTTP {}: {}'.format(self.resp.status_code, self.resp.text)

    def __init__(self, url, key):
        """
        :param url  endpoint url, eg: https://yourcompany.freshdesk.com
        :param key  API key
        """
        self.url = url
        self.key = key
        self.last_resp = None

        # Resources types
        self.customers = FreshDeskCustomers(self)
        self.contacts = FreshDeskContacts(self)

    def req(self, func, path, resource_type, params={}, **kwargs):
        abs_url = urlparse.urljoin(self.url, path)

        if func in (requests.patch, requests.put, requests.post):
            req_attrs = {
                'data': json.dumps({resource_type: kwargs}),
                'headers': {'Content-Type': 'application/json'}
            }
        else:
            req_attrs = {}

        resp =  func(
            abs_url,
            auth=(self.key, 'nopassword'),
            params=params,
            **req_attrs)
        self.last_resp = resp
        if resp.ok:
            # for those two types of requests, the output is empty
            if func in (requests.delete, requests.put):
                return resp.text
            else:
                # Either we get a list or a single object
                json_obj = resp.json()
                if isinstance(json_obj, (list, tuple)):
                    return [i[resource_type] for i in json_obj]
                else:
                    return json_obj[resource_type]
        else:
            raise self.APIError(resp)

class FreshDeskSolutionArticle(FreshDeskObjects):
    """ http://freshdesk.com/api#solution_article_attributes
    """
    api_name = 'article'
    wrapper_name = 'solution_article'

    # id is a dict with follwing keys : category, folder, [article]
    def api_endpoint(self, id):
        """
        :param id :dic
        """
        if api_name in id and 'folder' in id and 'category' in id:
            return '/solution/categories/{}/folders/{}/articles/{}s.json'.format(id['category'], id['folder'], api_name)
        # No article id means all articles
        elif 'folder' in id and 'category' in id:
            return '/solution/categories/{}/folders/{}.json'.format(id['category'], id['folder'])

    def create(self, id, **kwargs):
        return self.client.req(requests.post, self.api_endpoint(id), self.wrapper_name, **kwargs)
<commit_msg>Change urlparse import for python3<commit_after>import json
try:
    import urlparse
except ImportError:
    import urllib.parse as urlparse
import requests

# Freshdesk uses some exotic return codes

HTTP_ALREADY_EXISTS = 422

class FreshDeskObjects(object):
    """ Abstract class to wrap a type of freshdesk resource.
    """
    def __init__(self, client):
        self.client = client

    def api_endpoint(self, id=None):
        if id:
            return '/{}s/{}.json'.format(self.api_name, id)
        else:
            return '/{}s.json'.format(self.api_name)

    # CRUD methods

    def create(self, **kwargs):
        return self.client.req(
            requests.post, self.api_endpoint(), self.wrapper_name,
            **kwargs)

    def update(self, id, **kwargs):
        return self.client.req(
            requests.put, self.api_endpoint(id), self.wrapper_name,
            **kwargs)

    def delete(self, id):
        return self.client.req(
            requests.delete, self.api_endpoint(id), self.wrapper_name)

    def get(self, id):
        return self.client.req(
            requests.get, self.api_endpoint(id), self.wrapper_name)

    def get_list(self, remove_pagination=False, **params):
        """
        :type  remove_pagination boolean
        :param remove_pagination if True, will fetch all pages and present
                                 result as a single list (use with caution)
        :rtype                   a list of dicts
        """
        full_list = []
        page = 1
        while True:
            resp = self.client.req(
                requests.get, self.api_endpoint(), self.wrapper_name,
                params=params)
            full_list += resp
            if len(resp) <= 0 or not remove_pagination:
                break
            else:
                page += 1
                params['page'] = page

        return full_list

class FreshDeskContacts(FreshDeskObjects):
    """ http://freshdesk.com/api#user
    """
    api_name = 'contact'
    wrapper_name = 'user'

    # Status names (default is state=verified)
    DELETED    = 'deleted'
    ALL        = 'all'         # Do not include deleted accounts
    VERIFIED   = 'verified'
    UNVERIFIED = 'unverified'

    def create(self, name, email, **kwargs):
        return super(FreshDeskContacts, self).create(name=name, email=email, **kwargs)

    def create_or_enable(self, name, email, **kwargs):
        """ If creation fails because user exists in deleted state, restore it.

        Freshdesk do not allow two users with same email, even if the collision
        is with a deleted user.
        """
        try:
            return self.create(name=name, email=email, **kwargs)
        except FreshDeskClient.APIError as e:
            contacts = self.get_list(
                state='deleted', query='email is {}'.format(email))
            if ((len(contacts) > 0) and
                (e.resp.status_code == HTTP_ALREADY_EXISTS)):
                contact = contacts[0]
                self.update(contact['id'], name=name, deleted=False, **kwargs)
                # update local version as well
                contact[name] = name
                contact.update(kwargs)

                return contact
            else:
                raise

    def get_or_create(self, name, email, **kwargs):
        """ Try to get the contact by email, and creates it if it do not exist.

        :returns a boolean (true if created) and the object dict itself
        """
        try:
            obj = self.create_or_enable(name, email, **kwargs)
            created = True
        except FreshDeskClient.APIError as e:
            if e.resp.status_code == HTTP_ALREADY_EXISTS:
                contacts = self.get_list(
                    state='all', query='email is {}'.format(email))
                obj = contacts[0]
                created = False
            else:
                raise
        return created, obj

class FreshDeskCustomers(FreshDeskObjects):
    """ http://freshdesk.com/api#companies
    """
    api_name = 'customer'
    wrapper_name = api_name

    def create(self, name, **kwargs):
        return super(FreshDeskCustomers, self).create(name=name, **kwargs)

class FreshDeskClient(object):
    """ Simple wrapper arround freshdesk REST API

    This wrapper is non neutral, it includes Oasiswork policy about default
    settings.

        cli = FreshDeskClient('https://mycompany.freshdesk.com', 'qwertyui')
        cli.contacts.get_list(state='all')

    """
    class APIError(Exception):
        def __init__(self, resp):
            self.resp = resp
        def __str__(self):
            return 'HTTP {}: {}'.format(self.resp.status_code, self.resp.text)

    def __init__(self, url, key):
        """
        :param url  endpoint url, eg: https://yourcompany.freshdesk.com
        :param key  API key
        """
        self.url = url
        self.key = key
        self.last_resp = None

        # Resources types
        self.customers = FreshDeskCustomers(self)
        self.contacts = FreshDeskContacts(self)

    def req(self, func, path, resource_type, params={}, **kwargs):
        abs_url = urlparse.urljoin(self.url, path)

        if func in (requests.patch, requests.put, requests.post):
            req_attrs = {
                'data': json.dumps({resource_type: kwargs}),
                'headers': {'Content-Type': 'application/json'}
            }
        else:
            req_attrs = {}

        resp =  func(
            abs_url,
            auth=(self.key, 'nopassword'),
            params=params,
            **req_attrs)
        self.last_resp = resp
        if resp.ok:
            # for those two types of requests, the output is empty
            if func in (requests.delete, requests.put):
                return resp.text
            else:
                # Either we get a list or a single object
                json_obj = resp.json()
                if isinstance(json_obj, (list, tuple)):
                    return [i[resource_type] for i in json_obj]
                else:
                    return json_obj[resource_type]
        else:
            raise self.APIError(resp)

class FreshDeskSolutionArticle(FreshDeskObjects):
    """ http://freshdesk.com/api#solution_article_attributes
    """
    api_name = 'article'
    wrapper_name = 'solution_article'

    # id is a dict with follwing keys : category, folder, [article]
    def api_endpoint(self, id):
        """
        :param id :dic
        """
        if api_name in id and 'folder' in id and 'category' in id:
            return '/solution/categories/{}/folders/{}/articles/{}s.json'.format(id['category'], id['folder'], api_name)
        # No article id means all articles
        elif 'folder' in id and 'category' in id:
            return '/solution/categories/{}/folders/{}.json'.format(id['category'], id['folder'])

    def create(self, id, **kwargs):
        return self.client.req(requests.post, self.api_endpoint(id), self.wrapper_name, **kwargs)
<|endoftext|>


<commit_before># -*- coding: utf8 -*-
import logging
from time import sleep
from os import listdir
from os.path import isfile, join
from datetime import datetime

from fabric.operations import local

logger = logging.getLogger(__name__)

def local_run(command):
    """
    Makes sure both stderr and stdout is in the fabric.operations.local output
    """
    return local('%s 2>&1' % command, capture=True)

def detect_test_runners(build):
    try:
        files = _list_files(build.working_directory)
    except OSError, e:
        files = []
        logger.error('Could not read files in build %s: \n %s' % (build.id, e.message))
    return _detect_test_runners(files)

def _detect_test_runners(files):
    if 'Makefile' in files:
        return ['make test']
    if 'tox.ini' in files:
        return ['tox', 'flake8']
    if 'setup.py' in files:
        return ['python setup.py test', 'flake8']
    if 'manage.py' in files:
        return ['python manage.py test', 'flake8']
    if 'package.json' in files:
        return ['npm test']
    if 'build.sbt' in files:
        return ['sbt test']
    if 'Cargo.toml' in files:
        return ['cargo test']
    if '_config.yml' in files:
        return ['jekyll build']
    return []

def test__detect_test_runners():
    assert(_detect_test_runners([]) == [])
    assert(_detect_test_runners(['random file']) == [])
    files = ['_config.yml', 'Cargo.toml', 'build.sbt', 'package.json', 'manage.py', 'setup.py',
             'tox.ini', 'Makefile']
    assert(_detect_test_runners(files) == ['make test'])
    del files[len(files) - 1]
    assert(_detect_test_runners(files) == ['tox', 'flake8'])
    del files[len(files) - 1]
    assert(_detect_test_runners(files) == ['python setup.py test', 'flake8'])
    del files[len(files) - 1]
    assert(_detect_test_runners(files) == ['python manage.py test', 'flake8'])
    del files[len(files) - 1]
    assert(_detect_test_runners(files) == ['npm test'])
    del files[len(files) - 1]
    assert(_detect_test_runners(files) == ['sbt test'])
    del files[len(files) - 1]
    assert(_detect_test_runners(files) == ['cargo test'])
    del files[len(files) - 1]
    assert(_detect_test_runners(files) == ['jekyll build'])

def _list_files(path):
    return [f for f in listdir(path) if isfile(join(path, f))]

class CachedProperty(object):
    def __init__(self, func, name=None):
        self.func = func
        self.__doc__ = getattr(func, '__doc__')
        self.name = name or func.__name__

    def __get__(self, instance, type=None):
        if instance is None:
            return self
        res = instance.__dict__[self.name] = self.func(instance)
        return res

cached_property = CachedProperty

def test_cached_property():
    class A(object):
        @cached_property
        def func(self):
            return datetime.now().microsecond

    a = A()
    first = a.func
    sleep(0.1)
    last = a.func
    assert(first == last)
<commit_msg>helper: Add npm install before npm test<commit_after># -*- coding: utf8 -*-
import logging
from time import sleep
from os import listdir
from os.path import isfile, join
from datetime import datetime

from fabric.operations import local

logger = logging.getLogger(__name__)

def local_run(command):
    """
    Makes sure both stderr and stdout is in the fabric.operations.local output
    """
    return local('%s 2>&1' % command, capture=True)

def detect_test_runners(build):
    try:
        files = _list_files(build.working_directory)
    except OSError, e:
        files = []
        logger.error('Could not read files in build %s: \n %s' % (build.id, e.message))
    return _detect_test_runners(files)

def _detect_test_runners(files):
    if 'Makefile' in files:
        return ['make test']
    if 'tox.ini' in files:
        return ['tox', 'flake8']
    if 'setup.py' in files:
        return ['python setup.py test', 'flake8']
    if 'manage.py' in files:
        return ['python manage.py test', 'flake8']
    if 'package.json' in files:
        return ['npm install', 'npm test']
    if 'build.sbt' in files:
        return ['sbt test']
    if 'Cargo.toml' in files:
        return ['cargo test']
    if '_config.yml' in files:
        return ['jekyll build']
    return []

def test__detect_test_runners():
    assert(_detect_test_runners([]) == [])
    assert(_detect_test_runners(['random file']) == [])
    files = ['_config.yml', 'Cargo.toml', 'build.sbt', 'package.json', 'manage.py', 'setup.py',
             'tox.ini', 'Makefile']
    assert(_detect_test_runners(files) == ['make test'])
    del files[len(files) - 1]
    assert(_detect_test_runners(files) == ['tox', 'flake8'])
    del files[len(files) - 1]
    assert(_detect_test_runners(files) == ['python setup.py test', 'flake8'])
    del files[len(files) - 1]
    assert(_detect_test_runners(files) == ['python manage.py test', 'flake8'])
    del files[len(files) - 1]
    assert(_detect_test_runners(files) == ['npm test'])
    del files[len(files) - 1]
    assert(_detect_test_runners(files) == ['sbt test'])
    del files[len(files) - 1]
    assert(_detect_test_runners(files) == ['cargo test'])
    del files[len(files) - 1]
    assert(_detect_test_runners(files) == ['jekyll build'])

def _list_files(path):
    return [f for f in listdir(path) if isfile(join(path, f))]

class CachedProperty(object):
    def __init__(self, func, name=None):
        self.func = func
        self.__doc__ = getattr(func, '__doc__')
        self.name = name or func.__name__

    def __get__(self, instance, type=None):
        if instance is None:
            return self
        res = instance.__dict__[self.name] = self.func(instance)
        return res

cached_property = CachedProperty

def test_cached_property():
    class A(object):
        @cached_property
        def func(self):
            return datetime.now().microsecond

    a = A()
    first = a.func
    sleep(0.1)
    last = a.func
    assert(first == last)
<|endoftext|>


<commit_before># encoding: UTF-8
""" Rules functions """
import re
from _ctypes import ArgumentError
from datetime import date, datetime
import re
from calendar import prcal
from re import search
from idlelib.ReplaceDialog import replace

INT_REGEXP = '(0|[1-9]\d*)'
IS_INT = re.compile('^%s$' % INT_REGEXP)
IS_RANGE = re.compile('^%s\.\.%s$' % (INT_REGEXP, INT_REGEXP))

def to_int(text):
    """ Convert text to int
        Args:
            text (string): number in string format
        Throws:
            ArgumentError: text has unsupported format
        Returns:
            int
    """
    m = IS_INT.match(str(text).strip())
    if m:
        return int(m.group(1))
    raise ArgumentError('Invalid number string: "%s"' % text)

def to_range(text):
    """ Convert text to range tuple
        Args:
            text (string): min..max
        Throws:
            ArgumentError: text has unsupported format
        Returns:
            (int, int): (min, max)
    """
    m = IS_RANGE.match(str(text).strip())
    if m:
        min = int(m.group(1))
        max = int(m.group(2))
        if not max > min:
            raise ArgumentError('Invalid range string: max value (%d) seems to be grater than min (%d)' % (max, min), min, max)
        return (min, max)
    raise ArgumentError('Invalid range string %s' % text)

def within_f(value, range):
    """ Check is value in range
        Args:
            value (string):
            text: range
        Throws:
            ArgumentError
        Returns:
            boolean
    """
    (min, max) = to_range(range)
    value = to_int(value)
    return value >= min and value <= max

def in_f(set, find):
    """ Check is value in set
        Args:
            find (string): string to find
            set (string): comma separated list of value of ranges 1,2,8..10
        Throws:
            ArgumentError
        Returns:
            boolean
    """
    find = str(find).strip()
    for e in set.split(','):
        e = e.strip()
        if find == e:
            # find == set element
            return True
        elif IS_RANGE.match(e):
            # set element is range:
            if within_f(value = find, range = e):
                return True
    return False

def f_and(*args):
    """ Arg1 AND arg2 ..."""
    return reduce(lambda a, b: bool(a) and bool(b), list)

def f_or(*args):
    """ Arg1 or arg2 ... """
    reduce(lambda a, b: bool(a) or bool(b), list)

IS_PCRE = re.compile('^\/(.*)\/(\w*)$')

def build_flags(flags):
    """ Build python regular expression flags from PCRE flags
        Args:
            flags (string): regulare expression flags : ailms
        Returns:
            int:
    """
    ret = 0
    if flags is None:
        return ret
    flags = flags.upper()
    for flag in ('A','I','L','M','S'):
        if flag in flags:
            ret |= getattr(re, flag)
    return ret

def build_regexp(pattern, flags = None):
    """ Build regexp:
            pattern (string): pattern as python regular expression in PCRE (/pattern/flags)
        Returns:
            _sre.SRE_Pattern
    """
    flags = build_flags(flags)
    pcre = IS_PCRE.match(pattern)
    if pcre:
        pattern = pcre.group(1)
        flags = flags | build_flags(pcre.group(2))
    return re.compile(pattern.encode('utf-8'), flags)

def f_mod(l, r):
    return int(l) % int(r)

def f_any(*args):
    return any(args)

def f_all(*args):
    return all(args)

def f_eq(*args):
    if any([type(arg) is int for arg in args]):
        # convert all arguments to int if int argumnt exists
        try:
            args = [int(arg) for arg in args]
        except ValueError:
            # can not convert argument to int <-> !=
            return False
    return all([args[0] == arg for arg in args])

def cmp(arg1, arg2):
    if type(arg1) is date and type(arg2) is date:
        pass
    else:
        arg1 = float(arg1)
        arg2 = float(arg2)
    if arg1 == arg2:
        return '='
    elif arg1 > arg2:
        return '>'
    else:
        return '<'

SUPPORTED_FUNCTIONS = {
    # McCarthy's Elementary S-functions and Predicates
    'quote': lambda expr: expr,
    'car': lambda list: list[1],
    'cons': lambda e, cell: [e] + cell,
    'eq': lambda l, r: l == r,
    'atom': lambda expr: isinstance(expr, (type(None), str, int, float, bool)),
    # Tr8n Extensions
    '=': f_eq,  # ['=', 1, 2]
    '!=': lambda l, r: '=' != cmp(l, r),  # ['!=', 1, 2]
    '<': lambda l, r: '<' == cmp(l, r),  # ['<', 1, 2]
    '>': lambda l, r: '>' == cmp(l, r),  # ['>', 1, 2]
    '+': lambda l, r: float(l) + float(r),  # ['+', 1, 2]
    '-': lambda l, r: float(l) - float(r),  # ['-', 1, 2]
    '*': lambda l, r: float(l) * float(r),  # ['*', 1, 2]
    '%': f_mod,  # ['%', 14, 10]
    'mod': f_mod,  # ['mod', '@n', 10]
    '/': lambda l, r: (l * 1.0) / r,  # ['/', 1, 2]
    '!': lambda expr: not expr,  # ['!', ['true']]
    'not': lambda val: not val,  # ['not', ['true']]
    '&&': f_all,  # ['&&', [], [], ...]
    'and': f_all,  # ['and', [], [], ...]
    '::': f_any,  # ['::', [], [], ...]
    '||': f_any,
    'or': f_any,  # ['or', [], [], ...]
    'if': lambda c, t, f: t if c else f,# ['if', 'cond', 'true', 'false']
    'true': lambda: True,  # ['true']
    'false': lambda: False,  # ['false']
    'date': lambda date: date.strptime(date, '%Y-%m-%d'),# ['date', '2010-01-01']
    'today': lambda: date.today(),  # ['today']
    'time': lambda time: date.strptime(time, '%Y-%m-%d %H:%M:%S'),  # ['time', '2010-01-01 10:10:05']
    'now': lambda: date.now(),  # ['now']
    'append': lambda l, r: str(r) + str(l),  # ['append', 'world', 'hello ']
    'prepend': lambda l, r: str(l) + str(r),  # ['prepend', 'hello  ', 'world']
    'match': lambda pattern, string, flags = None:  re.search(build_regexp(pattern, flags), str(string).encode('utf-8')),  # ['match', /a/, 'abc']
    'in': in_f,  # ['in', '1,2,3,5..10,20..24', '@n']
    'within': within_f,  # ['within', '0..3', '@n']
    'replace': lambda search, replace, subject: build_regexp(search).sub(replace, str(subject).encode('utf-8')),
    'count': len,  # ['count', '@genders']
    'all': lambda of, value: all([el == value for el in of]),  # ['all', '@genders', 'male']
    'any': lambda of, value: any([el == value for el in of])  # ['any', '@genders', 'female']
}

<commit_msg>Fix imports<commit_after># encoding: UTF-8
""" Rules functions """
import re
from _ctypes import ArgumentError
from datetime import date, datetime
import re
from re import search

INT_REGEXP = '(0|[1-9]\d*)'
IS_INT = re.compile('^%s$' % INT_REGEXP)
IS_RANGE = re.compile('^%s\.\.%s$' % (INT_REGEXP, INT_REGEXP))

def to_int(text):
    """ Convert text to int
        Args:
            text (string): number in string format
        Throws:
            ArgumentError: text has unsupported format
        Returns:
            int
    """
    m = IS_INT.match(str(text).strip())
    if m:
        return int(m.group(1))
    raise ArgumentError('Invalid number string: "%s"' % text)

def to_range(text):
    """ Convert text to range tuple
        Args:
            text (string): min..max
        Throws:
            ArgumentError: text has unsupported format
        Returns:
            (int, int): (min, max)
    """
    m = IS_RANGE.match(str(text).strip())
    if m:
        min = int(m.group(1))
        max = int(m.group(2))
        if not max > min:
            raise ArgumentError('Invalid range string: max value (%d) seems to be grater than min (%d)' % (max, min), min, max)
        return (min, max)
    raise ArgumentError('Invalid range string %s' % text)

def within_f(value, range):
    """ Check is value in range
        Args:
            value (string):
            text: range
        Throws:
            ArgumentError
        Returns:
            boolean
    """
    (min, max) = to_range(range)
    value = to_int(value)
    return value >= min and value <= max

def in_f(set, find):
    """ Check is value in set
        Args:
            find (string): string to find
            set (string): comma separated list of value of ranges 1,2,8..10
        Throws:
            ArgumentError
        Returns:
            boolean
    """
    find = str(find).strip()
    for e in set.split(','):
        e = e.strip()
        if find == e:
            # find == set element
            return True
        elif IS_RANGE.match(e):
            # set element is range:
            if within_f(value = find, range = e):
                return True
    return False

def f_and(*args):
    """ Arg1 AND arg2 ..."""
    return reduce(lambda a, b: bool(a) and bool(b), list)

def f_or(*args):
    """ Arg1 or arg2 ... """
    reduce(lambda a, b: bool(a) or bool(b), list)

IS_PCRE = re.compile('^\/(.*)\/(\w*)$')

def build_flags(flags):
    """ Build python regular expression flags from PCRE flags
        Args:
            flags (string): regulare expression flags : ailms
        Returns:
            int:
    """
    ret = 0
    if flags is None:
        return ret
    flags = flags.upper()
    for flag in ('A','I','L','M','S'):
        if flag in flags:
            ret |= getattr(re, flag)
    return ret

def build_regexp(pattern, flags = None):
    """ Build regexp:
            pattern (string): pattern as python regular expression in PCRE (/pattern/flags)
        Returns:
            _sre.SRE_Pattern
    """
    flags = build_flags(flags)
    pcre = IS_PCRE.match(pattern)
    if pcre:
        pattern = pcre.group(1)
        flags = flags | build_flags(pcre.group(2))
    return re.compile(pattern.encode('utf-8'), flags)

def f_mod(l, r):
    return int(l) % int(r)

def f_any(*args):
    return any(args)

def f_all(*args):
    return all(args)

def f_eq(*args):
    if any([type(arg) is int for arg in args]):
        # convert all arguments to int if int argumnt exists
        try:
            args = [int(arg) for arg in args]
        except ValueError:
            # can not convert argument to int <-> !=
            return False
    return all([args[0] == arg for arg in args])

def cmp(arg1, arg2):
    if type(arg1) is date and type(arg2) is date:
        pass
    else:
        arg1 = float(arg1)
        arg2 = float(arg2)
    if arg1 == arg2:
        return '='
    elif arg1 > arg2:
        return '>'
    else:
        return '<'

SUPPORTED_FUNCTIONS = {
    # McCarthy's Elementary S-functions and Predicates
    'quote': lambda expr: expr,
    'car': lambda list: list[1],
    'cons': lambda e, cell: [e] + cell,
    'eq': lambda l, r: l == r,
    'atom': lambda expr: isinstance(expr, (type(None), str, int, float, bool)),
    # Tr8n Extensions
    '=': f_eq,  # ['=', 1, 2]
    '!=': lambda l, r: '=' != cmp(l, r),  # ['!=', 1, 2]
    '<': lambda l, r: '<' == cmp(l, r),  # ['<', 1, 2]
    '>': lambda l, r: '>' == cmp(l, r),  # ['>', 1, 2]
    '+': lambda l, r: float(l) + float(r),  # ['+', 1, 2]
    '-': lambda l, r: float(l) - float(r),  # ['-', 1, 2]
    '*': lambda l, r: float(l) * float(r),  # ['*', 1, 2]
    '%': f_mod,  # ['%', 14, 10]
    'mod': f_mod,  # ['mod', '@n', 10]
    '/': lambda l, r: (l * 1.0) / r,  # ['/', 1, 2]
    '!': lambda expr: not expr,  # ['!', ['true']]
    'not': lambda val: not val,  # ['not', ['true']]
    '&&': f_all,  # ['&&', [], [], ...]
    'and': f_all,  # ['and', [], [], ...]
    '::': f_any,  # ['::', [], [], ...]
    '||': f_any,
    'or': f_any,  # ['or', [], [], ...]
    'if': lambda c, t, f: t if c else f,# ['if', 'cond', 'true', 'false']
    'true': lambda: True,  # ['true']
    'false': lambda: False,  # ['false']
    'date': lambda date: date.strptime(date, '%Y-%m-%d'),# ['date', '2010-01-01']
    'today': lambda: date.today(),  # ['today']
    'time': lambda time: date.strptime(time, '%Y-%m-%d %H:%M:%S'),  # ['time', '2010-01-01 10:10:05']
    'now': lambda: date.now(),  # ['now']
    'append': lambda l, r: str(r) + str(l),  # ['append', 'world', 'hello ']
    'prepend': lambda l, r: str(l) + str(r),  # ['prepend', 'hello  ', 'world']
    'match': lambda pattern, string, flags = None:  re.search(build_regexp(pattern, flags), str(string).encode('utf-8')),  # ['match', /a/, 'abc']
    'in': in_f,  # ['in', '1,2,3,5..10,20..24', '@n']
    'within': within_f,  # ['within', '0..3', '@n']
    'replace': lambda search, replace, subject: build_regexp(search).sub(replace, str(subject).encode('utf-8')),
    'count': len,  # ['count', '@genders']
    'all': lambda of, value: all([el == value for el in of]),  # ['all', '@genders', 'male']
    'any': lambda of, value: any([el == value for el in of])  # ['any', '@genders', 'female']
}

<|endoftext|>