<commit_before>#!/usr/bin/python3
# pylint: disable=invalid-name,missing-docstring
#
# Copyright (C) 2021 Richard Hughes <richard@hughsie.com>
#
# SPDX-License-Identifier: LGPL-2.1+
#
# pylint: disable=too-many-instance-attributes,no-self-use

import os
import sys
import subprocess
import glob
from typing import Dict, Optional, List, Union

DEFAULT_BUILDDIR = ".ossfuzz"

class Builder:
    def __init__(self) -> None:

        self.cc = self._ensure_environ("CC", "gcc")
        self.cxx = self._ensure_environ("CXX", "g++")
        self.builddir = self._ensure_environ("WORK", os.path.realpath(DEFAULT_BUILDDIR))
        self.installdir = self._ensure_environ(
            "OUT", os.path.realpath(os.path.join(DEFAULT_BUILDDIR, "out"))
        )
        self.srcdir = self._ensure_environ("SRC", os.path.realpath(".."))
        self.ldflags = ["-lpthread", "-lresolv", "-ldl", "-lffi", "-lz"]

        # defined in env
        self.cflags = ["-Wno-deprecated-declarations"]
        if "CFLAGS" in os.environ:
            self.cflags += os.environ["CFLAGS"].split(" ")
        self.cxxflags = []
        if "CXXFLAGS" in os.environ:
            self.cxxflags += os.environ["CXXFLAGS"].split(" ")

        # set up shared / static
        os.environ["PKG_CONFIG"] = "pkg-config --static"
        if "PATH" in os.environ:
            os.environ["PATH"] = "{}:{}".format(
                os.environ["PATH"], os.path.join(self.builddir, "bin")
            )
        else:
            os.environ["PATH"] = os.path.join(self.builddir, "bin")
        os.environ["PKG_CONFIG_PATH"] = os.path.join(self.builddir, "lib", "pkgconfig")

        # writable
        os.makedirs(self.builddir, exist_ok=True)
        os.makedirs(self.installdir, exist_ok=True)

    def _ensure_environ(self, key: str, value: str) -> str:
        """set the environment unless already set"""
        if key not in os.environ:
            os.environ[key] = value
        return os.environ[key]

    def checkout_source(self, name: str, url: str, commit: Optional[str] = None) -> str:
        """checkout source tree, optionally to a specific commit"""
        srcdir_name = os.path.join(self.srcdir, name)
        if os.path.exists(srcdir_name):
            return srcdir_name
        subprocess.run(["git", "clone", url], cwd=self.srcdir, check=True)
        if commit:
            subprocess.run(["git", "checkout", commit], cwd=srcdir_name, check=True)
        return srcdir_name

    def build_meson_project(self, srcdir: str, argv) -> None:
        """configure and build the meson project"""
        srcdir_build = os.path.join(srcdir, DEFAULT_BUILDDIR)
        if not os.path.exists(srcdir_build):
            subprocess.run(
                [
                    "meson",
                    "--prefix",
                    self.builddir,
                    "--libdir",
                    "lib",
                    "--default-library",
                    "static",
                ]
                + argv
                + [DEFAULT_BUILDDIR],
                cwd=srcdir,
                check=True,
            )
            subprocess.run(["ninja", "install"], cwd=srcdir_build, check=True)

    def add_work_includedir(self, value: str) -> None:
        """add a CFLAG"""
        self.cflags.append("-I{}/{}".format(self.builddir, value))

    def add_src_includedir(self, value: str) -> None:
        """add a CFLAG"""
        self.cflags.append("-I{}/{}".format(self.srcdir, value))

    def add_build_ldflag(self, value: str) -> None:
        """add a LDFLAG"""
        self.ldflags.append(os.path.join(self.builddir, value))

    def substitute(self, src: str, replacements: Dict[str, str]) -> str:
        """map changes"""

        dst = os.path.basename(src).replace(".in", "")
        with open(os.path.join(self.srcdir, src), "r") as f:
            blob = f.read()
            for key in replacements:
                blob = blob.replace(key, replacements[key])
            with open(os.path.join(self.builddir, dst), "w") as out:
                out.write(blob)
        return dst

    def compile(self, src: str) -> str:
        """compile a specific source file"""
        argv = [self.cc]
        argv.extend(self.cflags)
        fullsrc = os.path.join(self.srcdir, src)
        if not os.path.exists(fullsrc):
            fullsrc = os.path.join(self.builddir, src)
        dst = os.path.basename(src).replace(".c", ".o")
        argv.extend(["-c", fullsrc, "-o", os.path.join(self.builddir, dst)])
        print("building {} into {}".format(src, dst))
        try:
            subprocess.run(argv, cwd=self.srcdir, check=True)
        except subprocess.CalledProcessError as e:
            print(e)
            sys.exit(1)
        return os.path.join(self.builddir, "{}".format(dst))

    def link(self, objs: List[str], dst: str) -> None:
        """link multiple obects into a binary"""
        argv = [self.cxx] + self.cxxflags
        for obj in objs:
            if obj.startswith("-"):
                argv.append(obj)
            else:
                argv.append(os.path.join(self.builddir, obj))
        argv += ["-o", os.path.join(self.installdir, dst)]
        argv += self.ldflags
        print("building {} into {}".format(",".join(objs), dst))
        subprocess.run(argv, cwd=self.srcdir, check=True)

    def write_header(
        self, dst: str, defines: Dict[str, Optional[Union[str, int]]]
    ) -> None:
        """write a header file"""
        dstdir = os.path.join(self.builddir, os.path.dirname(dst))
        os.makedirs(dstdir, exist_ok=True)
        print("writing {}".format(dst))
        with open(os.path.join(dstdir, os.path.basename(dst)), "w") as f:
            for key in defines:
                value = defines[key]
                if value is not None:
                    if isinstance(value, int):
                        f.write("#define {} {}\n".format(key, value))
                    else:
                        f.write('#define {} "{}"\n'.format(key, value))
                else:
                    f.write("#define {}\n".format(key))
        self.add_work_includedir(os.path.dirname(dst))

    def makezip(self, dst: str, globstr: str) -> None:
        """create a zip file archive from a glob"""
        argv = ["zip", "--junk-paths", os.path.join(self.installdir, dst)] + glob.glob(
            os.path.join(self.srcdir, globstr)
        )
        print("assembling {}".format(dst))
        subprocess.run(argv, cwd=self.srcdir, check=True)

    def grep_meson(self, src: str, token: str = "fuzzing") -> List[str]:
        """find source files tagged with a specific comment"""
        srcs = []
        with open(os.path.join(self.srcdir, src, "meson.build"), "r") as f:
            for line in f.read().split("\n"):
                if line.find(token) == -1:
                    continue

                # get rid of token
                line = line.split("#")[0]

                # get rid of variable
                try:
                    line = line.split("=")[1]
                except IndexError as _:
                    pass

                # get rid of whitespace
                for char in ["'", ",", " "]:
                    line = line.replace(char, "")

                # all done
                srcs.append(os.path.join(src, line))
        return srcs

class Fuzzer:
    def __init__(self, name, srcdir=None, globstr=None, pattern=None) -> None:

        self.name = name
        self.srcdir = srcdir or name
        self.globstr = globstr or "{}*".format(name)
        self.pattern = pattern or "{}-firmware".format(name)

    @property
    def new_gtype(self) -> str:
        return "fu_{}_new".format(self.pattern).replace("-", "_")

    @property
    def header(self) -> str:
        return "fu-{}.h".format(self.pattern)

def _build(bld: Builder) -> None:

    # GLib
    src = bld.checkout_source("glib", url="https://gitlab.gnome.org/GNOME/glib.git")
    bld.build_meson_project(
        src,
        [
            "-Dlibmount=disabled",
            "-Dselinux=disabled",
            "-Dnls=disabled",
            "-Dlibelf=disabled",
            "-Dbsymbolic_functions=false",
            "-Dtests=false",
            "-Dinternal_pcre=true",
        ],
    )
    bld.add_work_includedir("include/glib-2.0")
    bld.add_work_includedir("lib/glib-2.0/include")
    bld.add_build_ldflag("lib/libgio-2.0.a")
    bld.add_build_ldflag("lib/libgmodule-2.0.a")
    bld.add_build_ldflag("lib/libgobject-2.0.a")
    bld.add_build_ldflag("lib/libglib-2.0.a")
    bld.add_build_ldflag("lib/libgthread-2.0.a")

    # JSON-GLib
    src = bld.checkout_source(
        "json-glib", url="https://gitlab.gnome.org/GNOME/json-glib.git"
    )
    bld.build_meson_project(
        src, ["-Dgtk_doc=disabled", "-Dtests=false", "-Dintrospection=disabled"]
    )
    bld.add_work_includedir("include/json-glib-1.0/json-glib")
    bld.add_work_includedir("include/json-glib-1.0")
    bld.add_build_ldflag("lib/libjson-glib-1.0.a")

    # libxmlb
    src = bld.checkout_source("libxmlb", url="https://github.com/hughsie/libxmlb.git")
    bld.build_meson_project(
        src, ["-Dgtkdoc=false", "-Dintrospection=false", "-Dtests=false"]
    )
    bld.add_work_includedir("include/libxmlb-2")
    bld.add_work_includedir("include/libxmlb-2/libxmlb")
    bld.add_build_ldflag("lib/libxmlb.a")

    # write required headers
    bld.write_header("libfwupd/fwupd-version.h", {})
    bld.write_header(
        "config.h",
        {
            "FWUPD_DATADIR": "/tmp",
            "FWUPD_LOCALSTATEDIR": "/tmp",
            "FWUPD_PLUGINDIR": "/tmp",
            "FWUPD_SYSCONFDIR": "/tmp",
            "HAVE_REALPATH": None,
            "PACKAGE_NAME": "fwupd",
            "PACKAGE_VERSION": "0.0.0",
        },
    )

    # libfwupd + libfwupdplugin
    built_objs: List[str] = []
    bld.add_src_includedir("fwupd")
    for path in ["fwupd/libfwupd", "fwupd/libfwupdplugin"]:
        bld.add_src_includedir(path)
        for src in bld.grep_meson(path):
            built_objs.append(bld.compile(src))

    # dummy binary entrypoint
    if "LIB_FUZZING_ENGINE" in os.environ:
        built_objs.append(os.environ["LIB_FUZZING_ENGINE"])
    else:
        built_objs.append(bld.compile("fwupd/libfwupdplugin/fu-fuzzer-main.c"))

    # built in formats
    for fzr in [
        Fuzzer("dfuse"),
        Fuzzer("fmap"),
        Fuzzer("ihex"),
        Fuzzer("srec"),
        Fuzzer("efi-filesystem", pattern="efi-firmware-filesystem"),
        Fuzzer("efi-volume", pattern="efi-firmware-volume"),
        Fuzzer("ifd"),
    ]:
        src = bld.substitute(
            "fwupd/libfwupdplugin/fu-fuzzer-firmware.c.in",
            {
                "@FIRMWARENEW@": fzr.new_gtype,
                "@INCLUDE@": os.path.join("libfwupdplugin", fzr.header),
            },
        )
        bld.link([bld.compile(src)] + built_objs, "{}_fuzzer".format(fzr.name))
        bld.makezip(
            "{}_fuzzer_seed_corpus.zip".format(fzr.name),
            "fwupd/src/fuzzing/firmware/{}".format(fzr.globstr),
        )

    # plugins
    for fzr in [
        Fuzzer("acpi-phat", pattern="acpi-phat"),
        Fuzzer("bcm57xx"),
        Fuzzer("ccgx-dmc", srcdir="ccgx", globstr="ccgx-dmc*.bin"),
        Fuzzer("ccgx", globstr="ccgx*.cyacd"),
        Fuzzer("cros-ec"),
        Fuzzer("ebitdo"),
        Fuzzer("elantp"),
        Fuzzer("hailuck-kbd", srcdir="hailuck", globstr="ihex*"),
        Fuzzer("pixart", srcdir="pixart-rf", pattern="pxi-firmware"),
        Fuzzer("redfish-smbios", srcdir="redfish", pattern="redfish-smbios"),
        Fuzzer("solokey"),
        Fuzzer("synaprom", srcdir="synaptics-prometheus"),
        Fuzzer("synaptics-mst"),
        Fuzzer("synaptics-rmi"),
        Fuzzer("wacom-usb", pattern="wac-firmware", globstr="wacom*"),
    ]:
        fuzz_objs = []
        for obj in bld.grep_meson("fwupd/plugins/{}".format(fzr.srcdir)):
            fuzz_objs.append(bld.compile(obj))
        src = bld.substitute(
            "fwupd/libfwupdplugin/fu-fuzzer-firmware.c.in",
            {
                "@FIRMWARENEW@": fzr.new_gtype,
                "@INCLUDE@": os.path.join("plugins", fzr.srcdir, fzr.header),
            },
        )
        fuzz_objs.append(bld.compile(src))
        bld.link(fuzz_objs + built_objs, "{}_fuzzer".format(fzr.name))
        bld.makezip(
            "{}_fuzzer_seed_corpus.zip".format(fzr.name),
            "fwupd/src/fuzzing/firmware/{}".format(fzr.globstr),
        )

if __name__ == "__main__":
    _builder = Builder()
    _build(_builder)
    sys.exit(0)
<commit_msg>trivial: Fix oss-fuzz by installing missing deps<commit_after>#!/usr/bin/python3
# pylint: disable=invalid-name,missing-docstring
#
# Copyright (C) 2021 Richard Hughes <richard@hughsie.com>
#
# SPDX-License-Identifier: LGPL-2.1+
#
# pylint: disable=too-many-instance-attributes,no-self-use

import os
import sys
import subprocess
import glob
from typing import Dict, Optional, List, Union

DEFAULT_BUILDDIR = ".ossfuzz"

class Builder:
    def __init__(self) -> None:

        self.cc = self._ensure_environ("CC", "gcc")
        self.cxx = self._ensure_environ("CXX", "g++")
        self.builddir = self._ensure_environ("WORK", os.path.realpath(DEFAULT_BUILDDIR))
        self.installdir = self._ensure_environ(
            "OUT", os.path.realpath(os.path.join(DEFAULT_BUILDDIR, "out"))
        )
        self.srcdir = self._ensure_environ("SRC", os.path.realpath(".."))
        self.ldflags = ["-lpthread", "-lresolv", "-ldl", "-lffi", "-lz", "-llzma"]

        # defined in env
        self.cflags = ["-Wno-deprecated-declarations"]
        if "CFLAGS" in os.environ:
            self.cflags += os.environ["CFLAGS"].split(" ")
        self.cxxflags = []
        if "CXXFLAGS" in os.environ:
            self.cxxflags += os.environ["CXXFLAGS"].split(" ")

        # set up shared / static
        os.environ["PKG_CONFIG"] = "pkg-config --static"
        if "PATH" in os.environ:
            os.environ["PATH"] = "{}:{}".format(
                os.environ["PATH"], os.path.join(self.builddir, "bin")
            )
        else:
            os.environ["PATH"] = os.path.join(self.builddir, "bin")
        os.environ["PKG_CONFIG_PATH"] = os.path.join(self.builddir, "lib", "pkgconfig")

        # writable
        os.makedirs(self.builddir, exist_ok=True)
        os.makedirs(self.installdir, exist_ok=True)

    def _ensure_environ(self, key: str, value: str) -> str:
        """set the environment unless already set"""
        if key not in os.environ:
            os.environ[key] = value
        return os.environ[key]

    def checkout_source(self, name: str, url: str, commit: Optional[str] = None) -> str:
        """checkout source tree, optionally to a specific commit"""
        srcdir_name = os.path.join(self.srcdir, name)
        if os.path.exists(srcdir_name):
            return srcdir_name
        subprocess.run(["git", "clone", url], cwd=self.srcdir, check=True)
        if commit:
            subprocess.run(["git", "checkout", commit], cwd=srcdir_name, check=True)
        return srcdir_name

    def build_meson_project(self, srcdir: str, argv) -> None:
        """configure and build the meson project"""
        srcdir_build = os.path.join(srcdir, DEFAULT_BUILDDIR)
        if not os.path.exists(srcdir_build):
            subprocess.run(
                [
                    "meson",
                    "--prefix",
                    self.builddir,
                    "--libdir",
                    "lib",
                    "--default-library",
                    "static",
                ]
                + argv
                + [DEFAULT_BUILDDIR],
                cwd=srcdir,
                check=True,
            )
            subprocess.run(["ninja", "install"], cwd=srcdir_build, check=True)

    def add_work_includedir(self, value: str) -> None:
        """add a CFLAG"""
        self.cflags.append("-I{}/{}".format(self.builddir, value))

    def add_src_includedir(self, value: str) -> None:
        """add a CFLAG"""
        self.cflags.append("-I{}/{}".format(self.srcdir, value))

    def add_build_ldflag(self, value: str) -> None:
        """add a LDFLAG"""
        self.ldflags.append(os.path.join(self.builddir, value))

    def substitute(self, src: str, replacements: Dict[str, str]) -> str:
        """map changes"""

        dst = os.path.basename(src).replace(".in", "")
        with open(os.path.join(self.srcdir, src), "r") as f:
            blob = f.read()
            for key in replacements:
                blob = blob.replace(key, replacements[key])
            with open(os.path.join(self.builddir, dst), "w") as out:
                out.write(blob)
        return dst

    def compile(self, src: str) -> str:
        """compile a specific source file"""
        argv = [self.cc]
        argv.extend(self.cflags)
        fullsrc = os.path.join(self.srcdir, src)
        if not os.path.exists(fullsrc):
            fullsrc = os.path.join(self.builddir, src)
        dst = os.path.basename(src).replace(".c", ".o")
        argv.extend(["-c", fullsrc, "-o", os.path.join(self.builddir, dst)])
        print("building {} into {}".format(src, dst))
        try:
            subprocess.run(argv, cwd=self.srcdir, check=True)
        except subprocess.CalledProcessError as e:
            print(e)
            sys.exit(1)
        return os.path.join(self.builddir, "{}".format(dst))

    def link(self, objs: List[str], dst: str) -> None:
        """link multiple obects into a binary"""
        argv = [self.cxx] + self.cxxflags
        for obj in objs:
            if obj.startswith("-"):
                argv.append(obj)
            else:
                argv.append(os.path.join(self.builddir, obj))
        argv += ["-o", os.path.join(self.installdir, dst)]
        argv += self.ldflags
        print("building {} into {}".format(",".join(objs), dst))
        subprocess.run(argv, cwd=self.srcdir, check=True)

    def write_header(
        self, dst: str, defines: Dict[str, Optional[Union[str, int]]]
    ) -> None:
        """write a header file"""
        dstdir = os.path.join(self.builddir, os.path.dirname(dst))
        os.makedirs(dstdir, exist_ok=True)
        print("writing {}".format(dst))
        with open(os.path.join(dstdir, os.path.basename(dst)), "w") as f:
            for key in defines:
                value = defines[key]
                if value is not None:
                    if isinstance(value, int):
                        f.write("#define {} {}\n".format(key, value))
                    else:
                        f.write('#define {} "{}"\n'.format(key, value))
                else:
                    f.write("#define {}\n".format(key))
        self.add_work_includedir(os.path.dirname(dst))

    def makezip(self, dst: str, globstr: str) -> None:
        """create a zip file archive from a glob"""
        argv = ["zip", "--junk-paths", os.path.join(self.installdir, dst)] + glob.glob(
            os.path.join(self.srcdir, globstr)
        )
        print("assembling {}".format(dst))
        subprocess.run(argv, cwd=self.srcdir, check=True)

    def grep_meson(self, src: str, token: str = "fuzzing") -> List[str]:
        """find source files tagged with a specific comment"""
        srcs = []
        with open(os.path.join(self.srcdir, src, "meson.build"), "r") as f:
            for line in f.read().split("\n"):
                if line.find(token) == -1:
                    continue

                # get rid of token
                line = line.split("#")[0]

                # get rid of variable
                try:
                    line = line.split("=")[1]
                except IndexError as _:
                    pass

                # get rid of whitespace
                for char in ["'", ",", " "]:
                    line = line.replace(char, "")

                # all done
                srcs.append(os.path.join(src, line))
        return srcs

class Fuzzer:
    def __init__(self, name, srcdir=None, globstr=None, pattern=None) -> None:

        self.name = name
        self.srcdir = srcdir or name
        self.globstr = globstr or "{}*".format(name)
        self.pattern = pattern or "{}-firmware".format(name)

    @property
    def new_gtype(self) -> str:
        return "fu_{}_new".format(self.pattern).replace("-", "_")

    @property
    def header(self) -> str:
        return "fu-{}.h".format(self.pattern)

def _build(bld: Builder) -> None:

    # GLib
    src = bld.checkout_source("glib", url="https://gitlab.gnome.org/GNOME/glib.git")
    bld.build_meson_project(
        src,
        [
            "-Dlibmount=disabled",
            "-Dselinux=disabled",
            "-Dnls=disabled",
            "-Dlibelf=disabled",
            "-Dbsymbolic_functions=false",
            "-Dtests=false",
            "-Dinternal_pcre=true",
        ],
    )
    bld.add_work_includedir("include/glib-2.0")
    bld.add_work_includedir("lib/glib-2.0/include")
    bld.add_build_ldflag("lib/libgio-2.0.a")
    bld.add_build_ldflag("lib/libgmodule-2.0.a")
    bld.add_build_ldflag("lib/libgobject-2.0.a")
    bld.add_build_ldflag("lib/libglib-2.0.a")
    bld.add_build_ldflag("lib/libgthread-2.0.a")

    # JSON-GLib
    src = bld.checkout_source(
        "json-glib", url="https://gitlab.gnome.org/GNOME/json-glib.git"
    )
    bld.build_meson_project(
        src, ["-Dgtk_doc=disabled", "-Dtests=false", "-Dintrospection=disabled"]
    )
    bld.add_work_includedir("include/json-glib-1.0/json-glib")
    bld.add_work_includedir("include/json-glib-1.0")
    bld.add_build_ldflag("lib/libjson-glib-1.0.a")

    # libxmlb
    src = bld.checkout_source("libxmlb", url="https://github.com/hughsie/libxmlb.git")
    bld.build_meson_project(
        src, ["-Dgtkdoc=false", "-Dintrospection=false", "-Dtests=false"]
    )
    bld.add_work_includedir("include/libxmlb-2")
    bld.add_work_includedir("include/libxmlb-2/libxmlb")
    bld.add_build_ldflag("lib/libxmlb.a")

    # write required headers
    bld.write_header("libfwupd/fwupd-version.h", {})
    bld.write_header(
        "config.h",
        {
            "FWUPD_DATADIR": "/tmp",
            "FWUPD_LOCALSTATEDIR": "/tmp",
            "FWUPD_PLUGINDIR": "/tmp",
            "FWUPD_SYSCONFDIR": "/tmp",
            "HAVE_REALPATH": None,
            "PACKAGE_NAME": "fwupd",
            "PACKAGE_VERSION": "0.0.0",
        },
    )

    # libfwupd + libfwupdplugin
    built_objs: List[str] = []
    bld.add_src_includedir("fwupd")
    for path in ["fwupd/libfwupd", "fwupd/libfwupdplugin"]:
        bld.add_src_includedir(path)
        for src in bld.grep_meson(path):
            built_objs.append(bld.compile(src))

    # dummy binary entrypoint
    if "LIB_FUZZING_ENGINE" in os.environ:
        built_objs.append(os.environ["LIB_FUZZING_ENGINE"])
    else:
        built_objs.append(bld.compile("fwupd/libfwupdplugin/fu-fuzzer-main.c"))

    # built in formats
    for fzr in [
        Fuzzer("dfuse"),
        Fuzzer("fmap"),
        Fuzzer("ihex"),
        Fuzzer("srec"),
        Fuzzer("efi-filesystem", pattern="efi-firmware-filesystem"),
        Fuzzer("efi-volume", pattern="efi-firmware-volume"),
        Fuzzer("ifd"),
    ]:
        src = bld.substitute(
            "fwupd/libfwupdplugin/fu-fuzzer-firmware.c.in",
            {
                "@FIRMWARENEW@": fzr.new_gtype,
                "@INCLUDE@": os.path.join("libfwupdplugin", fzr.header),
            },
        )
        bld.link([bld.compile(src)] + built_objs, "{}_fuzzer".format(fzr.name))
        bld.makezip(
            "{}_fuzzer_seed_corpus.zip".format(fzr.name),
            "fwupd/src/fuzzing/firmware/{}".format(fzr.globstr),
        )

    # plugins
    for fzr in [
        Fuzzer("acpi-phat", pattern="acpi-phat"),
        Fuzzer("bcm57xx"),
        Fuzzer("ccgx-dmc", srcdir="ccgx", globstr="ccgx-dmc*.bin"),
        Fuzzer("ccgx", globstr="ccgx*.cyacd"),
        Fuzzer("cros-ec"),
        Fuzzer("ebitdo"),
        Fuzzer("elantp"),
        Fuzzer("hailuck-kbd", srcdir="hailuck", globstr="ihex*"),
        Fuzzer("pixart", srcdir="pixart-rf", pattern="pxi-firmware"),
        Fuzzer("redfish-smbios", srcdir="redfish", pattern="redfish-smbios"),
        Fuzzer("solokey"),
        Fuzzer("synaprom", srcdir="synaptics-prometheus"),
        Fuzzer("synaptics-mst"),
        Fuzzer("synaptics-rmi"),
        Fuzzer("wacom-usb", pattern="wac-firmware", globstr="wacom*"),
    ]:
        fuzz_objs = []
        for obj in bld.grep_meson("fwupd/plugins/{}".format(fzr.srcdir)):
            fuzz_objs.append(bld.compile(obj))
        src = bld.substitute(
            "fwupd/libfwupdplugin/fu-fuzzer-firmware.c.in",
            {
                "@FIRMWARENEW@": fzr.new_gtype,
                "@INCLUDE@": os.path.join("plugins", fzr.srcdir, fzr.header),
            },
        )
        fuzz_objs.append(bld.compile(src))
        bld.link(fuzz_objs + built_objs, "{}_fuzzer".format(fzr.name))
        bld.makezip(
            "{}_fuzzer_seed_corpus.zip".format(fzr.name),
            "fwupd/src/fuzzing/firmware/{}".format(fzr.globstr),
        )

if __name__ == "__main__":

    # install missing deps here rather than patching the Dockerfile in oss-fuzz
    try:
        subprocess.check_call(
            ["apt-get", "install", "-y", "liblzma-dev"], stdout=open(os.devnull, "wb")
        )
    except FileNotFoundError:
        pass
    except subprocess.CalledProcessError as e:
        print(e.output)
        sys.exit(1)

    _builder = Builder()
    _build(_builder)
    sys.exit(0)
<|endoftext|>


<commit_before>#!/usr/bin/python
"""
emailfilter.py -- Email tickets to Trac.

A simple MTA filter to create Trac tickets from inbound emails.

Copyright 2005, Daniel Lundin <daniel@edgewall.com>
Copyright 2005, Edgewall Software

The scripts reads emails from stdin and inserts directly into a Trac database.
MIME headers are mapped as follows:

    * From: => Reporter
    * Subject: => Summary
    * Body => Description

How to use
----------
 * Set TRAC_ENV_PATH to the path of your project's Trac environment
 * Configure script as a mail (pipe) filter with your MTA
    typically, this involves adding a line like this to /etc/aliases:
       somename: |/path/to/email2trac.py
    Check your MTA's documentation for specifics.

Todo
----
  * Configure target database through env variable?
  * Handle/discard HTML parts
  * Attachment support
"""

TRAC_ENV_PATH = '/var/trac/test'

import email
import sys

from trac.env import Environment
from trac.ticket import Ticket

class TicketEmailParser(object):

    env = None

    def __init__(self, env):
        self.env = env

    def parse(self, fp):
        msg = email.message_from_file(fp)
        tkt = Ticket(self.env)
        tkt['status'] = 'new'
        tkt['reporter'] = msg['from']
        tkt['summary'] = msg['subject']
        for part in msg.walk():
            if part.get_content_type() == 'text/plain':
                tkt['description'] = part.get_payload(decode=1).strip()

        if tkt.values.get('description'):
            tkt.insert()

if __name__ == '__main__':
    env = Environment(TRAC_ENV_PATH, create=0)
    tktparser = TicketEmailParser(env)
    tktparser.parse(sys.stdin)
<commit_msg>0.12dev: Added a comment that the script is only a starting point. Thanks to Gisbert Amm for the patch.<commit_after>#!/usr/bin/python
"""
emailfilter.py -- Email tickets to Trac.

A simple MTA filter to create Trac tickets from inbound emails.

Copyright 2005, Daniel Lundin <daniel@edgewall.com>
Copyright 2005, Edgewall Software

Please note:

This is only a starting point. See
 * http://trac.edgewall.org/ticket/5327 and
 * http://trac-hacks.org/intertrac/EmailtoTracScript
for more complete and advanced examples.

The scripts reads emails from stdin and inserts directly into a Trac database.
MIME headers are mapped as follows:

    * From: => Reporter
    * Subject: => Summary
    * Body => Description

How to use
----------
 * Set TRAC_ENV_PATH to the path of your project's Trac environment
 * Configure script as a mail (pipe) filter with your MTA
    typically, this involves adding a line like this to /etc/aliases:
       somename: |/path/to/email2trac.py
    Check your MTA's documentation for specifics.

Todo
----
  * Configure target database through env variable?
  * Handle/discard HTML parts
  * Attachment support
"""

TRAC_ENV_PATH = '/var/trac/test'

import email
import sys

from trac.env import Environment
from trac.ticket import Ticket

class TicketEmailParser(object):

    env = None

    def __init__(self, env):
        self.env = env

    def parse(self, fp):
        msg = email.message_from_file(fp)
        tkt = Ticket(self.env)
        tkt['status'] = 'new'
        tkt['reporter'] = msg['from']
        tkt['summary'] = msg['subject']
        for part in msg.walk():
            if part.get_content_type() == 'text/plain':
                tkt['description'] = part.get_payload(decode=1).strip()

        if tkt.values.get('description'):
            tkt.insert()

if __name__ == '__main__':
    env = Environment(TRAC_ENV_PATH, create=0)
    tktparser = TicketEmailParser(env)
    tktparser.parse(sys.stdin)
<|endoftext|>


<commit_before># Copyright (C) 2014 Edgewall Software
# This file is distributed under the same license as the Trac project.

import StringIO
import warnings

from trac.util.text import print_table, printout

def _svn_version():
    from svn import core
    version = (core.SVN_VER_MAJOR, core.SVN_VER_MINOR,
               core.SVN_VER_MICRO)
    return '%d.%d.%d' % version + core.SVN_VER_TAG

PACKAGES = [
    ("Python",            'sys.version'),
    ("Setuptools",        'setuptools.__version__'),
    ("Genshi",            'genshi.__version__'),
    ("Babel",             'babel.__version__'),
    ("sqlite3",           'sqlite3.version'),
    ("PySqlite",          'pysqlite2.dbapi2.version'),
    ("MySQLdb",           'MySQLdb.__version__'),
    ("Psycopg2",          'psycopg2.__version__'),
    ("SVN bindings",      _svn_version),
    ("Mercurial",         'mercurial.util.version()'),
    ("Pygments",          'pygments.__version__'),
    ("Pytz",              'pytz.__version__'),
    ("ConfigObj",         'configobj.__version__'),
    ("Docutils",          'docutils.__version__'),
    ("Twill",             'twill.__version__'),
    ("LXML",              'lxml.etree.__version__'),
    ("coverage",          'coverage.__version__'),
    ("figleaf",           'figleaf.__version__'),
]

def package_versions(packages, out=None):
    name_version_pairs = []
    for name, accessor in packages:
        version = resolve_accessor(accessor)
        name_version_pairs.append((name, version))
    print_table(name_version_pairs, ("Package", "Version"), ' : ', out)

def resolve_accessor(accessor):
    if isinstance(accessor, basestring):
        def fn():
            module, attr = accessor.rsplit('.', 1)
            version = attr.replace('()', '')
            version = getattr(__import__(module, {}, {}, [version]), version)
            if attr.endswith('()'):
                version = version()
            return version
    else:
        fn = accessor
    try:
        return fn()
    except Exception:
        return "not installed"

def shift(prefix, block):
    return '\n'.join(prefix + line for line in block.split('\n') if line)

def print_status():
    warnings.filterwarnings('ignore', '', DeprecationWarning) # Twill 0.9...
    buf = StringIO.StringIO()
    package_versions(PACKAGES, buf)
    printout(shift('  ', buf.getvalue()))

if __name__ == '__main__':
    print_status()
<commit_msg>0.12.8dev: simplified `resolve_accessor()` in `make_status.py`<commit_after># Copyright (C) 2014 Edgewall Software
# This file is distributed under the same license as the Trac project.

import StringIO
import warnings

from trac.util.text import print_table, printout

def _svn_version():
    from svn import core
    version = (core.SVN_VER_MAJOR, core.SVN_VER_MINOR,
               core.SVN_VER_MICRO)
    return '%d.%d.%d' % version + core.SVN_VER_TAG

PACKAGES = [
    ("Python",            'sys.version'),
    ("Setuptools",        'setuptools.__version__'),
    ("Genshi",            'genshi.__version__'),
    ("Babel",             'babel.__version__'),
    ("sqlite3",           'sqlite3.version'),
    ("PySqlite",          'pysqlite2.dbapi2.version'),
    ("MySQLdb",           'MySQLdb.__version__'),
    ("Psycopg2",          'psycopg2.__version__'),
    ("SVN bindings",      '__main__._svn_version()'),
    ("Mercurial",         'mercurial.util.version()'),
    ("Pygments",          'pygments.__version__'),
    ("Pytz",              'pytz.__version__'),
    ("ConfigObj",         'configobj.__version__'),
    ("Docutils",          'docutils.__version__'),
    ("Twill",             'twill.__version__'),
    ("LXML",              'lxml.etree.__version__'),
    ("coverage",          'coverage.__version__'),
    ("figleaf",           'figleaf.__version__'),
]

def package_versions(packages, out=None):
    name_version_pairs = []
    for name, accessor in packages:
        version = resolve_accessor(accessor)
        name_version_pairs.append((name, version))
    print_table(name_version_pairs, ("Package", "Version"), ' : ', out)

def resolve_accessor(accessor):
    try:
        module, attr = accessor.rsplit('.', 1)
        version = attr.replace('()', '')
        version = getattr(__import__(module, {}, {}, [version]), version)
        if attr.endswith('()'):
            version = version()
        return version
    except Exception:
        return "not installed"

def shift(prefix, block):
    return '\n'.join(prefix + line for line in block.split('\n') if line)

def print_status():
    warnings.filterwarnings('ignore', '', DeprecationWarning) # Twill 0.9...
    buf = StringIO.StringIO()
    package_versions(PACKAGES, buf)
    printout(shift('  ', buf.getvalue()))

if __name__ == '__main__':
    print_status()
<|endoftext|>


<commit_before>from wq.db.patterns import models
import datetime
import swapper
from django.db.models.signals import post_save
from django.core.exceptions import ImproperlyConfigured
from django.dispatch import receiver
from django.conf import settings
from collections import OrderedDict
from .compat import clone_field

swapper.set_app_prefix('vera', 'WQ')
MODELS = swapper.get_model_names(
    'vera', ('Site', 'Event', 'Report', 'ReportStatus', 'Parameter', 'Result')
)

VALID_REPORT_ORDER = getattr(settings, "WQ_VALID_REPORT_ORDER", ('-entered',))
# Base classes for Site-Event-Report-Attribute-Value pattern
# Extend these when swapping out default implementation (below)

class BaseSite(models.NaturalKeyModel):
    @property
    def valid_events(self):
        events = self.event_set.filter(
            report__status__is_valid=True
        ).values_list('pk', flat=True)
        # FIXME: events may be duplicated
        return self.event_set.filter(pk__in=events)

    class Meta:
        abstract = True

class BaseEvent(models.NaturalKeyModel):
    site = models.ForeignKey(MODELS['Site'], null=True, blank=True)

    @property
    def valid_reports(self):
        return self.report_set.filter(status__is_valid=True).order_by(
            *VALID_REPORT_ORDER
        )

    @property
    def vals(self):
        return OrderedDict([
            (a.type.natural_key()[0], a.value)
            for a in self.results
        ])

    # Valid results for this event
    @property
    def results(self):
        Result = swapper.load_model('vera', 'Result')
        return Result.objects.valid_results(report__event=self)

    @property
    def is_valid(self):
        return self.valid_reports.count() > 0

    class Meta:
        abstract = True

class ReportManager(models.RelatedModelManager):
    def create_report(self, event_key, values, **kwargs):
        Event = swapper.load_model('vera', 'Event')
        kwargs['event'] = Event.objects.find(*event_key)
        report = self.create(**kwargs)
        report.vals = values
        return report

class ValidReportManager(ReportManager):
    def get_queryset(self):
        qs = super(ValidReportManager, self)
        return qs.filter(status__is_valid=True).order_by(*VALID_REPORT_ORDER)

class BaseReport(models.RelatedModel):
    event = models.ForeignKey(MODELS['Event'])
    entered = models.DateTimeField(blank=True)
    user = models.ForeignKey(
        settings.AUTH_USER_MODEL, null=True, blank=True,

        # Needed for model swapping in tests
        related_name="%(app_label)s_%(class)s",
    )
    status = models.ForeignKey(MODELS['ReportStatus'], null=True, blank=True)

    objects = ReportManager()
    valid_objects = ValidReportManager()

    @property
    def is_valid(self):
        return self.status and self.status.is_valid

    @property
    def vals(self):
        vals = OrderedDict()
        for result in self.results.all():
            vals[result.type.natural_key()[0]] = result.value
        return vals

    @vals.setter
    def vals(self, vals):
        Parameter = swapper.load_model('vera', 'Parameter')
        keys = [(key,) for key in vals.keys()]
        params, success = Parameter.objects.resolve_keys(keys)
        if not success:
            missing = [
                name for name, params in params.items() if params is None
            ]
            raise TypeError(
                "Could not identify one or more parameters: %s!"
                % missing
            )

        for key, param in params.items():
            result, is_new = self.results.get_or_create(type=param)
            result.value = vals[key[0]]
            result.save()

    def save(self, *args, **kwargs):
        if not self.entered:
            self.entered = datetime.datetime.now()
        super(BaseReport, self).save(*args, **kwargs)

    def __unicode__(self):
        if self.pk is not None:
            return "%s according to %s" % (self.event, self.user)
        else:
            return "New Report"

    class Meta:
        abstract = True
        ordering = VALID_REPORT_ORDER

class BaseReportStatus(models.NaturalKeyModel):
    slug = models.SlugField()
    name = models.CharField(max_length=255)
    is_valid = models.BooleanField(default=False)

    def __unicode__(self):
        return self.name

    class Meta:
        abstract = True
        unique_together = [['slug']]

class BaseParameter(models.IdentifiedRelatedModel):
    name = models.CharField(max_length=255)
    is_numeric = models.BooleanField(default=False)
    units = models.CharField(max_length=50, null=True, blank=True)

    def __unicode__(self):
        if self.units:
            return u"%s (%s)" % (self.name, self.units)
        else:
            return self.name

    class Meta:
        abstract = True
        ordering = ('name',)

class ResultManager(models.Manager):
    def valid_results(self, **filter):
        Parameter = swapper.load_model('vera', 'Parameter')

        filter['report__status__is_valid'] = True
        filter['empty'] = False

        # DISTINCT ON event, then parameter, collapsing results from different
        # reports into one
        distinct = ['report__event__id'] + nest_ordering(
            'type', Parameter._meta.ordering
        ) + ['type__id']

        # ORDER BY distinct fields, then valid report order
        order = distinct + nest_ordering('report', VALID_REPORT_ORDER)

        return self.filter(
            **filter
        ).order_by(*order).distinct(*distinct).select_related('report')

class BaseResult(models.Model):
    type = models.ForeignKey(MODELS['Parameter'])
    report = models.ForeignKey(MODELS['Report'], related_name='results')
    value_numeric = models.FloatField(null=True, blank=True)
    value_text = models.TextField(null=True, blank=True)
    empty = models.BooleanField(default=False, db_index=True)

    objects = ResultManager()

    def is_empty(self, value):
        if value is None:
            return True
        if isinstance(value, basestring) and len(value.strip()) == 0:
            return True
        return False

    @property
    def value(self):
        if self.type.is_numeric:
            return self.value_numeric
        return self.value_text

    @value.setter
    def value(self, val):
        self.empty = self.is_empty(val)
        if self.type.is_numeric:
            if self.empty:
                self.value_numeric = None
            else:
                self.value_numeric = val
        else:
            self.value_text = val

    def __unicode__(self):
        return "%s -> %s: %s" % (self.report, self.type, self.value)

    class Meta:
        abstract = True
        ordering = ('type',)
        index_together = [
            ('type', 'report', 'empty'),
        ]

# Denormalize Events with Results from valid Reports
class EventResultManager(models.Manager):
    def set_for_event(self, event, delete=True):
        """
        Update EventResult cache for the given event.
        """
        if delete:
            self.filter(event=event).delete()
        for result in event.results:
            er = self.model(
                event=event,
                result=result
            )
            er.save()

    def set_for_events(self, delete=True, **event_filter):
        """
        Update EventResult cache for the given events.  The event query should
        be specified as query keyword arguments, rather than a queryset, so
        that a JOIN can be used instead of retrieving the results for each
        event separately.
        """

        # Delete existing EventResults (using denormalized Event fields)
        if delete:
            er_filter = {
                'event_' + key: val for key, val in event_filter.items()
            }
            self.filter(**er_filter).delete()

        # Filter results (using JOIN through Report to Event)
        Result = swapper.load_model('vera', 'Result')
        result_filter = {
            'report__event__' + key: val for key, val in event_filter.items()
        }
        ers = []
        for result in Result.objects.valid_results(**result_filter):
            er = self.model(
                event=result.report.event,
                result=result
            )
            er.denormalize()
            ers.append(er)
        self.bulk_create(ers)

    def set_all(self):
        """
        Reset the entire EventResult cache.
        """
        self.set_for_events()

class BaseEventResult(models.Model):
    """
    Denormalized event-result pairs (for valid reports)
    """
    id = models.PositiveIntegerField(primary_key=True)
    event = models.ForeignKey(MODELS['Event'])
    result = models.ForeignKey(MODELS['Result'])
    objects = EventResultManager()

    def result_value(self):
        if self.result_type.is_numeric:
            return self.result_value_numeric
        return self.result_value_text

    def __unicode__(self):
        return "%s -> %s: %s" % (
            self.event,
            self.result_type,
            self.result_value
        )

    def save(self, *args, **kwargs):
        if self.event_id is None or self.result_id is None:
            return
        self.denormalize()
        super(BaseEventResult, self).save(*args, **kwargs)

    def denormalize(self):
        """
        Denormalize all properties from the event and the result.
        """
        self.pk = self.result.pk

        def set_value(src, name):
            if field.primary_key:
                return
            obj = getattr(self, src)
            setattr(self, src + '_' + name, getattr(obj, name))

        for field in self.event._meta.fields:
            set_value('event', field.name)
        for field in self.result._meta.fields:
            set_value('result', field.name)

    class Meta:
        abstract = True

def create_eventresult_model(event_cls, result_cls,
                             base=BaseEventResult, swappable=False):
    """
    **Here be magic**

    EventResult needs to have all of the fields from Event and Result.
    In order to DRY (and since either of these classes may be swapped), we
    add the fields dynamically here.  If neither Event or Result are swapped,
    this function will be called below, otherwise the user should call
    this function in their models.py.
    """

    app = 'vera'
    module = base.__module__
    for cls in event_cls, result_cls, base:
        if cls._meta.app_label != 'vera':
            app = cls._meta.app_label
            module = cls.__module__

    class Meta(base.Meta):
        db_table = 'wq_eventresult'
        unique_together = ('event', 'result_type')
        app_label = app

    if swappable:
        Meta.swappable = swappable

    attrs = {
        'Meta': Meta,
        '__module__': module
    }

    def add_field(prefix, field):
        if field.primary_key:
            return
        attrs[prefix + '_' + field.name] = clone_field(field)

    for field in event_cls._meta.fields:
        add_field('event', field)

    for field in result_cls._meta.fields:
        add_field('result', field)

    EventResult = type(
        'EventResult',
        (base,),
        attrs
    )

    # Disconnect any existing receiver
    post_save.disconnect(dispatch_uid="eventresult_receiver")

    @receiver(post_save, weak=False, dispatch_uid="eventresult_receiver")
    def handler(sender, instance=None, **kwargs):
        event = find_event(instance)
        if event:
            EventResult.objects.set_for_event(event)

    return EventResult

# Default implementation of the above classes, can be swapped
class Site(BaseSite):
    latitude = models.FloatField(null=True, blank=True)
    longitude = models.FloatField(null=True, blank=True)

    def __unicode__(self):
        return "%s, %s" % (round(self.latitude, 3), round(self.longitude, 3))

    class Meta(BaseSite.Meta):
        db_table = 'wq_site'
        swappable = swapper.swappable_setting('vera', 'Site')
        unique_together = ('latitude', 'longitude')

class Event(BaseEvent):
    date = models.DateField()

    def __unicode__(self):
        return "%s on %s" % (self.site, self.date)

    class Meta(BaseEvent.Meta):
        db_table = 'wq_event'
        swappable = swapper.swappable_setting('vera', 'Event')
        unique_together = ('site', 'date')
        ordering = ('-date',)

class Report(BaseReport):
    class Meta(BaseReport.Meta):
        db_table = 'wq_report'
        swappable = swapper.swappable_setting('vera', 'Report')

class ReportStatus(BaseReportStatus):
    class Meta(BaseReportStatus.Meta):
        verbose_name_plural = 'report statuses'
        db_table = 'wq_reportstatus'
        swappable = swapper.swappable_setting('vera', 'ReportStatus')

class Parameter(BaseParameter):
    class Meta(BaseParameter.Meta):
        db_table = 'wq_parameter'
        swappable = swapper.swappable_setting('vera', 'Parameter')

class Result(BaseResult):
    class Meta(BaseResult.Meta):
        db_table = 'wq_result'
        swappable = swapper.swappable_setting('vera', 'Result')

EventResult = create_eventresult_model(
    Event, Result, swappable=swapper.swappable_setting('vera', 'EventResult')
)

if ((swapper.is_swapped('vera', 'Event')
        or swapper.is_swapped('vera', 'Result'))
        and not swapper.is_swapped('vera', 'EventResult')):
    raise ImproperlyConfigured(
        "Event or Result was swapped but EventResult was not!"
    )

def nest_ordering(prefix, ordering, ignore_reverse=False):
    nest_order = []
    for field in ordering:
        reverse = ''
        if field.startswith('-'):
            field = field[1:]
            if not ignore_reverse:
                reverse = '-'
        nest_order.append(reverse + prefix + '__' + field)
    return nest_order

def find_event(instance):
    if isinstance(instance, BaseEvent):
        return instance
    if isinstance(instance, BaseReport):
        return instance.event
    if isinstance(instance, BaseResult):
        return instance.report.event
    return None
<commit_msg>additional EventResult optimizations<commit_after>from wq.db.patterns import models
import datetime
import swapper
from django.db.models.signals import post_save
from django.core.exceptions import ImproperlyConfigured
from django.dispatch import receiver
from django.conf import settings
from collections import OrderedDict
from .compat import clone_field

swapper.set_app_prefix('vera', 'WQ')
MODELS = swapper.get_model_names(
    'vera', ('Site', 'Event', 'Report', 'ReportStatus', 'Parameter', 'Result')
)

VALID_REPORT_ORDER = getattr(settings, "WQ_VALID_REPORT_ORDER", ('-entered',))
# Base classes for Site-Event-Report-Attribute-Value pattern
# Extend these when swapping out default implementation (below)

class BaseSite(models.NaturalKeyModel):
    @property
    def valid_events(self):
        events = self.event_set.filter(
            report__status__is_valid=True
        ).values_list('pk', flat=True)
        # FIXME: events may be duplicated
        return self.event_set.filter(pk__in=events)

    class Meta:
        abstract = True

class BaseEvent(models.NaturalKeyModel):
    site = models.ForeignKey(MODELS['Site'], null=True, blank=True)

    @property
    def valid_reports(self):
        return self.report_set.filter(status__is_valid=True).order_by(
            *VALID_REPORT_ORDER
        )

    @property
    def vals(self):
        return OrderedDict([
            (a.type.natural_key()[0], a.value)
            for a in self.results
        ])

    # Valid results for this event
    @property
    def results(self):
        Result = swapper.load_model('vera', 'Result')
        return Result.objects.valid_results(report__event=self)

    @property
    def is_valid(self):
        return self.valid_reports.count() > 0

    class Meta:
        abstract = True

class ReportManager(models.RelatedModelManager):
    def create_report(self, event_key, values, **kwargs):
        Event = swapper.load_model('vera', 'Event')
        kwargs['event'] = Event.objects.find(*event_key)
        report = self.create(**kwargs)
        report.vals = values
        return report

class ValidReportManager(ReportManager):
    def get_queryset(self):
        qs = super(ValidReportManager, self)
        return qs.filter(status__is_valid=True).order_by(*VALID_REPORT_ORDER)

class BaseReport(models.RelatedModel):
    event = models.ForeignKey(MODELS['Event'])
    entered = models.DateTimeField(blank=True)
    user = models.ForeignKey(
        settings.AUTH_USER_MODEL, null=True, blank=True,

        # Needed for model swapping in tests
        related_name="%(app_label)s_%(class)s",
    )
    status = models.ForeignKey(MODELS['ReportStatus'], null=True, blank=True)

    objects = ReportManager()
    valid_objects = ValidReportManager()

    @property
    def is_valid(self):
        return self.status and self.status.is_valid

    @property
    def vals(self):
        vals = OrderedDict()
        for result in self.results.all():
            vals[result.type.natural_key()[0]] = result.value
        return vals

    @vals.setter
    def vals(self, vals):
        Parameter = swapper.load_model('vera', 'Parameter')
        keys = [(key,) for key in vals.keys()]
        params, success = Parameter.objects.resolve_keys(keys)
        if not success:
            missing = [
                name for name, params in params.items() if params is None
            ]
            raise TypeError(
                "Could not identify one or more parameters: %s!"
                % missing
            )

        for key, param in params.items():
            result, is_new = self.results.get_or_create(type=param)
            result.value = vals[key[0]]
            result.save()

    def save(self, *args, **kwargs):
        if not self.entered:
            self.entered = datetime.datetime.now()
        super(BaseReport, self).save(*args, **kwargs)

    def __unicode__(self):
        if self.pk is not None:
            return "%s according to %s" % (self.event, self.user)
        else:
            return "New Report"

    class Meta:
        abstract = True
        ordering = VALID_REPORT_ORDER

class BaseReportStatus(models.NaturalKeyModel):
    slug = models.SlugField()
    name = models.CharField(max_length=255)
    is_valid = models.BooleanField(default=False)

    def __unicode__(self):
        return self.name

    class Meta:
        abstract = True
        unique_together = [['slug']]

class BaseParameter(models.IdentifiedRelatedModel):
    name = models.CharField(max_length=255)
    is_numeric = models.BooleanField(default=False)
    units = models.CharField(max_length=50, null=True, blank=True)

    def __unicode__(self):
        if self.units:
            return u"%s (%s)" % (self.name, self.units)
        else:
            return self.name

    class Meta:
        abstract = True
        ordering = ('name',)

class ResultManager(models.Manager):
    def valid_results(self, **filter):
        Parameter = swapper.load_model('vera', 'Parameter')

        filter['report__status__is_valid'] = True
        filter['empty'] = False

        # DISTINCT ON event, then parameter, collapsing results from different
        # reports into one
        distinct = ['report__event__id'] + nest_ordering(
            'type', Parameter._meta.ordering
        ) + ['type__id']

        # ORDER BY distinct fields, then valid report order
        order = distinct + nest_ordering('report', VALID_REPORT_ORDER)

        return self.filter(
            **filter
        ).order_by(*order).distinct(*distinct).select_related('report')

class BaseResult(models.Model):
    type = models.ForeignKey(MODELS['Parameter'])
    report = models.ForeignKey(MODELS['Report'], related_name='results')
    value_numeric = models.FloatField(null=True, blank=True)
    value_text = models.TextField(null=True, blank=True)
    empty = models.BooleanField(default=False, db_index=True)

    objects = ResultManager()

    def is_empty(self, value):
        if value is None:
            return True
        if isinstance(value, basestring) and len(value.strip()) == 0:
            return True
        return False

    @property
    def value(self):
        if self.type.is_numeric:
            return self.value_numeric
        return self.value_text

    @value.setter
    def value(self, val):
        self.empty = self.is_empty(val)
        if self.type.is_numeric:
            if self.empty:
                self.value_numeric = None
            else:
                self.value_numeric = val
        else:
            self.value_text = val

    def __unicode__(self):
        return "%s -> %s: %s" % (self.report, self.type, self.value)

    class Meta:
        abstract = True
        ordering = ('type',)
        index_together = [
            ('type', 'report', 'empty'),
        ]

# Denormalize Events with Results from valid Reports
class EventResultManager(models.Manager):
    def set_for_event(self, event, delete=True):
        """
        Update EventResult cache for the given event.
        """
        if delete:
            self.filter(event=event).delete()
        for result in event.results:
            er = self.model(
                event=event,
                result=result
            )
            er.save()

    def set_for_events(self, delete=True, **event_filter):
        """
        Update EventResult cache for the given events.  The event query should
        be specified as query keyword arguments, rather than a queryset, so
        that a JOIN can be used instead of retrieving the results for each
        event separately.
        """

        # Delete existing EventResults (using denormalized Event fields)
        if delete:
            er_filter = {
                'event_' + key: val for key, val in event_filter.items()
            }
            self.filter(**er_filter).delete()

        # Filter results (using JOIN through Report to Event)
        Result = swapper.load_model('vera', 'Result')
        result_filter = {
            'report__event__' + key: val for key, val in event_filter.items()
        }
        ers = []
        results = Result.objects.valid_results(
            **result_filter
        ).select_related('report__event')
        for result in results:
            er = self.model(
                event=result.report.event,
                result=result
            )
            er.denormalize()
            ers.append(er)
        self.bulk_create(ers)

    def set_all(self):
        """
        Reset the entire EventResult cache.
        """
        self.set_for_events()

class BaseEventResult(models.Model):
    """
    Denormalized event-result pairs (for valid reports)
    """
    id = models.PositiveIntegerField(primary_key=True)
    event = models.ForeignKey(MODELS['Event'])
    result = models.ForeignKey(MODELS['Result'])
    objects = EventResultManager()

    def result_value(self):
        if self.result_type.is_numeric:
            return self.result_value_numeric
        return self.result_value_text

    def __unicode__(self):
        return "%s -> %s: %s" % (
            self.event,
            self.result_type,
            self.result_value
        )

    def save(self, *args, **kwargs):
        if self.event_id is None or self.result_id is None:
            return
        self.denormalize()
        super(BaseEventResult, self).save(*args, **kwargs)

    def denormalize(self):
        """
        Denormalize all properties from the event and the result.
        """
        self.pk = self.result.pk

        def set_value(src, field):
            if field.primary_key:
                return
            name = field.name
            if field.rel:
                name += "_id"
            obj = getattr(self, src)
            setattr(self, src + '_' + name, getattr(obj, name))

        for field in self.event._meta.fields:
            set_value('event', field)
        for field in self.result._meta.fields:
            set_value('result', field)

    class Meta:
        abstract = True

def create_eventresult_model(event_cls, result_cls,
                             base=BaseEventResult, swappable=False):
    """
    **Here be magic**

    EventResult needs to have all of the fields from Event and Result.
    In order to DRY (and since either of these classes may be swapped), we
    add the fields dynamically here.  If neither Event or Result are swapped,
    this function will be called below, otherwise the user should call
    this function in their models.py.
    """

    app = 'vera'
    module = base.__module__
    for cls in event_cls, result_cls, base:
        if cls._meta.app_label != 'vera':
            app = cls._meta.app_label
            module = cls.__module__

    class Meta(base.Meta):
        db_table = 'wq_eventresult'
        unique_together = ('event', 'result_type')
        app_label = app

    if swappable:
        Meta.swappable = swappable

    attrs = {
        'Meta': Meta,
        '__module__': module
    }

    def add_field(prefix, field):
        if field.primary_key:
            return
        attrs[prefix + '_' + field.name] = clone_field(field)

    for field in event_cls._meta.fields:
        add_field('event', field)

    for field in result_cls._meta.fields:
        add_field('result', field)

    EventResult = type(
        'EventResult',
        (base,),
        attrs
    )

    # Disconnect any existing receiver
    post_save.disconnect(dispatch_uid="eventresult_receiver")

    @receiver(post_save, weak=False, dispatch_uid="eventresult_receiver")
    def handler(sender, instance=None, **kwargs):
        event = find_event(instance)
        if event:
            EventResult.objects.set_for_event(event)

    return EventResult

# Default implementation of the above classes, can be swapped
class Site(BaseSite):
    latitude = models.FloatField(null=True, blank=True)
    longitude = models.FloatField(null=True, blank=True)

    def __unicode__(self):
        return "%s, %s" % (round(self.latitude, 3), round(self.longitude, 3))

    class Meta(BaseSite.Meta):
        db_table = 'wq_site'
        swappable = swapper.swappable_setting('vera', 'Site')
        unique_together = ('latitude', 'longitude')

class Event(BaseEvent):
    date = models.DateField()

    def __unicode__(self):
        return "%s on %s" % (self.site, self.date)

    class Meta(BaseEvent.Meta):
        db_table = 'wq_event'
        swappable = swapper.swappable_setting('vera', 'Event')
        unique_together = ('site', 'date')
        ordering = ('-date',)

class Report(BaseReport):
    class Meta(BaseReport.Meta):
        db_table = 'wq_report'
        swappable = swapper.swappable_setting('vera', 'Report')

class ReportStatus(BaseReportStatus):
    class Meta(BaseReportStatus.Meta):
        verbose_name_plural = 'report statuses'
        db_table = 'wq_reportstatus'
        swappable = swapper.swappable_setting('vera', 'ReportStatus')

class Parameter(BaseParameter):
    class Meta(BaseParameter.Meta):
        db_table = 'wq_parameter'
        swappable = swapper.swappable_setting('vera', 'Parameter')

class Result(BaseResult):
    class Meta(BaseResult.Meta):
        db_table = 'wq_result'
        swappable = swapper.swappable_setting('vera', 'Result')

EventResult = create_eventresult_model(
    Event, Result, swappable=swapper.swappable_setting('vera', 'EventResult')
)

if ((swapper.is_swapped('vera', 'Event')
        or swapper.is_swapped('vera', 'Result'))
        and not swapper.is_swapped('vera', 'EventResult')):
    raise ImproperlyConfigured(
        "Event or Result was swapped but EventResult was not!"
    )

def nest_ordering(prefix, ordering, ignore_reverse=False):
    nest_order = []
    for field in ordering:
        reverse = ''
        if field.startswith('-'):
            field = field[1:]
            if not ignore_reverse:
                reverse = '-'
        nest_order.append(reverse + prefix + '__' + field)
    return nest_order

def find_event(instance):
    if isinstance(instance, BaseEvent):
        return instance
    if isinstance(instance, BaseReport):
        return instance.event
    if isinstance(instance, BaseResult):
        return instance.report.event
    return None
<|endoftext|>


<commit_before>#Wall_Follow.py
#when run returns no module long_range_ir

from tamproxy import Timer
import os, sys
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from control import GoStraight
from constants import *

class WallFollow():

    ## Initialize a WallFollowing.
    #
    # @param left   A Motor representing the left motor.
    # @param right  A Motor representing the right motor.
    # @param timer  A Timer for moderating data taking.
    def __init__(self, left, right, timer, ir0, ir1, ir2, ir3):
        self.leftMotor = left
        self.rightMotor = right
        self.timer = Timer()
        self.timer.reset()
        self.ir0 = ir0
        self.ir1 = ir1
        self.ir2 = ir2
        self.ir3 = ir3

        # Number of values to record
        self.recordLen = 10
        # Record of values from youngest to oldest.
        self.record = []

        # Tweak values as needed
        self.kp = 0.9
        self.ki = -0.00005
        self.kd = -0.2

    ## get distance value from ir sensors
    # returns distance from edge of bot to wall
    #
    # if statement determines which wall is nearest

    def distance(self):
        # TODO: This is hacky. Fix it to be nice, or at least well-docced.

        # comment below only uses front right IR
        # no corner problem
        # issue: drives right when approachin wall and can't follow it

        # uses both front IRs returns value of nearest one
        # calls corner if too close to wall with both IRs

        if self.ir1.read_ir() + self.ir0.read_ir() < 90:

            self.corner(self)

        if self.ir1.read_ir()  < self.ir0.read_ir():
            #right wall nearer
            return (self.ir1.read_ir())
        else:
            #left wall nearer
            return -(self.ir0.read_ir())

    ## Given a distance value from distance make bot move to be 14 cm from wall.
    #
    # @param distance a number proportional to the distance between the edge of robot and wall
    #
    # @param speed  A value from -255 to 255 that corresponds to the general
    #               speed of the robot.

    def followWall(self, distance, speed):

        if self.timer.millis() > 100:
            self.timer.reset()

            # error value
            # 50 from hypotenuse of a 45,45,90
            err = 50 - distance

            # Integrate over the last several timesteps.
            self.record.insert(0, err)
            if len(self.record) > self.recordLen:
                self.record.pop()

            # Take the derivative over recorded history.
            deriv = self.record[0] - self.record[-1] if len(self.record) > 1 else 0

            power = self.kp * err + self.kd * deriv + self.ki * sum(self.record)

            self.leftMotor.write ((speed - power) > 0, min(abs(speed - power), 255))
            self.rightMotor.write((speed + power) > 0, min(abs(speed + power), 255))

    ## Makes bot turn spinup(counter clockwise if too close to wall)
    # will stop once IR0 is significantly smaller in value than IR1
    # once reaching corner will follow wall on right, need to fix for left
    # todo
    def corner(self):
        #need to tweak motors
        #todo

        self.leftMotor.write (1, min(30), 255)
        self.rightMotor.write(0, min(30), 255)

        if self.timer.millis() > 100:
            self.timer.reset()

            # once right IR is 30 cm less in value, bot will resume following wall.

            if (self.ir1.read_ir() + 30) < self.ir0.read_ir():

                self.followWall(self.distance(),-30)

    ## Reinitialize this class to start taking data over.
    def reset(self):
        self.timer.reset()
        self.record = []<commit_msg>moved if statement to wall follow<commit_after>#Wall_Follow.py
#when run returns no module long_range_ir

from tamproxy import Timer
import os, sys
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from control import GoStraight
from constants import *

class WallFollow():

    ## Initialize a WallFollowing.
    #
    # @param left   A Motor representing the left motor.
    # @param right  A Motor representing the right motor.
    # @param timer  A Timer for moderating data taking.
    def __init__(self, left, right, timer, ir0, ir1, ir2, ir3):
        self.leftMotor = left
        self.rightMotor = right
        self.timer = Timer()
        self.timer.reset()
        self.ir0 = ir0
        self.ir1 = ir1
        self.ir2 = ir2
        self.ir3 = ir3

        # Number of values to record
        self.recordLen = 10
        # Record of values from youngest to oldest.
        self.record = []

        # Tweak values as needed
        self.kp = 0.9
        self.ki = -0.00005
        self.kd = -0.2

    ## get distance value from ir sensors
    # returns distance from edge of bot to wall
    #
    # if statement determines which wall is nearest

    def distance(self):
        # TODO: This is hacky. Fix it to be nice, or at least well-docced.

        # issue: drives right when approachin wall and can't follow it
        # uses both front IRs returns distance of nearest wall
        if self.ir1.read_ir()  < self.ir0.read_ir():
            #right wall nearer
            return (self.ir1.read_ir())
        else:
            #left wall nearer
            return -(self.ir0.read_ir())

    ## Given a distance value from distance make bot move to be 14 cm from wall.
    #
    # @param distance a number proportional to the distance between the edge of robot and wall
    #
    # @param speed  A value from -255 to 255 that corresponds to the general
    #               speed of the robot.

    def followWall(self, distance, speed):

        if self.timer.millis() > 100:
            self.timer.reset()

            if self.ir1.read_ir() + self.ir0.read_ir() < 90:
                self.corner(self)

            else:

                # error value
                # 50 from hypotenuse of a 45,45,90
                err = 50 - distance

                # Integrate over the last several timesteps.
                self.record.insert(0, err)

                if len(self.record) > self.recordLen:
                    self.record.pop()

                # Take the derivative over recorded history.
                deriv = self.record[0] - self.record[-1] if len(self.record) > 1 else 0

                power = self.kp * err + self.kd * deriv + self.ki * sum(self.record)

                self.leftMotor.write ((speed - power) > 0, min(abs(speed - power), 255))
                self.rightMotor.write((speed + power) > 0, min(abs(speed + power), 255))

    ## Makes bot turn spinup(counter clockwise if too close to wall)
    # will stop once IR0 is significantly smaller in value than IR1
    # once reaching corner will follow wall on right, need to fix for left
    # todo
    def corner(self):
        #need to tweak motors
        #todo

        self.leftMotor.write (1, min(30), 255)
        self.rightMotor.write(0, min(30), 255)

        if self.timer.millis() > 100:
            self.timer.reset()

            # once right IR is 30 cm less in value, bot will resume following wall.

            if (self.ir1.read_ir() + 30) < self.ir0.read_ir():

                self.followWall(self.distance(),-30)

    ## Reinitialize this class to start taking data over.
    def reset(self):
        self.timer.reset()
        self.record = []<|endoftext|>


<commit_before>import os
import time
import json
import hashlib
import github
from github import Github
import github_client
from githubwriter import GithubWriter
from pprint import pprint
from gitdata import GitData
from ConfigParser import SafeConfigParser

# NexSON validation
from nexson_validator import WarningCodes, create_validation_nexson, prepare_annotation, add_or_replace_annotation

@request.restful()
def v1():
    "The OpenTree API v1"
    response.view = 'generic.json'

    # CORS support for cross-domain API requests (from anywhere)
    response.headers['Access-Control-Allow-Origin'] = "*"
    response.headers['Access-Control-Allow-Credentials'] = 'true'
    response.headers['Access-Control-Max-Age'] = 86400  # cache for a day

    app_name = "api"
    conf = SafeConfigParser({})
    if os.path.isfile("%s/applications/%s/private/localconfig" % (os.path.abspath('.'), app_name,)):
        conf.read("%s/applications/%s/private/localconfig" % (os.path.abspath('.'), app_name,))
    else:
        conf.read("%s/applications/%s/private/config" % (os.path.abspath('.'), app_name,))

    repo_path = conf.get("apis","repo_path")

    def __validate(nexson):
        '''Returns three objects:
            an annotation dict (NexSON formmatted),
            the validation_log object created when NexSON validation was performed, and
            the object of class NexSON which was created from nexson. This object will
                alias parts of the nexson dict that is passed in as an argument.
        '''
        # stub function for hooking into NexSON validation
        codes_to_skip = [WarningCodes.UNVALIDATED_ANNOTATION]
        script_name = 'api.opentreeoflife.org/validate' # TODO
        validation_log, nexson_obj = create_validation_nexson(nexson, codes_to_skip, retain_deprecated=True)
        annotation = prepare_annotation(validation_log,
                                        author_name=script_name,
                                        annotation_label="Open Tree NexSON validation")
        return annotation, validation_log, nexson_obj

    def GET(resource,resource_id,jsoncallback=None,callback=None,_=None,**kwargs):
        "OpenTree API methods relating to reading"
        valid_resources = ('study', 'search')

        if not resource in valid_resources:
            raise HTTP(400, 'Resource requested not in list of valid resources: %s' % valid_resources)

        # support JSONP request from another domain
        if jsoncallback or callback:
            response.view = 'generic.jsonp'

        # fetch using the GitHub API auth-token for a logged-in curator
        auth_token = kwargs.get('auth_token', 'ANONYMOUS')
        if auth_token == 'ANONYMOUS':
            # non-web callers might be using an HTTP header ("Authorization: token abc123def456")
            auth_header = request.env.get('http_authorization', None) or request.wsgi.environ.get('HTTP_AUTHORIZATION', None)
            if auth_header:
                auth_token = auth_header.split()[1]

        # return the correct nexson of study_id, using the specified view
        try:
            gd = GitData(repo=repo_path)
            study_nexson = gd.fetch_study(resource_id)
            return dict(FULL_RESPONSE=study_nexson)
        except Exception, e:
            return 'ERROR fetching study:\n%s' % e

    def POST(resource, resource_id=None, _method='POST', **kwargs):
        "OTOL API methods relating to creating (and importing) resources"
        gd = GitData(repo=repo_path)

        # support JSONP request from another domain
        if kwargs.get('jsoncallback',None) or kwargs.get('callback',None):
            response.view = 'generic.jsonp'

        # check for HTTP method override (passed on query string)
        if _method == 'PUT':
            PUT(resource, resource_id, kwargs)
        elif _method == 'DELETE':
            DELETE(resource, resource_id, kwargs)
        # elif _method == 'PATCH': ...

        if not resource=='study': raise HTTP(400, 'resource != study')

        # we're creating a new study (possibly with import instructions in the payload)
        cc0_agreement = kwargs.get('cc0_agreement', '')
        import_option = kwargs.get('import_option', '')
        treebase_id = kwargs.get('treebase_id', '')
        dryad_DOI = kwargs.get('dryad_DOI', '')
        import_option = kwargs.get('import_option', '')

        return kwargs
        # TODO:
        # assign a new ID for this study, create its folder in repo(?)
        # forward ID and info to treemachine, expect to get study JSON back
        # IF treemachine returns JSON, save as {ID}.json and return URL as '201 Created'
        # or perhaps '303 See other' w/ redirect?
        # (should we do this on a WIP branch? save empty folder in 'master'?)
        # IF treemachine throws an error, return error info as '500 Internal Server Error'

    def PUT(resource, resource_id=None, **kwargs):
        gd = GitData(repo=repo_path)
        "OTOL API methods relating to updating existing resources"
        #TODO, need to make this spawn a thread to do the second commit rather than block
        block_until_annotation_commit = True
        # support JSONP request from another domain
        if kwargs.get('jsoncallback',None) or kwargs.get('callback',None):
            response.view = 'generic.jsonp'
        if not resource=='study': raise HTTP(400, 'resource != study')

        if resource_id < 0 : raise HTTP(400, 'invalid resource_id: must be a postive integer')

        # this is the GitHub API auth-token for a logged-in curator
        auth_token   = kwargs.get('auth_token','')
        gh           = Github(auth_token)

        # use the Github Oauth token to get a name/email if not specified
        author_name  = kwargs.get('author_name',gh.get_user().name)
        author_email = kwargs.get('author_email',gh.get_user().email)

        if not auth_token:
            raise HTTP(400,"You must authenticate before updating via the OpenTree API")

        try:
            nexson = kwargs.get('nexson', {})
            if not isinstance(nexson, dict):
                nexson = json.loads(nexson)
        except:
            raise HTTP(400, 'NexSON must be valid JSON')

        annotation, validation_log, rich_nexson = __validate(nexson)
        if validation_log.errors:
            raise HTTP(400, json.dumps(annotation))

        # sort the keys of the POSTed NexSON and indent 0 spaces
        nexson = json.dumps(nexson, sort_keys=True, indent=0)
        # We compare sha1's instead of the actual data to reduce memory use
        # when comparing large studies
        posted_nexson_sha1 = hashlib.sha1(nexson).hexdigest()
        nexson_sha1        = hashlib.sha1( gd.fetch_study(resource_id) ).hexdigest()

        # the POSTed data is the same as what we currently have, do nothing and return successfully
        if posted_nexson_sha1 == nexson_sha1:
            return { "error": 0, "description": "success, nothing to update" };
        else:
            gd = GitData(repo=repo_path)

            branch_friendly_name = "".join([c for c in author_name if c.isalpha()])
            branch_name          = "%s_study_%s" % (branch_friendly_name, resource_id)

            def do_commit(file_content):
                author  = "%s <%s>" % (author_name, author_email)

                new_sha = gd.write_study(resource_id,file_content,branch_name,author)

                # actually push the changes to Github
                gd.push()

                # What other useful information should be returned on a successful write?
                return {
                    "error": 0,
                    "branch_name": branch_name,
                    "description": "Updated study #%s" % resource_id,
                    "sha":  new_sha
                }

            unadulterated_content_commit = do_commit(nexson)
            if unadulterated_content_commit['error'] != 0:
                return unadulterated_content_commit
            if block_until_annotation_commit:
                # add the annotation and commit the resulting blob...
                # This currently causes a TypeError: 'NexSON' object is not subscriptable
                add_or_replace_annotation(rich_nexson, annotation)
                nexson = json.dumps(rich_nexson._raw, sort_keys=True, indent=0)
                return do_commit(nexson)
            else:
                return unadulterated_content_commit

    def DELETE(resource, resource_id=None, **kwargs):
        "OTOL API methods relating to deleting existing resources"
        # support JSONP request from another domain
        if kwargs.get('jsoncallback',None) or kwargs.get('callback',None):
            response.view = 'generic.jsonp'

        if not resource=='study': raise HTTP(400, 'resource != study')

    def OPTIONS(args, **kwargs):
        "A simple method for approving CORS preflight request"
        if request.env.http_access_control_request_method:
             response.headers['Access-Control-Allow-Methods'] = request.env.http_access_control_request_method
        if request.env.http_access_control_request_headers:
             response.headers['Access-Control-Allow-Headers'] = request.env.http_access_control_request_headers
        raise HTTP(200)

    return locals()
<commit_msg>Fix the 'NexSON' object is not subscriptable bug<commit_after>import os
import time
import json
import hashlib
import github
from github import Github
import github_client
from githubwriter import GithubWriter
from pprint import pprint
from gitdata import GitData
from ConfigParser import SafeConfigParser

# NexSON validation
from nexson_validator import WarningCodes, create_validation_nexson, prepare_annotation, add_or_replace_annotation

@request.restful()
def v1():
    "The OpenTree API v1"
    response.view = 'generic.json'

    # CORS support for cross-domain API requests (from anywhere)
    response.headers['Access-Control-Allow-Origin'] = "*"
    response.headers['Access-Control-Allow-Credentials'] = 'true'
    response.headers['Access-Control-Max-Age'] = 86400  # cache for a day

    app_name = "api"
    conf = SafeConfigParser({})
    if os.path.isfile("%s/applications/%s/private/localconfig" % (os.path.abspath('.'), app_name,)):
        conf.read("%s/applications/%s/private/localconfig" % (os.path.abspath('.'), app_name,))
    else:
        conf.read("%s/applications/%s/private/config" % (os.path.abspath('.'), app_name,))

    repo_path = conf.get("apis","repo_path")

    def __validate(nexson):
        '''Returns three objects:
            an annotation dict (NexSON formmatted),
            the validation_log object created when NexSON validation was performed, and
            the object of class NexSON which was created from nexson. This object will
                alias parts of the nexson dict that is passed in as an argument.
        '''
        # stub function for hooking into NexSON validation
        codes_to_skip = [WarningCodes.UNVALIDATED_ANNOTATION]
        script_name = 'api.opentreeoflife.org/validate' # TODO
        validation_log, nexson_obj = create_validation_nexson(nexson, codes_to_skip, retain_deprecated=True)
        annotation = prepare_annotation(validation_log,
                                        author_name=script_name,
                                        annotation_label="Open Tree NexSON validation")
        return annotation, validation_log, nexson_obj

    def GET(resource,resource_id,jsoncallback=None,callback=None,_=None,**kwargs):
        "OpenTree API methods relating to reading"
        valid_resources = ('study', 'search')

        if not resource in valid_resources:
            raise HTTP(400, 'Resource requested not in list of valid resources: %s' % valid_resources)

        # support JSONP request from another domain
        if jsoncallback or callback:
            response.view = 'generic.jsonp'

        # fetch using the GitHub API auth-token for a logged-in curator
        auth_token = kwargs.get('auth_token', 'ANONYMOUS')
        if auth_token == 'ANONYMOUS':
            # non-web callers might be using an HTTP header ("Authorization: token abc123def456")
            auth_header = request.env.get('http_authorization', None) or request.wsgi.environ.get('HTTP_AUTHORIZATION', None)
            if auth_header:
                auth_token = auth_header.split()[1]

        # return the correct nexson of study_id, using the specified view
        try:
            gd = GitData(repo=repo_path)
            study_nexson = gd.fetch_study(resource_id)
            return dict(FULL_RESPONSE=study_nexson)
        except Exception, e:
            return 'ERROR fetching study:\n%s' % e

    def POST(resource, resource_id=None, _method='POST', **kwargs):
        "OTOL API methods relating to creating (and importing) resources"
        gd = GitData(repo=repo_path)

        # support JSONP request from another domain
        if kwargs.get('jsoncallback',None) or kwargs.get('callback',None):
            response.view = 'generic.jsonp'

        # check for HTTP method override (passed on query string)
        if _method == 'PUT':
            PUT(resource, resource_id, kwargs)
        elif _method == 'DELETE':
            DELETE(resource, resource_id, kwargs)
        # elif _method == 'PATCH': ...

        if not resource=='study': raise HTTP(400, 'resource != study')

        # we're creating a new study (possibly with import instructions in the payload)
        cc0_agreement = kwargs.get('cc0_agreement', '')
        import_option = kwargs.get('import_option', '')
        treebase_id = kwargs.get('treebase_id', '')
        dryad_DOI = kwargs.get('dryad_DOI', '')
        import_option = kwargs.get('import_option', '')

        return kwargs
        # TODO:
        # assign a new ID for this study, create its folder in repo(?)
        # forward ID and info to treemachine, expect to get study JSON back
        # IF treemachine returns JSON, save as {ID}.json and return URL as '201 Created'
        # or perhaps '303 See other' w/ redirect?
        # (should we do this on a WIP branch? save empty folder in 'master'?)
        # IF treemachine throws an error, return error info as '500 Internal Server Error'

    def PUT(resource, resource_id=None, **kwargs):
        gd = GitData(repo=repo_path)
        "OTOL API methods relating to updating existing resources"
        #TODO, need to make this spawn a thread to do the second commit rather than block
        block_until_annotation_commit = True
        # support JSONP request from another domain
        if kwargs.get('jsoncallback',None) or kwargs.get('callback',None):
            response.view = 'generic.jsonp'
        if not resource=='study': raise HTTP(400, 'resource != study')

        if resource_id < 0 : raise HTTP(400, 'invalid resource_id: must be a postive integer')

        # this is the GitHub API auth-token for a logged-in curator
        auth_token   = kwargs.get('auth_token','')
        gh           = Github(auth_token)

        # use the Github Oauth token to get a name/email if not specified
        author_name  = kwargs.get('author_name',gh.get_user().name)
        author_email = kwargs.get('author_email',gh.get_user().email)

        if not auth_token:
            raise HTTP(400,"You must authenticate before updating via the OpenTree API")

        try:
            nexson = kwargs.get('nexson', {})
            if not isinstance(nexson, dict):
                nexson = json.loads(nexson)
        except:
            raise HTTP(400, 'NexSON must be valid JSON')

        annotation, validation_log, rich_nexson = __validate(nexson)
        if validation_log.errors:
            raise HTTP(400, json.dumps(annotation))

        # sort the keys of the POSTed NexSON and indent 0 spaces
        nexson = json.dumps(nexson, sort_keys=True, indent=0)
        # We compare sha1's instead of the actual data to reduce memory use
        # when comparing large studies
        posted_nexson_sha1 = hashlib.sha1(nexson).hexdigest()
        nexson_sha1        = hashlib.sha1( gd.fetch_study(resource_id) ).hexdigest()

        # the POSTed data is the same as what we currently have, do nothing and return successfully
        if posted_nexson_sha1 == nexson_sha1:
            return { "error": 0, "description": "success, nothing to update" };
        else:
            gd = GitData(repo=repo_path)

            branch_friendly_name = "".join([c for c in author_name if c.isalpha()])
            branch_name          = "%s_study_%s" % (branch_friendly_name, resource_id)

            def do_commit(file_content):
                author  = "%s <%s>" % (author_name, author_email)

                new_sha = gd.write_study(resource_id,file_content,branch_name,author)

                # actually push the changes to Github
                gd.push()

                # What other useful information should be returned on a successful write?
                return {
                    "error": 0,
                    "branch_name": branch_name,
                    "description": "Updated study #%s" % resource_id,
                    "sha":  new_sha
                }

            unadulterated_content_commit = do_commit(nexson)
            if unadulterated_content_commit['error'] != 0:
                return unadulterated_content_commit
            if block_until_annotation_commit:
                # add the annotation and commit the resulting blob...
                add_or_replace_annotation(rich_nexson._raw, annotation)
                nexson = json.dumps(rich_nexson._raw, sort_keys=True, indent=0)
                return do_commit(nexson)
            else:
                return unadulterated_content_commit

    def DELETE(resource, resource_id=None, **kwargs):
        "OTOL API methods relating to deleting existing resources"
        # support JSONP request from another domain
        if kwargs.get('jsoncallback',None) or kwargs.get('callback',None):
            response.view = 'generic.jsonp'

        if not resource=='study': raise HTTP(400, 'resource != study')

    def OPTIONS(args, **kwargs):
        "A simple method for approving CORS preflight request"
        if request.env.http_access_control_request_method:
             response.headers['Access-Control-Allow-Methods'] = request.env.http_access_control_request_method
        if request.env.http_access_control_request_headers:
             response.headers['Access-Control-Allow-Headers'] = request.env.http_access_control_request_headers
        raise HTTP(200)

    return locals()
<|endoftext|>


<commit_before>import os
import time
import json
import hashlib
import github_client
from pprint import pprint
from github import Github

@request.restful()
def index():
    response.view = 'generic.json'

    def GET(resource,resource_id,jsoncallback=None,callback=None,_=None,**kwargs):
        if not resource=='study': raise HTTP(400, 'resource != study [GET]')

        # support JSONP request from another domain
        if jsoncallback or callback:
            response.view = 'generic.jsonp'

        # fetch using the GitHub API auth-token for a logged-in curator
        auth_token = kwargs.get('auth_token', 'ANONYMOUS')
        if auth_token == 'ANONYMOUS':
            # non-web callers might be using an HTTP header ("Authorization: token abc123def456")
            auth_header = request.env.get('http_authorization', None) or request.wsgi.environ.get('HTTP_AUTHORIZATION', None)
            if auth_header:
                auth_token = auth_header.split()[1]

        # return the correct nexson of study_id, using the specified view
        try:
            # TODO: use readlines() in some way, to handle humongous files?
            return dict(FULL_RESPONSE=github_client.fetch_study(resource_id, auth_token))
        except Exception, e:
            return 'ERROR fetching study:\n%s' % e

    def POST(resource, resource_id=None, **kwargs):
        # support JSONP request from another domain
        if kwargs.get('jsoncallback',None) or kwargs.get('callback',None):
            response.view = 'generic.jsonp'

        if not resource=='study': raise HTTP(400, 'resource != study')

        # TODO: use presence or absence of resource_id as a cue to creation vs.
        # updates? Or better yet, support method overrides (on POST requests)
        # for PUT, PATCH, DELETE and redirect as needed:
        # http://www.vinaysahni.com/best-practices-for-a-pragmatic-restful-api#method-override
        #
        if False:
            if resource_id:
                # this is an update(?) of an existing study, use code below
                pass
            else:
                # we're creating a new study (possibly with import instructions in the payload)
                cc0_agreement = kwargs.get('cc0_agreement', '')
                import_option = kwargs.get('import_option', '')
                treebase_id = kwargs.get('treebase_id', '')
                dryad_DOI = kwargs.get('dryad_DOI', '')
                import_option = kwargs.get('import_option', '')

                return kwargs
                # TODO: assign a new ID for this study, create its folder in repo(?)

                # forward ID and info to treemachine, expect to get study JSON back

                # IF treemachine returns JSON, save as {ID}.json and return URL as '201 Created'
                # or perhaps '303 See other' w/ redirect?
                # (should we do this on a WIP branch? save empty folder in 'master'?)

                # IF treemachine throws an error, return error info as '500 Internal Server Error'

        if resource_id < 0 : raise HTTP(400, 'invalid resource_id: must be a postive integer')

        author_name  = kwargs.get('author_name','')
        author_email = kwargs.get('author_email','')
        # this is the GitHub API auth-token for a logged-in curator
        auth_token   = kwargs.get('auth_token','')

        if not auth_token:
            raise HTTP(400,"You must authenticate to before updating via the OTOL API")

        # Connect to the Github v3 API via with this OAuth token
        gh = Github(oauth_token)

        try:
            nexson        = json.loads( kwargs.get('nexson','{}') )
        except:
            raise HTTP(400, 'NexSON must be valid JSON')

        # sort the keys of the POSTed NexSON and indent 4 spaces
        nexson = json.dumps(nexson, sort_keys=True, indent=4)

        _update_treenexus()

        # We compare sha1's instead of the actual data to reduce memory use
        # when comparing large studies
        posted_nexson_sha1 = hashlib.sha1(nexson).hexdigest()
        nexson_sha1        = hashlib.sha1( _get_nexson(resource_id, auth_token) ).hexdigest()

        # the POSTed data is the same as what we have on disk, do nothing and return successfully
        if posted_nexson_sha1 == nexson_sha1:
            return { error: 0, description: "success" };
        else:
            # we have new data
            # TODO: Use http://jacquev6.github.io/PyGithub/github_objects/Repository.html#github.Repository.Repository.create_git_commit
            return dict()
    return locals()

def _update_treenexus():
    """Update the treenexus git submodule"""
    # submodule update must be run from the top-level dir of our repo
    rc = os.system("cd ..; git submodule update")
    if rc:
        raise HTTP(400,"Unable to update local treenexus.git")

<commit_msg>Rename index method in our controller to v1 to make it clear it is v1 of the API<commit_after>import os
import time
import json
import hashlib
import github_client
from pprint import pprint
from github import Github

@request.restful()
def v1():
    "The OTOL API v1"
    response.view = 'generic.json'

    def GET(resource,resource_id,jsoncallback=None,callback=None,_=None,**kwargs):
        "OTOL API methods relating to reading"
        if not resource=='study': raise HTTP(400, 'resource != study [GET]')

        # support JSONP request from another domain
        if jsoncallback or callback:
            response.view = 'generic.jsonp'

        # fetch using the GitHub API auth-token for a logged-in curator
        auth_token = kwargs.get('auth_token', 'ANONYMOUS')
        if auth_token == 'ANONYMOUS':
            # non-web callers might be using an HTTP header ("Authorization: token abc123def456")
            auth_header = request.env.get('http_authorization', None) or request.wsgi.environ.get('HTTP_AUTHORIZATION', None)
            if auth_header:
                auth_token = auth_header.split()[1]

        # return the correct nexson of study_id, using the specified view
        try:
            # TODO: use readlines() in some way, to handle humongous files?
            return dict(FULL_RESPONSE=github_client.fetch_study(resource_id, auth_token))
        except Exception, e:
            return 'ERROR fetching study:\n%s' % e

    def POST(resource, resource_id=None, **kwargs):
        "OTOL API methods relating to writing"
        # support JSONP request from another domain
        if kwargs.get('jsoncallback',None) or kwargs.get('callback',None):
            response.view = 'generic.jsonp'

        if not resource=='study': raise HTTP(400, 'resource != study')

        # TODO: use presence or absence of resource_id as a cue to creation vs.
        # updates? Or better yet, support method overrides (on POST requests)
        # for PUT, PATCH, DELETE and redirect as needed:
        # http://www.vinaysahni.com/best-practices-for-a-pragmatic-restful-api#method-override
        #
        if False:
            if resource_id:
                # this is an update(?) of an existing study, use code below
                pass
            else:
                # we're creating a new study (possibly with import instructions in the payload)
                cc0_agreement = kwargs.get('cc0_agreement', '')
                import_option = kwargs.get('import_option', '')
                treebase_id = kwargs.get('treebase_id', '')
                dryad_DOI = kwargs.get('dryad_DOI', '')
                import_option = kwargs.get('import_option', '')

                return kwargs
                # TODO: assign a new ID for this study, create its folder in repo(?)

                # forward ID and info to treemachine, expect to get study JSON back

                # IF treemachine returns JSON, save as {ID}.json and return URL as '201 Created'
                # or perhaps '303 See other' w/ redirect?
                # (should we do this on a WIP branch? save empty folder in 'master'?)

                # IF treemachine throws an error, return error info as '500 Internal Server Error'

        if resource_id < 0 : raise HTTP(400, 'invalid resource_id: must be a postive integer')

        author_name  = kwargs.get('author_name','')
        author_email = kwargs.get('author_email','')
        # this is the GitHub API auth-token for a logged-in curator
        auth_token   = kwargs.get('auth_token','')

        if not auth_token:
            raise HTTP(400,"You must authenticate to before updating via the OTOL API")

        # Connect to the Github v3 API via with this OAuth token
        gh = Github(oauth_token)

        try:
            nexson        = json.loads( kwargs.get('nexson','{}') )
        except:
            raise HTTP(400, 'NexSON must be valid JSON')

        # sort the keys of the POSTed NexSON and indent 4 spaces
        nexson = json.dumps(nexson, sort_keys=True, indent=4)

        _update_treenexus()

        # We compare sha1's instead of the actual data to reduce memory use
        # when comparing large studies
        posted_nexson_sha1 = hashlib.sha1(nexson).hexdigest()
        nexson_sha1        = hashlib.sha1( _get_nexson(resource_id, auth_token) ).hexdigest()

        # the POSTed data is the same as what we have on disk, do nothing and return successfully
        if posted_nexson_sha1 == nexson_sha1:
            return { error: 0, description: "success" };
        else:
            # we have new data
            # TODO: Use http://jacquev6.github.io/PyGithub/github_objects/Repository.html#github.Repository.Repository.create_git_commit
            return dict()
    return locals()

def _update_treenexus():
    """Update the treenexus git submodule"""
    # submodule update must be run from the top-level dir of our repo
    rc = os.system("cd ..; git submodule update")
    if rc:
        raise HTTP(400,"Unable to update local treenexus.git")

<|endoftext|>