<commit_before># Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

import argparse
import contextlib
import hashlib
import os
import shutil
import subprocess
import sys
import tarfile

def get(url, path, verbose=False):
    print("downloading " + url)
    sha_url = url + ".sha256"
    sha_path = path + ".sha256"
    for _url, _path in ((url, path), (sha_url, sha_path)):
        # see http://serverfault.com/questions/301128/how-to-download
        if sys.platform == 'win32':
            run(["PowerShell.exe", "/nologo", "-Command",
                 "(New-Object System.Net.WebClient)"
                 ".DownloadFile('{}', '{}')".format(_url, _path)],
                verbose=verbose)
        else:
            run(["curl", "-o", _path, _url], verbose=verbose)
    print("verifying " + path)
    with open(path, "rb") as f:
        found = hashlib.sha256(f.read()).hexdigest()
    with open(sha_path, "r") as f:
        expected, _ = f.readline().split()
    if found != expected:
        err = ("invalid checksum:\n"
               "    found:    {}\n"
               "    expected: {}".format(found, expected))
        if verbose:
            raise RuntimeError(err)
        sys.exit(err)

def unpack(tarball, dst, verbose=False, match=None):
    print("extracting " + tarball)
    fname = os.path.basename(tarball).replace(".tar.gz", "")
    with contextlib.closing(tarfile.open(tarball)) as tar:
        for p in tar.getnames():
            if "/" not in p:
                continue
            name = p.replace(fname + "/", "", 1)
            if match is not None and not name.startswith(match):
                continue
            name = name[len(match) + 1:]

            fp = os.path.join(dst, name)
            if verbose:
                print("  extracting " + p)
            tar.extract(p, dst)
            tp = os.path.join(dst, p)
            if os.path.isdir(tp) and os.path.exists(fp):
                continue
            shutil.move(tp, fp)
    shutil.rmtree(os.path.join(dst, fname))

def run(args, verbose=False):
    if verbose:
        print("running: " + ' '.join(args))
    sys.stdout.flush()
    # Use Popen here instead of call() as it apparently allows powershell on
    # Windows to not lock up waiting for input presumably.
    ret = subprocess.Popen(args)
    code = ret.wait()
    if code != 0:
        err = "failed to run: " + ' '.join(args)
        if verbose:
            raise RuntimeError(err)
        sys.exit(err)

def stage0_data(rust_root):
    nightlies = os.path.join(rust_root, "src/stage0.txt")
    with open(nightlies, 'r') as nightlies:
        data = {}
        for line in nightlies.read().split("\n"):
            if line.startswith("#") or line == '':
                continue
            a, b = line.split(": ", 1)
            data[a] = b
        return data

class RustBuild:
    def download_stage0(self):
        cache_dst = os.path.join(self.build_dir, "cache")
        rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
        cargo_cache = os.path.join(cache_dst, self.stage0_cargo_date())
        if not os.path.exists(rustc_cache):
            os.makedirs(rustc_cache)
        if not os.path.exists(cargo_cache):
            os.makedirs(cargo_cache)

        if self.rustc().startswith(self.bin_root()) and \
           (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
            if os.path.exists(self.bin_root()):
                shutil.rmtree(self.bin_root())
            channel = self.stage0_rustc_channel()
            filename = "rust-std-" + channel + "-" + self.build + ".tar.gz"
            url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
            tarball = os.path.join(rustc_cache, filename)
            if not os.path.exists(tarball):
                get(url + "/" + filename, tarball, verbose=self.verbose)
            unpack(tarball, self.bin_root(),
                   match="rust-std-" + self.build,
                   verbose=self.verbose)

            filename = "rustc-" + channel + "-" + self.build + ".tar.gz"
            url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
            tarball = os.path.join(rustc_cache, filename)
            if not os.path.exists(tarball):
                get(url + "/" + filename, tarball, verbose=self.verbose)
            unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
            with open(self.rustc_stamp(), 'w') as f:
                f.write(self.stage0_rustc_date())

        if self.cargo().startswith(self.bin_root()) and \
           (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
            channel = self.stage0_cargo_channel()
            filename = "cargo-" + channel + "-" + self.build + ".tar.gz"
            url = "https://static.rust-lang.org/cargo-dist/" + self.stage0_cargo_date()
            tarball = os.path.join(cargo_cache, filename)
            if not os.path.exists(tarball):
                get(url + "/" + filename, tarball, verbose=self.verbose)
            unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
            with open(self.cargo_stamp(), 'w') as f:
                f.write(self.stage0_cargo_date())

    def stage0_cargo_date(self):
        return self._cargo_date

    def stage0_cargo_channel(self):
        return self._cargo_channel

    def stage0_rustc_date(self):
        return self._rustc_date

    def stage0_rustc_channel(self):
        return self._rustc_channel

    def rustc_stamp(self):
        return os.path.join(self.bin_root(), '.rustc-stamp')

    def cargo_stamp(self):
        return os.path.join(self.bin_root(), '.cargo-stamp')

    def rustc_out_of_date(self):
        if not os.path.exists(self.rustc_stamp()):
            return True
        with open(self.rustc_stamp(), 'r') as f:
            return self.stage0_rustc_date() != f.read()

    def cargo_out_of_date(self):
        if not os.path.exists(self.cargo_stamp()):
            return True
        with open(self.cargo_stamp(), 'r') as f:
            return self.stage0_cargo_date() != f.read()

    def bin_root(self):
        return os.path.join(self.build_dir, self.build, "stage0")

    def get_toml(self, key):
        for line in self.config_toml.splitlines():
            if line.startswith(key + ' ='):
                return self.get_string(line)
        return None

    def get_mk(self, key):
        for line in iter(self.config_mk.splitlines()):
            if line.startswith(key):
                return line[line.find(':=') + 2:].strip()
        return None

    def cargo(self):
        config = self.get_toml('cargo')
        if config:
            return config
        return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())

    def rustc(self):
        config = self.get_toml('rustc')
        if config:
            return config
        config = self.get_mk('CFG_LOCAL_RUST')
        if config:
            return config + '/bin/rustc' + self.exe_suffix()
        return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())

    def get_string(self, line):
        start = line.find('"')
        end = start + 1 + line[start+1:].find('"')
        return line[start+1:end]

    def exe_suffix(self):
        if sys.platform == 'win32':
            return '.exe'
        else:
            return ''

    def build_bootstrap(self):
        env = os.environ.copy()
        env["CARGO_TARGET_DIR"] = os.path.join(self.build_dir, "bootstrap")
        env["RUSTC"] = self.rustc()
        env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
        env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
        env["PATH"] = os.path.join(self.bin_root(), "bin") + \
                      os.pathsep + env["PATH"]
        self.run([self.cargo(), "build", "--manifest-path",
                  os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")],
                 env)

    def run(self, args, env):
        proc = subprocess.Popen(args, env = env)
        ret = proc.wait()
        if ret != 0:
            sys.exit(ret)

    def build_triple(self):
        config = self.get_toml('build')
        if config:
            return config
        config = self.get_mk('CFG_BUILD')
        if config:
            return config
        try:
            ostype = subprocess.check_output(['uname', '-s']).strip()
            cputype = subprocess.check_output(['uname', '-m']).strip()
        except FileNotFoundError:
            if sys.platform == 'win32':
                return 'x86_64-pc-windows-msvc'
            else:
                err = "uname not found"
                if self.verbose:
                    raise Exception(err)
                sys.exit(err)

        # Darwin's `uname -s` lies and always returns i386. We have to use
        # sysctl instead.
        if ostype == 'Darwin' and cputype == 'i686':
            sysctl = subprocess.check_output(['sysctl', 'hw.optional.x86_64'])
            if sysctl.contains(': 1'):
                cputype = 'x86_64'

        # The goal here is to come up with the same triple as LLVM would,
        # at least for the subset of platforms we're willing to target.
        if ostype == 'Linux':
            ostype = 'unknown-linux-gnu'
        elif ostype == 'FreeBSD':
            ostype = 'unknown-freebsd'
        elif ostype == 'DragonFly':
            ostype = 'unknown-dragonfly'
        elif ostype == 'Bitrig':
            ostype = 'unknown-bitrig'
        elif ostype == 'OpenBSD':
            ostype = 'unknown-openbsd'
        elif ostype == 'NetBSD':
            ostype = 'unknown-netbsd'
        elif ostype == 'Darwin':
            ostype = 'apple-darwin'
        elif ostype.startswith('MINGW'):
            # msys' `uname` does not print gcc configuration, but prints msys
            # configuration. so we cannot believe `uname -m`:
            # msys1 is always i686 and msys2 is always x86_64.
            # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
            # MINGW64 on x86_64.
            ostype = 'pc-windows-gnu'
            cputype = 'i686'
            if os.environ.get('MSYSTEM') == 'MINGW64':
                cputype = 'x86_64'
        elif ostype.startswith('MSYS'):
            ostype = 'pc-windows-gnu'
        elif ostype.startswith('CYGWIN_NT'):
            cputype = 'i686'
            if ostype.endswith('WOW64'):
                cputype = 'x86_64'
            ostype = 'pc-windows-gnu'
        else:
            err = "unknown OS type: " + ostype
            if self.verbose:
                raise ValueError(err)
            sys.exit(err)

        if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
            cputype = 'i686'
        elif cputype in {'xscale', 'arm'}:
            cputype = 'arm'
        elif cputype == 'armv7l':
            cputype = 'arm'
            ostype += 'eabihf'
        elif cputype == 'aarch64':
            cputype = 'aarch64'
        elif cputype in {'powerpc', 'ppc', 'ppc64'}:
            cputype = 'powerpc'
        elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
            cputype = 'x86_64'
        else:
            err = "unknown cpu type: " + cputype
            if self.verbose:
                raise ValueError(err)
            sys.exit(err)

        return cputype + '-' + ostype

def main():
    parser = argparse.ArgumentParser(description='Build rust')
    parser.add_argument('--config')
    parser.add_argument('-v', '--verbose', action='store_true')

    args = [a for a in sys.argv if a != '-h']
    args, _ = parser.parse_known_args(args)

    # Configure initial bootstrap
    rb = RustBuild()
    rb.config_toml = ''
    rb.config_mk = ''
    rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
    rb.build_dir = os.path.join(os.getcwd(), "build")
    rb.verbose = args.verbose

    try:
        with open(args.config or 'config.toml') as config:
            rb.config_toml = config.read()
    except:
        pass
    try:
        rb.config_mk = open('config.mk').read()
    except:
        pass

    data = stage0_data(rb.rust_root)
    rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
    rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)

    # Fetch/build the bootstrap
    rb.build = rb.build_triple()
    rb.download_stage0()
    sys.stdout.flush()
    rb.build_bootstrap()
    sys.stdout.flush()

    # Run the bootstrap
    args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
    args.append('--src')
    args.append(rb.rust_root)
    args.append('--build')
    args.append(rb.build)
    args.extend(sys.argv[1:])
    env = os.environ.copy()
    env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
    rb.run(args, env)

if __name__ == '__main__':
    main()
<commit_msg>Get a file during bootstrap to a temp location.<commit_after># Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

import argparse
import contextlib
import hashlib
import os
import shutil
import subprocess
import sys
import tarfile
import tempfile

def get(url, path, verbose=False):
    sha_url = url + ".sha256"
    temp_file = tempfile.NamedTemporaryFile(delete=False)
    temp_path = temp_file.name
    sha_file = tempfile.NamedTemporaryFile(suffix=".sha256", delete=True)
    sha_path = sha_file.name
    download(sha_path, sha_url, temp_path, url, verbose)
    verify(sha_path, temp_path, verbose)
    sha_file.close()
    print("moving " + temp_path + " to " + path)
    shutil.move(temp_path, path)
    temp_file.close()

def download(sha_path, sha_url, temp_path, url, verbose):
    for _url, _path in ((url, temp_path), (sha_url, sha_path)):
        print("downloading " + _url + " to " + _path)
        # see http://serverfault.com/questions/301128/how-to-download
        if sys.platform == 'win32':
            run(["PowerShell.exe", "/nologo", "-Command",
                 "(New-Object System.Net.WebClient)"
                 ".DownloadFile('{}', '{}')".format(_url, _path)],
                verbose=verbose)
        else:
            run(["curl", "-o", _path, _url], verbose=verbose)

def verify(sha_path, temp_path, verbose):
    print("verifying " + temp_path)
    with open(temp_path, "rb") as f:
        found = hashlib.sha256(f.read()).hexdigest()
    with open(sha_path, "r") as f:
        expected, _ = f.readline().split()
    if found != expected:
        err = ("invalid checksum:\n"
               "    found:    {}\n"
               "    expected: {}".format(found, expected))
        if verbose:
            raise RuntimeError(err)
        sys.exit(err)

def unpack(tarball, dst, verbose=False, match=None):
    print("extracting " + tarball)
    fname = os.path.basename(tarball).replace(".tar.gz", "")
    with contextlib.closing(tarfile.open(tarball)) as tar:
        for p in tar.getnames():
            if "/" not in p:
                continue
            name = p.replace(fname + "/", "", 1)
            if match is not None and not name.startswith(match):
                continue
            name = name[len(match) + 1:]

            fp = os.path.join(dst, name)
            if verbose:
                print("  extracting " + p)
            tar.extract(p, dst)
            tp = os.path.join(dst, p)
            if os.path.isdir(tp) and os.path.exists(fp):
                continue
            shutil.move(tp, fp)
    shutil.rmtree(os.path.join(dst, fname))

def run(args, verbose=False):
    if verbose:
        print("running: " + ' '.join(args))
    sys.stdout.flush()
    # Use Popen here instead of call() as it apparently allows powershell on
    # Windows to not lock up waiting for input presumably.
    ret = subprocess.Popen(args)
    code = ret.wait()
    if code != 0:
        err = "failed to run: " + ' '.join(args)
        if verbose:
            raise RuntimeError(err)
        sys.exit(err)

def stage0_data(rust_root):
    nightlies = os.path.join(rust_root, "src/stage0.txt")
    with open(nightlies, 'r') as nightlies:
        data = {}
        for line in nightlies.read().split("\n"):
            if line.startswith("#") or line == '':
                continue
            a, b = line.split(": ", 1)
            data[a] = b
        return data

class RustBuild:
    def download_stage0(self):
        cache_dst = os.path.join(self.build_dir, "cache")
        rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
        cargo_cache = os.path.join(cache_dst, self.stage0_cargo_date())
        if not os.path.exists(rustc_cache):
            os.makedirs(rustc_cache)
        if not os.path.exists(cargo_cache):
            os.makedirs(cargo_cache)

        if self.rustc().startswith(self.bin_root()) and \
           (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
            if os.path.exists(self.bin_root()):
                shutil.rmtree(self.bin_root())
            channel = self.stage0_rustc_channel()
            filename = "rust-std-" + channel + "-" + self.build + ".tar.gz"
            url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
            tarball = os.path.join(rustc_cache, filename)
            if not os.path.exists(tarball):
                get(url + "/" + filename, tarball, verbose=self.verbose)
            unpack(tarball, self.bin_root(),
                   match="rust-std-" + self.build,
                   verbose=self.verbose)

            filename = "rustc-" + channel + "-" + self.build + ".tar.gz"
            url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
            tarball = os.path.join(rustc_cache, filename)
            if not os.path.exists(tarball):
                get(url + "/" + filename, tarball, verbose=self.verbose)
            unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
            with open(self.rustc_stamp(), 'w') as f:
                f.write(self.stage0_rustc_date())

        if self.cargo().startswith(self.bin_root()) and \
           (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
            channel = self.stage0_cargo_channel()
            filename = "cargo-" + channel + "-" + self.build + ".tar.gz"
            url = "https://static.rust-lang.org/cargo-dist/" + self.stage0_cargo_date()
            tarball = os.path.join(cargo_cache, filename)
            if not os.path.exists(tarball):
                get(url + "/" + filename, tarball, verbose=self.verbose)
            unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
            with open(self.cargo_stamp(), 'w') as f:
                f.write(self.stage0_cargo_date())

    def stage0_cargo_date(self):
        return self._cargo_date

    def stage0_cargo_channel(self):
        return self._cargo_channel

    def stage0_rustc_date(self):
        return self._rustc_date

    def stage0_rustc_channel(self):
        return self._rustc_channel

    def rustc_stamp(self):
        return os.path.join(self.bin_root(), '.rustc-stamp')

    def cargo_stamp(self):
        return os.path.join(self.bin_root(), '.cargo-stamp')

    def rustc_out_of_date(self):
        if not os.path.exists(self.rustc_stamp()):
            return True
        with open(self.rustc_stamp(), 'r') as f:
            return self.stage0_rustc_date() != f.read()

    def cargo_out_of_date(self):
        if not os.path.exists(self.cargo_stamp()):
            return True
        with open(self.cargo_stamp(), 'r') as f:
            return self.stage0_cargo_date() != f.read()

    def bin_root(self):
        return os.path.join(self.build_dir, self.build, "stage0")

    def get_toml(self, key):
        for line in self.config_toml.splitlines():
            if line.startswith(key + ' ='):
                return self.get_string(line)
        return None

    def get_mk(self, key):
        for line in iter(self.config_mk.splitlines()):
            if line.startswith(key):
                return line[line.find(':=') + 2:].strip()
        return None

    def cargo(self):
        config = self.get_toml('cargo')
        if config:
            return config
        return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())

    def rustc(self):
        config = self.get_toml('rustc')
        if config:
            return config
        config = self.get_mk('CFG_LOCAL_RUST')
        if config:
            return config + '/bin/rustc' + self.exe_suffix()
        return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())

    def get_string(self, line):
        start = line.find('"')
        end = start + 1 + line[start+1:].find('"')
        return line[start+1:end]

    def exe_suffix(self):
        if sys.platform == 'win32':
            return '.exe'
        else:
            return ''

    def build_bootstrap(self):
        env = os.environ.copy()
        env["CARGO_TARGET_DIR"] = os.path.join(self.build_dir, "bootstrap")
        env["RUSTC"] = self.rustc()
        env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
        env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
        env["PATH"] = os.path.join(self.bin_root(), "bin") + \
                      os.pathsep + env["PATH"]
        self.run([self.cargo(), "build", "--manifest-path",
                  os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")],
                 env)

    def run(self, args, env):
        proc = subprocess.Popen(args, env = env)
        ret = proc.wait()
        if ret != 0:
            sys.exit(ret)

    def build_triple(self):
        config = self.get_toml('build')
        if config:
            return config
        config = self.get_mk('CFG_BUILD')
        if config:
            return config
        try:
            ostype = subprocess.check_output(['uname', '-s']).strip()
            cputype = subprocess.check_output(['uname', '-m']).strip()
        except FileNotFoundError:
            if sys.platform == 'win32':
                return 'x86_64-pc-windows-msvc'
            else:
                err = "uname not found"
                if self.verbose:
                    raise Exception(err)
                sys.exit(err)

        # Darwin's `uname -s` lies and always returns i386. We have to use
        # sysctl instead.
        if ostype == 'Darwin' and cputype == 'i686':
            sysctl = subprocess.check_output(['sysctl', 'hw.optional.x86_64'])
            if sysctl.contains(': 1'):
                cputype = 'x86_64'

        # The goal here is to come up with the same triple as LLVM would,
        # at least for the subset of platforms we're willing to target.
        if ostype == 'Linux':
            ostype = 'unknown-linux-gnu'
        elif ostype == 'FreeBSD':
            ostype = 'unknown-freebsd'
        elif ostype == 'DragonFly':
            ostype = 'unknown-dragonfly'
        elif ostype == 'Bitrig':
            ostype = 'unknown-bitrig'
        elif ostype == 'OpenBSD':
            ostype = 'unknown-openbsd'
        elif ostype == 'NetBSD':
            ostype = 'unknown-netbsd'
        elif ostype == 'Darwin':
            ostype = 'apple-darwin'
        elif ostype.startswith('MINGW'):
            # msys' `uname` does not print gcc configuration, but prints msys
            # configuration. so we cannot believe `uname -m`:
            # msys1 is always i686 and msys2 is always x86_64.
            # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
            # MINGW64 on x86_64.
            ostype = 'pc-windows-gnu'
            cputype = 'i686'
            if os.environ.get('MSYSTEM') == 'MINGW64':
                cputype = 'x86_64'
        elif ostype.startswith('MSYS'):
            ostype = 'pc-windows-gnu'
        elif ostype.startswith('CYGWIN_NT'):
            cputype = 'i686'
            if ostype.endswith('WOW64'):
                cputype = 'x86_64'
            ostype = 'pc-windows-gnu'
        else:
            err = "unknown OS type: " + ostype
            if self.verbose:
                raise ValueError(err)
            sys.exit(err)

        if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
            cputype = 'i686'
        elif cputype in {'xscale', 'arm'}:
            cputype = 'arm'
        elif cputype == 'armv7l':
            cputype = 'arm'
            ostype += 'eabihf'
        elif cputype == 'aarch64':
            cputype = 'aarch64'
        elif cputype in {'powerpc', 'ppc', 'ppc64'}:
            cputype = 'powerpc'
        elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
            cputype = 'x86_64'
        else:
            err = "unknown cpu type: " + cputype
            if self.verbose:
                raise ValueError(err)
            sys.exit(err)

        return cputype + '-' + ostype

def main():
    parser = argparse.ArgumentParser(description='Build rust')
    parser.add_argument('--config')
    parser.add_argument('-v', '--verbose', action='store_true')

    args = [a for a in sys.argv if a != '-h']
    args, _ = parser.parse_known_args(args)

    # Configure initial bootstrap
    rb = RustBuild()
    rb.config_toml = ''
    rb.config_mk = ''
    rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
    rb.build_dir = os.path.join(os.getcwd(), "build")
    rb.verbose = args.verbose

    try:
        with open(args.config or 'config.toml') as config:
            rb.config_toml = config.read()
    except:
        pass
    try:
        rb.config_mk = open('config.mk').read()
    except:
        pass

    data = stage0_data(rb.rust_root)
    rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
    rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)

    # Fetch/build the bootstrap
    rb.build = rb.build_triple()
    rb.download_stage0()
    sys.stdout.flush()
    rb.build_bootstrap()
    sys.stdout.flush()

    # Run the bootstrap
    args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
    args.append('--src')
    args.append(rb.rust_root)
    args.append('--build')
    args.append(rb.build)
    args.extend(sys.argv[1:])
    env = os.environ.copy()
    env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
    rb.run(args, env)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import unittest
import tempfile
import shutil
import uuid

import groundstation.node
import groundstation.transfer.response
import groundstation.transfer.request
from groundstation.station import Station

class MockStream(list):
    def enqueue(self, *args, **kwargs):
        self.append(*args, **kwargs)

def MockTERMINATE():
    pass

class MockStation(object):
    def __init__(self, **kwargs):
        self.tmpdir = tempfile.mkdtemp()
        self.node = groundstation.node.Node()
        self.station = Station(self.tmpdir, self.node)
        self.stream = MockStream()
        self.TERMINATE = MockTERMINATE
        self.id = "test_station"

        if 'origin' in kwargs:
            self.origin = kwargs['origin']
        else:
            self.origin = uuid.uuid1()

    def _Response(self, *args, **kwargs):
        kwargs['station'] = self.station
        return groundstation.transfer.response.Response(*args, **kwargs)

    def _Request(self, *args, **kwargs):
        kwargs['station'] = self.station
        return groundstation.transfer.request.Request(*args, **kwargs)

    def __del__(self):
        shutil.rmtree(self.tmpdir)

    def teardown(self):
        pass

class StationHandlerTestCase(unittest.TestCase):
    def setUp(self):
        self.station = MockStation()
<commit_msg>Implement some opt ins for tests to back onto real objects<commit_after>import unittest
import tempfile
import shutil
import uuid

import groundstation.node
import groundstation.transfer.response
import groundstation.transfer.request
from groundstation.station import Station

class MockStream(list):
    def enqueue(self, *args, **kwargs):
        self.append(*args, **kwargs)

def MockTERMINATE():
    pass

class MockStation(object):
    def __init__(self, **kwargs):
        self.tmpdir = tempfile.mkdtemp()
        self.node = groundstation.node.Node()
        self.station = Station(self.tmpdir, self.node)
        self.stream = MockStream()
        self.TERMINATE = MockTERMINATE
        self.id = "test_station"

        if 'origin' in kwargs:
            self.origin = kwargs['origin']
        else:
            self.origin = uuid.uuid1()

    def _Response(self, *args, **kwargs):
        kwargs['station'] = self.station
        return groundstation.transfer.response.Response(*args, **kwargs)

    def _Request(self, *args, **kwargs):
        kwargs['station'] = self.station
        return groundstation.transfer.request.Request(*args, **kwargs)

    def __del__(self):
        shutil.rmtree(self.tmpdir)

    def teardown(self):
        pass

    def set_real_id(self, value):
        self.id = uuid.uuid1()

    def set_real_terminate(self, value):
        sys.stderr.write("mockterminatesetter")
        if value:
            sys.stderr.write("Setting real terminate")
            self.TERMINATE = lambda: groundstation.transfer.request.TERMINATE(self)
        else:
            sys.stderr.write("Using mock terminate")
            self.TERMINATE = MockTERMINATE

    def set_real_register(self, value):
        self.station.station.registry.register(self.station)

class StationHandlerTestCase(unittest.TestCase):
    def setUp(self):
        self.station = MockStation()
<|endoftext|>


<commit_before>#!/usr/bin/python2.4
#
# Copyright (c) 2006-2007 rPath, Inc.
#
# All rights reserved
#

import os, sys
import testhelp

import threading

from jobslave import jobhandler, slave

class DummyConnection(object):
    def __init__(self, *args, **kwargs):
        self.sent = []
        self.listeners = []
        self.subscriptions = []
        self.unsubscriptions = []
        self.acks = []

    def send(self, dest, message):
        self.sent.insert(0, (dest, message))

    def receive(self, message):
        for listener in self.listeners:
            listener.receive(message)

    def subscribe(self, dest, ack = 'auto'):
        assert ack == 'client', 'Queue will not be able to refuse a message'
        self.subscriptions.insert(0, dest)

    def unsubscribe(self, dest):
        self.unsubscriptions.insert(0, dest)

    def addlistener(self, listener):
        if listener not in self.listeners:
            self.listeners.append(listener)

    def dellistener(self, listener):
        if listener in self.listeners:
            self.listeners.remove(listener)

    def start(self):
        pass

    def ack(self, messageId):
        self.acks.append(messageId)

    def insertMessage(self, message):
        message = 'message-id: dummy-message\n\n\n' + message
        self.receive(message)

    def disconnect(self):
        pass

class DummyResponse(object):
    response = DummyConnection()

class ThreadedJobSlave(slave.JobSlave, threading.Thread):
    def __init__(self, *args, **kwargs):
        threading.Thread.__init__(self)
        slave.JobSlave.__init__(self, *args, **kwargs)

    def disconnect(self):
        self.imageServer.running = False

class JobSlaveHelper(testhelp.TestCase):
    def setUp(self):
        testhelp.TestCase.setUp(self)

        self.slaveCfg = slave.SlaveConfig()
        self.slaveCfg.configLine('TTL 0')
        self.slaveCfg.configLine('imageTimeout 0')
        self.slaveCfg.configLine('namespace test')
        self.slaveCfg.configLine('nodeName testMaster:testSlave')
        self.slaveCfg.configLine('jobQueueName job3.0.0:x86')
        self.jobSlave = ThreadedJobSlave(self.slaveCfg)

    def tearDown(self):
        self.jobSlave.imageServer.stop()
        testhelp.TestCase.tearDown(self)

    def getHandler(self, buildType):
        return jobhandler.getHandler( \
            {'serialVersion': 1,
             'type' : 'build',
             'project' : {'name': 'Foo',
                          'hostname' : 'foo',
                          'label': 'foo.rpath.local@rpl:devel',
                          'conaryCfg': ''},
             'UUID' : 'mint.rpath.local-build-25',
             'troveName' : 'group-core',
             'troveVersion' : '/conary.rpath.com@rpl:1/0:1.0.1-1-1',
             'troveFlavor': '1#x86',
             'data' : {'jsversion': '3.0.0'},
             'outputQueue': 'test',
             'buildType' : buildType},
            self.jobSlave)

    def supressOutput(self, func, *args, **kwargs):
        oldErr = os.dup(sys.stderr.fileno())
        oldOut = os.dup(sys.stdout.fileno())
        fd = os.open(os.devnull, os.W_OK)
        os.dup2(fd, sys.stderr.fileno())
        os.dup2(fd, sys.stdout.fileno())
        os.close(fd)
        try:
            return func(*args, **kwargs)
        finally:
            os.dup2(oldErr, sys.stderr.fileno())
            os.dup2(oldOut, sys.stdout.fileno())
<commit_msg>fix up finished dir<commit_after>#!/usr/bin/python2.4
#
# Copyright (c) 2006-2007 rPath, Inc.
#
# All rights reserved
#

import os, sys
import testhelp

import threading
import tempfile

from jobslave import jobhandler, slave, constants, generators
from conary.lib import util

class DummyConnection(object):
    def __init__(self, *args, **kwargs):
        self.sent = []
        self.listeners = []
        self.subscriptions = []
        self.unsubscriptions = []
        self.acks = []

    def send(self, dest, message):
        self.sent.insert(0, (dest, message))

    def receive(self, message):
        for listener in self.listeners:
            listener.receive(message)

    def subscribe(self, dest, ack = 'auto'):
        assert ack == 'client', 'Queue will not be able to refuse a message'
        self.subscriptions.insert(0, dest)

    def unsubscribe(self, dest):
        self.unsubscriptions.insert(0, dest)

    def addlistener(self, listener):
        if listener not in self.listeners:
            self.listeners.append(listener)

    def dellistener(self, listener):
        if listener in self.listeners:
            self.listeners.remove(listener)

    def start(self):
        pass

    def ack(self, messageId):
        self.acks.append(messageId)

    def insertMessage(self, message):
        message = 'message-id: dummy-message\n\n\n' + message
        self.receive(message)

    def disconnect(self):
        pass

class DummyResponse(object):
    response = DummyConnection()

class ThreadedJobSlave(slave.JobSlave, threading.Thread):
    def __init__(self, *args, **kwargs):
        threading.Thread.__init__(self)
        slave.JobSlave.__init__(self, *args, **kwargs)

    def disconnect(self):
        self.imageServer.running = False

class JobSlaveHelper(testhelp.TestCase):
    def setUp(self):
        testhelp.TestCase.setUp(self)

        self.slaveCfg = slave.SlaveConfig()
        self.slaveCfg.configLine('TTL 0')
        self.slaveCfg.configLine('imageTimeout 0')
        self.slaveCfg.configLine('namespace test')
        self.slaveCfg.configLine('nodeName testMaster:testSlave')
        self.slaveCfg.configLine('jobQueueName job3.0.0:x86')
        self.jobSlave = ThreadedJobSlave(self.slaveCfg)

        self.finishedDir = tempfile.mkdtemp(prefix="jobslave-test")
        generators.constants.finishedDir = self.finishedDir

    def tearDown(self):
        util.rmtree(self.finishedDir)
        self.jobSlave.imageServer.stop()
        testhelp.TestCase.tearDown(self)

    def getHandler(self, buildType):
        return jobhandler.getHandler( \
            {'serialVersion': 1,
             'type' : 'build',
             'project' : {'name': 'Foo',
                          'hostname' : 'foo',
                          'label': 'foo.rpath.local@rpl:devel',
                          'conaryCfg': ''},
             'UUID' : 'mint.rpath.local-build-25',
             'troveName' : 'group-core',
             'troveVersion' : '/conary.rpath.com@rpl:1/0:1.0.1-1-1',
             'troveFlavor': '1#x86',
             'data' : {'jsversion': '3.0.0'},
             'outputQueue': 'test',
             'buildType' : buildType},
            self.jobSlave)

    def supressOutput(self, func, *args, **kwargs):
        oldErr = os.dup(sys.stderr.fileno())
        oldOut = os.dup(sys.stdout.fileno())
        fd = os.open(os.devnull, os.W_OK)
        os.dup2(fd, sys.stderr.fileno())
        os.dup2(fd, sys.stdout.fileno())
        os.close(fd)
        try:
            return func(*args, **kwargs)
        finally:
            os.dup2(oldErr, sys.stderr.fileno())
            os.dup2(oldOut, sys.stdout.fileno())
<|endoftext|>


<commit_before>#!/usr/bin/env python3
from __future__ import print_function
import subprocess
import os

if __name__ == "__main__":
    # Spawn lots of process
    for i in range(0, 10):
        cmd = ["sleep", str(1 + i % 2)]
        proc = subprocess.Popen(cmd)
<commit_msg>Randomize tests a little<commit_after>#!/usr/bin/env python3
from __future__ import print_function
import subprocess
import os
import random

if __name__ == "__main__":
    # Spawn lots of process
    for i in range(0, 100):
        cmd = ["sleep", str(1 + i % 2 + random.random())]
        proc = subprocess.Popen(cmd)
<|endoftext|>


<commit_before>
import unittest

import os
import sys
import platform
import atexit

import requests
import cookielib
from cookielib import IPV4_RE

import io
import csv
import json
import urllib

#import logging
#logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
#cookielib.debug = True

_scheme = os.getenv('TEST_SCHEME', 'https')
_server = os.getenv('TEST_HOSTNAME', platform.uname()[1])
_server_url = "%s://%s" % (_scheme, _server)

if os.getenv('TEST_SSL_VERIFY', 'true').lower() == 'false':
    _verify = False
else:
    _verify = True

# this will be the dynamically generated catalog ID and corresponding path
cid = None
cpath = None

def dump_cookies(cookies, preamble):
    sys.stderr.write(preamble)
    for c in cookies:
        sys.stderr.write('  %r\n' % c)
    sys.stderr.write('end\n')

class TestSession (requests.Session):
    """Our extended version of requests.Session for test client stubs.

       We have a built-in idea of the server and catalog URL and
       override all the testing HTTP methods to supply a bare path and
       prepend these to get a full URL for each access.

       An absolute path e.g. "/ermrest" will be appended to the bare
       server URL.

       A relative path e.g. "schema" will be appended to the catalog
       URL.

    """

    def _test_mount(self, cookiefilename=None, reportname=None):
        """Mount the statically configured test server, configure a cookiestore."""
        requests.Session.mount(
            self, _server_url + '/',
            requests.adapters.HTTPAdapter(max_retries=5)
        )

        self.verify = _verify

        if cookiefilename:
            cj = cookielib.MozillaCookieJar(cookiefilename)
            cj.load(ignore_expires=True)
            for cookie in cj:
                kwargs={
                    "domain": cookie.domain,
                    "path": cookie.path,
                }
                if cookie.domain.find('.') == -1 and not IPV4_RE.search(cookie.domain):
                    # mangle this the same way cookielib does or domain matching will fail!
                    kwargs['domain'] = cookie.domain + '.local'
                if cookie.expires:
                    kwargs['expires'] = cookie.expires

                self.cookies.set(
                    cookie.name,
                    cookie.value,
                    **kwargs
                )

        if reportname:
            dump_cookies(
                self.cookies,
                ('Created %s' + (' with cookies %s:\n' % cookiefilename if cookiefilename else '\n')) % reportname
            )

    def _path2url(self, path):
        if path == '' or path[0] != '/':
            path = "%s/%s" % (cpath, path)
        return _server_url + path

    def get_client_id(self):
        r = self.get('/authn/session')
        if r.status_code == 200:
            return r.json()["client"]["id"]
        else:
            return None

    def head(self, path, **kwargs):
        return requests.Session.head(self, self._path2url(path), **kwargs)

    def get(self, path, **kwargs):
        return requests.Session.get(self, self._path2url(path), **kwargs)

    def _fixup_kwargs(self, kwargs):
        if 'json' in kwargs and kwargs['json'] is None and 'data' not in kwargs:
            kwargs['data'] = 'null'
            kwargs['headers'] = dict(kwargs.get('headers', {}))
            kwargs['headers'].update({'content-type': 'application/json'})

    def put(self, path, **kwargs):
        self._fixup_kwargs(kwargs)
        return requests.Session.put(self, self._path2url(path), **kwargs)

    def post(self, path, **kwargs):
        self._fixup_kwargs(kwargs)
        return requests.Session.post(self, self._path2url(path), **kwargs)

    def delete(self, path, **kwargs):
        return requests.Session.delete(self, self._path2url(path), **kwargs)

# setup the primary session (privileged user)
primary_session = TestSession()
_primary_cookies = os.getenv('TEST_COOKIES1')
assert _primary_cookies, "TEST_COOKIES1 must be a cookie file name"
primary_session._test_mount(_primary_cookies, 'primary session')
primary_client_id = primary_session.get_client_id()
sys.stderr.write('Using primary_session with client ID %r.\n' % primary_client_id)

# setup the secondary session (less privileged user) if possible
secondary_session = TestSession()
_secondary_cookies = os.getenv('TEST_COOKIES2')
if _secondary_cookies:
    secondary_session._test_mount(_secondary_cookies, 'secondary session')
    secondary_client_id = secondary_session.get_client_id()
    if secondary_client_id is None:
        secondary_session = None
    else:
        assert primary_client_id != secondary_client_id, "TEST_COOKIES1 and TEST_COOKIES2 must provide distinct client IDs"
        sys.stderr.write('Using secondary_session with client ID %r.\n' % secondary_client_id)
else:
    sys.stderr.write('Disabling secondary_session due to missing TEST_COOKIES2.\n\n')
    secondary_session = None
    secondary_client_id = None

catalog_acls = {
    "owner": [primary_client_id],
    "write": [],
    "insert": [],
    "update": [],
    "delete": [],
    "select": ["*"],
    "create": [],
    "enumerate": ["*"]
}

sys.stderr.write('Creating test catalog... ')
try:
    _r = primary_session.post('/ermrest/catalog')
    _r.raise_for_status()
    cid = _r.json()['id']
    cpath = "/ermrest/catalog/%s" % cid
    primary_session.put('acl', json=catalog_acls).raise_for_status()
    sys.stderr.write('OK.\n\n')
except Exception, e:
    sys.stderr.write('ERROR: %s.\n\n' % e)
    sys.stderr.write('REQUEST: %r %r\n%r\n\n\nRESPONSE: %r\n%r\n%r\n' % (
        _r.request.method,
        _r.request.url,
        _r.request.headers,
        _r.status_code,
        _r.headers,
        _r.content
    ))
    raise e

# setup the anonymous session (no authentication) if possible
anonymous_session = TestSession()
anonymous_session._test_mount(reportname='anonymous session')
try:
    anonymous_session.get('').raise_for_status()
except Exception, e:
    sys.stderr.write('Disabling anonymous_session due to error: %s\n\n' % e)
    anonymous_session = None

@atexit.register
def _cleanup():
    if cid is not None:
        sys.stderr.write('\nOn exit, deleting %s... ' % cpath)
        r = primary_session.delete(cpath)
        try:
            r.raise_for_status()
            sys.stderr.write('OK.\n')
        except Exception, e:
            sys.stderr.write('ERROR: %s.\n' % e)

class ErmrestTest (unittest.TestCase):
    session = primary_session

    def assertJsonEqual(self, actual, expected):
        if type(expected) in (str, unicode):
            self.assertIn(type(actual), (str, unicode))
        else:
            self.assertIsInstance(actual, type(expected))
        if type(actual) is list:
            self.assertEqual(len(actual), len(expected))
            for i in range(len(actual)):
                self.assertJsonEqual(actual[i], expected[i])
        elif type(actual) is dict:
            self.assertEqual(set(actual.keys()), set(expected.keys()))
            for k in actual.keys():
                self.assertJsonEqual(actual[k], expected[k])
        else:
            self.assertEqual(actual, expected)

    def assertHttp(self, r, status=200, content_type=None, hdrs={}):
        try:
            if status is None:
                pass
            elif type(status) is int:
                self.assertEqual(r.status_code, status)
            else:
                self.assertIn(r.status_code, status)
            if content_type is None:
                pass
            elif type(content_type) in (str, unicode):
                self.assertEqual(r.headers.get('content-type')[0:len(content_type)], content_type)
            else:
                self.assertIn(r.headers.get('content-type'), content_type)
                self.assertDictContainsSubset(hdrs, r.headers)
        except:
            sys.stderr.write('\n%s %s\n%s\n%s\n%s\n%s\n%s\n\n' % (
                r.request.method,
                r.request.url,
                r.request.headers,
                r.request.body,
                r.status_code,
                r.headers,
                r.content
                ))
            raise

def array_to_csv(a, json_array=False):
    assert type(a) is list
    keys = set()
    for row in a:
        keys.update(set(row.keys()))
    keys = list(keys)

    def wrap(v):
        if v is None:
            return ''
        elif type(v) is dict:
            return json.dumps(v)
        elif type(v) is list:
            if json_array:
                return json.dumps(v)
            else:
                return '{%s}' % (','.join([
                    '"%s"' % wrap(x).replace('"', '\\"') for x in v
                ]))
        else:
            return "%s" % v

    output = io.BytesIO()
    writer = csv.writer(output)
    writer.writerow(keys)
    for row in a:
        writer.writerow([ wrap(row[k]) for k in keys ])
    return output.getvalue()

def urlquote(s):
    return urllib.quote(s, safe="")
<commit_msg>turn off verbose cookiestore logging<commit_after>
import unittest

import os
import sys
import platform
import atexit

import requests
import cookielib
from cookielib import IPV4_RE

import io
import csv
import json
import urllib

#import logging
#logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
#cookielib.debug = True

_scheme = os.getenv('TEST_SCHEME', 'https')
_server = os.getenv('TEST_HOSTNAME', platform.uname()[1])
_server_url = "%s://%s" % (_scheme, _server)

if os.getenv('TEST_SSL_VERIFY', 'true').lower() == 'false':
    _verify = False
else:
    _verify = True

# this will be the dynamically generated catalog ID and corresponding path
cid = None
cpath = None

def dump_cookies(cookies, preamble):
    sys.stderr.write(preamble)
    for c in cookies:
        sys.stderr.write('  %r\n' % c)
    sys.stderr.write('end\n')

class TestSession (requests.Session):
    """Our extended version of requests.Session for test client stubs.

       We have a built-in idea of the server and catalog URL and
       override all the testing HTTP methods to supply a bare path and
       prepend these to get a full URL for each access.

       An absolute path e.g. "/ermrest" will be appended to the bare
       server URL.

       A relative path e.g. "schema" will be appended to the catalog
       URL.

    """

    def _test_mount(self, cookiefilename=None, reportname=None):
        """Mount the statically configured test server, configure a cookiestore."""
        requests.Session.mount(
            self, _server_url + '/',
            requests.adapters.HTTPAdapter(max_retries=5)
        )

        self.verify = _verify

        if cookiefilename:
            cj = cookielib.MozillaCookieJar(cookiefilename)
            cj.load(ignore_expires=True)
            for cookie in cj:
                kwargs={
                    "domain": cookie.domain,
                    "path": cookie.path,
                }
                if cookie.domain.find('.') == -1 and not IPV4_RE.search(cookie.domain):
                    # mangle this the same way cookielib does or domain matching will fail!
                    kwargs['domain'] = cookie.domain + '.local'
                if cookie.expires:
                    kwargs['expires'] = cookie.expires

                self.cookies.set(
                    cookie.name,
                    cookie.value,
                    **kwargs
                )

        if reportname:
            preamble = ('Created %s' + (' with cookies %s:\n' % cookiefilename if cookiefilename else '\n')) % reportname
            if False:
                dump_cookies(self.cookies, preamble)
            else:
                sys.stderr.write(preamble)

    def _path2url(self, path):
        if path == '' or path[0] != '/':
            path = "%s/%s" % (cpath, path)
        return _server_url + path

    def get_client_id(self):
        r = self.get('/authn/session')
        if r.status_code == 200:
            return r.json()["client"]["id"]
        else:
            return None

    def head(self, path, **kwargs):
        return requests.Session.head(self, self._path2url(path), **kwargs)

    def get(self, path, **kwargs):
        return requests.Session.get(self, self._path2url(path), **kwargs)

    def _fixup_kwargs(self, kwargs):
        if 'json' in kwargs and kwargs['json'] is None and 'data' not in kwargs:
            kwargs['data'] = 'null'
            kwargs['headers'] = dict(kwargs.get('headers', {}))
            kwargs['headers'].update({'content-type': 'application/json'})

    def put(self, path, **kwargs):
        self._fixup_kwargs(kwargs)
        return requests.Session.put(self, self._path2url(path), **kwargs)

    def post(self, path, **kwargs):
        self._fixup_kwargs(kwargs)
        return requests.Session.post(self, self._path2url(path), **kwargs)

    def delete(self, path, **kwargs):
        return requests.Session.delete(self, self._path2url(path), **kwargs)

# setup the primary session (privileged user)
primary_session = TestSession()
_primary_cookies = os.getenv('TEST_COOKIES1')
assert _primary_cookies, "TEST_COOKIES1 must be a cookie file name"
primary_session._test_mount(_primary_cookies, 'primary session')
primary_client_id = primary_session.get_client_id()
sys.stderr.write('Using primary_session with client ID %r.\n' % primary_client_id)

# setup the secondary session (less privileged user) if possible
secondary_session = TestSession()
_secondary_cookies = os.getenv('TEST_COOKIES2')
if _secondary_cookies:
    secondary_session._test_mount(_secondary_cookies, 'secondary session')
    secondary_client_id = secondary_session.get_client_id()
    if secondary_client_id is None:
        secondary_session = None
    else:
        assert primary_client_id != secondary_client_id, "TEST_COOKIES1 and TEST_COOKIES2 must provide distinct client IDs"
        sys.stderr.write('Using secondary_session with client ID %r.\n' % secondary_client_id)
else:
    sys.stderr.write('Disabling secondary_session due to missing TEST_COOKIES2.\n\n')
    secondary_session = None
    secondary_client_id = None

catalog_acls = {
    "owner": [primary_client_id],
    "write": [],
    "insert": [],
    "update": [],
    "delete": [],
    "select": ["*"],
    "create": [],
    "enumerate": ["*"]
}

sys.stderr.write('Creating test catalog... ')
try:
    _r = primary_session.post('/ermrest/catalog')
    _r.raise_for_status()
    cid = _r.json()['id']
    cpath = "/ermrest/catalog/%s" % cid
    primary_session.put('acl', json=catalog_acls).raise_for_status()
    sys.stderr.write('OK.\n\n')
except Exception, e:
    sys.stderr.write('ERROR: %s.\n\n' % e)
    sys.stderr.write('REQUEST: %r %r\n%r\n\n\nRESPONSE: %r\n%r\n%r\n' % (
        _r.request.method,
        _r.request.url,
        _r.request.headers,
        _r.status_code,
        _r.headers,
        _r.content
    ))
    raise e

# setup the anonymous session (no authentication) if possible
anonymous_session = TestSession()
anonymous_session._test_mount(reportname='anonymous session')
try:
    anonymous_session.get('').raise_for_status()
except Exception, e:
    sys.stderr.write('Disabling anonymous_session due to error: %s\n\n' % e)
    anonymous_session = None

@atexit.register
def _cleanup():
    if cid is not None:
        sys.stderr.write('\nOn exit, deleting %s... ' % cpath)
        r = primary_session.delete(cpath)
        try:
            r.raise_for_status()
            sys.stderr.write('OK.\n')
        except Exception, e:
            sys.stderr.write('ERROR: %s.\n' % e)

class ErmrestTest (unittest.TestCase):
    session = primary_session

    def assertJsonEqual(self, actual, expected):
        if type(expected) in (str, unicode):
            self.assertIn(type(actual), (str, unicode))
        else:
            self.assertIsInstance(actual, type(expected))
        if type(actual) is list:
            self.assertEqual(len(actual), len(expected))
            for i in range(len(actual)):
                self.assertJsonEqual(actual[i], expected[i])
        elif type(actual) is dict:
            self.assertEqual(set(actual.keys()), set(expected.keys()))
            for k in actual.keys():
                self.assertJsonEqual(actual[k], expected[k])
        else:
            self.assertEqual(actual, expected)

    def assertHttp(self, r, status=200, content_type=None, hdrs={}):
        try:
            if status is None:
                pass
            elif type(status) is int:
                self.assertEqual(r.status_code, status)
            else:
                self.assertIn(r.status_code, status)
            if content_type is None:
                pass
            elif type(content_type) in (str, unicode):
                self.assertEqual(r.headers.get('content-type')[0:len(content_type)], content_type)
            else:
                self.assertIn(r.headers.get('content-type'), content_type)
                self.assertDictContainsSubset(hdrs, r.headers)
        except:
            sys.stderr.write('\n%s %s\n%s\n%s\n%s\n%s\n%s\n\n' % (
                r.request.method,
                r.request.url,
                r.request.headers,
                r.request.body,
                r.status_code,
                r.headers,
                r.content
                ))
            raise

def array_to_csv(a, json_array=False):
    assert type(a) is list
    keys = set()
    for row in a:
        keys.update(set(row.keys()))
    keys = list(keys)

    def wrap(v):
        if v is None:
            return ''
        elif type(v) is dict:
            return json.dumps(v)
        elif type(v) is list:
            if json_array:
                return json.dumps(v)
            else:
                return '{%s}' % (','.join([
                    '"%s"' % wrap(x).replace('"', '\\"') for x in v
                ]))
        else:
            return "%s" % v

    output = io.BytesIO()
    writer = csv.writer(output)
    writer.writerow(keys)
    for row in a:
        writer.writerow([ wrap(row[k]) for k in keys ])
    return output.getvalue()

def urlquote(s):
    return urllib.quote(s, safe="")
<|endoftext|>


<commit_before># /test/test_acceptance.py
#
# Acceptance tests for travis-bump-version.
#
# See /LICENCE.md for Copyright information
"""Acceptance tests for travis-bump-version."""

import sys

from mock import Mock, call

from nose_parameterized import parameterized

from six import StringIO

from testtools import ExpectedException, TestCase
from testtools.matchers import Contains, MatchesAll, Not

import travisbumpversion.main

def run(*args, **kwargs):
    """Run, converting kwargs to switches and passing args."""
    arguments = list(args)
    for key, value in kwargs.items():
        arguments.append("--" + key.replace("_", "-"))
        if not isinstance(value, list):
            value = [value]

        arguments.extend(value)

    try:
        travisbumpversion.main.main(arguments)
    finally:
        sys.stderr.seek(0)
        sys.stdout.seek(0)

def _format_with_args(func, num, params):
    """Format docstring using provided arguments."""
    del num

    return func.__doc__.format(*params[0])

class TestAcceptance(TestCase):

    """Acceptance tests for travis-bump-version."""

    def __init__(self, *args, **kwargs):
        """Initialize instance variables."""
        super(TestAcceptance, self).__init__(*args, **kwargs)
        self.last_commit_message_body = Mock()
        self.last_tag_from_git = Mock()
        self.call_bumpversion = Mock()
        self.push_commit_and_tags = Mock()

    def setUp(self):
        """Set up acceptance tests, patch out functions calling subprocess."""
        super(TestAcceptance, self).setUp()
        self.patch(travisbumpversion.main,
                   "_last_commit_message_body",
                   self.last_commit_message_body)
        self.patch(travisbumpversion.main,
                   "_last_tag_from_git",
                   self.last_tag_from_git)
        self.patch(travisbumpversion.main,
                   "_call_bumpversion",
                   self.call_bumpversion)
        self.patch(travisbumpversion.main,
                   "_push_commit_and_tags",
                   self.push_commit_and_tags)
        self.patch(sys, "exit", Mock())
        self.patch(sys, "stdout", StringIO())
        self.patch(sys, "stderr", StringIO())

    def test_bail_when_not_passing_api_token(self):
        """Raise when failing to pass --api-token."""
        with ExpectedException(RuntimeError):
            sys.exit.side_effect = RuntimeError("")
            run("file", repo="user/repo")

        output = sys.stderr.read()
        self.assertThat(output, MatchesAll(Contains("""--api-token"""),
                                           Contains("""required""")))
        self.assertEqual(call(2), sys.exit.call_args_list[0])

    def test_bail_when_not_passing_repo(self):
        """Raise when failing to pass --repo."""
        with ExpectedException(RuntimeError):
            sys.exit.side_effect = RuntimeError("")
            run("file", api_token="token")

        output = sys.stderr.read()
        self.assertThat(output, MatchesAll(Contains("""--repo"""),
                                           Contains("""required""")))
        self.assertEqual(call(2), sys.exit.call_args_list[0])

    def test_bail_when_not_passing_files(self):
        """Raise when failing to pass list of files."""
        run(api_token="token", repo="user/repo")
        self.assertThat(sys.stderr.read(),
                        Contains("""Need to specify some files"""))
        self.assertEqual(call(1), sys.exit.call_args_list[0])

    def test_bail_when_not_last_tag_doesnt_start_with_v(self):
        """Raise when last tag doesn't start with v."""
        self.last_tag_from_git.return_value = "x0.0.1"
        run("file", api_token="token", repo="user/repo")
        self.assertThat(sys.stderr.read(),
                        Contains("""Release a tag"""))
        self.assertEqual(call(1), sys.exit.call_args_list[0])

    def test_use_patch_by_default_for_empty_body(self):
        """Use level patch by default for empty commit message body."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = ""
        run("file", api_token="token", repo="user/repo")
        self.call_bumpversion.assert_called_with("patch", ["file"])
        self.assertEqual(call(0), sys.exit.call_args_list[0])

    def test_use_patch_by_default_for_body_with_no_code(self):
        """Use level patch by default commit message body with no code."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = "Title\n\nBody"
        run("file", api_token="token", repo="user/repo")
        self.call_bumpversion.assert_called_with("patch", ["file"])
        self.assertEqual(call(0), sys.exit.call_args_list[0])

    def test_use_patch_level_for_malformed_bumpversion_code(self):
        """Use level patch when bumpversion code is malformed."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = "T\n\nbumpversion: mjr"
        run("file", api_token="token", repo="user/repo")
        self.call_bumpversion.assert_called_with("patch", ["file"])
        self.assertEqual(call(0), sys.exit.call_args_list[0])

    def test_complain_for_malformed_bumpversion_code(self):
        """Complain when bumpversion code is malformed."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = "T\n\nbumpversion: mjr"
        run("file", api_token="token", repo="user/repo")
        self.assertThat(sys.stderr.read(), Contains("""Invalid code mjr"""))

    @parameterized.expand(("major", "minor", "patch"),
                          testcase_func_doc=_format_with_args)
    def test_use_valid_bumpversion_code(self, lvl):
        """Use level {0} when bumpversion code is set to {0}."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = "T\n\nbumpversion: " + lvl
        run("file", api_token="token", repo="user/repo")
        self.call_bumpversion.assert_called_with(lvl, ["file"])
        self.assertEqual(call(0), sys.exit.call_args_list[0])

    def test_push_commit_and_tags_success(self):
        """Push commit and tags with provided token and repo."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = ""
        run("file", api_token="token", repo="user/repo")
        self.push_commit_and_tags.assert_called_with("token", "user/repo")
        self.assertEqual(call(0), sys.exit.call_args_list[0])

    def test_push_commit_and_tags_complain_on_failure(self):
        """Push commit and tags with provided token and repo."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = ""
        error = RuntimeError("""Failed""")
        self.push_commit_and_tags.side_effect = error
        run("file", api_token="token", repo="user/repo")
        self.assertThat(sys.stderr.read(), Contains("""Failed to push"""))
        self.assertEqual(call(0), sys.exit.call_args_list[0])

    def test_push_commit_fail_contains_git_reasons(self):
        """Failed push reports why git push failed."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = ""
        error = RuntimeError("""Git reasons""")
        self.push_commit_and_tags.side_effect = error
        run("file", api_token="token", repo="user/repo")
        self.assertThat(sys.stderr.read(), Contains("""Git reasons"""))
        self.assertEqual(call(0), sys.exit.call_args_list[0])

    def test_push_failure_dont_disclose_api_key(self):
        """Don't disclose API keys on push failure."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = ""
        error = RuntimeError("""Failed token""")
        self.push_commit_and_tags.side_effect = error
        run("file", api_token="token", repo="user/repo")
        stderr_output = sys.stderr.read()
        self.assertThat(stderr_output, Not(Contains("token")))
        self.assertThat(stderr_output, Contains("[redacted]"))
        self.assertEqual(call(0), sys.exit.call_args_list[0])
<commit_msg>test_acceptance: Fix linter errors<commit_after># /test/test_acceptance.py
#
# Acceptance tests for travis-bump-version.
#
# See /LICENCE.md for Copyright information
"""Acceptance tests for travis-bump-version."""

import sys

from mock import Mock, call

from nose_parameterized import parameterized

from six import StringIO

from testtools import ExpectedException, TestCase
from testtools.matchers import Contains, MatchesAll, Not

import travisbumpversion.main

def run(*args, **kwargs):
    """Run, converting kwargs to switches and passing args."""
    arguments = list(args)
    for key, value in kwargs.items():
        arguments.append("--" + key.replace("_", "-"))
        if not isinstance(value, list):
            value = [value]

        arguments.extend(value)

    try:
        travisbumpversion.main.main(arguments)
    finally:
        sys.stderr.seek(0)
        sys.stdout.seek(0)

def _format_with_args(func, num, params):
    """Format docstring using provided arguments."""
    del num

    return func.__doc__.format(*params[0])

class TestAcceptance(TestCase):
    """Acceptance tests for travis-bump-version."""

    def __init__(self, *args, **kwargs):
        """Initialize instance variables."""
        super(TestAcceptance, self).__init__(*args, **kwargs)
        self.last_commit_message_body = Mock()
        self.last_tag_from_git = Mock()
        self.call_bumpversion = Mock()
        self.push_commit_and_tags = Mock()

    def setUp(self):
        """Set up acceptance tests, patch out functions calling subprocess."""
        super(TestAcceptance, self).setUp()
        self.patch(travisbumpversion.main,
                   "_last_commit_message_body",
                   self.last_commit_message_body)
        self.patch(travisbumpversion.main,
                   "_last_tag_from_git",
                   self.last_tag_from_git)
        self.patch(travisbumpversion.main,
                   "_call_bumpversion",
                   self.call_bumpversion)
        self.patch(travisbumpversion.main,
                   "_push_commit_and_tags",
                   self.push_commit_and_tags)
        self.patch(sys, "exit", Mock())
        self.patch(sys, "stdout", StringIO())
        self.patch(sys, "stderr", StringIO())

    def test_bail_when_not_passing_api_token(self):
        """Raise when failing to pass --api-token."""
        with ExpectedException(RuntimeError):
            sys.exit.side_effect = RuntimeError("")
            run("file", repo="user/repo")

        output = sys.stderr.read()
        self.assertThat(output, MatchesAll(Contains("""--api-token"""),
                                           Contains("""required""")))
        self.assertEqual(call(2), sys.exit.call_args_list[0])

    def test_bail_when_not_passing_repo(self):
        """Raise when failing to pass --repo."""
        with ExpectedException(RuntimeError):
            sys.exit.side_effect = RuntimeError("")
            run("file", api_token="token")

        output = sys.stderr.read()
        self.assertThat(output, MatchesAll(Contains("""--repo"""),
                                           Contains("""required""")))
        self.assertEqual(call(2), sys.exit.call_args_list[0])

    def test_bail_when_not_passing_files(self):
        """Raise when failing to pass list of files."""
        run(api_token="token", repo="user/repo")
        self.assertThat(sys.stderr.read(),
                        Contains("""Need to specify some files"""))
        self.assertEqual(call(1), sys.exit.call_args_list[0])

    def test_bail_when_not_last_tag_doesnt_start_with_v(self):
        """Raise when last tag doesn't start with v."""
        self.last_tag_from_git.return_value = "x0.0.1"
        run("file", api_token="token", repo="user/repo")
        self.assertThat(sys.stderr.read(),
                        Contains("""Release a tag"""))
        self.assertEqual(call(1), sys.exit.call_args_list[0])

    def test_use_patch_by_default_for_empty_body(self):
        """Use level patch by default for empty commit message body."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = ""
        run("file", api_token="token", repo="user/repo")
        self.call_bumpversion.assert_called_with("patch", ["file"])
        self.assertEqual(call(0), sys.exit.call_args_list[0])

    def test_use_patch_by_default_for_body_with_no_code(self):
        """Use level patch by default commit message body with no code."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = "Title\n\nBody"
        run("file", api_token="token", repo="user/repo")
        self.call_bumpversion.assert_called_with("patch", ["file"])
        self.assertEqual(call(0), sys.exit.call_args_list[0])

    def test_use_patch_level_for_malformed_bumpversion_code(self):
        """Use level patch when bumpversion code is malformed."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = "T\n\nbumpversion: mjr"
        run("file", api_token="token", repo="user/repo")
        self.call_bumpversion.assert_called_with("patch", ["file"])
        self.assertEqual(call(0), sys.exit.call_args_list[0])

    def test_complain_for_malformed_bumpversion_code(self):
        """Complain when bumpversion code is malformed."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = "T\n\nbumpversion: mjr"
        run("file", api_token="token", repo="user/repo")
        self.assertThat(sys.stderr.read(), Contains("""Invalid code mjr"""))

    @parameterized.expand(("major", "minor", "patch"),
                          testcase_func_doc=_format_with_args)
    def test_use_valid_bumpversion_code(self, lvl):
        """Use level {0} when bumpversion code is set to {0}."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = "T\n\nbumpversion: " + lvl
        run("file", api_token="token", repo="user/repo")
        self.call_bumpversion.assert_called_with(lvl, ["file"])
        self.assertEqual(call(0), sys.exit.call_args_list[0])

    def test_push_commit_and_tags_success(self):
        """Push commit and tags with provided token and repo."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = ""
        run("file", api_token="token", repo="user/repo")
        self.push_commit_and_tags.assert_called_with("token", "user/repo")
        self.assertEqual(call(0), sys.exit.call_args_list[0])

    def test_push_commit_and_tags_complain_on_failure(self):
        """Push commit and tags with provided token and repo."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = ""
        error = RuntimeError("""Failed""")
        self.push_commit_and_tags.side_effect = error
        run("file", api_token="token", repo="user/repo")
        self.assertThat(sys.stderr.read(), Contains("""Failed to push"""))
        self.assertEqual(call(0), sys.exit.call_args_list[0])

    def test_push_commit_fail_contains_git_reasons(self):
        """Failed push reports why git push failed."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = ""
        error = RuntimeError("""Git reasons""")
        self.push_commit_and_tags.side_effect = error
        run("file", api_token="token", repo="user/repo")
        self.assertThat(sys.stderr.read(), Contains("""Git reasons"""))
        self.assertEqual(call(0), sys.exit.call_args_list[0])

    def test_push_failure_dont_disclose_api_key(self):
        """Don't disclose API keys on push failure."""
        self.last_tag_from_git.return_value = "v0.0.1"
        self.last_commit_message_body.return_value = ""
        error = RuntimeError("""Failed token""")
        self.push_commit_and_tags.side_effect = error
        run("file", api_token="token", repo="user/repo")
        stderr_output = sys.stderr.read()
        self.assertThat(stderr_output, Not(Contains("token")))
        self.assertThat(stderr_output, Contains("[redacted]"))
        self.assertEqual(call(0), sys.exit.call_args_list[0])
<|endoftext|>


<commit_before>import store_fixture
import groundstation.store

class TestGitStore(store_fixture.StoreTestCase):
    storeClass = groundstation.store.git_store.GitStore
<commit_msg>Add testcase for database initialization<commit_after>import os

import store_fixture
import groundstation.store

class TestGitStore(store_fixture.StoreTestCase):
    storeClass = groundstation.store.git_store.GitStore

    def test_creates_required_dirs(self):
        for d in groundstation.store.git_store.GitStore.required_dirs:
            path = os.path.join(self.path, d)
            self.assertTrue(os.path.exists(path))
            self.assertTrue(os.path.isdir(path))
<|endoftext|>


<commit_before>import colorsys

# RGB
def rgb(c1, c2, c3):
    return ((c1 * 127.5) + 127.5, (c2 * 127.5) + 127.5, (c3 * 127.5) + 127.5)

# HSL
def hsl(c1, c2, c3):
    (r, g, b) = colorsys.hls_to_rgb((c1 * 0.5) + 0.5, (c3 * 0.5) + 0.5, (c2 * 0.5) + 0.5)
    return (r * 255.0, g * 255.0, b * 255.0)

colors = [rgb, hsl]
colorsLength = len(colors)
<commit_msg>add color generator<commit_after>import colorsys
from ..random import random

# RGB
def rgb(c1, c2, c3):
    return ((c1 * 127.5) + 127.5, (c2 * 127.5) + 127.5, (c3 * 127.5) + 127.5)

# HSL
def hsl(c1, c2, c3):
    (r, g, b) = colorsys.hls_to_rgb((c1 * 0.5) + 0.5, (c3 * 0.5) + 0.5, (c2 * 0.5) + 0.5)
    return (r * 255.0, g * 255.0, b * 255.0)

# Generate
def generateColors():
    color1 = float(random()) / float(0xFFFFFFFFFFFFFFFF)
    return color1, (color1 + (float(random()) / (7.0 * float(0xFFFFFFFFFFFFFFFF)))) % 1.0

colors = [rgb, hsl]
colorsLength = len(colors)
<|endoftext|>