<commit_before>#!/usr/bin/env python
#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

# About this script:
#   This script is intended to simplify creating Subversion releases, by
#   automating as much as is possible.  It works well with our Apache
#   infrastructure, and should make rolling, posting, and announcing
#   releases dirt simple.
#
#   This script may be run on a number of platforms, but it is intended to
#   be run on people.apache.org.  As such, it may have dependencies (such
#   as Python version) which may not be common, but are guaranteed to be
#   available on people.apache.org.

# Stuff we need
import os
import sys
import shutil
import urllib2
import tarfile
import logging
import subprocess
import argparse       # standard in Python 2.7

# Our required / recommended versions
autoconf_ver = '2.68'
libtool_ver = '2.4'
swig_ver = '2.0.4'

# Some constants
repos = 'http://svn.apache.org/repos/asf/subversion'

#----------------------------------------------------------------------
# Utility functions

def get_prefix(base_dir):
    return os.path.join(base_dir, 'prefix')

def get_tempdir(base_dir):
    return os.path.join(base_dir, 'tempdir')

def get_nullfile():
    # This is certainly not cross platform
    return open('/dev/null', 'w')

def run_script(verbose, script):
    if verbose:
        stdout = None
        stderr = None
    else:
        stdout = get_nullfile()
        stderr = subprocess.STDOUT

    for l in script.split('\n'):
        subprocess.check_call(l.split(), stdout=stdout, stderr=stderr)

def download_file(url, target):
    response = urllib2.urlopen(url)
    target_file = open(target, 'w')
    target_file.write(response.read())

#----------------------------------------------------------------------
# Cleaning up the environment

def cleanup(base_dir, args):
    'Remove generated files and folders.'
    logging.info('Cleaning')

    shutil.rmtree(get_prefix(base_dir), True)
    shutil.rmtree(get_tempdir(base_dir), True)

#----------------------------------------------------------------------
# Creating an environment to roll the release

class RollDep(object):
    'The super class for each of the build dependencies.'
    def __init__(self, base_dir, use_existing, verbose):
        self._base_dir = base_dir
        self._use_existing = use_existing
        self._verbose = verbose

    def _test_version(self, cmd):
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
        (stdout, stderr) = proc.communicate()
        rc = proc.wait()
        if rc: return ''

        return stdout.split('\n')

    def build(self):
        if not hasattr(self, '_extra_configure_flags'):
            self._extra_configure_flags = ''
        cwd = os.getcwd()
        tempdir = get_tempdir(self._base_dir)
        tarball = os.path.join(tempdir, self._filebase + '.tar.gz')

        if os.path.exists(tarball):
            if not self._use_existing:
                raise RuntimeError('autoconf tarball "%s" already exists'
                                                                    % tarball)
            logging.info('Using existing %s.tar.gz' % self._filebase)
        else:
            logging.info('Fetching %s' % self._filebase)
            download_file(self._url, tarball)

        # Extract tarball
        tarfile.open(tarball).extractall(tempdir)

        logging.info('Building ' + self.label)
        os.chdir(os.path.join(tempdir, self._filebase))
        run_script(self._verbose,
                   '''./configure --prefix=%s %s
                      make
                      make install''' % (get_prefix(self._base_dir),
                                         self._extra_configure_flags))

        os.chdir(cwd)

class AutoconfDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'autoconf'
        self._filebase = 'autoconf-' + autoconf_ver
        self._url = 'http://ftp.gnu.org/gnu/autoconf/%s.tar.gz' % self._filebase

    def have_usable(self):
        output = self._test_version(['autoconf', '-V'])
        if not output: return False

        version = output[0].split()[-1:][0]
        return version == autoconf_ver

    def use_system(self):
        if not self._use_existing: return False
        return self.have_usable()

class LibtoolDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'libtool'
        self._filebase = 'libtool-' + libtool_ver
        self._url = 'http://ftp.gnu.org/gnu/libtool/%s.tar.gz' % self._filebase

    def have_usable(self):
        output = self._test_version(['libtool', '--version'])
        if not output: return False

        version = output[0].split()[-1:][0]
        return version == libtool_ver

    def use_system(self):
        # We unconditionally return False here, to avoid using a borked
        # system libtool (I'm looking at you, Debian).
        return False

class SwigDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose, sf_mirror):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'swig'
        self._filebase = 'swig-' + swig_ver
        self._url = 'http://sourceforge.net/projects/swig/files/swig/%(swig)s/%(swig)s.tar.gz/download?use_mirror=%(sf_mirror)s' % \
            { 'swig' : self._filebase,
              'sf_mirror' : sf_mirror }
        self._extra_configure_flags = '--without-pcre'

    def have_usable(self):
        output = self._test_version(['swig', '-version'])
        if not output: return False

        version = output[1].split()[-1:][0]
        return version == swig_ver

    def use_system(self):
        if not self._use_existing: return False
        return have_usable()

def build_env(base_dir, args):
    'Download prerequisites for a release and prepare the environment.'
    logging.info('Creating release environment')

    try:
        os.mkdir(get_prefix(base_dir))
        os.mkdir(get_tempdir(base_dir))
    except OSError:
        if not args.use_existing:
            raise

    autoconf = AutoconfDep(base_dir, args.use_existing, args.verbose)
    libtool = LibtoolDep(base_dir, args.use_existing, args.verbose)
    swig = SwigDep(base_dir, args.use_existing, args.verbose, args.sf_mirror)

    # iterate over our rolling deps, and build them if needed
    for dep in [autoconf, libtool, swig]:
        if dep.use_system():
            logging.info('Using system %s' % dep.label)
        else:
            dep.build()

#----------------------------------------------------------------------
# Create release artifacts

def roll_tarballs(base_dir, args):
    'Create the release artifacts.'
    version_base = args.version.split('-')[0]
    version_extra = args.version.split('-')[1]

    if args.branch:
        branch = args.branch
    else:
        branch = version_base[:-1] + 'x'

    logging.info('Rolling release %s from branch %s@%d' % (args.version,
                                                           branch, args.revnum))

    # Ensure we've got the appropriate rolling dependencies available
    autoconf = AutoconfDep(base_dir, False, args.verbose)
    libtool = LibtoolDep(base_dir, False, args.verbose)
    swig = SwigDep(base_dir, False, args.verbose, None)

    for dep in [autoconf, libtool, swig]:
        if not dep.have_usable():
           raise RuntimeError('Cannot find usable %s' % dep.label)

def announce(base_dir, args):
    'Write the release announcement.'

#----------------------------------------------------------------------
# Main entry point for argument parsing and handling

def main():
    'Parse arguments, and drive the appropriate subcommand.'

    # Setup our main parser
    parser = argparse.ArgumentParser(
                            description='Create an Apache Subversion release.')
    parser.add_argument('--clean', action='store_true', default=False,
                   help='Remove any directories previously created by %(prog)s')
    parser.add_argument('--verbose', action='store_true', default=False,
                   help='Increase output verbosity')
    parser.add_argument('--base-dir', default=os.getcwd(),
                   help='''The directory in which to create needed files and
                           folders.  The default is the current working
                           directory.''')
    subparsers = parser.add_subparsers(title='subcommands')

    # Setup the parser for the build-env subcommand
    subparser = subparsers.add_parser('build-env',
                    help='''Download release prerequisistes, including autoconf,
                            libtool, and swig.''')
    subparser.set_defaults(func=build_env)
    subparser.add_argument('--sf-mirror', default='softlayer',
                    help='''The mirror to use for downloading files from
                            SourceForge.  If in the EU, you may want to use
                            'kent' for this value.''')
    subparser.add_argument('--use-existing', action='store_true', default=False,
                    help='''Attempt to use existing build dependencies before
                            downloading and building a private set.''')

    # Setup the parser for the roll subcommand
    subparser = subparsers.add_parser('roll',
                    help='''Create the release artifacts.''')
    subparser.set_defaults(func=roll_tarballs)
    subparser.add_argument('version',
                    help='''The release label, such as '1.7.0-alpha1'.''')
    subparser.add_argument('revnum', type=int,
                    help='''The revision number to base the release on.''')
    subparser.add_argument('--branch',
                    help='''The branch to base the release on.''')

    # A meta-target
    subparser = subparsers.add_parser('clean',
                    help='''The same as the '--clean' switch, but as a
                            separate subcommand.''')
    subparser.set_defaults(func=cleanup)

    # Parse the arguments
    args = parser.parse_args()

    # first, process any global operations
    if args.clean:
        cleanup(args.base_dir, args)

    # Set up logging
    logger = logging.getLogger()
    if args.verbose:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    # Fix up our path so we can use our installed versions
    os.environ['PATH'] = os.path.join(get_prefix(args.base_dir), 'bin') + ':' \
                                                            + os.environ['PATH']

    # finally, run the subcommand, and give it the parsed arguments
    args.func(args.base_dir, args)

if __name__ == '__main__':
    main()
<commit_msg>Teach release.py to take the place of roll.sh.  The 'roll' subcommand now confirms the rolling environment is correct, shell's out to dist.sh, and then moves the tarballs into the correct location, calculating sha1 sums along the way.<commit_after>#!/usr/bin/env python
#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

# About this script:
#   This script is intended to simplify creating Subversion releases, by
#   automating as much as is possible.  It works well with our Apache
#   infrastructure, and should make rolling, posting, and announcing
#   releases dirt simple.
#
#   This script may be run on a number of platforms, but it is intended to
#   be run on people.apache.org.  As such, it may have dependencies (such
#   as Python version) which may not be common, but are guaranteed to be
#   available on people.apache.org.

# Stuff we need
import os
import sys
import shutil
import urllib2
import hashlib
import tarfile
import logging
import subprocess
import argparse       # standard in Python 2.7

# Our required / recommended versions
autoconf_ver = '2.68'
libtool_ver = '2.4'
swig_ver = '2.0.4'

# Some constants
repos = 'http://svn.apache.org/repos/asf/subversion'

#----------------------------------------------------------------------
# Utility functions

def get_prefix(base_dir):
    return os.path.join(base_dir, 'prefix')

def get_tempdir(base_dir):
    return os.path.join(base_dir, 'tempdir')

def get_deploydir(base_dir):
    return os.path.join(base_dir, 'deploy')

def get_nullfile():
    # This is certainly not cross platform
    return open('/dev/null', 'w')

def run_script(verbose, script):
    if verbose:
        stdout = None
        stderr = None
    else:
        stdout = get_nullfile()
        stderr = subprocess.STDOUT

    for l in script.split('\n'):
        subprocess.check_call(l.split(), stdout=stdout, stderr=stderr)

def download_file(url, target):
    response = urllib2.urlopen(url)
    target_file = open(target, 'w')
    target_file.write(response.read())

#----------------------------------------------------------------------
# Cleaning up the environment

def cleanup(base_dir, args):
    'Remove generated files and folders.'
    logging.info('Cleaning')

    shutil.rmtree(get_prefix(base_dir), True)
    shutil.rmtree(get_tempdir(base_dir), True)
    shutil.rmtree(get_deploydir(base_dir), True)

#----------------------------------------------------------------------
# Creating an environment to roll the release

class RollDep(object):
    'The super class for each of the build dependencies.'
    def __init__(self, base_dir, use_existing, verbose):
        self._base_dir = base_dir
        self._use_existing = use_existing
        self._verbose = verbose

    def _test_version(self, cmd):
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
        (stdout, stderr) = proc.communicate()
        rc = proc.wait()
        if rc: return ''

        return stdout.split('\n')

    def build(self):
        if not hasattr(self, '_extra_configure_flags'):
            self._extra_configure_flags = ''
        cwd = os.getcwd()
        tempdir = get_tempdir(self._base_dir)
        tarball = os.path.join(tempdir, self._filebase + '.tar.gz')

        if os.path.exists(tarball):
            if not self._use_existing:
                raise RuntimeError('autoconf tarball "%s" already exists'
                                                                    % tarball)
            logging.info('Using existing %s.tar.gz' % self._filebase)
        else:
            logging.info('Fetching %s' % self._filebase)
            download_file(self._url, tarball)

        # Extract tarball
        tarfile.open(tarball).extractall(tempdir)

        logging.info('Building ' + self.label)
        os.chdir(os.path.join(tempdir, self._filebase))
        run_script(self._verbose,
                   '''./configure --prefix=%s %s
                      make
                      make install''' % (get_prefix(self._base_dir),
                                         self._extra_configure_flags))

        os.chdir(cwd)

class AutoconfDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'autoconf'
        self._filebase = 'autoconf-' + autoconf_ver
        self._url = 'http://ftp.gnu.org/gnu/autoconf/%s.tar.gz' % self._filebase

    def have_usable(self):
        output = self._test_version(['autoconf', '-V'])
        if not output: return False

        version = output[0].split()[-1:][0]
        return version == autoconf_ver

    def use_system(self):
        if not self._use_existing: return False
        return self.have_usable()

class LibtoolDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'libtool'
        self._filebase = 'libtool-' + libtool_ver
        self._url = 'http://ftp.gnu.org/gnu/libtool/%s.tar.gz' % self._filebase

    def have_usable(self):
        output = self._test_version(['libtool', '--version'])
        if not output: return False

        version = output[0].split()[-1:][0]
        return version == libtool_ver

    def use_system(self):
        # We unconditionally return False here, to avoid using a borked
        # system libtool (I'm looking at you, Debian).
        return False

class SwigDep(RollDep):
    def __init__(self, base_dir, use_existing, verbose, sf_mirror):
        RollDep.__init__(self, base_dir, use_existing, verbose)
        self.label = 'swig'
        self._filebase = 'swig-' + swig_ver
        self._url = 'http://sourceforge.net/projects/swig/files/swig/%(swig)s/%(swig)s.tar.gz/download?use_mirror=%(sf_mirror)s' % \
            { 'swig' : self._filebase,
              'sf_mirror' : sf_mirror }
        self._extra_configure_flags = '--without-pcre'

    def have_usable(self):
        output = self._test_version(['swig', '-version'])
        if not output: return False

        version = output[1].split()[-1:][0]
        return version == swig_ver

    def use_system(self):
        if not self._use_existing: return False
        return have_usable()

def build_env(base_dir, args):
    'Download prerequisites for a release and prepare the environment.'
    logging.info('Creating release environment')

    try:
        os.mkdir(get_prefix(base_dir))
        os.mkdir(get_tempdir(base_dir))
    except OSError:
        if not args.use_existing:
            raise

    autoconf = AutoconfDep(base_dir, args.use_existing, args.verbose)
    libtool = LibtoolDep(base_dir, args.use_existing, args.verbose)
    swig = SwigDep(base_dir, args.use_existing, args.verbose, args.sf_mirror)

    # iterate over our rolling deps, and build them if needed
    for dep in [autoconf, libtool, swig]:
        if dep.use_system():
            logging.info('Using system %s' % dep.label)
        else:
            dep.build()

#----------------------------------------------------------------------
# Create release artifacts

def roll_tarballs(base_dir, args):
    'Create the release artifacts.'
    extns = ['zip', 'tar.gz', 'tar.bz2']
    version_base = args.version.split('-')[0]
    version_extra = args.version.split('-')[1]

    if args.branch:
        branch = args.branch
    else:
        branch = version_base[:-1] + 'x'

    logging.info('Rolling release %s from branch %s@%d' % (args.version,
                                                           branch, args.revnum))

    # Ensure we've got the appropriate rolling dependencies available
    autoconf = AutoconfDep(base_dir, False, args.verbose)
    libtool = LibtoolDep(base_dir, False, args.verbose)
    swig = SwigDep(base_dir, False, args.verbose, None)

    for dep in [autoconf, libtool, swig]:
        if not dep.have_usable():
           raise RuntimeError('Cannot find usable %s' % dep.label)

    # Make sure CHANGES is sync'd
    if not branch == 'trunk':
        trunk_CHANGES = '%s/trunk/CHANGES@%d' % (repos, args.revnum)
        branch_CHANGES = '%s/branches/%s/CHANGES@%d' % (repos, branch,
                                                        args.revnum)
        proc = subprocess.Popen(['svn', 'diff', '--summarize', branch_CHANGES,
                                   trunk_CHANGES],
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT)
        (stdout, stderr) = proc.communicate()
        proc.wait()

        if stdout:
            raise RuntimeError('CHANGES not synced between trunk and branch')

    # Create the output directory
    if not os.path.exists(get_deploydir(base_dir)):
        os.mkdir(get_deploydir(base_dir))

    # For now, just delegate to dist.sh to create the actual artifacts
    if version_extra:
        if version_extra.startswith('alpha'):
            extra_args = '-alpha %s' % version_extra[5:]
        elif version_extra.startswith('beta'):
            extra_args = '-beta %s' % version_extra[4:]
        elif version_extra.startswith('rc'):
            extra_args = '-rc %s' % version_extra[2:]
    logging.info('Building UNIX tarballs')
    run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d %s'
                     % (sys.path[0], version_base, branch, args.revnum,
                        extra_args) )
    logging.info('Buildling Windows tarballs')
    run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d -zip %s'
                     % (sys.path[0], version_base, branch, args.revnum,
                        extra_args) )

    # Move the results to the deploy directory
    logging.info('Moving resulting artifacts')
    for e in extns:
        shutil.move('subversion-%s.%s' % (args.version, e),
                    get_deploydir(base_dir))
    shutil.move('svn_version.h.dist', get_deploydir(base_dir))

    # Create sha1 sums
    logging.info('Calculating checksums')
    for e in extns:
        filename = os.path.join(get_deploydir(base_dir), 'subversion-%s.%s' %
                                                         (args.version, e))
        m = hashlib.sha1()
        m.update(open(filename, 'r').read())
        open(filename + '.sha1', 'w').write(m.hexdigest())

    # And we're done!

def announce(base_dir, args):
    'Write the release announcement.'

#----------------------------------------------------------------------
# Main entry point for argument parsing and handling

def main():
    'Parse arguments, and drive the appropriate subcommand.'

    # Setup our main parser
    parser = argparse.ArgumentParser(
                            description='Create an Apache Subversion release.')
    parser.add_argument('--clean', action='store_true', default=False,
                   help='Remove any directories previously created by %(prog)s')
    parser.add_argument('--verbose', action='store_true', default=False,
                   help='Increase output verbosity')
    parser.add_argument('--base-dir', default=os.getcwd(),
                   help='''The directory in which to create needed files and
                           folders.  The default is the current working
                           directory.''')
    subparsers = parser.add_subparsers(title='subcommands')

    # Setup the parser for the build-env subcommand
    subparser = subparsers.add_parser('build-env',
                    help='''Download release prerequisistes, including autoconf,
                            libtool, and swig.''')
    subparser.set_defaults(func=build_env)
    subparser.add_argument('--sf-mirror', default='softlayer',
                    help='''The mirror to use for downloading files from
                            SourceForge.  If in the EU, you may want to use
                            'kent' for this value.''')
    subparser.add_argument('--use-existing', action='store_true', default=False,
                    help='''Attempt to use existing build dependencies before
                            downloading and building a private set.''')

    # Setup the parser for the roll subcommand
    subparser = subparsers.add_parser('roll',
                    help='''Create the release artifacts.''')
    subparser.set_defaults(func=roll_tarballs)
    subparser.add_argument('version',
                    help='''The release label, such as '1.7.0-alpha1'.''')
    subparser.add_argument('revnum', type=int,
                    help='''The revision number to base the release on.''')
    subparser.add_argument('--branch',
                    help='''The branch to base the release on.''')

    # A meta-target
    subparser = subparsers.add_parser('clean',
                    help='''The same as the '--clean' switch, but as a
                            separate subcommand.''')
    subparser.set_defaults(func=cleanup)

    # Parse the arguments
    args = parser.parse_args()

    # first, process any global operations
    if args.clean:
        cleanup(args.base_dir, args)

    # Set up logging
    logger = logging.getLogger()
    if args.verbose:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    # Fix up our path so we can use our installed versions
    os.environ['PATH'] = os.path.join(get_prefix(args.base_dir), 'bin') + ':' \
                                                            + os.environ['PATH']

    # finally, run the subcommand, and give it the parsed arguments
    args.func(args.base_dir, args)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import inspect
from collections import Mapping, OrderedDict
from functools import partial

from ..utils.orderedtype import OrderedType
from .argument import Argument, to_arguments
from .structures import NonNull
from .unmountedtype import UnmountedType

base_type = type

def source_resolver(source, root, args, context, info):
    resolved = getattr(root, source, None)
    if inspect.isfunction(resolved):
        return resolved()
    return resolved

class Field(OrderedType):

    def __init__(self, type, args=None, resolver=None, source=None,
                 deprecation_reason=None, name=None, description=None,
                 required=False, _creation_counter=None, default_value=None,
                 **extra_args):
        super(Field, self).__init__(_creation_counter=_creation_counter)
        assert not args or isinstance(args, Mapping), (
            'Arguments in a field have to be a mapping, received "{}".'
        ).format(args)
        assert not (source and resolver), (
            'A Field cannot have a source and a resolver in at the same time.'
        )
        assert not callable(default_value), (
            'The default value can not be a function but received "{}".'
        ).format(base_type(default_value))

        if required:
            type = NonNull(type)

        # Check if name is actually an argument of the field
        if isinstance(name, (Argument, UnmountedType)):
            extra_args['name'] = name
            name = None

        # Check if source is actually an argument of the field
        if isinstance(source, (Argument, UnmountedType)):
            extra_args['source'] = source
            source = None

        self.name = name
        self._type = type
        self.args = to_arguments(args or OrderedDict(), extra_args)
        if source:
            resolver = partial(source_resolver, source)
        self.resolver = resolver
        self.deprecation_reason = deprecation_reason
        self.description = description
        self.default_value = default_value

    @property
    def type(self):
        if inspect.isfunction(self._type):
            return self._type()
        return self._type

    def get_resolver(self, parent_resolver):
        return self.resolver or parent_resolver
<commit_msg>fix so partial functions can be passed to field, solves issue #316<commit_after>import inspect
from collections import Mapping, OrderedDict
from functools import partial

from ..utils.orderedtype import OrderedType
from .argument import Argument, to_arguments
from .structures import NonNull
from .unmountedtype import UnmountedType

base_type = type

def source_resolver(source, root, args, context, info):
    resolved = getattr(root, source, None)
    if inspect.isfunction(resolved):
        return resolved()
    return resolved

class Field(OrderedType):

    def __init__(self, type, args=None, resolver=None, source=None,
                 deprecation_reason=None, name=None, description=None,
                 required=False, _creation_counter=None, default_value=None,
                 **extra_args):
        super(Field, self).__init__(_creation_counter=_creation_counter)
        assert not args or isinstance(args, Mapping), (
            'Arguments in a field have to be a mapping, received "{}".'
        ).format(args)
        assert not (source and resolver), (
            'A Field cannot have a source and a resolver in at the same time.'
        )
        assert not callable(default_value), (
            'The default value can not be a function but received "{}".'
        ).format(base_type(default_value))

        if required:
            type = NonNull(type)

        # Check if name is actually an argument of the field
        if isinstance(name, (Argument, UnmountedType)):
            extra_args['name'] = name
            name = None

        # Check if source is actually an argument of the field
        if isinstance(source, (Argument, UnmountedType)):
            extra_args['source'] = source
            source = None

        self.name = name
        self._type = type
        self.args = to_arguments(args or OrderedDict(), extra_args)
        if source:
            resolver = partial(source_resolver, source)
        self.resolver = resolver
        self.deprecation_reason = deprecation_reason
        self.description = description
        self.default_value = default_value

    @property
    def type(self):
        if inspect.isfunction(self._type) or type(self._type) is partial:
            return self._type()
        return self._type

    def get_resolver(self, parent_resolver):
        return self.resolver or parent_resolver
<|endoftext|>


<commit_before>load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

ENVOY_COMMIT = "ff3b5e3ce3f8916aab88b573678826e6fd95e01c"  # March 22, 2021
ENVOY_SHA = "683601fd7f003c50f8783b2d576f68d7c02d57c7ad123b88e6caf45cc3292dd9"

HDR_HISTOGRAM_C_VERSION = "0.11.2"  # October 12th, 2020
HDR_HISTOGRAM_C_SHA = "637f28b5f64de2e268131e4e34e6eef0b91cf5ff99167db447d9b2825eae6bad"

def nighthawk_dependencies():
    http_archive(
        name = "envoy",
        sha256 = ENVOY_SHA,
        strip_prefix = "envoy-%s" % ENVOY_COMMIT,
        # // clang-format off: Envoy's format check: Only repository_locations.bzl may contains URL references
        url = "https://github.com/envoyproxy/envoy/archive/%s.tar.gz" % ENVOY_COMMIT,
        # // clang-format on
    )
    http_archive(
        name = "dep_hdrhistogram_c",
        build_file_content = """
cc_library(
    name = "hdrhistogram_c",
    srcs = [
        "src/hdr_encoding.c",
        "src/hdr_histogram.c",
        "src/hdr_histogram_log.c",
        "src/hdr_interval_recorder.c",
        "src/hdr_thread.c",
        "src/hdr_time.c",
        "src/hdr_writer_reader_phaser.c",
    ],
    hdrs = [
        "src/hdr_atomic.h",
        "src/hdr_encoding.h",
        "src/hdr_endian.h",
        "src/hdr_histogram.h",
        "src/hdr_histogram_log.h",
        "src/hdr_interval_recorder.h",
        "src/hdr_tests.h",
        "src/hdr_thread.h",
        "src/hdr_time.h",
        "src/hdr_writer_reader_phaser.h",
    ],
    copts = [
        "-std=gnu99",
        "-Wno-implicit-function-declaration",
        "-Wno-error",
    ],
    deps = ["//external:zlib"],
    visibility = ["//visibility:public"],
)
  """,
        sha256 = HDR_HISTOGRAM_C_SHA,
        strip_prefix = "HdrHistogram_c-%s" % HDR_HISTOGRAM_C_VERSION,
        # // clang-format off
        url = "https://github.com/HdrHistogram/HdrHistogram_c/archive/%s.tar.gz" % HDR_HISTOGRAM_C_VERSION,
        # // clang-format on
    )
<commit_msg>Update Envoy to 486cd88 (Mar 28th 2021) (#667)<commit_after>load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

ENVOY_COMMIT = "486cd88789625afe786de80d0b3f3b07576f3557"  # March 28, 2021
ENVOY_SHA = "595282e9e7565d2957083b44e0c1fccc7a09b36397d46b149194871b47ae55b4"

HDR_HISTOGRAM_C_VERSION = "0.11.2"  # October 12th, 2020
HDR_HISTOGRAM_C_SHA = "637f28b5f64de2e268131e4e34e6eef0b91cf5ff99167db447d9b2825eae6bad"

def nighthawk_dependencies():
    http_archive(
        name = "envoy",
        sha256 = ENVOY_SHA,
        strip_prefix = "envoy-%s" % ENVOY_COMMIT,
        # // clang-format off: Envoy's format check: Only repository_locations.bzl may contains URL references
        url = "https://github.com/envoyproxy/envoy/archive/%s.tar.gz" % ENVOY_COMMIT,
        # // clang-format on
    )
    http_archive(
        name = "dep_hdrhistogram_c",
        build_file_content = """
cc_library(
    name = "hdrhistogram_c",
    srcs = [
        "src/hdr_encoding.c",
        "src/hdr_histogram.c",
        "src/hdr_histogram_log.c",
        "src/hdr_interval_recorder.c",
        "src/hdr_thread.c",
        "src/hdr_time.c",
        "src/hdr_writer_reader_phaser.c",
    ],
    hdrs = [
        "src/hdr_atomic.h",
        "src/hdr_encoding.h",
        "src/hdr_endian.h",
        "src/hdr_histogram.h",
        "src/hdr_histogram_log.h",
        "src/hdr_interval_recorder.h",
        "src/hdr_tests.h",
        "src/hdr_thread.h",
        "src/hdr_time.h",
        "src/hdr_writer_reader_phaser.h",
    ],
    copts = [
        "-std=gnu99",
        "-Wno-implicit-function-declaration",
        "-Wno-error",
    ],
    deps = ["//external:zlib"],
    visibility = ["//visibility:public"],
)
  """,
        sha256 = HDR_HISTOGRAM_C_SHA,
        strip_prefix = "HdrHistogram_c-%s" % HDR_HISTOGRAM_C_VERSION,
        # // clang-format off
        url = "https://github.com/HdrHistogram/HdrHistogram_c/archive/%s.tar.gz" % HDR_HISTOGRAM_C_VERSION,
        # // clang-format on
    )
<|endoftext|>


<commit_before>import autograd.numpy as np
from autograd.scipy.misc import logsumexp

def relu(x):
    """
    Rectified Linear Unit
    """
    return x * (x > 0)

def softmax(x, axis=0):
    """
    The softmax function normalizes everything to between 0 and 1.
    """
    return np.exp(x - logsumexp(x, axis=axis, keepdims=True))

def tanh(x):
    """
    tanh non-linearity
    """
    return np.tanh(x)
<commit_msg>added logistic nonlinearity<commit_after>import autograd.numpy as np
from autograd.scipy.misc import logsumexp

def relu(x):
    """
    Rectified Linear Unit
    """
    return x * (x > 0)

def softmax(x, axis=0):
    """
    The softmax function normalizes everything to between 0 and 1.
    """
    return np.exp(x - logsumexp(x, axis=axis, keepdims=True))

def tanh(x):
    """
    tanh non-linearity
    """
    return np.tanh(x)

def logistic(x):
    """
    logistic nonlinearity
    """
    return np.log(1 / (1 + np.exp(x)))
<|endoftext|>


<commit_before>import os
import re
import socket
import time
import traceback

try:
    from collections import defaultdict
except ImportError:
    from compat.defaultdict import defaultdict

from datetime import datetime
from glob import glob

try:
    from xml.etree.ElementTree import ElementTree
except ImportError:
    from elementtree import ElementTree

from util import get_hostname

class Continue(Exception):
    pass

class Hudson(object):
    key = 'Hudson'
    datetime_format = '%Y-%m-%d_%H-%M-%S'

    def __init__(self):
        self.high_watermarks = None

    def _extract_timestamp(self, dir_name):
        try:
            # Parse the timestamp from the directory name
            date_str = os.path.basename(dir_name)
            time_tuple = time.strptime(date_str, self.datetime_format)
            return time.mktime(time_tuple)
        except ValueError:
            raise Exception("Error with build directory name, not a parsable date: %s" % (dir_name))

    def _get_build_metadata(self, dir_name):
        # Read the build.xml metadata file that Hudson generates
        build_metadata = os.path.join(dir_name, 'build.xml')

        if not os.access(build_metadata, os.R_OK):
            raise Continue("Can't read build file at %s" % (build_metadata))
        else:
            tree = ElementTree()
            tree.parse(build_metadata)

            keys = ['result', 'number', 'duration']

            kv_pairs = ((k, tree.find(k)) for k in keys)
            d = dict([(k, v.text)
                        for k, v in kv_pairs
                        if v is not None])
            return d

    def _get_build_results(self, logger, job_dir):
        job_name = os.path.basename(job_dir)

        try:
            dirs = glob(os.path.join(job_dir, 'builds', '*_*'))
            if len(dirs) > 0:
                dirs = sorted(dirs, reverse=True)
                # We try to get the last valid build
                for index in xrange(0, len(dirs) - 1):
                    dir_name = dirs[index]
                    timestamp = self._extract_timestamp(dir_name)
                    # Check if it's a new build
                    if timestamp > self.high_watermarks[job_name]:
                        # If we can't get build metadata, we try the previous one
                        try:
                            build_metadata = self._get_build_metadata(dir_name)
                        except:
                            continue

                        output = {
                                'job_name':     job_name,
                                'timestamp':    timestamp,
                                'event_type':   'build result'
                            }
                        output.update(build_metadata)
                        self.high_watermarks[job_name] = timestamp
                        yield output
                    # If it not a new build, stop here
                    else:
                        break
        except Exception, e:
            log.error("Error while working on job %s, exception: %s" % (job_name, e))

    def check(self, logger, agentConfig):
        if self.high_watermarks is None:
            # On the first run of check(), prime the high_watermarks dict
            # so that we only send events that occured after the agent
            # started.
            # (Setting high_watermarks in the next statement prevents
            #  any kind of infinite loop (assuming nothing ever sets
            #  high_watermarks to None again!))
            self.high_watermarks = defaultdict(lambda: 0)
            self.check(logger, agentConfig)

        hudson_home = agentConfig.get('hudson_home', None)

        if not hudson_home:
            return False

        job_dirs = glob(os.path.join(hudson_home, 'jobs', '*'))

        build_events = []

        for job_dir in job_dirs:
            for output in self._get_build_results(logger, job_dir):
                output['api_key'] = agentConfig['api_key']
                output['host'] = get_hostname(agentConfig)
                build_events.append(output)

        return build_events

if __name__ == '__main__':
    import logging
    import sys

    hudson_home, apiKey = sys.argv[1:3]

    logger = logging.getLogger('ddagent.checks.hudson')
    logger.setLevel(logging.INFO)
    logger.addHandler(logging.StreamHandler())
    hudson = Hudson()
    while True:
        print hudson.check(logger,
                           {'hudson_home': hudson_home,
                            'api_key': apiKey})
        time.sleep(5)
<commit_msg>Fix variable name<commit_after>import os
import re
import socket
import time
import traceback

try:
    from collections import defaultdict
except ImportError:
    from compat.defaultdict import defaultdict

from datetime import datetime
from glob import glob

try:
    from xml.etree.ElementTree import ElementTree
except ImportError:
    from elementtree import ElementTree

from util import get_hostname

class Continue(Exception):
    pass

class Hudson(object):
    key = 'Hudson'
    datetime_format = '%Y-%m-%d_%H-%M-%S'

    def __init__(self):
        self.high_watermarks = None

    def _extract_timestamp(self, dir_name):
        try:
            # Parse the timestamp from the directory name
            date_str = os.path.basename(dir_name)
            time_tuple = time.strptime(date_str, self.datetime_format)
            return time.mktime(time_tuple)
        except ValueError:
            raise Exception("Error with build directory name, not a parsable date: %s" % (dir_name))

    def _get_build_metadata(self, dir_name):
        # Read the build.xml metadata file that Hudson generates
        build_metadata = os.path.join(dir_name, 'build.xml')

        if not os.access(build_metadata, os.R_OK):
            raise Continue("Can't read build file at %s" % (build_metadata))
        else:
            tree = ElementTree()
            tree.parse(build_metadata)

            keys = ['result', 'number', 'duration']

            kv_pairs = ((k, tree.find(k)) for k in keys)
            d = dict([(k, v.text)
                        for k, v in kv_pairs
                        if v is not None])
            return d

    def _get_build_results(self, logger, job_dir):
        job_name = os.path.basename(job_dir)

        try:
            dirs = glob(os.path.join(job_dir, 'builds', '*_*'))
            if len(dirs) > 0:
                dirs = sorted(dirs, reverse=True)
                # We try to get the last valid build
                for index in xrange(0, len(dirs) - 1):
                    dir_name = dirs[index]
                    timestamp = self._extract_timestamp(dir_name)
                    # Check if it's a new build
                    if timestamp > self.high_watermarks[job_name]:
                        # If we can't get build metadata, we try the previous one
                        try:
                            build_metadata = self._get_build_metadata(dir_name)
                        except:
                            continue

                        output = {
                                'job_name':     job_name,
                                'timestamp':    timestamp,
                                'event_type':   'build result'
                            }
                        output.update(build_metadata)
                        self.high_watermarks[job_name] = timestamp
                        yield output
                    # If it not a new build, stop here
                    else:
                        break
        except Exception, e:
            logger.error("Error while working on job %s, exception: %s" % (job_name, e))

    def check(self, logger, agentConfig):
        if self.high_watermarks is None:
            # On the first run of check(), prime the high_watermarks dict
            # so that we only send events that occured after the agent
            # started.
            # (Setting high_watermarks in the next statement prevents
            #  any kind of infinite loop (assuming nothing ever sets
            #  high_watermarks to None again!))
            self.high_watermarks = defaultdict(lambda: 0)
            self.check(logger, agentConfig)

        hudson_home = agentConfig.get('hudson_home', None)

        if not hudson_home:
            return False

        job_dirs = glob(os.path.join(hudson_home, 'jobs', '*'))

        build_events = []

        for job_dir in job_dirs:
            for output in self._get_build_results(logger, job_dir):
                output['api_key'] = agentConfig['api_key']
                output['host'] = get_hostname(agentConfig)
                build_events.append(output)

        return build_events

if __name__ == '__main__':
    import logging
    import sys

    hudson_home, apiKey = sys.argv[1:3]

    logger = logging.getLogger('ddagent.checks.hudson')
    logger.setLevel(logging.INFO)
    logger.addHandler(logging.StreamHandler())
    hudson = Hudson()
    while True:
        print hudson.check(logger,
                           {'hudson_home': hudson_home,
                            'api_key': apiKey})
        time.sleep(5)
<|endoftext|>


<commit_before>import os, unittest
import pandas as pd
from sqlalchemy import create_engine
from igf_data.igfdb.igfTables import Base, Project, Project_attribute, Sample
from igf_data.igfdb.baseadaptor import BaseAdaptor
from igf_data.igfdb.projectadaptor import ProjectAdaptor
from igf_data.igfdb.useradaptor import UserAdaptor
from igf_data.igfdb.sampleadaptor import SampleAdaptor
from igf_data.utils.dbutils import read_dbconf_json

class Projectadaptor_test1(unittest.TestCase):
  def setUp(self):
    self.dbconfig='data/dbconfig.json'
    dbparam=read_dbconf_json(self.dbconfig)
    base=BaseAdaptor(**dbparam)
    self.engine=base.engine
    self.dbname=dbparam['dbname']
    Base.metadata.create_all(self.engine)
    self.session_class=base.get_session_class()
    project_data=[{'project_igf_id':'IGFP0001_test_22-8-2017_rna',
                   'project_name':'test_22-8-2017_rna',
                   'description':'Its project 1',
                   'project_deadline':'Before August 2017',
                   'comments':'Some samples are treated with drug X',
                 },
                 {'project_igf_id':'IGFP0002_test_22-8-2017_rna',
                   'project_name':'test_23-8-2017_rna',
                   'description':'Its project 2',
                   'project_deadline':'Before August 2017',
                   'comments':'Some samples are treated with drug X',
                 }]
    base.start_session()
    pa=ProjectAdaptor(**{'session':base.session})
    pa.store_project_and_attribute_data(data=project_data)
    sa=SampleAdaptor(**{'session': base.session})
    sample_data=[{'sample_igf_id':'IGFS001','project_igf_id':'IGFP0001_test_22-8-2017_rna',},
                 {'sample_igf_id':'IGFS002','project_igf_id':'IGFP0001_test_22-8-2017_rna',},
                 {'sample_igf_id':'IGFS003','project_igf_id':'IGFP0001_test_22-8-2017_rna',},
                 {'sample_igf_id':'IGFS004','project_igf_id':'IGFP0001_test_22-8-2017_rna','status':'FAILED',},
                ]
    sa.store_sample_and_attribute_data(data=sample_data)
    base.close_session()

  def tearDown(self):
    Base.metadata.drop_all(self.engine)
    os.remove(self.dbname)

  def test_fetch_project_samples(self):
    pa=ProjectAdaptor(**{'session_class':self.session_class})
    pa.start_session()
    sample1=pa.fetch_project_samples(project_igf_id='IGFP0001_test_22-8-2017_rna',output_mode='dataframe')
    self.assertEqual(len(sample1.index),3)
    sample2=pa.fetch_project_samples(project_igf_id='IGFP0002_test_22-8-2017_rna',output_mode='dataframe')
    self.assertEqual(len(sample2.index),0)
    sample3=pa.fetch_project_samples(project_igf_id='IGFP0001_test_22-8-2017_rna',
                                     only_active=False,
                                     output_mode='dataframe')
    self.assertEqual(len(sample3.index),4)
    pa.close_session()

  def test_count_project_samples(self):
    pa=ProjectAdaptor(**{'session_class':self.session_class})
    pa.start_session()
    sample1=pa.count_project_samples(project_igf_id='IGFP0001_test_22-8-2017_rna')
    self.assertEqual(sample1,3)
    sample2=pa.count_project_samples(project_igf_id='IGFP0002_test_22-8-2017_rna')
    self.assertEqual(sample2,0)
    sample3=pa.count_project_samples(project_igf_id='IGFP0001_test_22-8-2017_rna',
                                     only_active=False)
    self.assertEqual(sample3,4)
    pa.close_session()

class Projectadaptor_test2(unittest.TestCase):
  def setUp(self):
    self.dbconfig='data/dbconfig.json'
    dbparam=read_dbconf_json(self.dbconfig)
    base=BaseAdaptor(**dbparam)
    self.engine=base.engine
    self.dbname=dbparam['dbname']
    Base.metadata.create_all(self.engine)
    self.session_class=base.get_session_class()
    project_data=[{'project_igf_id':'IGFP0001_test_22-8-2017_rna',
                   'project_name':'test_22-8-2017_rna',
                   'description':'Its project 1',
                   'project_deadline':'Before August 2017',
                   'comments':'Some samples are treated with drug X',
                 },
                 {'project_igf_id':'IGFP0002_test_22-8-2017_rna',
                   'project_name':'test_23-8-2017_rna',
                   'description':'Its project 2',
                   'project_deadline':'Before August 2017',
                   'comments':'Some samples are treated with drug X'}]
    user_data=[{'name':'UserA',
                'email_id':'usera@ic.ac.uk',
                'username':'usera'}]
    project_user_data=[{'project_igf_id': 'IGFP0001_test_22-8-2017_rna',
                        'email_id': 'usera@ic.ac.uk',
                        'data_authority':True},
                       {'project_igf_id': 'IGFP0002_test_22-8-2017_rna',
                        'email_id': 'usera@ic.ac.uk'}]
    base.start_session()
    ua=UserAdaptor(**{'session':base.session})
    ua.store_user_data(data=user_data)
    pa=ProjectAdaptor(**{'session':base.session})
    pa.store_project_and_attribute_data(data=project_data)
    pa.assign_user_to_project(data=project_user_data)
    base.close_session()

  def tearDown(self):
    Base.metadata.drop_all(self.engine)
    os.remove(self.dbname)

  def test_check_data_authority_for_project(self):
    pa=ProjectAdaptor(**{'session_class':self.session_class})
    pa.start_session()
    pa_results1=pa.check_data_authority_for_project(project_igf_id='IGFP0001_test_22-8-2017_rna')
    self.assertTrue(pa_results1)
    pa_results2=pa.check_data_authority_for_project(project_igf_id='IGFP0002_test_22-8-2017_rna')
    self.assertFalse(pa_results2)
    pa.close_session()

  def test_fetch_data_authority_for_project(self):
    pa=ProjectAdaptor(**{'session_class':self.session_class})
    pa.start_session()
    pa_results1=pa.fetch_data_authority_for_project(project_igf_id='IGFP0001_test_22-8-2017_rna')
    self.assertEqual(pa_results1.email_id,'usera@ic.ac.uk')
    pa_results2=pa.fetch_data_authority_for_project(project_igf_id='IGFP0002_test_22-8-2017_rna')
    self.assertEqual(pa_results2, None)
    pa.close_session()

if __name__ == '__main__':
  unittest.main()<commit_msg>updated unit test for new adapter method<commit_after>import os, unittest
import pandas as pd
from sqlalchemy import create_engine
from igf_data.igfdb.igfTables import Base, Project, Project_attribute, Sample
from igf_data.igfdb.baseadaptor import BaseAdaptor
from igf_data.igfdb.projectadaptor import ProjectAdaptor
from igf_data.igfdb.useradaptor import UserAdaptor
from igf_data.igfdb.sampleadaptor import SampleAdaptor
from igf_data.utils.dbutils import read_dbconf_json

class Projectadaptor_test1(unittest.TestCase):
  def setUp(self):
    self.dbconfig='data/dbconfig.json'
    dbparam=read_dbconf_json(self.dbconfig)
    base=BaseAdaptor(**dbparam)
    self.engine=base.engine
    self.dbname=dbparam['dbname']
    Base.metadata.create_all(self.engine)
    self.session_class=base.get_session_class()
    project_data=[{'project_igf_id':'IGFP0001_test_22-8-2017_rna',
                   'project_name':'test_22-8-2017_rna',
                   'description':'Its project 1',
                   'project_deadline':'Before August 2017',
                   'comments':'Some samples are treated with drug X',
                 },
                 {'project_igf_id':'IGFP0002_test_22-8-2017_rna',
                   'project_name':'test_23-8-2017_rna',
                   'description':'Its project 2',
                   'project_deadline':'Before August 2017',
                   'comments':'Some samples are treated with drug X',
                 }]
    base.start_session()
    pa=ProjectAdaptor(**{'session':base.session})
    pa.store_project_and_attribute_data(data=project_data)
    sa=SampleAdaptor(**{'session': base.session})
    sample_data=[{'sample_igf_id':'IGFS001','project_igf_id':'IGFP0001_test_22-8-2017_rna',},
                 {'sample_igf_id':'IGFS002','project_igf_id':'IGFP0001_test_22-8-2017_rna',},
                 {'sample_igf_id':'IGFS003','project_igf_id':'IGFP0001_test_22-8-2017_rna',},
                 {'sample_igf_id':'IGFS004','project_igf_id':'IGFP0001_test_22-8-2017_rna','status':'FAILED',},
                ]
    sa.store_sample_and_attribute_data(data=sample_data)
    base.close_session()

  def tearDown(self):
    Base.metadata.drop_all(self.engine)
    os.remove(self.dbname)

  def test_fetch_project_samples(self):
    pa=ProjectAdaptor(**{'session_class':self.session_class})
    pa.start_session()
    sample1=pa.fetch_project_samples(project_igf_id='IGFP0001_test_22-8-2017_rna',output_mode='dataframe')
    self.assertEqual(len(sample1.index),3)
    sample2=pa.fetch_project_samples(project_igf_id='IGFP0002_test_22-8-2017_rna',output_mode='dataframe')
    self.assertEqual(len(sample2.index),0)
    sample3=pa.fetch_project_samples(project_igf_id='IGFP0001_test_22-8-2017_rna',
                                     only_active=False,
                                     output_mode='dataframe')
    self.assertEqual(len(sample3.index),4)
    pa.close_session()

  def test_count_project_samples(self):
    pa=ProjectAdaptor(**{'session_class':self.session_class})
    pa.start_session()
    sample1=pa.count_project_samples(project_igf_id='IGFP0001_test_22-8-2017_rna')
    self.assertEqual(sample1,3)
    sample2=pa.count_project_samples(project_igf_id='IGFP0002_test_22-8-2017_rna')
    self.assertEqual(sample2,0)
    sample3=pa.count_project_samples(project_igf_id='IGFP0001_test_22-8-2017_rna',
                                     only_active=False)
    self.assertEqual(sample3,4)
    pa.close_session()

class Projectadaptor_test2(unittest.TestCase):
  def setUp(self):
    self.dbconfig='data/dbconfig.json'
    dbparam=read_dbconf_json(self.dbconfig)
    base=BaseAdaptor(**dbparam)
    self.engine=base.engine
    self.dbname=dbparam['dbname']
    Base.metadata.create_all(self.engine)
    self.session_class=base.get_session_class()
    project_data=[{'project_igf_id':'IGFP0001_test_22-8-2017_rna',
                   'project_name':'test_22-8-2017_rna',
                   'description':'Its project 1',
                   'project_deadline':'Before August 2017',
                   'comments':'Some samples are treated with drug X',
                 },
                 {'project_igf_id':'IGFP0002_test_22-8-2017_rna',
                   'project_name':'test_23-8-2017_rna',
                   'description':'Its project 2',
                   'project_deadline':'Before August 2017',
                   'comments':'Some samples are treated with drug X'}]
    user_data=[{'name':'UserA',
                'email_id':'usera@ic.ac.uk',
                'username':'usera'}]
    project_user_data=[{'project_igf_id': 'IGFP0001_test_22-8-2017_rna',
                        'email_id': 'usera@ic.ac.uk',
                        'data_authority':True},
                       {'project_igf_id': 'IGFP0002_test_22-8-2017_rna',
                        'email_id': 'usera@ic.ac.uk'}]
    base.start_session()
    ua=UserAdaptor(**{'session':base.session})
    ua.store_user_data(data=user_data)
    pa=ProjectAdaptor(**{'session':base.session})
    pa.store_project_and_attribute_data(data=project_data)
    pa.assign_user_to_project(data=project_user_data)
    base.close_session()

  def tearDown(self):
    Base.metadata.drop_all(self.engine)
    os.remove(self.dbname)

  def test_check_data_authority_for_project(self):
    pa=ProjectAdaptor(**{'session_class':self.session_class})
    pa.start_session()
    pa_results1=pa.check_data_authority_for_project(project_igf_id='IGFP0001_test_22-8-2017_rna')
    self.assertTrue(pa_results1)
    pa_results2=pa.check_data_authority_for_project(project_igf_id='IGFP0002_test_22-8-2017_rna')
    self.assertFalse(pa_results2)
    pa.close_session()

  def test_fetch_data_authority_for_project(self):
    pa=ProjectAdaptor(**{'session_class':self.session_class})
    pa.start_session()
    pa_results1=pa.fetch_data_authority_for_project(project_igf_id='IGFP0001_test_22-8-2017_rna')
    self.assertEqual(pa_results1.email_id,'usera@ic.ac.uk')
    pa_results2=pa.fetch_data_authority_for_project(project_igf_id='IGFP0002_test_22-8-2017_rna')
    self.assertEqual(pa_results2, None)
    pa.close_session()

  def test_fetch_all_project_igf_ids(self):
    pa = ProjectAdaptor(**{'session_class':self.session_class})
    pa.start_session()
    project_list = pa.fetch_all_project_igf_ids()
    pa.close_session()
    self.assertTrue('IGFP0002_test_22-8-2017_rna' in project_list['project_igf_id'].values)
    self.assertTrue('IGFP0001_test_22-8-2017_rna' in project_list['project_igf_id'].values)
    self.assertEqual(len(project_list['project_igf_id'].values), 2)

if __name__ == '__main__':
  unittest.main()<|endoftext|>


<commit_before>from checks import Check, gethostname
from fnmatch import fnmatch
import os
import time

class Cacti(Check):
    CFUNC_TO_AGGR = {
        'AVERAGE': 'avg',
        'MAXIMUM': 'max',
        'MINIMUM': 'min'
    }

    CACTI_TO_DD = {
        'hdd_free': 'system.disk.free',
        'hdd_used': 'system.disk.used',
        'swap_free': 'system.swap.free',
        'load_1min': 'system.load.1',
        'load_5min': 'system.load.5',
        'load_15min': 'system.load.15',
        'mem_buffers': 'system.mem.buffers',
        'proc': 'system.proc.running',
        'users': 'system.users.current',
        'mem_swap': 'system.swap.free',
        'ping': 'system.ping.latency'
    }

    def __init__(self, logger):
        Check.__init__(self, logger)
        self.db = None
        self.rrd_path = None
        self.logger = logger
        self.last_ts = {}

    def _add_stat(self, name, value, agentConfig):
        ''' For collecting stats on Cacti checks '''
        self.stats.append(
            (name, time.time(), value, {'host_name': gethostname(agentConfig)})
        )

    def _fetch_rrd_meta(self, agentConfig, whitelist):
        ''' Return a list of list of dicts with host_name, host_desc, device_name, and rrd_path '''
        def _in_whitelist(rrd):
            path = rrd.replace('<path_rra>/','')
            for p in whitelist:
                if fnmatch(path, p):
                    return True
            return False

        c = self.db.cursor()
        c.execute("""
                SELECT
                    h.hostname as host_name,
                    dl.snmp_index as device_name,
                    dt.data_source_path as rrd_path
                FROM data_local dl
                    JOIN host h on dl.host_id = h.id
                    JOIN data_template_data dt on dt.local_data_id = dl.id
                WHERE dt.data_source_path IS NOT NULL
                AND dt.data_source_path != ''
            """)
        res = []
        for host_name, device_name, rrd_path in c.fetchall():
            if not whitelist or _in_whitelist(rrd_path):
                if host_name in ('localhost', '127.0.0.1'):
                    host_name = gethostname(agentConfig)
                res.append({
                    'host_name': host_name,
                    'device_name': device_name or None,
                    'rrd_path': rrd_path.replace('<path_rra>', self.rrd_path)
                })

        # Collect stats
        self._add_stat('cacti.rrd.count', len(res), agentConfig)
        num_hosts = len(set([r['host_name'] for r in res]))
        self._add_stat('cacti.hosts.count', num_hosts, agentConfig)

        return res

    def _format_metric_name(self, m_name, cfunc):
        ''' Format a cacti metric name into a Datadog-friendly name'''
        try:
            aggr = Cacti.CFUNC_TO_AGGR[cfunc]
        except KeyError:
            aggr = cfunc.lower()

        try:
            m_name = Cacti.CACTI_TO_DD[m_name]
            if aggr != 'avg':
                m_name += '.%s' % (aggr)
            return m_name
        except KeyError:
            return "cacti.%s.%s" % (m_name.lower(), aggr)

    def _consolidation_funcs(self, rrd_path, rrdtool):
        ''' Determine the available consolidation functions for this rrd '''
        import rrdtool

        info = rrdtool.info(rrd_path)
        funcs = []
        for k,v in info.items():
            if k.endswith('.cf'):
                funcs.append(v)
        return funcs

    def _read_rrd(self, rrd_path, host_name, device_name):
        import rrdtool

        metrics = []
        c_funcs = self._consolidation_funcs(rrd_path, rrdtool)
        start = self.last_ts.get(rrd_path, 0)

        for c in c_funcs:
            try:
                fetched = rrdtool.fetch(rrd_path, c, '--start', str(start))
            except rrdtool.error:
                # Start time was out of range, return empty list
                return []

            # Extract the data
            (start_ts, end_ts, interval) = fetched[0]
            metric_names = fetched[1]
            points = fetched[2]
            for k, m_name in enumerate(metric_names):
                m_name = self._format_metric_name(m_name, c)
                for i, p in enumerate(points):
                    ts = start_ts + (i * interval)

                    # Add the metric to our list
                    metrics.append((m_name, ts, p[k], {'host_name': host_name, 'device_name': device_name}))

            # Update the last timestamp
            self.last_ts[rrd_path] = end_ts
            return metrics

        try:
            self.logger.debug("Cacti check start")
            if  'cacti_mysql_server' in agentConfig \
                and 'cacti_mysql_user' in agentConfig \
                and 'cacti_rrd_path' in agentConfig \
                and agentConfig['cacti_mysql_server'] != '' \
                and agentConfig['cacti_mysql_user'] != '' \
                and agentConfig['cacti_rrd_path'] != '':

                # Connect to MySQL
                try:
                    import MySQLdb
                    self.db = MySQLdb.connect(agentConfig['cacti_mysql_server'], agentConfig['cacti_mysql_user'],
                            agentConfig.get('cacti_mysql_pass', ''), db="cacti")
                except ImportError, e:
                    self.logger.exception("Cannot import MySQLdb")
                    return False
                except MySQLdb.OperationalError:
                    self.logger.exception('MySQL connection error')
                    return False
                self.logger.debug("Connected to MySQL to fetch Cacti metadata")

                try:
                    import rrdtool
                except ImportError:
                    self.logger.exception("Cannot import rrdtool")
                    return False

                # Clear stats for this check
                self.stats = []

                # Get whitelist patterns, if available
                patterns = []
                whitelist = agentConfig.get('cacti_rrd_whitelist', None)
                if whitelist:
                    if not os.path.isfile(whitelist) or not os.access(whitelist, os.R_OK):
                        # Don't run the check if the whitelist is unavailable
                        self.logger.exception("Unable to read whitelist file at %s" % (whitelist))
                        return False

                    wl = None
                    try:
                        wl = open(whitelist)
                        for line in wl:
                            patterns.append(line.strip())
                        wl.close()
                    except:
                        self.logger.exception("Cannot open whitelist file")
                        return False

                # Fetch RRD metadata
                self.rrd_path = agentConfig['cacti_rrd_path']
                rrd_meta = self._fetch_rrd_meta(agentConfig, patterns)

                metrics = []
                for rrd in rrd_meta:
                    metrics.extend(
                        self._read_rrd(rrd['rrd_path'], rrd['host_name'], rrd['device_name'])
                    )

                self._add_stat('cacti.metrics.count', len(metrics), agentConfig)

                # Add the Cacti stats to the payload
                metrics.extend(self.stats)

                return metrics
            else:
                return False

        except:
            self.logger.exception("Cannot check Cacti")
            return False
<commit_msg>Removed check method by mistake #fatfinger<commit_after>from checks import Check, gethostname
from fnmatch import fnmatch
import os
import time

class Cacti(Check):
    CFUNC_TO_AGGR = {
        'AVERAGE': 'avg',
        'MAXIMUM': 'max',
        'MINIMUM': 'min'
    }

    CACTI_TO_DD = {
        'hdd_free': 'system.disk.free',
        'hdd_used': 'system.disk.used',
        'swap_free': 'system.swap.free',
        'load_1min': 'system.load.1',
        'load_5min': 'system.load.5',
        'load_15min': 'system.load.15',
        'mem_buffers': 'system.mem.buffers',
        'proc': 'system.proc.running',
        'users': 'system.users.current',
        'mem_swap': 'system.swap.free',
        'ping': 'system.ping.latency'
    }

    def __init__(self, logger):
        Check.__init__(self, logger)
        self.db = None
        self.rrd_path = None
        self.logger = logger
        self.last_ts = {}

    def _add_stat(self, name, value, agentConfig):
        ''' For collecting stats on Cacti checks '''
        self.stats.append(
            (name, time.time(), value, {'host_name': gethostname(agentConfig)})
        )

    def _fetch_rrd_meta(self, agentConfig, whitelist):
        ''' Return a list of list of dicts with host_name, host_desc, device_name, and rrd_path '''
        def _in_whitelist(rrd):
            path = rrd.replace('<path_rra>/','')
            for p in whitelist:
                if fnmatch(path, p):
                    return True
            return False

        c = self.db.cursor()
        c.execute("""
                SELECT
                    h.hostname as host_name,
                    dl.snmp_index as device_name,
                    dt.data_source_path as rrd_path
                FROM data_local dl
                    JOIN host h on dl.host_id = h.id
                    JOIN data_template_data dt on dt.local_data_id = dl.id
                WHERE dt.data_source_path IS NOT NULL
                AND dt.data_source_path != ''
            """)
        res = []
        for host_name, device_name, rrd_path in c.fetchall():
            if not whitelist or _in_whitelist(rrd_path):
                if host_name in ('localhost', '127.0.0.1'):
                    host_name = gethostname(agentConfig)
                res.append({
                    'host_name': host_name,
                    'device_name': device_name or None,
                    'rrd_path': rrd_path.replace('<path_rra>', self.rrd_path)
                })

        # Collect stats
        self._add_stat('cacti.rrd.count', len(res), agentConfig)
        num_hosts = len(set([r['host_name'] for r in res]))
        self._add_stat('cacti.hosts.count', num_hosts, agentConfig)

        return res

    def _format_metric_name(self, m_name, cfunc):
        ''' Format a cacti metric name into a Datadog-friendly name'''
        try:
            aggr = Cacti.CFUNC_TO_AGGR[cfunc]
        except KeyError:
            aggr = cfunc.lower()

        try:
            m_name = Cacti.CACTI_TO_DD[m_name]
            if aggr != 'avg':
                m_name += '.%s' % (aggr)
            return m_name
        except KeyError:
            return "cacti.%s.%s" % (m_name.lower(), aggr)

    def _consolidation_funcs(self, rrd_path, rrdtool):
        ''' Determine the available consolidation functions for this rrd '''
        import rrdtool

        info = rrdtool.info(rrd_path)
        funcs = []
        for k,v in info.items():
            if k.endswith('.cf'):
                funcs.append(v)
        return funcs

    def _read_rrd(self, rrd_path, host_name, device_name):
        import rrdtool

        metrics = []
        c_funcs = self._consolidation_funcs(rrd_path, rrdtool)
        start = self.last_ts.get(rrd_path, 0)

        for c in c_funcs:
            try:
                fetched = rrdtool.fetch(rrd_path, c, '--start', str(start))
            except rrdtool.error:
                # Start time was out of range, return empty list
                return []

            # Extract the data
            (start_ts, end_ts, interval) = fetched[0]
            metric_names = fetched[1]
            points = fetched[2]
            for k, m_name in enumerate(metric_names):
                m_name = self._format_metric_name(m_name, c)
                for i, p in enumerate(points):
                    ts = start_ts + (i * interval)

                    # Add the metric to our list
                    metrics.append((m_name, ts, p[k], {'host_name': host_name, 'device_name': device_name}))

            # Update the last timestamp
            self.last_ts[rrd_path] = end_ts
            return metrics

    def check(self, agentConfig):
        "Check entry point"
        try:
            self.logger.debug("Cacti check start")
            if  'cacti_mysql_server' in agentConfig \
                and 'cacti_mysql_user' in agentConfig \
                and 'cacti_rrd_path' in agentConfig \
                and agentConfig['cacti_mysql_server'] != '' \
                and agentConfig['cacti_mysql_user'] != '' \
                and agentConfig['cacti_rrd_path'] != '':

                # Connect to MySQL
                try:
                    import MySQLdb
                    self.db = MySQLdb.connect(agentConfig['cacti_mysql_server'], agentConfig['cacti_mysql_user'],
                            agentConfig.get('cacti_mysql_pass', ''), db="cacti")
                except ImportError, e:
                    self.logger.exception("Cannot import MySQLdb")
                    return False
                except MySQLdb.OperationalError:
                    self.logger.exception('MySQL connection error')
                    return False
                self.logger.debug("Connected to MySQL to fetch Cacti metadata")

                try:
                    import rrdtool
                except ImportError:
                    self.logger.exception("Cannot import rrdtool")
                    return False

                # Clear stats for this check
                self.stats = []

                # Get whitelist patterns, if available
                patterns = []
                whitelist = agentConfig.get('cacti_rrd_whitelist', None)
                if whitelist:
                    if not os.path.isfile(whitelist) or not os.access(whitelist, os.R_OK):
                        # Don't run the check if the whitelist is unavailable
                        self.logger.exception("Unable to read whitelist file at %s" % (whitelist))
                        return False

                    wl = None
                    try:
                        wl = open(whitelist)
                        for line in wl:
                            patterns.append(line.strip())
                        wl.close()
                    except:
                        self.logger.exception("Cannot open whitelist file")
                        return False

                # Fetch RRD metadata
                self.rrd_path = agentConfig['cacti_rrd_path']
                rrd_meta = self._fetch_rrd_meta(agentConfig, patterns)

                metrics = []
                for rrd in rrd_meta:
                    metrics.extend(
                        self._read_rrd(rrd['rrd_path'], rrd['host_name'], rrd['device_name'])
                    )

                self._add_stat('cacti.metrics.count', len(metrics), agentConfig)

                # Add the Cacti stats to the payload
                metrics.extend(self.stats)

                return metrics
            else:
                return False

        except:
            self.logger.exception("Cannot check Cacti")
            return False
<|endoftext|>


<commit_before>import httplib
import urllib2

from util import json, headers

class RabbitMq(object):
    def check(self, logger, agentConfig):

        if 'rabbitmq_status_url' not in agentConfig or \
           'rabbitmq_user' not in agentConfig or \
           'rabbitmq_pass' not in agentConfig or \
            agentConfig['rabbitmq_status_url'] == 'http://www.example.com:55672/json':
            return False

        try:
            logger.debug('getRabbitMQStatus: attempting authentication setup')
            manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
            manager.add_password(None, agentConfig['rabbitmq_status_url'], agentConfig['rabbitmq_user'], agentConfig['rabbitmq_pass'])
            handler = urllib2.HTTPBasicAuthHandler(manager)
            opener = urllib2.build_opener(handler)
            urllib2.install_opener(opener)

            logger.debug('getRabbitMQStatus: attempting urlopen')
            req = urllib2.Request(agentConfig['rabbitmq_status_url'], None, headers(agentConfig))

            # Do the request, log any errors
            request = urllib2.urlopen(req)
            response = request.read()

            return json.loads(response)
        except:
            logger.exception('Unable to get RabbitMQ status')
            return False
<commit_msg>Remove old rabbitmq check<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2019-2020 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

from test_framework.test_framework import SyscoinTestFramework
from test_framework.util import assert_equal, assert_raises_rpc_error, set_node_times, disconnect_nodes, connect_nodes, bump_node_times
from test_framework.messages import COIN
import time
ZDAG_NOT_FOUND = -1
ZDAG_STATUS_OK = 0
ZDAG_WARNING_RBF = 1
ZDAG_WARNING_NOT_ZDAG_TX = 2
ZDAG_WARNING_SIZE_OVER_POLICY = 3
ZDAG_MAJOR_CONFLICT = 4
MAX_INITIAL_BROADCAST_DELAY = 15 * 60 # 15 minutes in seconds
class AssetZDAGTest(SyscoinTestFramework):
    def set_test_params(self):
        self.setup_clean_chain = True
        self.num_nodes = 4
        self.rpc_timeout = 240
        self.extra_args = [['-assetindex=1'],['-assetindex=1'],['-assetindex=1'],['-assetindex=1']]

    def run_test(self):
        self.nodes[0].generate(200)
        self.sync_blocks()
        self.burn_zdag_doublespend()
        self.basic_zdag_doublespend()

    def basic_zdag_doublespend(self):
        self.basic_asset()
        self.nodes[0].generate(1)
        newaddress2 = self.nodes[1].getnewaddress()
        newaddress3 = self.nodes[1].getnewaddress()
        newaddress1 = self.nodes[0].getnewaddress()
        self.nodes[2].importprivkey(self.nodes[1].dumpprivkey(newaddress2))
        self.nodes[0].assetsend(self.asset, newaddress1, int(2*COIN))
        # create 2 utxo's so below newaddress1 recipient of 0.5 COIN uses 1 and the newaddress3 recipient on node3 uses the other on dbl spend
        self.nodes[0].sendtoaddress(newaddress2, 1)
        self.nodes[0].sendtoaddress(newaddress2, 1)
        self.nodes[0].generate(1)
        self.sync_blocks()
        out =  self.nodes[0].listunspent(query_options={'assetGuid': self.asset, 'minimumAmountAsset': 0.5})
        assert_equal(len(out), 1)
        out =  self.nodes[2].listunspent()
        assert_equal(len(out), 2)
        # send 2 asset UTXOs to newaddress2 same logic as explained above about dbl spend
        self.nodes[0].assetallocationsend(self.asset, newaddress2, int(0.4*COIN))
        self.nodes[0].assetallocationsend(self.asset, newaddress2, int(1*COIN))
        self.nodes[0].generate(1)
        self.sync_blocks()
        # should have 2 sys utxos and 2 asset utxos
        out =  self.nodes[2].listunspent()
        assert_equal(len(out), 4)
        # this will use 1 sys utxo and 1 asset utxo and send it to change address owned by node2
        self.nodes[1].assetallocationsend(self.asset, newaddress1, int(0.4*COIN))
        self.sync_mempools(timeout=30)
        # node3 should have 2 less utxos because they were sent to change on node2
        out =  self.nodes[2].listunspent(minconf=0)
        assert_equal(len(out), 2)
        tx1 = self.nodes[1].assetallocationsend(self.asset, newaddress1, int(1*COIN))['txid']
        # dbl spend
        tx2 = self.nodes[2].assetallocationsend(self.asset, newaddress1, int(0.9*COIN))['txid']
        # use tx2 to build tx3
        tx3 = self.nodes[2].assetallocationsend(self.asset, newaddress1, int(0.05*COIN))['txid']
        # use tx2 to build tx4
        tx4 = self.nodes[2].assetallocationsend(self.asset, newaddress1, int(0.025*COIN))['txid']
        self.sync_all()
        for i in range(3):
            assert_equal(self.nodes[i].assetallocationverifyzdag(tx1)['status'], ZDAG_MAJOR_CONFLICT)
            # ensure the tx2 made it to mempool, should propogate dbl-spend first time
            assert_equal(self.nodes[i].assetallocationverifyzdag(tx2)['status'], ZDAG_MAJOR_CONFLICT)
            # will conflict because its using tx2 which is in conflict state
            assert_equal(self.nodes[i].assetallocationverifyzdag(tx3)['status'], ZDAG_MAJOR_CONFLICT)
            # will conflict because its using tx3 which uses tx2 which is in conflict state
            assert_equal(self.nodes[i].assetallocationverifyzdag(tx4)['status'], ZDAG_MAJOR_CONFLICT)
        self.nodes[0].generate(1)
        self.sync_blocks()
        tx2inchain = False
        for i in range(3):
            try:
                self.nodes[i].getrawtransaction(tx1)
            except:
                tx2inchain = True
                continue
            assert_equal(self.nodes[i].assetallocationverifyzdag(tx1)['status'], ZDAG_NOT_FOUND)
            assert_equal(self.nodes[i].assetallocationverifyzdag(tx2)['status'], ZDAG_NOT_FOUND)
            assert_equal(self.nodes[i].assetallocationverifyzdag(tx3)['status'], ZDAG_NOT_FOUND)
            assert_equal(self.nodes[i].assetallocationverifyzdag(tx4)['status'], ZDAG_NOT_FOUND)
            assert_raises_rpc_error(-5, 'No such mempool transaction', self.nodes[i].getrawtransaction, tx2)
            assert_raises_rpc_error(-5, 'No such mempool transaction', self.nodes[i].getrawtransaction, tx3)
            assert_raises_rpc_error(-5, 'No such mempool transaction', self.nodes[i].getrawtransaction, tx4)
        out =  self.nodes[0].listunspent(query_options={'assetGuid': self.asset, 'minimumAmountAsset':0,'maximumAmountAsset':0})
        assert_equal(len(out), 1)
        out =  self.nodes[0].listunspent(query_options={'assetGuid': self.asset, 'minimumAmountAsset':0.4,'maximumAmountAsset':0.4})
        assert_equal(len(out), 1)
        out =  self.nodes[0].listunspent(query_options={'assetGuid': self.asset, 'minimumAmountAsset':0.6,'maximumAmountAsset':0.6})
        assert_equal(len(out), 1)
        out =  self.nodes[0].listunspent(query_options={'assetGuid': self.asset, 'minimumAmountAsset':1.0,'maximumAmountAsset':1.0})
        if tx2inchain is True:
            assert_equal(len(out), 0)
            out =  self.nodes[0].listunspent(query_options={'assetGuid': self.asset})
            assert_equal(len(out), 6)
        else:
            assert_equal(len(out), 1)
            out =  self.nodes[0].listunspent(query_options={'assetGuid': self.asset})
            assert_equal(len(out), 4)

    # a = 1, b = 0.2, c = 0.1, a->b (0.2), b->a(0.2),  a->c(0.2), c->a(0.2), burn a(0.5), burn a(0.5), b->c(0.2), burn c(0.3) (a=0, b=0, c=0 and burn=1.3)
    def burn_zdag_doublespend(self):
        self.basic_asset()
        self.nodes[0].generate(1)
        useraddress2 = self.nodes[1].getnewaddress()
        useraddress3 = self.nodes[2].getnewaddress()
        useraddress4 = self.nodes[3].getnewaddress()
        useraddress1 = self.nodes[0].getnewaddress()
        self.nodes[3].importprivkey(self.nodes[0].dumpprivkey(useraddress1))
        self.nodes[0].assetsend(self.asset, newaddress1, int(2*COIN))

        self.nodes[0].sendtoaddress(useraddress2, 1)
        self.nodes[0].sendtoaddress(useraddress3, 1)
        self.nodes[0].generate(1)
        self.sync_blocks()
        self.nodes[0].assetsendmany(self.asset,[{address: useraddress1,amount:int(1.5*COIN)},{address: useraddress2,amount:int(0.4*COIN)},{address: useraddress3,amount:int(0.5*COIN)}])

        self.nodes[0].assetallocationsend(self.asset, useraddress2, int(0.2*COIN))
        self.nodes[1].assetallocationsend(self.asset, useraddress1, int(0.2*COIN))
        self.nodes[0].assetallocationsend(self.asset, useraddress3, int(0.2*COIN))
        self.nodes[2].assetallocationsend(self.asset, useraddress1, int(0.2*COIN))

        self.nodes[0].assetallocationburn(self.asset, int(0.5*COIN), "0x931d387731bbbc988b312206c74f77d004d6b84b")
        self.sync_mempools(timeout=30)
        self.nodes[0].assetallocationburn(self.asset, int(0.5*COIN), "0x931d387731bbbc988b312206c74f77d004d6b84b")
        # dbl spend
        txdblspend = self.nodes[3].assetallocationburn(self.asset, int(0.5*COIN), "0x931d387731bbbc988b312206c74f77d004d6b84b")["txid"]

        self.nodes[1].assetallocationsend(self.asset, useraddress3, int(0.2*COIN))
        self.nodes[2].assetallocationburn(self.asset, int(0.3*COIN), "")
        self.sync_mempools(timeout=30)
        # node1/node2/node3 shouldn't have dbl spend tx because no RBF and not zdag tx
        assert_raises_rpc_error(-5, 'No such mempool transaction', self.nodes[0].getrawtransaction, txdblspend)
        assert_raises_rpc_error(-5, 'No such mempool transaction', self.nodes[1].getrawtransaction, txdblspend)
        assert_raises_rpc_error(-5, 'No such mempool transaction', self.nodes[2].getrawtransaction, txdblspend)
        self.nodes[3].getrawtransaction(txdblspend)
        self.nodes[0].generate(1)
        self.sync_blocks()
        # after block, even node4 should have removed conflicting tx
        assert_raises_rpc_error(-5, 'No such mempool transaction', self.nodes[3].getrawtransaction, txdblspend)

    def basic_asset(self):
        self.asset = self.nodes[0].assetnew('1', "TST", "asset description", "0x9f90b5093f35aeac5fbaeb591f9c9de8e2844a46", 8, 1000*COIN, 10000*COIN, 31, {})['asset_guid']

if __name__ == '__main__':
    AssetZDAGTest().main()
<commit_msg>fix address name<commit_after>#!/usr/bin/env python3
# Copyright (c) 2019-2020 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

from test_framework.test_framework import SyscoinTestFramework
from test_framework.util import assert_equal, assert_raises_rpc_error, set_node_times, disconnect_nodes, connect_nodes, bump_node_times
from test_framework.messages import COIN
import time
ZDAG_NOT_FOUND = -1
ZDAG_STATUS_OK = 0
ZDAG_WARNING_RBF = 1
ZDAG_WARNING_NOT_ZDAG_TX = 2
ZDAG_WARNING_SIZE_OVER_POLICY = 3
ZDAG_MAJOR_CONFLICT = 4
MAX_INITIAL_BROADCAST_DELAY = 15 * 60 # 15 minutes in seconds
class AssetZDAGTest(SyscoinTestFramework):
    def set_test_params(self):
        self.setup_clean_chain = True
        self.num_nodes = 4
        self.rpc_timeout = 240
        self.extra_args = [['-assetindex=1'],['-assetindex=1'],['-assetindex=1'],['-assetindex=1']]

    def run_test(self):
        self.nodes[0].generate(200)
        self.sync_blocks()
        self.burn_zdag_doublespend()
        self.basic_zdag_doublespend()

    def basic_zdag_doublespend(self):
        self.basic_asset()
        self.nodes[0].generate(1)
        newaddress2 = self.nodes[1].getnewaddress()
        newaddress3 = self.nodes[1].getnewaddress()
        newaddress1 = self.nodes[0].getnewaddress()
        self.nodes[2].importprivkey(self.nodes[1].dumpprivkey(newaddress2))
        self.nodes[0].assetsend(self.asset, newaddress1, int(2*COIN))
        # create 2 utxo's so below newaddress1 recipient of 0.5 COIN uses 1 and the newaddress3 recipient on node3 uses the other on dbl spend
        self.nodes[0].sendtoaddress(newaddress2, 1)
        self.nodes[0].sendtoaddress(newaddress2, 1)
        self.nodes[0].generate(1)
        self.sync_blocks()
        out =  self.nodes[0].listunspent(query_options={'assetGuid': self.asset, 'minimumAmountAsset': 0.5})
        assert_equal(len(out), 1)
        out =  self.nodes[2].listunspent()
        assert_equal(len(out), 2)
        # send 2 asset UTXOs to newaddress2 same logic as explained above about dbl spend
        self.nodes[0].assetallocationsend(self.asset, newaddress2, int(0.4*COIN))
        self.nodes[0].assetallocationsend(self.asset, newaddress2, int(1*COIN))
        self.nodes[0].generate(1)
        self.sync_blocks()
        # should have 2 sys utxos and 2 asset utxos
        out =  self.nodes[2].listunspent()
        assert_equal(len(out), 4)
        # this will use 1 sys utxo and 1 asset utxo and send it to change address owned by node2
        self.nodes[1].assetallocationsend(self.asset, newaddress1, int(0.4*COIN))
        self.sync_mempools(timeout=30)
        # node3 should have 2 less utxos because they were sent to change on node2
        out =  self.nodes[2].listunspent(minconf=0)
        assert_equal(len(out), 2)
        tx1 = self.nodes[1].assetallocationsend(self.asset, newaddress1, int(1*COIN))['txid']
        # dbl spend
        tx2 = self.nodes[2].assetallocationsend(self.asset, newaddress1, int(0.9*COIN))['txid']
        # use tx2 to build tx3
        tx3 = self.nodes[2].assetallocationsend(self.asset, newaddress1, int(0.05*COIN))['txid']
        # use tx2 to build tx4
        tx4 = self.nodes[2].assetallocationsend(self.asset, newaddress1, int(0.025*COIN))['txid']
        self.sync_all()
        for i in range(3):
            assert_equal(self.nodes[i].assetallocationverifyzdag(tx1)['status'], ZDAG_MAJOR_CONFLICT)
            # ensure the tx2 made it to mempool, should propogate dbl-spend first time
            assert_equal(self.nodes[i].assetallocationverifyzdag(tx2)['status'], ZDAG_MAJOR_CONFLICT)
            # will conflict because its using tx2 which is in conflict state
            assert_equal(self.nodes[i].assetallocationverifyzdag(tx3)['status'], ZDAG_MAJOR_CONFLICT)
            # will conflict because its using tx3 which uses tx2 which is in conflict state
            assert_equal(self.nodes[i].assetallocationverifyzdag(tx4)['status'], ZDAG_MAJOR_CONFLICT)
        self.nodes[0].generate(1)
        self.sync_blocks()
        tx2inchain = False
        for i in range(3):
            try:
                self.nodes[i].getrawtransaction(tx1)
            except:
                tx2inchain = True
                continue
            assert_equal(self.nodes[i].assetallocationverifyzdag(tx1)['status'], ZDAG_NOT_FOUND)
            assert_equal(self.nodes[i].assetallocationverifyzdag(tx2)['status'], ZDAG_NOT_FOUND)
            assert_equal(self.nodes[i].assetallocationverifyzdag(tx3)['status'], ZDAG_NOT_FOUND)
            assert_equal(self.nodes[i].assetallocationverifyzdag(tx4)['status'], ZDAG_NOT_FOUND)
            assert_raises_rpc_error(-5, 'No such mempool transaction', self.nodes[i].getrawtransaction, tx2)
            assert_raises_rpc_error(-5, 'No such mempool transaction', self.nodes[i].getrawtransaction, tx3)
            assert_raises_rpc_error(-5, 'No such mempool transaction', self.nodes[i].getrawtransaction, tx4)
        out =  self.nodes[0].listunspent(query_options={'assetGuid': self.asset, 'minimumAmountAsset':0,'maximumAmountAsset':0})
        assert_equal(len(out), 1)
        out =  self.nodes[0].listunspent(query_options={'assetGuid': self.asset, 'minimumAmountAsset':0.4,'maximumAmountAsset':0.4})
        assert_equal(len(out), 1)
        out =  self.nodes[0].listunspent(query_options={'assetGuid': self.asset, 'minimumAmountAsset':0.6,'maximumAmountAsset':0.6})
        assert_equal(len(out), 1)
        out =  self.nodes[0].listunspent(query_options={'assetGuid': self.asset, 'minimumAmountAsset':1.0,'maximumAmountAsset':1.0})
        if tx2inchain is True:
            assert_equal(len(out), 0)
            out =  self.nodes[0].listunspent(query_options={'assetGuid': self.asset})
            assert_equal(len(out), 6)
        else:
            assert_equal(len(out), 1)
            out =  self.nodes[0].listunspent(query_options={'assetGuid': self.asset})
            assert_equal(len(out), 4)

    # a = 1, b = 0.2, c = 0.1, a->b (0.2), b->a(0.2),  a->c(0.2), c->a(0.2), burn a(0.5), burn a(0.5), b->c(0.2), burn c(0.3) (a=0, b=0, c=0 and burn=1.3)
    def burn_zdag_doublespend(self):
        self.basic_asset()
        self.nodes[0].generate(1)
        useraddress2 = self.nodes[1].getnewaddress()
        useraddress3 = self.nodes[2].getnewaddress()
        useraddress4 = self.nodes[3].getnewaddress()
        useraddress1 = self.nodes[0].getnewaddress()
        self.nodes[3].importprivkey(self.nodes[0].dumpprivkey(useraddress1))
        self.nodes[0].assetsend(self.asset, useraddress1, int(2*COIN))

        self.nodes[0].sendtoaddress(useraddress2, 1)
        self.nodes[0].sendtoaddress(useraddress3, 1)
        self.nodes[0].generate(1)
        self.sync_blocks()
        self.nodes[0].assetsendmany(self.asset,[{address: useraddress1,amount:int(1.5*COIN)},{address: useraddress2,amount:int(0.4*COIN)},{address: useraddress3,amount:int(0.5*COIN)}])

        self.nodes[0].assetallocationsend(self.asset, useraddress2, int(0.2*COIN))
        self.nodes[1].assetallocationsend(self.asset, useraddress1, int(0.2*COIN))
        self.nodes[0].assetallocationsend(self.asset, useraddress3, int(0.2*COIN))
        self.nodes[2].assetallocationsend(self.asset, useraddress1, int(0.2*COIN))

        self.nodes[0].assetallocationburn(self.asset, int(0.5*COIN), "0x931d387731bbbc988b312206c74f77d004d6b84b")
        self.sync_mempools(timeout=30)
        self.nodes[0].assetallocationburn(self.asset, int(0.5*COIN), "0x931d387731bbbc988b312206c74f77d004d6b84b")
        # dbl spend
        txdblspend = self.nodes[3].assetallocationburn(self.asset, int(0.5*COIN), "0x931d387731bbbc988b312206c74f77d004d6b84b")["txid"]

        self.nodes[1].assetallocationsend(self.asset, useraddress3, int(0.2*COIN))
        self.nodes[2].assetallocationburn(self.asset, int(0.3*COIN), "")
        self.sync_mempools(timeout=30)
        # node1/node2/node3 shouldn't have dbl spend tx because no RBF and not zdag tx
        assert_raises_rpc_error(-5, 'No such mempool transaction', self.nodes[0].getrawtransaction, txdblspend)
        assert_raises_rpc_error(-5, 'No such mempool transaction', self.nodes[1].getrawtransaction, txdblspend)
        assert_raises_rpc_error(-5, 'No such mempool transaction', self.nodes[2].getrawtransaction, txdblspend)
        self.nodes[3].getrawtransaction(txdblspend)
        self.nodes[0].generate(1)
        self.sync_blocks()
        # after block, even node4 should have removed conflicting tx
        assert_raises_rpc_error(-5, 'No such mempool transaction', self.nodes[3].getrawtransaction, txdblspend)

    def basic_asset(self):
        self.asset = self.nodes[0].assetnew('1', "TST", "asset description", "0x9f90b5093f35aeac5fbaeb591f9c9de8e2844a46", 8, 1000*COIN, 10000*COIN, 31, {})['asset_guid']

if __name__ == '__main__':
    AssetZDAGTest().main()
<|endoftext|>